Support for prebuilt packages in the SDK (swiftlang#7337)

This should allow satisfying a package dependency with a package in the
SDK (or toolchain) based on metadata. If the given prebuilt library is
version-compatible, we essentially elide the dependency from the graph.
If it is not, we will fetch and build the dependency as normal. If a
library with associated metadata is linked without a corresponding
package dependency, we'll emit a warning.

Author: neonichu

Package.swift

@@ -217,7 +217,10 @@ let package = Package(
             /** Primitive Package model objects */
             name: "PackageModel",
             dependencies: ["Basics"],
-            exclude: ["CMakeLists.txt", "README.md"]
+            exclude: ["CMakeLists.txt", "README.md"],
+            resources: [
+                .copy("InstalledLibrariesSupport/provided-libraries.json"),
+            ]
         ),
 
         .target(

Sources/Build/BuildOperation.swift

@@ -102,6 +102,12 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// Alternative path to search for pkg-config `.pc` files.
     private let pkgConfigDirectories: [AbsolutePath]
 
+    /// Map of dependency package identities by root packages that depend on them.
+    private let dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]]
+
+    /// Map of  root package identities by target names which are declared in them.
+    private let rootPackageIdentityByTargetName: [String: PackageIdentity]
+
     public init(
         productsBuildParameters: BuildParameters,
         toolsBuildParameters: BuildParameters,
@@ -110,6 +116,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         pluginConfiguration: PluginConfiguration? = .none,
         additionalFileRules: [FileRuleDescription],
         pkgConfigDirectories: [AbsolutePath],
+        dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]],
+        targetsByRootPackageIdentity: [PackageIdentity: [String]],
         outputStream: OutputByteStream,
         logLevel: Basics.Diagnostic.Severity,
         fileSystem: Basics.FileSystem,
@@ -129,6 +137,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.additionalFileRules = additionalFileRules
         self.pluginConfiguration = pluginConfiguration
         self.pkgConfigDirectories = pkgConfigDirectories
+        self.dependenciesByRootPackageIdentity = dependenciesByRootPackageIdentity
+        self.rootPackageIdentityByTargetName = (try? Dictionary<String, PackageIdentity>(throwingUniqueKeysWithValues: targetsByRootPackageIdentity.lazy.flatMap { e in e.value.map { ($0, e.key) } })) ?? [:]
         self.outputStream = outputStream
         self.logLevel = logLevel
         self.fileSystem = fileSystem
@@ -248,6 +258,79 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         }
     }
 
+    private static var didEmitUnexpressedDependencies = false
+
+    private func detectUnexpressedDependencies() {
+        return self.detectUnexpressedDependencies(
+            // Note: once we switch from the toolchain global metadata, we will have to ensure we can match the right metadata used during the build.
+            availableLibraries: self.productsBuildParameters.toolchain.providedLibraries,
+            targetDependencyMap: self.buildDescription.targetDependencyMap
+        )
+    }
+
+    // TODO: Currently this function will only match frameworks.
+    internal func detectUnexpressedDependencies(
+        availableLibraries: [LibraryMetadata],
+        targetDependencyMap: [String: [String]]?
+    ) {
+        // Ensure we only emit these once, regardless of how many builds are being done.
+        guard !Self.didEmitUnexpressedDependencies else {
+            return
+        }
+        Self.didEmitUnexpressedDependencies = true
+
+        let availableFrameworks = Dictionary<String, PackageIdentity>(uniqueKeysWithValues: availableLibraries.compactMap {
+            if let identity = Set($0.identities.map(\.identity)).spm_only {
+                return ("\($0.productName!).framework", identity)
+            } else {
+                return nil
+            }
+        })
+
+        targetDependencyMap?.keys.forEach { targetName in
+            let c99name = targetName.spm_mangledToC99ExtendedIdentifier()
+            // Since we're analysing post-facto, we don't know which parameters are the correct ones.
+            let possibleTempsPaths = [productsBuildParameters, toolsBuildParameters].map {
+                $0.buildPath.appending(component: "\(c99name).build")
+            }
+
+            let usedSDKDependencies: [String] = Set(possibleTempsPaths).flatMap { possibleTempsPath in
+                guard let contents = try? self.fileSystem.readFileContents(possibleTempsPath.appending(component: "\(c99name).d")) else {
+                    return [String]()
+                }
+
+                // FIXME: We need a real makefile deps parser here...
+                let deps = contents.description.split(whereSeparator: { $0.isWhitespace })
+                return deps.filter {
+                    !$0.hasPrefix(possibleTempsPath.parentDirectory.pathString)
+                }.compactMap {
+                    try? AbsolutePath(validating: String($0))
+                }.compactMap {
+                    return $0.components.first(where: { $0.hasSuffix(".framework") })
+                }
+            }
+
+            let dependencies: [PackageIdentity]
+            if let rootPackageIdentity = self.rootPackageIdentityByTargetName[targetName] {
+                dependencies = self.dependenciesByRootPackageIdentity[rootPackageIdentity] ?? []
+            } else {
+                dependencies = []
+            }
+
+            Set(usedSDKDependencies).forEach {
+                if availableFrameworks.keys.contains($0) {
+                    if let availableFrameworkPackageIdentity = availableFrameworks[$0], !dependencies.contains(
+                        availableFrameworkPackageIdentity
+                    ) {
+                        observabilityScope.emit(
+                            warning: "target '\(targetName)' has an unexpressed depedency on '\(availableFrameworkPackageIdentity)'"
+                        )
+                    }
+                }
+            }
+        }
+    }
+
     /// Perform a build using the given build description and subset.
     public func build(subset: BuildSubset) throws {
         guard !self.productsBuildParameters.shouldSkipBuilding else {
@@ -284,6 +367,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
         let duration = buildStartTime.distance(to: .now())
 
+        self.detectUnexpressedDependencies()
+
         let subsetDescriptor: String?
         switch subset {
         case .product(let productName):
@@ -466,6 +551,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 packageGraphLoader: { return graph },
                 additionalFileRules: self.additionalFileRules,
                 pkgConfigDirectories: self.pkgConfigDirectories,
+                dependenciesByRootPackageIdentity: [:],
+                targetsByRootPackageIdentity: [:],
                 outputStream: self.outputStream,
                 logLevel: self.logLevel,
                 fileSystem: self.fileSystem,

Sources/CMakeLists.txt

@@ -6,6 +6,7 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
+add_compile_definitions(SKIP_RESOURCE_SUPPORT)
 add_compile_definitions(USE_IMPL_ONLY_IMPORTS)
 
 add_subdirectory(SPMSQLite3)

Sources/Commands/PackageTools/EditCommands.swift

@@ -72,6 +72,7 @@ extension SwiftPackageTool {
                 packageName: packageName,
                 forceRemove: shouldForceRemove,
                 root: swiftTool.getWorkspaceRoot(),
+                availableLibraries: swiftTool.getHostToolchain().providedLibraries,
                 observabilityScope: swiftTool.observabilityScope
             )
         }

Sources/CoreCommands/BuildSystemSupport.swift

@@ -32,6 +32,7 @@ private struct NativeBuildSystemFactory: BuildSystemFactory {
         logLevel: Diagnostic.Severity?,
         observabilityScope: ObservabilityScope?
     ) throws -> any BuildSystem {
+        let rootPackageInfo = try swiftTool.getRootPackageInformation()
         let testEntryPointPath = productsBuildParameters?.testingParameters.testProductStyle.explicitlySpecifiedEntryPointPath
         return try BuildOperation(
             productsBuildParameters: try productsBuildParameters ?? self.swiftTool.productsBuildParameters,
@@ -50,6 +51,8 @@ private struct NativeBuildSystemFactory: BuildSystemFactory {
             ),
             additionalFileRules: FileRuleDescription.swiftpmFileTypes,
             pkgConfigDirectories: self.swiftTool.options.locations.pkgConfigDirectories,
+            dependenciesByRootPackageIdentity: rootPackageInfo.dependecies,
+            targetsByRootPackageIdentity: rootPackageInfo.targets,
             outputStream: outputStream ?? self.swiftTool.outputStream,
             logLevel: logLevel ?? self.swiftTool.logLevel,
             fileSystem: self.swiftTool.fileSystem,

Sources/CoreCommands/SwiftTool.swift

@@ -48,7 +48,7 @@ import var TSCBasic.stderrStream
 import class TSCBasic.TerminalController
 import class TSCBasic.ThreadSafeOutputByteStream
 
-import var TSCUtility.verbosity
+import TSCUtility // cannot be scoped because of `String.spm_mangleToC99ExtendedIdentifier()`
 
 typealias Diagnostic = Basics.Diagnostic
 
@@ -460,6 +460,29 @@ public final class SwiftTool {
         return workspace
     }
 
+    public func getRootPackageInformation() throws -> (dependecies: [PackageIdentity: [PackageIdentity]], targets: [PackageIdentity: [String]]) {
+        let workspace = try self.getActiveWorkspace()
+        let root = try self.getWorkspaceRoot()
+        let rootManifests = try temp_await {
+            workspace.loadRootManifests(
+                packages: root.packages,
+                observabilityScope: self.observabilityScope,
+                completion: $0
+            )
+        }
+
+        var identities = [PackageIdentity: [PackageIdentity]]()
+        var targets = [PackageIdentity: [String]]()
+
+        rootManifests.forEach {
+            let identity = PackageIdentity(path: $0.key)
+            identities[identity] = $0.value.dependencies.map(\.identity)
+            targets[identity] = $0.value.targets.map { $0.name.spm_mangledToC99ExtendedIdentifier() }
+        }
+
+        return (identities, targets)
+    }
+
     private func getEditsDirectory() throws -> AbsolutePath {
         // TODO: replace multiroot-data-file with explicit overrides
         if let multiRootPackageDataFile = options.locations.multirootPackageDataFile {
@@ -581,6 +604,7 @@ public final class SwiftTool {
                 explicitProduct: explicitProduct,
                 forceResolvedVersions: options.resolver.forceResolvedVersions,
                 testEntryPointPath: testEntryPointPath,
+                availableLibraries: self.getHostToolchain().providedLibraries,
                 observabilityScope: self.observabilityScope
             )
 

Sources/PackageGraph/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(PackageGraph
   PackageGraphRoot.swift
   PackageModel+Extensions.swift
   PackageRequirement.swift
+  PrebuiltPackageContainer.swift
   PinsStore.swift
   Resolution/PubGrub/Assignment.swift
   Resolution/PubGrub/ContainerProvider.swift

Sources/PackageGraph/PackageGraph+Loading.swift

@@ -32,6 +32,7 @@ extension PackageGraph {
         customPlatformsRegistry: PlatformRegistry? = .none,
         customXCTestMinimumDeploymentTargets: [PackageModel.Platform: PlatformVersion]? = .none,
         testEntryPointPath: AbsolutePath? = nil,
+        availableLibraries: [LibraryMetadata],
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope
     ) throws -> PackageGraph {
@@ -159,6 +160,7 @@ extension PackageGraph {
             unsafeAllowedPackages: unsafeAllowedPackages,
             platformRegistry: customPlatformsRegistry ?? .default,
             platformVersionProvider: platformVersionProvider,
+            availableLibraries: availableLibraries,
             fileSystem: fileSystem,
             observabilityScope: observabilityScope
         )
@@ -243,6 +245,7 @@ private func createResolvedPackages(
     unsafeAllowedPackages: Set<PackageReference>,
     platformRegistry: PlatformRegistry,
     platformVersionProvider: PlatformVersionProvider,
+    availableLibraries: [LibraryMetadata],
     fileSystem: FileSystem,
     observabilityScope: ObservabilityScope
 ) throws -> [ResolvedPackage] {
@@ -528,18 +531,24 @@ private func createResolvedPackages(
                         }.map {$0.targets}.flatMap{$0}.filter { t in
                             t.name != productRef.name
                         }
-                        
-                        // Find a product name from the available product dependencies that is most similar to the required product name.
-                        let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
-                        let error = PackageGraphError.productDependencyNotFound(
-                            package: package.identity.description,
-                            targetName: targetBuilder.target.name,
-                            dependencyProductName: productRef.name,
-                            dependencyPackageName: productRef.package,
-                            dependencyProductInDecl: !declProductsAsDependency.isEmpty,
-                            similarProductName: bestMatchedProductName
-                        )
-                        packageObservabilityScope.emit(error)
+
+                        let identitiesAvailableInSDK = availableLibraries.flatMap { $0.identities.map { $0.identity } }
+                        // TODO: Do we have to care about "name" vs. identity here?
+                        if let name = productRef.package, identitiesAvailableInSDK.contains(PackageIdentity.plain(name)) {
+                            // Do not emit any diagnostic.
+                        } else {
+                            // Find a product name from the available product dependencies that is most similar to the required product name.
+                            let bestMatchedProductName = bestMatch(for: productRef.name, from: Array(allTargetNames))
+                            let error = PackageGraphError.productDependencyNotFound(
+                                package: package.identity.description,
+                                targetName: targetBuilder.target.name,
+                                dependencyProductName: productRef.name,
+                                dependencyPackageName: productRef.package,
+                                dependencyProductInDecl: !declProductsAsDependency.isEmpty,
+                                similarProductName: bestMatchedProductName
+                            )
+                            packageObservabilityScope.emit(error)
+                        }
                     }
                     continue
                 }

Sources/PackageGraph/PrebuiltPackageContainer.swift

@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Basics
+import PackageModel
+import struct TSCUtility.Version
+
+/// A package container that can represent a prebuilt library from a package.
+public struct PrebuiltPackageContainer: PackageContainer {
+    private let chosenIdentity: LibraryMetadata.Identity
+    private let metadata: LibraryMetadata
+
+    public init(metadata: LibraryMetadata) throws {
+        self.metadata = metadata
+
+        // FIXME: Unclear what is supposed to happen if we have multiple identities.
+        if let identity = metadata.identities.first {
+            self.chosenIdentity = identity
+        } else {
+            let name = metadata.productName.map { "'\($0)' " } ?? ""
+            throw InternalError("provided library \(name)does not specifiy any identities")
+        }
+    }
+
+    public var package: PackageReference {
+        return .init(identity: chosenIdentity.identity, kind: chosenIdentity.kind)
+    }
+
+    public func isToolsVersionCompatible(at version: Version) -> Bool {
+        return true
+    }
+
+    public func toolsVersion(for version: Version) throws -> ToolsVersion {
+        return .v4
+    }
+
+    public func toolsVersionsAppropriateVersionsDescending() throws -> [Version] {
+        return try versionsAscending()
+    }
+
+    public func versionsAscending() throws -> [Version] {
+        return [.init(stringLiteral: metadata.version)]
+    }
+
+    public func getDependencies(at version: Version, productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
+        return []
+    }
+
+    public func getDependencies(at revision: String, productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
+        return []
+    }
+
+    public func getUnversionedDependencies(productFilter: ProductFilter) throws -> [PackageContainerConstraint] {
+        return []
+    }
+
+    public func loadPackageReference(at boundVersion: BoundVersion) throws -> PackageReference {
+        return package
+    }
+}