[Traits] Add experimental flags to enable specific or all traits, disable default traits (#7694)

# Motivation
When building, testing or running a root package we want users to be
able to change the traits enabled in the packages. This allows them to
build binaries with different traits enabled or run tests with different
traits.

# Modification
This PR adds trait options to a few commands and wires them through to
the ModuleGraph loading and the build description caching. The latter is
important to trigger rebuilds when the trait configuration has changed.
Along the way I refactored the `GraphLoadingNode` and externalised the
logic to calculate traits since I need slightly different logic for the
root nodes and for the dependency nodes.

# Result
`swift build/run` now support passing trait configuration.

---------

Co-authored-by: Max Desiatov <[email protected]>

Author: FranzBusch

Sources/Build/BuildOperation.swift

@@ -55,6 +55,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// The path to scratch space (.build) directory.
     let scratchDirectory: AbsolutePath
 
+    /// The trait configuration for this build operation.
+    private let traitConfiguration: TraitConfiguration?
+
     /// The llbuild build system reference previously created
     /// via `createBuildSystem` call.
     private var current: (buildSystem: SPMLLBuild.BuildSystem, tracker: LLBuildProgressTracker)?
@@ -109,13 +112,49 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// Map of  root package identities by target names which are declared in them.
     private let rootPackageIdentityByTargetName: [String: PackageIdentity]
 
-    public init(
+    public convenience init(
+        productsBuildParameters: BuildParameters,
+        toolsBuildParameters: BuildParameters,
+        cacheBuildManifest: Bool,
+        packageGraphLoader: @escaping () throws -> ModulesGraph,
+        pluginConfiguration: PluginConfiguration? = .none,
+        scratchDirectory: AbsolutePath,
+        additionalFileRules: [FileRuleDescription],
+        pkgConfigDirectories: [AbsolutePath],
+        dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]],
+        targetsByRootPackageIdentity: [PackageIdentity: [String]],
+        outputStream: OutputByteStream,
+        logLevel: Basics.Diagnostic.Severity,
+        fileSystem: Basics.FileSystem,
+        observabilityScope: ObservabilityScope
+    ) {
+        self.init(
+            productsBuildParameters: productsBuildParameters,
+            toolsBuildParameters: toolsBuildParameters,
+            cacheBuildManifest: cacheBuildManifest,
+            packageGraphLoader: packageGraphLoader,
+            pluginConfiguration: pluginConfiguration,
+            scratchDirectory: scratchDirectory,
+            traitConfiguration: nil,
+            additionalFileRules: additionalFileRules,
+            pkgConfigDirectories: pkgConfigDirectories,
+            dependenciesByRootPackageIdentity: dependenciesByRootPackageIdentity,
+            targetsByRootPackageIdentity: targetsByRootPackageIdentity,
+            outputStream: outputStream,
+            logLevel: logLevel,
+            fileSystem: fileSystem,
+            observabilityScope: observabilityScope
+        )
+    }
+
+    package init(
         productsBuildParameters: BuildParameters,
         toolsBuildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> ModulesGraph,
         pluginConfiguration: PluginConfiguration? = .none,
         scratchDirectory: AbsolutePath,
+        traitConfiguration: TraitConfiguration?,
         additionalFileRules: [FileRuleDescription],
         pkgConfigDirectories: [AbsolutePath],
         dependenciesByRootPackageIdentity: [PackageIdentity: [PackageIdentity]],
@@ -139,6 +178,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.additionalFileRules = additionalFileRules
         self.pluginConfiguration = pluginConfiguration
         self.scratchDirectory = scratchDirectory
+        self.traitConfiguration = traitConfiguration
         self.pkgConfigDirectories = pkgConfigDirectories
         self.dependenciesByRootPackageIdentity = dependenciesByRootPackageIdentity
         self.rootPackageIdentityByTargetName = (try? Dictionary<String, PackageIdentity>(throwingUniqueKeysWithValues: targetsByRootPackageIdentity.lazy.flatMap { e in e.value.map { ($0, e.key) } })) ?? [:]
@@ -171,7 +211,13 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                             throw InternalError("could not find build descriptor at \(buildDescriptionPath)")
                         }
                         // return the build description that's on disk.
-                        return try BuildDescription.load(fileSystem: self.fileSystem, path: buildDescriptionPath)
+                        let buildDescription = try BuildDescription.load(fileSystem: self.fileSystem, path: buildDescriptionPath)
+
+                        // We need to check that the build has same traits enabled for the cached build operation
+                        // match otherwise we have to re-plan.
+                        if buildDescription.traitConfiguration == self.traitConfiguration {
+                            return buildDescription
+                        }
                     }
                 } catch {
                     // since caching is an optimization, warn about failing to load the cached version
@@ -614,6 +660,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 cacheBuildManifest: false,
                 packageGraphLoader: { graph },
                 scratchDirectory: pluginsBuildParameters.dataPath,
+                traitConfiguration: self.traitConfiguration,
                 additionalFileRules: self.additionalFileRules,
                 pkgConfigDirectories: self.pkgConfigDirectories,
                 dependenciesByRootPackageIdentity: [:],
@@ -737,6 +784,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
         let (buildDescription, buildManifest) = try BuildDescription.create(
             with: plan,
+            traitConfiguration: self.traitConfiguration,
             disableSandboxForPluginCommands: self.pluginConfiguration?.disableSandbox ?? false,
             fileSystem: self.fileSystem,
             observabilityScope: self.observabilityScope
@@ -906,6 +954,7 @@ extension BuildOperation {
 extension BuildDescription {
     static func create(
         with plan: BuildPlan,
+        traitConfiguration: TraitConfiguration?,
         disableSandboxForPluginCommands: Bool,
         fileSystem: Basics.FileSystem,
         observabilityScope: ObservabilityScope
@@ -932,7 +981,8 @@ extension BuildDescription {
             testEntryPointCommands: testEntryPointCommands,
             copyCommands: copyCommands,
             writeCommands: writeCommands,
-            pluginDescriptions: plan.pluginDescriptions
+            pluginDescriptions: plan.pluginDescriptions,
+            traitConfiguration: traitConfiguration
         )
         try fileSystem.createDirectory(
             plan.destinationBuildParameters.buildDescriptionPath.parentDirectory,

Sources/Build/LLBuildDescription.swift

@@ -14,6 +14,7 @@ import Basics
 import Foundation
 import LLBuildManifest
 import SPMBuildCore
+import PackageGraph
 
 import struct TSCBasic.ByteString
 
@@ -60,6 +61,9 @@ public struct BuildDescription: Codable {
     /// Distilled information about any plugins defined in the package.
     let pluginDescriptions: [PluginBuildDescription]
 
+    /// The enabled traits of the root package.
+    let traitConfiguration: TraitConfiguration?
+
     public init(
         plan: BuildPlan,
         swiftCommands: [LLBuildManifest.CmdName: SwiftCompilerTool],
@@ -69,6 +73,30 @@ public struct BuildDescription: Codable {
         copyCommands: [LLBuildManifest.CmdName: CopyTool],
         writeCommands: [LLBuildManifest.CmdName: WriteAuxiliaryFile],
         pluginDescriptions: [PluginBuildDescription]
+    ) throws {
+        try self.init(
+            plan: plan,
+            swiftCommands: swiftCommands,
+            swiftFrontendCommands: swiftFrontendCommands,
+            testDiscoveryCommands: testDiscoveryCommands,
+            testEntryPointCommands: testEntryPointCommands,
+            copyCommands: copyCommands,
+            writeCommands: writeCommands,
+            pluginDescriptions: pluginDescriptions,
+            traitConfiguration: nil
+        )
+    }
+
+    package init(
+        plan: BuildPlan,
+        swiftCommands: [LLBuildManifest.CmdName: SwiftCompilerTool],
+        swiftFrontendCommands: [LLBuildManifest.CmdName: SwiftFrontendTool],
+        testDiscoveryCommands: [LLBuildManifest.CmdName: TestDiscoveryTool],
+        testEntryPointCommands: [LLBuildManifest.CmdName: TestEntryPointTool],
+        copyCommands: [LLBuildManifest.CmdName: CopyTool],
+        writeCommands: [LLBuildManifest.CmdName: WriteAuxiliaryFile],
+        pluginDescriptions: [PluginBuildDescription],
+        traitConfiguration: TraitConfiguration?
     ) throws {
         self.swiftCommands = swiftCommands
         self.swiftFrontendCommands = swiftFrontendCommands
@@ -78,6 +106,7 @@ public struct BuildDescription: Codable {
         self.writeCommands = writeCommands
         self.explicitTargetDependencyImportCheckingMode = plan.destinationBuildParameters.driverParameters
             .explicitTargetDependencyImportCheckingMode
+        self.traitConfiguration = traitConfiguration
         self.targetDependencyMap = try plan.targets
             .reduce(into: [TargetName: [TargetName]]()) { partial, targetBuildDescription in
                 let deps = try targetBuildDescription.target.recursiveDependencies(

Sources/Commands/PackageCommands/APIDiff.swift

@@ -67,6 +67,9 @@ struct APIDiff: SwiftCommand {
             help: "One or more targets to include in the API comparison. If present, only the specified targets (and any products specified using `--products`) will be compared.")
     var targets: [String] = []
 
+    @OptionGroup(visibility: .hidden)
+    package var traits: TraitOptions
+
     @Option(name: .customLong("baseline-dir"),
             help: "The path to a directory used to store API baseline files. If unspecified, a temporary directory will be used.")
     var overrideBaselineDir: AbsolutePath?
@@ -83,7 +86,11 @@ struct APIDiff: SwiftCommand {
         let baselineRevision = try repository.resolveRevision(identifier: treeish)
 
         // We turn build manifest caching off because we need the build plan.
-        let buildSystem = try swiftCommandState.createBuildSystem(explicitBuildSystem: .native, cacheBuildManifest: false)
+        let buildSystem = try swiftCommandState.createBuildSystem(
+            explicitBuildSystem: .native,
+            traitConfiguration: .init(traitOptions: self.traits),
+            cacheBuildManifest: false
+        )
 
         let packageGraph = try buildSystem.getPackageGraph()
         let modulesToDiff = try determineModulesToDiff(

Sources/Commands/PackageCommands/DumpCommands.swift

@@ -47,7 +47,12 @@ struct DumpSymbolGraph: SwiftCommand {
         // Build the current package.
         //
         // We turn build manifest caching off because we need the build plan.
-        let buildSystem = try swiftCommandState.createBuildSystem(explicitBuildSystem: .native, cacheBuildManifest: false)
+        let buildSystem = try swiftCommandState.createBuildSystem(
+            explicitBuildSystem: .native,
+            // We are enabling all traits for dumping the symbol graph.
+            traitConfiguration: .init(enableAllTraits: true),
+            cacheBuildManifest: false
+        )
         try buildSystem.build()
 
         // Configure the symbol graph extractor.

Sources/Commands/PackageCommands/InstalledPackages.swift

@@ -80,7 +80,7 @@ extension SwiftPackageCommand {
                 throw StringError("\(productToInstall.name) is already installed at \(existingPkg.path)")
             }
 
-            try tool.createBuildSystem(explicitProduct: productToInstall.name)
+            try tool.createBuildSystem(explicitProduct: productToInstall.name, traitConfiguration: .init())
                 .build(subset: .product(productToInstall.name))
 
             let binPath = try tool.productsBuildParameters.buildPath.appending(component: productToInstall.name)

Sources/Commands/PackageCommands/PluginCommand.swift

@@ -321,6 +321,7 @@ struct PluginCommand: SwiftCommand {
         // Build or bring up-to-date any executable host-side tools on which this plugin depends. Add them and any binary dependencies to the tool-names-to-path map.
         let buildSystem = try swiftCommandState.createBuildSystem(
             explicitBuildSystem: .native,
+            traitConfiguration: .init(),
             cacheBuildManifest: false,
             productsBuildParameters: swiftCommandState.productsBuildParameters,
             toolsBuildParameters: buildParameters,

Sources/Commands/Snippets/Cards/SnippetCard.swift

@@ -93,7 +93,7 @@ struct SnippetCard: Card {
 
     func runExample() throws {
         print("Building '\(snippet.path)'\n")
-        let buildSystem = try swiftCommandState.createBuildSystem(explicitProduct: snippet.name)
+        let buildSystem = try swiftCommandState.createBuildSystem(explicitProduct: snippet.name, traitConfiguration: .init())
         try buildSystem.build(subset: .product(snippet.name))
         let executablePath = try swiftCommandState.productsBuildParameters.buildPath.appending(component: snippet.name)
         if let exampleTarget = try buildSystem.getPackageGraph().module(for: snippet.name, destination: .destination) {

Sources/Commands/SwiftBuildCommand.swift

@@ -97,6 +97,10 @@ struct BuildCommandOptions: ParsableArguments {
     @Option(help: "Build the specified product")
     var product: String?
 
+    /// Specifies the traits to build.
+    @OptionGroup(visibility: .hidden)
+    package var traits: TraitOptions
+
     /// If should link the Swift stdlib statically.
     @Flag(name: .customLong("static-swift-stdlib"), inversion: .prefixedNo, help: "Link Swift stdlib statically")
     public var shouldLinkStaticSwiftStdlib: Bool = false
@@ -140,7 +144,10 @@ public struct SwiftBuildCommand: AsyncSwiftCommand {
 
         if options.printManifestGraphviz {
             // FIXME: Doesn't seem ideal that we need an explicit build operation, but this concretely uses the `LLBuildManifest`.
-            guard let buildOperation = try swiftCommandState.createBuildSystem(explicitBuildSystem: .native) as? BuildOperation else {
+            guard let buildOperation = try swiftCommandState.createBuildSystem(
+                explicitBuildSystem: .native,
+                traitConfiguration: .init(traitOptions: self.options.traits)
+            ) as? BuildOperation else {
                 throw StringError("asked for native build system but did not get it")
             }
             let buildManifest = try buildOperation.getBuildManifest()
@@ -198,6 +205,7 @@ public struct SwiftBuildCommand: AsyncSwiftCommand {
     ) throws {
         let buildSystem = try swiftCommandState.createBuildSystem(
             explicitProduct: options.product,
+            traitConfiguration: .init(traitOptions: self.options.traits),
             shouldLinkStaticSwiftStdlib: options.shouldLinkStaticSwiftStdlib,
             productsBuildParameters: productsBuildParameters,
             toolsBuildParameters: toolsBuildParameters,

Sources/Commands/SwiftRunCommand.swift

@@ -83,6 +83,10 @@ struct RunCommandOptions: ParsableArguments {
     @Argument(help: "The executable to run", completion: .shellCommand("swift package completion-tool list-executables"))
     var executable: String?
 
+    /// Specifies the traits to build the product with.
+    @OptionGroup(visibility: .hidden)
+    package var traits: TraitOptions
+
     /// The arguments to pass to the executable.
     @Argument(parsing: .captureForPassthrough,
               help: "The arguments to pass to the executable")
@@ -130,6 +134,7 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
             // FIXME: We need to implement the build tool invocation closure here so that build tool plugins work with the REPL. rdar://86112934
             let buildSystem = try swiftCommandState.createBuildSystem(
                 explicitBuildSystem: .native,
+                traitConfiguration: .init(traitOptions: self.options.traits),
                 cacheBuildManifest: false,
                 packageGraphLoader: graphLoader
             )
@@ -149,7 +154,10 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
 
         case .debugger:
             do {
-                let buildSystem = try swiftCommandState.createBuildSystem(explicitProduct: options.executable)
+                let buildSystem = try swiftCommandState.createBuildSystem(
+                    explicitProduct: options.executable,
+                    traitConfiguration: .init(traitOptions: self.options.traits)
+                )
                 let productName = try findProductName(in: buildSystem.getPackageGraph())
                 if options.shouldBuildTests {
                     try buildSystem.build(subset: .allIncludingTests)
@@ -191,7 +199,10 @@ public struct SwiftRunCommand: AsyncSwiftCommand {
             }
 
             do {
-                let buildSystem = try swiftCommandState.createBuildSystem(explicitProduct: options.executable)
+                let buildSystem = try swiftCommandState.createBuildSystem(
+                    explicitProduct: options.executable,
+                    traitConfiguration: .init(traitOptions: self.options.traits)
+                )
                 let productName = try findProductName(in: buildSystem.getPackageGraph())
                 if options.shouldBuildTests {
                     try buildSystem.build(subset: .allIncludingTests)

Sources/Commands/SwiftTestCommand.swift

@@ -1377,6 +1377,8 @@ private func buildTestsIfNeeded(
     testProduct: String?
 ) throws -> [BuiltTestProduct] {
     let buildSystem = try swiftCommandState.createBuildSystem(
+        // TODO: Will support traits in test in a follow up PR
+        traitConfiguration: .init(),
         productsBuildParameters: productsBuildParameters,
         toolsBuildParameters: toolsBuildParameters
     )

Sources/Commands/Utilities/APIDigester.swift

@@ -139,6 +139,7 @@ struct APIDigesterBaselineDumper {
         // FIXME: We need to implement the build tool invocation closure here so that build tool plugins work with the APIDigester. rdar://86112934
         let buildSystem = try swiftCommandState.createBuildSystem(
             explicitBuildSystem: .native,
+            traitConfiguration: .init(),
             cacheBuildManifest: false,
             productsBuildParameters: productsBuildParameters,
             toolsBuildParameters: toolsBuildParameters,

Sources/Commands/Utilities/PluginDelegate.swift

@@ -161,6 +161,7 @@ final class PluginDelegate: PluginInvocationDelegate {
         let buildSystem = try swiftCommandState.createBuildSystem(
             explicitBuildSystem: .native,
             explicitProduct: explicitProduct,
+            traitConfiguration: .init(),
             cacheBuildManifest: false,
             productsBuildParameters: buildParameters,
             outputStream: outputStream,
@@ -223,7 +224,10 @@ final class PluginDelegate: PluginInvocationDelegate {
         var toolsBuildParameters = try swiftCommandState.toolsBuildParameters
         toolsBuildParameters.testingParameters.enableTestability = true
         toolsBuildParameters.testingParameters.enableCodeCoverage = parameters.enableCodeCoverage
-        let buildSystem = try swiftCommandState.createBuildSystem(toolsBuildParameters: toolsBuildParameters)
+        let buildSystem = try swiftCommandState.createBuildSystem(
+            traitConfiguration: .init(),
+            toolsBuildParameters: toolsBuildParameters
+        )
         try buildSystem.build(subset: .allIncludingTests)
 
         // Clean out the code coverage directory that may contain stale `profraw` files from a previous run of
@@ -381,7 +385,11 @@ final class PluginDelegate: PluginInvocationDelegate {
         // while building.
 
         // Create a build system for building the target., skipping the the cache because we need the build plan.
-        let buildSystem = try swiftCommandState.createBuildSystem(explicitBuildSystem: .native, cacheBuildManifest: false)
+        let buildSystem = try swiftCommandState.createBuildSystem(
+            explicitBuildSystem: .native,
+            traitConfiguration: .init(),
+            cacheBuildManifest: false
+        )
 
         // Find the target in the build operation's package graph; it's an error if we don't find it.
         let packageGraph = try buildSystem.getPackageGraph()