Reapply "Include Swift Build support in second stage bootstrap builds" (swiftlang#8373)

This reverts commit 6a2e45f.

- Stop setting SWIFTPM_NO_SWBUILD_DEPENDENCY in the bootstrap script
- Copy resource bundles required by Swift Build into the toolchain when
installing

There is one additional fix to the bootstrap script to allow overwriting
directories when installing, to fix the incremental buildbot CI
configuration.

Author: owenv

Sources/CoreCommands/BuildSystemSupport.swift

@@ -121,6 +121,7 @@ private struct SwiftBuildSystemFactory: BuildSystemFactory {
                     explicitProduct: explicitProduct
                 )
             },
+            packageManagerResourcesDirectory: swiftCommandState.packageManagerResourcesDirectory,
             outputStream: outputStream ?? self.swiftCommandState.outputStream,
             logLevel: logLevel ?? self.swiftCommandState.logLevel,
             fileSystem: self.swiftCommandState.fileSystem,

Sources/CoreCommands/Options.swift

@@ -146,6 +146,12 @@ public struct LocationOptions: ParsableArguments {
         completion: .directory
     )
     public var pkgConfigDirectories: [AbsolutePath] = []
+    
+    @Option(
+        help: .init("Specify alternate path to search for resources required for SwiftPM to operate. (default: <Toolchain Directory>/usr/share/pm)", visibility: .hidden),
+        completion: .directory
+    )
+    public var packageManagerResourcesDirectory: AbsolutePath?
 
     @Flag(name: .customLong("ignore-lock"), help: .hidden)
     public var ignoreLock: Bool = false

Sources/CoreCommands/SwiftCommandState.swift

@@ -225,6 +225,9 @@ public final class SwiftCommandState {
 
     /// Path to the shared configuration directory
     public let sharedConfigurationDirectory: AbsolutePath
+    
+    /// Path to the package manager's own resources directory.
+    public let packageManagerResourcesDirectory: AbsolutePath?
 
     /// Path to the cross-compilation Swift SDKs directory.
     public let sharedSwiftSDKsDirectory: AbsolutePath
@@ -372,6 +375,17 @@ public final class SwiftCommandState {
                 warning: "`--experimental-swift-sdks-path` is deprecated and will be removed in a future version of SwiftPM. Use `--swift-sdks-path` instead."
             )
         }
+        
+        if let packageManagerResourcesDirectory = options.locations.packageManagerResourcesDirectory {
+            self.packageManagerResourcesDirectory = packageManagerResourcesDirectory
+        } else if let cwd = localFileSystem.currentWorkingDirectory {
+            self.packageManagerResourcesDirectory = try? AbsolutePath(validating: CommandLine.arguments[0], relativeTo: cwd)
+                .parentDirectory.parentDirectory.appending(components: ["share", "pm"])
+        } else {
+            self.packageManagerResourcesDirectory = try? AbsolutePath(validating: CommandLine.arguments[0])
+                .parentDirectory.parentDirectory.appending(components: ["share", "pm"])
+        }
+        
         self.sharedSwiftSDKsDirectory = try fileSystem.getSharedSwiftSDKsDirectory(
             explicitDirectory: options.locations.swiftSDKsDirectory ?? options.locations.deprecatedSwiftSDKsDirectory
         )

Sources/SwiftBuildSupport/SwiftBuildSystem.swift

@@ -62,12 +62,13 @@ func withService(
 func withSession(
     service: SWBBuildService,
     name: String,
+    packageManagerResourcesDirectory: Basics.AbsolutePath?,
     body: @escaping (
         _ session: SWBBuildServiceSession,
         _ diagnostics: [SwiftBuild.SwiftBuildMessage.DiagnosticInfo]
     ) async throws -> Void
 ) async throws {
-    switch await service.createSession(name: name, cachePath: nil, inferiorProductsPath: nil, environment: nil) {
+    switch await service.createSession(name: name, resourceSearchPaths: packageManagerResourcesDirectory.map { [$0.pathString] } ?? [], cachePath: nil, inferiorProductsPath: nil, environment: nil) {
     case (.success(let session), let diagnostics):
         do {
             try await body(session, diagnostics)
@@ -159,6 +160,7 @@ private final class PlanningOperationDelegate: SWBPlanningOperationDelegate, Sen
 public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
     private let buildParameters: BuildParameters
     private let packageGraphLoader: () async throws -> ModulesGraph
+    private let packageManagerResourcesDirectory: Basics.AbsolutePath?
     private let logLevel: Basics.Diagnostic.Severity
     private var packageGraph: AsyncThrowingValueMemoizer<ModulesGraph> = .init()
     private var pifBuilder: AsyncThrowingValueMemoizer<PIFBuilder> = .init()
@@ -209,13 +211,15 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
     public init(
         buildParameters: BuildParameters,
         packageGraphLoader: @escaping () async throws -> ModulesGraph,
+        packageManagerResourcesDirectory: Basics.AbsolutePath?,
         outputStream: OutputByteStream,
         logLevel: Basics.Diagnostic.Severity,
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope
     ) throws {
         self.buildParameters = buildParameters
         self.packageGraphLoader = packageGraphLoader
+        self.packageManagerResourcesDirectory = packageManagerResourcesDirectory
         self.outputStream = outputStream
         self.logLevel = logLevel
         self.fileSystem = fileSystem
@@ -257,7 +261,7 @@ public final class SwiftBuildSystem: SPMBuildCore.BuildSystem {
             )
 
             do {
-                try await withSession(service: service, name: self.buildParameters.pifManifest.pathString) { session, _ in
+                try await withSession(service: service, name: self.buildParameters.pifManifest.pathString, packageManagerResourcesDirectory: self.packageManagerResourcesDirectory) { session, _ in
                     self.outputStream.send("Building for \(self.buildParameters.configuration == .debug ? "debugging" : "production")...\n")
 
                     // Load the workspace, and set the system information to the default

Sources/swift-bootstrap/main.swift

@@ -361,6 +361,7 @@ struct SwiftBootstrapBuildTool: AsyncParsableCommand {
                 return try SwiftBuildSystem(
                     buildParameters: buildParameters,
                     packageGraphLoader: asyncUnsafePackageGraphLoader,
+                    packageManagerResourcesDirectory: nil,
                     outputStream: TSCBasic.stdoutStream,
                     logLevel: logLevel,
                     fileSystem: self.fileSystem,

Utilities/bootstrap

@@ -544,6 +544,11 @@ def install_swiftpm(prefix, args):
     dest = os.path.join(prefix, "lib", "swift", "pm", "PluginAPI")
     install_dylib(args, "PackagePlugin", dest, ["PackagePlugin"])
 
+    # Install resource bundles produced during the build.
+    for file in os.listdir(args.bin_dir):
+        if file.endswith('.bundle') or file.endswith('.resources'):
+            install_binary(args, file, os.path.join(os.path.join(prefix, "share"), "pm"))
+
 
 # Helper function that installs a dynamic library and a set of modules to a particular directory.
 @log_entry_exit
@@ -587,7 +592,7 @@ def install_file(args, src, destination, destination_is_directory=True, ignored_
 
     logging.info("Installing %s to %s", src, dest)
     if os.path.isdir(src):
-        shutil.copytree(src, dest, ignore=shutil.ignore_patterns(*ignored_patterns))
+        shutil.copytree(src, dest, ignore=shutil.ignore_patterns(*ignored_patterns), dirs_exist_ok=True)
     else:
         shutil.copy2(src, dest)
 
@@ -851,7 +856,6 @@ def get_swiftpm_env_cmd(args):
 
     if args.llbuild_link_framework:
         env_cmd.append("SWIFTPM_LLBUILD_FWK=1")
-    env_cmd.append("SWIFTPM_NO_SWBUILD_DEPENDENCY=1")
     env_cmd.append("SWIFTCI_USE_LOCAL_DEPS=1")
     env_cmd.append("SWIFTPM_MACOS_DEPLOYMENT_TARGET=%s" % g_macos_deployment_target)