Teach Driver to generate swift-frontend invocations to build prebuilt module cache

We should be able to generate prebuilt module cache by reusing
the infrastructure libSwiftDriver already provides. More specifically, we could
(1) write a dummy Swift file imports every frameworks/libraries in an SDK,
(2) use module dependency scanner in the driver to generate module building commands,
and (3) use MultiJobExecutor to execute those commands topologically.

This PR is for (2).

Author: nkcsgexi

Sources/SwiftDriver/Jobs/Planning.swift

@@ -496,6 +496,103 @@ extension Driver {
     return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs()
   }
 
+
+  public mutating func generatePrebuitModuleGenerationJobs(prebuiltModuleDir: VirtualPath) throws -> [Job] {
+    guard let sdkPath = frontendTargetInfo.sdkPath?.path else {
+      // cannot proceed without SDK path
+      return []
+    }
+    // Run the dependency scanner and update the dependency oracle with the results
+    let dependencyGraph = try gatherModuleDependencies()
+    var jobs: [Job] = []
+    var inputPathsCache: [String: [TypedVirtualPath]] = [:]
+
+    // Get the .swiftinterface path of the module in SDK
+    func getInputPaths(_ module: String) throws -> [TypedVirtualPath] {
+      if let cachedPaths = inputPathsCache[module] {
+        return cachedPaths
+      }
+      func cache(_ results: [TypedVirtualPath]) -> [TypedVirtualPath] {
+        inputPathsCache[module] = results
+        return results
+      }
+      guard let info = dependencyGraph.modules[.swift(module)] else {
+        return cache([])
+      }
+      if case .swift(let details) = info.details {
+        if let interfacePath = details.moduleInterfacePath {
+          var results: [TypedVirtualPath] = []
+          let currentPath = TypedVirtualPath(file: interfacePath.path, type: .swiftInterface)
+          try localFileSystem.getDirectoryContents(currentPath.file.parentDirectory.absolutePath!).forEach {
+            let rawPath = try VirtualPath.init(path: $0)
+            if rawPath.extension == "swiftinterface" {
+              results.append(TypedVirtualPath(file: rawPath.intern(), type: .swiftInterface))
+            }
+          }
+          return cache(results)
+        }
+      }
+      return cache([])
+    }
+
+    // Get the .swiftmodule path of the prebuilt module in the toolchain
+    func getOutputPath(_ module: String, _ inputPath: TypedVirtualPath) -> TypedVirtualPath {
+      let baseName = inputPath.file.basenameWithoutExt
+      return TypedVirtualPath(file: prebuiltModuleDir
+        .appending(component: module + ".swiftmodule")
+        .appending(component: baseName + ".swiftmodule").intern(),
+                              type: .swiftModule)
+    }
+
+    func getDependenciesPaths(_ module: String, _ inputPath: TypedVirtualPath) throws -> [TypedVirtualPath] {
+      var results: [TypedVirtualPath] = []
+      let info = dependencyGraph.modules[.swift(module)]!
+      guard let dependencies = info.directDependencies else {
+        return results
+      }
+      for dep in dependencies {
+        let moduleName = dep.moduleName
+        if let input = try getInputPaths(moduleName).first(where: { $0.file.basename == inputPath.file.basename }) {
+          results.append(getOutputPath(moduleName, input))
+        }
+      }
+      return results
+    }
+
+    let moduleInfo = dependencyGraph.mainModule
+    if let dependencies = moduleInfo.directDependencies {
+      for dep in dependencies {
+        let moduleName = dep.moduleName
+        let inputPaths = try getInputPaths(moduleName)
+        for inputPath in inputPaths {
+          let outputPath = getOutputPath(moduleName, inputPath)
+          var commandLine: [Job.ArgTemplate] = []
+          commandLine.appendFlag(.compileModuleFromInterface)
+          commandLine.appendFlag(.sdk)
+          commandLine.append(.path(VirtualPath.lookup(sdkPath)))
+          commandLine.appendFlag(.prebuiltModuleCachePath)
+          commandLine.appendPath(prebuiltModuleDir)
+          commandLine.appendFlag(.moduleName)
+          commandLine.appendFlag(moduleName)
+          commandLine.appendFlag(.o)
+          commandLine.appendPath(outputPath.file)
+          commandLine.appendPath(inputPath.file)
+          jobs.append(Job(
+            moduleName: moduleName,
+            kind: .compile,
+            tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+            commandLine: commandLine,
+            inputs: try getDependenciesPaths(moduleName, inputPath),
+            primaryInputs: [],
+            outputs: [outputPath]
+          ))
+        }
+      }
+    }
+
+    return jobs
+  }
+
   private mutating func gatherModuleDependencies()
   throws -> InterModuleDependencyGraph {
     var dependencyGraph = try performDependencyScan()

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/A.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A
+import Swift
+public func FuncA() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/A.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A
+import Swift
+public func FuncA() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/E.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name E
+import Swift
+public func FuncE() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/E.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,4 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name E
+import Swift
+public func FuncE() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/F.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,5 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name F
+import Swift
+import A
+public func FuncF() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/F.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,5 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name F
+import Swift
+import A
+public func FuncF() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/G.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,5 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name G
+import Swift
+import E
+public func FuncG() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/G.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,5 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name G
+import Swift
+import E
+public func FuncG() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/H.swiftmodule/arm64e-apple-macos.swiftinterface

@@ -0,0 +1,8 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name H
+import Swift
+import A
+import E
+import F
+import G
+public func FuncH() { }

TestInputs/ExplicitModuleBuilds/Swift/mock-sdk/usr/lib/swift/H.swiftmodule/x86_64-apple-macos.swiftinterface

@@ -0,0 +1,8 @@
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name H
+import Swift
+import A
+import E
+import F
+import G
+public func FuncH() { }

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -567,39 +567,38 @@ final class ExplicitModuleBuildTests: XCTestCase {
     #endif
   }
 
-  /// Test the libSwiftScan dependency scanning.
-  func testDependencyScanning() throws {
-    // Just instantiating to get at the toolchain path
-    let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
-                                   "-module-name", "testDependencyScanning",
-                                   "test.swift"])
+  func getStdlibShimsPaths(_ driver: Driver) throws -> (AbsolutePath, AbsolutePath) {
     let toolchainRootPath: AbsolutePath = try driver.toolchain.getToolPath(.swiftCompiler)
                                                             .parentDirectory // bin
                                                             .parentDirectory // toolchain root
-
-    let stdLibPath: AbsolutePath
-    let shimsPath: AbsolutePath
-    // On Darwin, use the SDK's stdlib, to make sure the test is more likely to actually
-    // find one.
     if driver.targetTriple.isDarwin {
-      let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers:         [Driver.stderrDiagnosticsHandler]),
+      let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
                                              processSet: ProcessSet(),
                                              fileSystem: localFileSystem,
                                              env: ProcessEnv.vars)
       let sdkPath = try executor.checkNonZeroExit(
         args: "xcrun", "-sdk", "macosx", "--show-sdk-path").spm_chomp()
-      stdLibPath = AbsolutePath(sdkPath).appending(component: "usr")
-                                        .appending(component: "lib")
-                                        .appending(component: "swift")
-      shimsPath = stdLibPath.appending(component: "shims")
-    } else {
-      stdLibPath = toolchainRootPath.appending(component: "lib")
+      let stdLibPath = AbsolutePath(sdkPath).appending(component: "usr")
+        .appending(component: "lib")
         .appending(component: "swift")
-        .appending(component: driver.targetTriple.osNameUnversioned)
-      shimsPath = toolchainRootPath.appending(component: "lib")
-        .appending(component: "swift")
-        .appending(component: "shims")
+      return (stdLibPath, stdLibPath.appending(component: "shims"))
+    } else {
+      return (toolchainRootPath.appending(component: "lib")
+                .appending(component: "swift")
+                .appending(component: driver.targetTriple.osNameUnversioned),
+              toolchainRootPath.appending(component: "lib")
+                .appending(component: "swift")
+                .appending(component: "shims"))
     }
+  }
+
+  /// Test the libSwiftScan dependency scanning.
+  func testDependencyScanning() throws {
+    // Just instantiating to get at the toolchain path
+    let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
+                                   "-module-name", "testDependencyScanning",
+                                   "test.swift"])
+    let (stdLibPath, shimsPath) = try getStdlibShimsPaths(driver)
 
     XCTAssertTrue(localFileSystem.exists(stdLibPath),
                   "expected Swift StdLib at: \(stdLibPath.description)")
@@ -732,4 +731,35 @@ final class ExplicitModuleBuildTests: XCTestCase {
     XCTAssertEqual(moduleMap[1].sourceInfoPath!.path.description, "B.swiftsourceinfo")
     XCTAssertEqual(moduleMap[1].isFramework, false)
   }
+
+  func testPrebuiltModuleGenerationJobs() throws {
+    try withTemporaryDirectory { path in
+      let main = path.appending(component: "testPrebuiltModuleGenerationJobs.swift")
+      try localFileSystem.writeFileContents(main) {
+        $0 <<< "import A\n"
+        $0 <<< "import E\n"
+        $0 <<< "import F\n"
+        $0 <<< "import G\n"
+        $0 <<< "import H\n"
+      }
+      let packageRootPath = URL(fileURLWithPath: #file).pathComponents
+        .prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
+      let testInputsPath = packageRootPath + "/TestInputs"
+      let mockSDKPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift/mock-sdk"
+      let dummyDriver = try Driver(args:["swiftc"])
+      let (stdLibPath, shimsPath) = try getStdlibShimsPaths(dummyDriver)
+
+      var driver = try Driver(args: ["swiftc", main.pathString,
+                                     "-sdk", mockSDKPath,
+                                     "-I", stdLibPath.description,
+                                     "-I", shimsPath.description
+                                    ])
+      let jobs = try driver.generatePrebuitModuleGenerationJobs(prebuiltModuleDir: VirtualPath(path: "/tmp/"))
+        .filter { $0.moduleName != "Swift" && $0.moduleName != "SwiftOnoneSupport"}
+      XCTAssertTrue(jobs.count == 10)
+      XCTAssertTrue(jobs.allSatisfy {$0.outputs.count == 1})
+      XCTAssertTrue(jobs.allSatisfy {$0.kind == .compile})
+      XCTAssertTrue(jobs.allSatisfy {$0.commandLine.contains(.flag("-compile-module-from-interface"))})
+    }
+  }
 }