Move the SDK EMB cache out of ModuleCache.noindex (swiftlang#1416)

Add SDK_EXPLICIT_MODULES_OUTPUT_PATH and use it for
-sdk-module-cache-path instead of MODULE_CACHE_DIR.

Putting the scanning and EBM cache in the same directory was confusing
when looking at logs. This defaults to
$(DERIVED_DATA_DIR)/SDKExplicitPrecompiledModules

rdar://175038966

Author: Bigcheese

Sources/SWBBuildSystem/BuildOperation.swift

@@ -1660,6 +1660,7 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act
             let targetSettings = operation.requestContext.getCachedSettings(operation.request.parameters, target: target.target)
             directories.insert(targetSettings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH))
             directories.insert(targetSettings.globalScope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH))
+            directories.insert(targetSettings.globalScope.evaluate(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH))
             if let value = TimeInterval(targetSettings.globalScope.evaluate(BuiltinMacros.CLANG_MODULES_PRUNE_AFTER)) {
                 pruneAfter = value
             }

Sources/SWBCore/ClangSerializedDiagnostics.swift

@@ -27,6 +27,7 @@ public struct LibclangDiagnosticAttachmentInfo: SerializableCodable, Sendable {
             BuiltinMacros.MODULE_CACHE_DIR,
             BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH,
             BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH,
+            BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH,
         ].compactMap {
             scope.evaluate($0).nilIfEmpty
         }

Sources/SWBCore/Settings/BuiltinMacros.swift

@@ -532,6 +532,7 @@ public final class BuiltinMacros {
     public static let CLANG_EXPLICIT_MODULES_IGNORE_LIBCLANG_VERSION_MISMATCH = BuiltinMacros.declareBooleanMacro("CLANG_EXPLICIT_MODULES_IGNORE_LIBCLANG_VERSION_MISMATCH")
     public static let CLANG_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declarePathMacro("CLANG_EXPLICIT_MODULES_OUTPUT_PATH")
     public static let SWIFT_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declarePathMacro("SWIFT_EXPLICIT_MODULES_OUTPUT_PATH")
+    public static let SDK_EXPLICIT_MODULES_OUTPUT_PATH = BuiltinMacros.declarePathMacro("SDK_EXPLICIT_MODULES_OUTPUT_PATH")
     public static let CLANG_EXPLICIT_MODULES_ENABLE_REPRODUCER_FOR_ERRORS = BuiltinMacros.declareBooleanMacro("_EXPERIMENTAL_CLANG_EXPLICIT_MODULES_ENABLE_REPRODUCER_FOR_ERRORS")
     public static let CLANG_EXPLICIT_MODULES_REPRODUCER_OUTPUT_PATH = BuiltinMacros.declarePathMacro("CLANG_CRASH_DIAGNOSTICS_DIR")
     public static let CLANG_ENABLE_COMPILE_CACHE = BuiltinMacros.declareBooleanMacro("CLANG_ENABLE_COMPILE_CACHE")
@@ -1581,6 +1582,7 @@ public final class BuiltinMacros {
         CLANG_EXPLICIT_MODULES_IGNORE_LIBCLANG_VERSION_MISMATCH,
         CLANG_EXPLICIT_MODULES_OUTPUT_PATH,
         SWIFT_EXPLICIT_MODULES_OUTPUT_PATH,
+        SDK_EXPLICIT_MODULES_OUTPUT_PATH,
         CLANG_EXPLICIT_MODULES_ENABLE_REPRODUCER_FOR_ERRORS,
         CLANG_EXPLICIT_MODULES_REPRODUCER_OUTPUT_PATH,
         CLANG_EXTRACT_API_EXEC,

Sources/SWBCore/Settings/Settings.swift

@@ -537,6 +537,9 @@ final class WorkspaceSettings: Sendable {
         // Add shared output path for Swift explicit modules
         table.push(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH, Static { BuiltinMacros.namespace.parseString("$(OBJROOT)/SwiftExplicitPrecompiledModules") })
 
+        // Shared output path for SDK explicit modules. Lives adjacent to ModuleCache.noindex under DerivedData so it is shared across projects (SDK PCMs are invariant across projects that share an SDK and toolchain).
+        table.push(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH, Static { BuiltinMacros.namespace.parseString("$(DERIVED_DATA_DIR)/SDKExplicitPrecompiledModules") })
+
         // Add default values for the compilation caching plugin (off-by-default).
         table.push(BuiltinMacros.COMPILATION_CACHE_PLUGIN_PATH, Static { BuiltinMacros.namespace.parseString("$(DEVELOPER_USR_DIR)/lib/libToolchainCASPlugin.dylib") })
 
@@ -4111,8 +4114,9 @@ private class SettingsBuilder: ProjectMatchLookup {
                 derivedDataPath = workspaceContext.workspaceSettings.getCacheRoot()
             }
 
-            // If there is no DerivedData then intentionally remove the MODULE_CACHE_DIR setting (which is defined in terms of it).
+            // If there is no DerivedData then intentionally remove the MODULE_CACHE_DIR and SDK_EXPLICIT_MODULES_OUTPUT_PATH settings (which are defined in terms of it).
             table.push(BuiltinMacros.MODULE_CACHE_DIR, literal: "")
+            table.push(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH, literal: "")
         }
 
         table.push(BuiltinMacros.DERIVED_DATA_DIR, BuiltinMacros.namespace.parseString(derivedDataPath!.str))

Sources/SWBCore/SpecImplementations/Tools/SwiftCompiler.swift

@@ -1443,9 +1443,11 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi
                         // Specify the Clang scanner cache separately as a shared cache among different projects
                         args.append(contentsOf: ["-clang-scanner-module-cache-path", moduleCacheDir.str])
                     }
-                    if LibSwiftDriver.supportsDriverFlag(spelled: "-sdk-module-cache-path"),
-                       !moduleCacheDir.isEmpty {
-                        args.append(contentsOf: ["-sdk-module-cache-path", moduleCacheDir.str])
+                    if LibSwiftDriver.supportsDriverFlag(spelled: "-sdk-module-cache-path") {
+                        let sdkExplicitModulesDir = cbc.scope.evaluate(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH)
+                        if !sdkExplicitModulesDir.isEmpty {
+                            args.append(contentsOf: ["-sdk-module-cache-path", sdkExplicitModulesDir.str])
+                        }
                     }
                 }
             } else {

Sources/SWBCore/WorkspaceContext.swift

@@ -356,7 +356,7 @@ public final class WorkspaceContext: Sendable {
     }
 
     public var cacheDirectoryMacros: [PathMacroDeclaration] {
-        return [BuiltinMacros.MODULE_CACHE_DIR, BuiltinMacros.COMPILATION_CACHE_CAS_PATH, BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.SHARED_PRECOMPS_DIR]
+        return [BuiltinMacros.MODULE_CACHE_DIR, BuiltinMacros.COMPILATION_CACHE_CAS_PATH, BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH, BuiltinMacros.SHARED_PRECOMPS_DIR]
     }
 
     /// The path to the module session file for the workspace given a set of build parameters.

Sources/SWBTaskExecution/BuildDescriptionManager.swift

@@ -214,6 +214,7 @@ package final class BuildDescriptionManager: Sendable {
                 settings.globalScope.evaluate(BuiltinMacros.MODULE_CACHE_DIR),
                 settings.globalScope.evaluate(BuiltinMacros.SWIFT_EXPLICIT_MODULES_OUTPUT_PATH),
                 settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH),
+                settings.globalScope.evaluate(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH),
             ]
 
             artifactInfoPerTarget[target] = settings.productType?.artifactInfo(in: settings.globalScope)

Tests/SWBBuildSystemTests/CleanOperationTests.swift

@@ -266,7 +266,7 @@ fileprivate struct CleanOperationTests: CoreBasedTests {
             let buildRequest = BuildRequest(parameters: parameters, buildTargets: buildTargets, continueBuildingAfterErrors: true, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false)
             try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
                 results.checkTasks(.matchRuleType("CreateBuildDirectory")) { tasks in
-                    #expect(tasks.count == 12)
+                    #expect(tasks.count == 13)
                 }
 
                 results.checkNoTask(.matchRuleType("CreateBuildDirectory"))

Tests/SWBCoreTests/SettingsTests.swift

@@ -403,6 +403,8 @@ import SWBTestSupport
                     #expect(settings.globalScope.evaluate(BuiltinMacros.OBJROOT) == Path("\(project.sourceRoot.str)/build"))
                     #expect(settings.globalScope.evaluate(BuiltinMacros.CONFIGURATION_BUILD_DIR) == Path("\(project.sourceRoot.str)/build/Config1"))
                     #expect(settings.globalScope.evaluate(BuiltinMacros.CLANG_EXPLICIT_MODULES_OUTPUT_PATH).str == "\(project.sourceRoot.str)/build/ExplicitPrecompiledModules")
+                    // SDK_EXPLICIT_MODULES_OUTPUT_PATH is defined in terms of DERIVED_DATA_DIR, so it is explicitly cleared when there is no DerivedData (as with MODULE_CACHE_DIR above).
+                    #expect(settings.tableForTesting.lookupMacro(BuiltinMacros.SDK_EXPLICIT_MODULES_OUTPUT_PATH)?.expression.stringRep == "")
                 }
 
                 // check that we get the right value for VERSION_INFO_STRING, which validates we parsed it correctly.

Tests/SWBTaskConstructionTests/SwiftTaskConstructionTests.swift

@@ -1248,6 +1248,7 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests {
                         "GCC_GENERATE_DEBUGGING_SYMBOLS": "NO",
                         "SWIFT_ENABLE_EXPLICIT_MODULES": "YES",
                         "SWIFT_EXPLICIT_MODULES_OUTPUT_PATH": "/path/to/ExplicitModules",
+                        "SDK_EXPLICIT_MODULES_OUTPUT_PATH": "/path/to/SDKExplicitModules",
                     ]),
             ],
             targets: [
@@ -1281,7 +1282,7 @@ fileprivate struct SwiftTaskConstructionTests: CoreBasedTests {
                         task.checkCommandLineContains(["-clang-scanner-module-cache-path", "/path/to/DerivedData/ModuleCache.noindex"])
                     }
                     if LibSwiftDriver.supportsDriverFlag(spelled: "-sdk-module-cache-path") {
-                        task.checkCommandLineContains(["-sdk-module-cache-path", "/path/to/DerivedData/ModuleCache.noindex"])
+                        task.checkCommandLineContains(["-sdk-module-cache-path", "/path/to/SDKExplicitModules"])
                     }
 
                     // Check module cache pruning options are passed.