Add synthetic modules-with-macros graph benchmarks

Benchmarks/Benchmarks/PackageGraphBenchmarks/PackageGraphBenchmarks.swift

@@ -27,15 +27,15 @@ let benchmarks = {
     let parsedValue = Int(envVar) {
         modulesGraphDepth = parsedValue
     } else {
-        modulesGraphDepth = 100
+        modulesGraphDepth = 150
     }
 
     let modulesGraphWidth: Int
     if let envVar = ProcessInfo.processInfo.environment["SWIFTPM_BENCHMARK_MODULES_GRAPH_WIDTH"],
     let parsedValue = Int(envVar) {
         modulesGraphWidth = parsedValue
     } else {
-        modulesGraphWidth = 100
+        modulesGraphWidth = 150
     }
 
     let packagesGraphDepth: Int
@@ -46,8 +46,6 @@ let benchmarks = {
         packagesGraphDepth = 10
     }
 
-    let noopObservability = ObservabilitySystem.NOOP
-
     // Benchmarks computation of a resolved graph of modules for a package using `Workspace` as an entry point. It runs PubGrub to get
     // resolved concrete versions of dependencies, assigning all modules and products to each other as corresponding dependencies
     // with their build triples, but with the build plan not yet constructed. In this benchmark specifically we're loading `Package.swift`
@@ -67,55 +65,120 @@ let benchmarks = {
         let workspace = try Workspace(fileSystem: localFileSystem, location: .init(forRootPackage: path, fileSystem: localFileSystem))
 
         for _ in benchmark.scaledIterations {
-            try workspace.loadPackageGraph(rootPath: path, observabilityScope: noopObservability)
+            try workspace.loadPackageGraph(rootPath: path, observabilityScope: ObservabilitySystem.NOOP)
         }
     }
 
-
-    // Benchmarks computation of a resolved graph of modules for a synthesized package using `loadModulesGraph` as an
-    // entry point, which almost immediately delegates to `ModulesGraph.load` under the hood.
+    // Benchmarks computation of a resolved graph of modules for a trivial synthesized package using `loadModulesGraph`
+    // as an entry point, which almost immediately delegates to `ModulesGraph.load` under the hood.
     Benchmark(
         "SyntheticModulesGraph",
         configuration: .init(
             metrics: defaultMetrics,
             maxDuration: .seconds(10),
             thresholds: [
-                .mallocCountTotal: .init(absolute: [.p90: 2500]),
-                .syscalls: .init(absolute: [.p90: 0]),
+                .mallocCountTotal: .init(absolute: [.p90: 17000]),
+                .syscalls: .init(absolute: [.p90: 5]),
             ]
         )
     ) { benchmark in
-        let targets = try (0..<modulesGraphWidth).map { i in
-            try TargetDescription(name: "Target\(i)", dependencies: (0..<min(i, modulesGraphDepth)).map {
-                .target(name: "Target\($0)")
-            })
-        }
-        let fileSystem = InMemoryFileSystem(
-            emptyFiles: targets.map { "/benchmark/Sources/\($0.name)/empty.swift" }
+        try syntheticModulesGraph(
+            benchmark,
+            modulesGraphDepth: modulesGraphDepth,
+            modulesGraphWidth: modulesGraphWidth
         )
-        let rootPackagePath = try AbsolutePath(validating: "/benchmark")
+    }
 
-        let manifest = Manifest(
-            displayName: "benchmark",
-            path: rootPackagePath,
-            packageKind: .root(rootPackagePath),
-            packageLocation: rootPackagePath.pathString,
-            defaultLocalization: nil,
-            platforms: [],
-            version: nil,
-            revision: nil,
-            toolsVersion: .v5_10,
-            pkgConfig: nil,
-            providers: nil,
-            cLanguageStandard: nil,
-            cxxLanguageStandard: nil,
-            swiftLanguageVersions: nil
+    // Benchmarks computation of a resolved graph of modules for a synthesized package that includes macros,
+    // using `loadModulesGraph` as an entry point, which almost immediately delegates to `ModulesGraph.load` under
+    // the hood.
+    Benchmark(
+        "SyntheticModulesGraphWithMacros",
+        configuration: .init(
+            metrics: defaultMetrics,
+            maxDuration: .seconds(10),
+            thresholds: [
+                .mallocCountTotal: .init(absolute: [.p90: 8000]),
+                .syscalls: .init(absolute: [.p90: 5]),
+            ]
         )
+    ) { benchmark in
+        try syntheticModulesGraph(
+            benchmark, 
+            modulesGraphDepth: modulesGraphDepth, 
+            modulesGraphWidth: modulesGraphWidth,
+            includeMacros: true
+        )
+    }
+}
 
-        for _ in benchmark.scaledIterations {
-            try blackHole(
-                loadModulesGraph(fileSystem: fileSystem, manifests: [manifest], observabilityScope: noopObservability)
-            )
+func syntheticModulesGraph(
+    _ benchmark: Benchmark, 
+    modulesGraphDepth: Int, 
+    modulesGraphWidth: Int, 
+    includeMacros: Bool = false
+) throws {
+    // If macros are included, modules are split in three parts:
+    // 1. top-level modules
+    // 2. macros
+    // 3. dependencies of macros
+    let macrosDenominator = includeMacros ? 3 : 1
+    let libraryModules: [TargetDescription] = try (0..<(modulesGraphWidth / macrosDenominator)).map { i -> TargetDescription in
+        let dependencies = (0..<min(i, modulesGraphDepth / macrosDenominator)).flatMap { i -> [TargetDescription.Dependency] in
+            if includeMacros {
+                [.target(name: "Module\(i)"), .target(name: "Macros\(i)")]
+            } else {
+                [.target(name: "Module\(i)")]
+            }
+        }
+        return try TargetDescription(name: "Module\(i)", dependencies: dependencies)
+    }
+
+    let macrosModules: [TargetDescription]
+    let macrosDependenciesModules: [TargetDescription]
+    if includeMacros {
+        macrosModules = try (0..<modulesGraphWidth / macrosDenominator).map { i in
+            try TargetDescription(name: "Macros\(i)", dependencies: (0..<min(i, modulesGraphDepth)).map {
+                .target(name: "MacrosDependency\($0)")
+            })
+        }
+        macrosDependenciesModules = try (0..<modulesGraphWidth / macrosDenominator).map { i in
+            try TargetDescription(name: "MacrosDependency\(i)")
         }
+    } else {
+        macrosModules = []
+        macrosDependenciesModules = []
+    }
+
+    let modules = libraryModules + macrosModules + macrosDependenciesModules
+    let fileSystem = InMemoryFileSystem(
+        emptyFiles: modules.map {
+            "/benchmark/Sources/\($0.name)/empty.swift"
+        }
+    )
+    let rootPackagePath = try AbsolutePath(validating: "/benchmark")
+
+    let manifest = Manifest(
+        displayName: "benchmark",
+        path: rootPackagePath,
+        packageKind: .root(rootPackagePath),
+        packageLocation: rootPackagePath.pathString,
+        defaultLocalization: nil,
+        platforms: [],
+        version: nil,
+        revision: nil,
+        toolsVersion: .v5_10,
+        pkgConfig: nil,
+        providers: nil,
+        cLanguageStandard: nil,
+        cxxLanguageStandard: nil,
+        swiftLanguageVersions: nil,
+        targets: modules
+    )
+
+    for _ in benchmark.scaledIterations {
+        try blackHole(
+            loadModulesGraph(fileSystem: fileSystem, manifests: [manifest], observabilityScope: ObservabilitySystem.NOOP)
+        )
     }
 }

Benchmarks/README.md

@@ -26,7 +26,7 @@ built with Swift 5.10. To record new thresholds, run the following command:
 ```
 swift package --allow-writing-to-package-directory benchmark \
   --format metricP90AbsoluteThresholds \
-  --path "Thresholds/$([[ $(uname) == Darwin ]] && echo macosx || echo linux)-$(uname -m)"
+  --path "Thresholds/$([[ $(uname) == Darwin ]] && echo macos || echo linux)-$(uname -m)"
 ```
 
 To verify that recorded thresholds do not exceeded given relative or absolute values (passed as `thresholds` arguments

Benchmarks/Thresholds/macos-arm64/PackageGraphBenchmarks.SwiftPMWorkspaceModulesGraph.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 11255,
+  "syscalls" : 1497
+}

Benchmarks/Thresholds/macos-arm64/PackageGraphBenchmarks.SyntheticModulesGraph.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 15679,
+  "syscalls" : 0
+}

Benchmarks/Thresholds/macos-arm64/PackageGraphBenchmarks.SyntheticModulesGraphWithMacros.p90.json

@@ -0,0 +1,4 @@
+{
+  "mallocCountTotal" : 7743,
+  "syscalls" : 0
+}

Tests/SPMBuildCoreTests/PluginInvocationTests.swift

@@ -11,6 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 import Basics
+
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
 @testable import PackageGraph
 import PackageLoading
 import PackageModel

Tests/SourceKitLSPAPITests/SourceKitLSPAPITests.swift

@@ -12,7 +12,10 @@
 
 import Basics
 import Build
+
+@_spi(DontAdoptOutsideOfSwiftPMExposedForBenchmarksAndTestsOnly)
 import PackageGraph
+
 import PackageModel
 import SourceKitLSPAPI
 import SPMTestSupport