Merge pull request swiftlang#34133 from porglezomp-misc/twist-it-bop-it-sccache-it

[build-script] Add a flag for sccache

Author: shahmishal

README.md

@@ -111,6 +111,8 @@ following (non-exhaustive) set of useful options::
 - ``--test``: Test the toolchain after it has been compiled. This is off by default.
 - ``--distcc``: Use distcc to speed up the build by distributing the c++ part of
   the swift build. This is off by default.
+- ``--sccache``: Use sccache to speed up subsequent builds of the compiler by
+  caching more c++ build artifacts. This is off by default.
 
 More options may be added over time. Please pass ``--help`` to
 ``build-toolchain`` to see the full set of options.

docs/DevelopmentTips.md

@@ -33,9 +33,11 @@ Compilation times for the compiler and the standard library can be agonizing, es
 ```
 $ brew install sccache
 $ sccache --start-server
-$ ./swift/utils/build-script MY_ARGS --cmake-c-launcher $(which sccache) --cmake-cxx-launcher $(which sccache)
+$ ./swift/utils/build-script MY_ARGS --sccache
 ```
 
+If you want to always use sccache, you can `export SWIFT_USE_SCCACHE=1` and the build script will pick it up.
+
 Given the size of artifacts generated, you might also want to bump the cache size from the default 10GB to something larger, say by putting `export SCCACHE_CACHE_SIZE="50G"` in your dotfile(s).
 
 You can run some compiles to see if it is actually doing something by running `sccache --show-stats`. Depending on the exact compilation task you're running, you might see very different cache hit rates. For example, `sccache` is particularly effective if you're rebuilding LLVM, which doesn't change so frequently from the Swift compiler's perspective. On the other hand, if you're changing the compiler's AST, the cache hit rate is likely to be much lower.

docs/HowToGuides/GettingStarted.md

@@ -246,15 +246,13 @@ Phew, that's a lot to digest! Now let's proceed to the actual build itself!
      ```sh
      utils/build-script --skip-build-benchmarks \
        --skip-ios --skip-watchos --skip-tvos --swift-darwin-supported-archs "x86_64" \
-       --cmake-c-launcher="$(which sccache)" --cmake-cxx-launcher="$(which sccache)" \
-       --release-debuginfo --test
+       --sccache --release-debuginfo --test
      ```
    - Via Xcode:
      ```sh
      utils/build-script --skip-build-benchmarks \
        --skip-ios --skip-watchos --skip-tvos --swift-darwin-supported-archs "x86_64" \
-       --cmake-c-launcher="$(which sccache)" --cmake-cxx-launcher="$(which sccache)" \
-       --release-debuginfo --test \
+       --sccache --release-debuginfo --test \
        --xcode
      ```
    This will create a directory

utils/build-script

@@ -187,6 +187,11 @@ def validate_arguments(toolchain, args):
             fatal_error(
                 "can't find distcc-pump (please install distcc-pump)")
 
+    if args.sccache:
+        if toolchain.sccache is None:
+            fatal_error(
+                "can't find sccache (please install sccache)")
+
     if args.host_target is None or args.stdlib_deployment_targets is None:
         fatal_error("unknown operating system")
 
@@ -523,6 +528,9 @@ class BuildScriptInvocation(object):
                 "--distcc",
                 "--distcc-pump=%s" % toolchain.distcc_pump
             ]
+        if args.sccache:
+            args.cmake_c_launcher = toolchain.sccache
+            args.cmake_cxx_launcher = toolchain.sccache
 
         # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do
         # NOT pass them to build-script-impl.
@@ -1064,6 +1072,12 @@ def parse_preset_args():
         action=argparse.actions.StoreTrueAction,
         nargs=argparse.Nargs.OPTIONAL,
         default=os.environ.get('USE_DISTCC') == '1')
+    parser.add_argument(
+        "--sccache",
+        help="use sccache",
+        action=argparse.actions.StoreTrueAction,
+        nargs=argparse.Nargs.OPTIONAL,
+        default=os.environ.get('SWIFT_USE_SCCACHE') == '1')
     parser.add_argument(
         "--cmake-c-launcher",
         help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER",
@@ -1157,6 +1171,10 @@ def main_preset():
         fatal_error(
             '--distcc can not be used with' +
             ' --cmake-c-launcher or --cmake-cxx-launcher')
+    if args.sccache and (args.cmake_c_launcher or args.cmake_cxx_launcher):
+        fatal_error(
+            '--sccache can not be used with' +
+            ' --cmake-c-launcher or --cmake-cxx-launcher')
 
     build_script_args = [sys.argv[0]]
     if args.dry_run:
@@ -1167,6 +1185,8 @@ def main_preset():
     build_script_args += preset_args
     if args.distcc:
         build_script_args += ["--distcc"]
+    if args.sccache:
+        build_script_args += ["--sccache"]
     if args.build_jobs:
         build_script_args += ["--jobs", str(args.build_jobs)]
     if args.swiftsyntax_install_prefix:

utils/build-toolchain

@@ -85,13 +85,7 @@ while [ $# -ne 0 ]; do
       DISTCC_FLAG="--distcc"
   ;;
     --sccache)
-      SCCACHE=$(which sccache)
-      if [[ -z "${SCCACHE}" ]]; then
-          echo "Error! Asked to use sccache, but could not find sccache in PATH?!"
-          usage
-          exit 1
-      fi
-      SCCACHE_FLAG="--cmake-c-launcher=${SCCACHE} --cmake-cxx-launcher=${SCCACHE}"
+      SCCACHE_FLAG="--sccache"
   ;;
     --preset-file)
       shift

utils/build_swift/build_swift/driver_arguments.py

@@ -387,6 +387,9 @@ def create_argument_parser():
     option('--distcc', toggle_true,
            default=os.environ.get('USE_DISTCC') == '1',
            help='use distcc in pump mode')
+    option('--sccache', toggle_true,
+           default=os.environ.get('SWIFT_USE_SCCACHE') == '1',
+           help='use sccache')
     option('--enable-asan', toggle_true,
            help='enable Address Sanitizer')
     option('--enable-ubsan', toggle_true,

utils/build_swift/tests/expected_options.py

@@ -138,6 +138,7 @@
         defaults.DARWIN_DEPLOYMENT_VERSION_WATCHOS,
     'darwin_xcrun_toolchain': None,
     'distcc': False,
+    'sccache': False,
     'dry_run': False,
     'enable_asan': False,
     'enable_experimental_differentiable_programming': True,
@@ -500,6 +501,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--build-swift-static-stdlib'),
     EnableOption('--build-swift-stdlib-unittest-extra'),
     EnableOption('--distcc'),
+    EnableOption('--sccache'),
     EnableOption('--enable-asan'),
     EnableOption('--enable-experimental-differentiable-programming'),
     EnableOption('--enable-experimental-concurrency'),

utils/swift_build_support/swift_build_support/cmake.py

@@ -126,6 +126,10 @@ def common_options(self):
             define("CMAKE_C_COMPILER_LAUNCHER:PATH", toolchain.distcc)
             define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", toolchain.distcc)
 
+        if args.sccache:
+            define("CMAKE_C_COMPILER_LAUNCHER:PATH", toolchain.sccache)
+            define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", toolchain.sccache)
+
         if args.cmake_c_launcher:
             define("CMAKE_C_COMPILER_LAUNCHER:PATH", args.cmake_c_launcher)
         if args.cmake_cxx_launcher:

utils/swift_build_support/swift_build_support/toolchain.py

@@ -62,6 +62,7 @@ def _getter(self):
 _register("llvm_cov", "llvm-cov")
 _register("lipo", "lipo")
 _register("libtool", "libtool")
+_register("sccache", "sccache")
 _register("swiftc", "swiftc")
 
 

utils/swift_build_support/tests/test_cmake.py

@@ -33,6 +33,15 @@ def mock_distcc_path(self):
             executable = 'mock-distcc'
         return os.path.join(os.path.dirname(__file__), executable)
 
+    def mock_sccache_path(self):
+        """Return a path string of a mock sccache executable
+        """
+        if platform.system() == 'Windows':
+            executable = 'sccache.cmd'
+        else:
+            executable = 'sccache'
+        return os.path.join(os.path.dirname(__file__), executable)
+
     def default_args(self):
         """Return new args object with default values
         """
@@ -46,6 +55,7 @@ def default_args(self):
                          enable_sanitize_coverage=False,
                          export_compile_commands=False,
                          distcc=False,
+                         sccache=False,
                          cmake_generator="Ninja",
                          cmake_c_launcher=None,
                          cmake_cxx_launcher=None,
@@ -73,6 +83,8 @@ def cmake(self, args):
         toolchain.libtool = args.host_libtool
         if args.distcc:
             toolchain.distcc = self.mock_distcc_path()
+        if args.sccache:
+            toolchain.sccache = self.mock_sccache_path()
         toolchain.ninja = self.which_ninja(args)
         return CMake(args=args, toolchain=toolchain)
 
@@ -222,6 +234,20 @@ def test_common_options_distcc(self):
              "-DCMAKE_LIBTOOL:PATH=/path/to/libtool",
              "-DCMAKE_MAKE_PROGRAM=" + self.which_ninja(args)])
 
+    def test_common_options_sccache(self):
+        args = self.default_args()
+        args.sccache = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.common_options()),
+            ["-G", "Ninja",
+             "-DCMAKE_C_COMPILER_LAUNCHER:PATH=" + self.mock_sccache_path(),
+             "-DCMAKE_CXX_COMPILER_LAUNCHER:PATH=" + self.mock_sccache_path(),
+             "-DCMAKE_C_COMPILER:PATH=/path/to/clang",
+             "-DCMAKE_CXX_COMPILER:PATH=/path/to/clang++",
+             "-DCMAKE_LIBTOOL:PATH=/path/to/libtool",
+             "-DCMAKE_MAKE_PROGRAM=" + self.which_ninja(args)])
+
     def test_common_options_launcher(self):
         args = self.default_args()
         cmake_c_launcher = "/path/to/c_launcher"

utils/swift_build_support/tests/test_toolchain.py

@@ -68,6 +68,9 @@ def test_misc_tools(self):
         self.assertTrue(tc.distcc_pump is None or
                         os.path.basename(tc.distcc_pump) == 'pump' or
                         os.path.basename(tc.distcc_pump) == 'distcc-pump')
+        # sccache
+        self.assertTrue(tc.sccache is None or
+                        os.path.basename(tc.sccache) == 'sccache')
 
     def test_find_tool(self):
         tc = host_toolchain()