build-script

#!/usr/bin/env python

# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors


"""
The ultimate tool for building Swift.
"""


from __future__ import absolute_import, print_function, unicode_literals

import json
import os
import pipes
import platform
import sys
import time

from build_swift.build_swift import argparse
from build_swift.build_swift import defaults
from build_swift.build_swift import driver_arguments
from build_swift.build_swift import migration
from build_swift.build_swift import presets
from build_swift.build_swift.constants import BUILD_SCRIPT_IMPL_PATH
from build_swift.build_swift.constants import SWIFT_BUILD_ROOT
from build_swift.build_swift.constants import SWIFT_REPO_NAME
from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT

import six

from swift_build_support.swift_build_support import build_graph
from swift_build_support.swift_build_support import products
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support import targets
from swift_build_support.swift_build_support import workspace
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.host_specific_configuration \
    import HostSpecificConfiguration
from swift_build_support.swift_build_support.targets import StdlibDeploymentTarget
from swift_build_support.swift_build_support.toolchain import host_toolchain


# -----------------------------------------------------------------------------
# Constants

# TODO: Remove this constant, it's really not helpful.
HOME = os.environ.get("HOME", "/")


# -----------------------------------------------------------------------------
# Helpers

def print_note(message, stream=sys.stdout):
    """Writes a diagnostic message to the given stream. By default this
    function outputs to stdout.
    """

    stream.write('[{}] NOTE: {}\n'.format(sys.argv[0], message))
    stream.flush()


def fatal_error(message, stream=sys.stderr):
    """Writes a message to the given stream and exits. By default this
    function outputs to stderr.
    """

    stream.write('[{}] ERROR: {}\n'.format(sys.argv[0], message))
    stream.flush()
    sys.exit(1)


def clean_delay():
    """Provide a short delay so accidentally invoked clean builds can be
    canceled.
    """

    sys.stdout.write('Starting clean build in  ')
    for i in range(3, 0, -1):
        sys.stdout.write('\b%d' % i)
        sys.stdout.flush()
        time.sleep(1)
    print('\b\b\b\bnow.')


def exit_rejecting_arguments(message, parser=None):
    print(message, file=sys.stderr)
    if parser:
        parser.print_usage(sys.stderr)
    sys.exit(2)  # 2 is the same as `argparse` error exit code.


def initialize_runtime_environment():
    """Change the program environment for building.
    """

    # Set an appropriate default umask.
    os.umask(0o022)

    # Unset environment variables that might affect how tools behave.
    for v in [
            'MAKEFLAGS',
            'SDKROOT',
            'MACOSX_DEPLOYMENT_TARGET',
            'IPHONEOS_DEPLOYMENT_TARGET',
            'TVOS_DEPLOYMENT_TARGET',
            'WATCHOS_DEPLOYMENT_TARGET']:
        os.environ.pop(v, None)

    # Set NINJA_STATUS to format ninja output
    os.environ['NINJA_STATUS'] = '[%f/%t][%p][%es] '


class JSONDumper(json.JSONEncoder):
    def __init__(self, *args, **kwargs):
        json.JSONEncoder.__init__(
            self, indent=2, separators=(',', ': '), sort_keys=True,
            *args, **kwargs)

    def default(self, o):
        if hasattr(o, '__dict__'):
            return vars(o)
        return six.text_type(o)


def print_xcodebuild_versions(file=sys.stdout):
    """
    Print the host machine's `xcodebuild` version, as well as version
    information for all available SDKs.
    """
    version = shell.capture(
        ['xcodebuild', '-version'], dry_run=False, echo=False).rstrip()
    # Allow non-zero exit codes.  Under certain obscure circumstances
    # xcodebuild can exit with a non-zero exit code even when the SDK is
    # usable.
    sdks = shell.capture(
        ['xcodebuild', '-version', '-sdk'], dry_run=False, echo=False,
        allow_non_zero_exit=True).rstrip()
    fmt = '{version}\n\n--- SDK versions ---\n{sdks}\n'

    print(fmt.format(version=version, sdks=sdks), file=file)
    file.flush()


def tar(source, destination):
    """
    Create a gzip archive of the file at 'source' at the given
    'destination' path.
    """
    # We do not use `tarfile` here because:
    #  - We wish to support LZMA2 compression while also supporting Python 2.7.
    #  - We wish to explicitly set the owner and group of the archive.
    args = ['tar', '-c', '-z', '-f', destination]

    if platform.system() != 'Darwin' and platform.system() != 'Windows':
        args += ['--owner=0', '--group=0']

    # Discard stderr output such as 'tar: Failed to open ...'. We'll detect
    # these cases using the exit code, which should cause 'shell.call' to
    # raise.
    shell.call(args + [source], stderr=shell.DEVNULL)


# -----------------------------------------------------------------------------
# Argument Validation

def validate_arguments(toolchain, args):
    if toolchain.cc is None or toolchain.cxx is None:
        fatal_error(
            "can't find clang (please install clang-3.5 or a "
            "later version)")

    if toolchain.cmake is None:
        fatal_error("can't find CMake (please install CMake)")

    if args.distcc:
        if toolchain.distcc is None:
            fatal_error(
                "can't find distcc (please install distcc)")
        if toolchain.distcc_pump is None:
            fatal_error(
                "can't find distcc-pump (please install distcc-pump)")

    if args.host_target is None or args.stdlib_deployment_targets is None:
        fatal_error("unknown operating system")

    if args.symbols_package:
        if not os.path.isabs(args.symbols_package):
            print(
                '--symbols-package must be an absolute path '
                '(was \'{}\')'.format(args.symbols_package))
            return 1
        if not args.install_symroot:
            fatal_error(
                "--install-symroot is required when specifying "
                "--symbols-package.")

    if args.android:
        if args.android_ndk is None or \
                args.android_api_level is None or \
                args.android_icu_uc is None or \
                args.android_icu_uc_include is None or \
                args.android_icu_i18n is None or \
                args.android_icu_i18n_include is None or \
                args.android_icu_data is None:
            fatal_error(
                "when building for Android, --android-ndk, "
                "--android-api-level, --android-icu-uc, "
                "--android-icu-uc-include, --android-icu-i18n, "
                "--android-icu-i18n-include, and --android-icu-data "
                "must be specified")

    targets_needing_toolchain = [
        'build_indexstoredb',
        'build_playgroundsupport',
        'build_pythonkit',
        'build_sourcekitlsp',
        'build_tensorflow_swift_apis',
        'build_toolchainbenchmarks',
        'build_swift_inspect',
        'tsan_libdispatch_test',
    ]
    has_target_needing_toolchain = \
        bool(sum(getattr(args, x) for x in targets_needing_toolchain))
    if args.legacy_impl and has_target_needing_toolchain:
        fatal_error(
            "--legacy-impl is incompatible with building packages needing "
            "a toolchain (%s)" % ", ".join(targets_needing_toolchain))


def default_stdlib_deployment_targets(args):
    """
    Return targets for the Swift stdlib, based on the build machine.
    If the build machine is not one of the recognized ones, return None.
    """

    host_target = StdlibDeploymentTarget.host_target()
    if host_target is None:
        return None

    # OS X build machines configure all Darwin platforms by default.
    # Put iOS native targets last so that we test them last
    # (it takes a long time).
    if host_target == StdlibDeploymentTarget.OSX.x86_64:
        targets = [host_target]
        if args.build_ios and args.build_ios_simulator:
            targets.extend(StdlibDeploymentTarget.iOSSimulator.targets)
        if args.build_ios and args.build_ios_device:
            targets.extend(StdlibDeploymentTarget.iOS.targets)
        if args.build_tvos and args.build_tvos_simulator:
            targets.extend(StdlibDeploymentTarget.AppleTVSimulator.targets)
        if args.build_tvos and args.build_tvos_device:
            targets.extend(StdlibDeploymentTarget.AppleTV.targets)
        if args.build_watchos and args.build_watchos_simulator:
            targets.extend(StdlibDeploymentTarget
                           .AppleWatchSimulator.targets)
        if args.build_watchos and args.build_watchos_device:
            targets.extend(StdlibDeploymentTarget.AppleWatch.targets)
        return targets
    else:
        # All other machines only configure their host stdlib by default.
        return [host_target]


def apply_default_arguments(toolchain, args):
    # Infer if ninja is required
    ninja_required = (
        args.build_foundation or
        args.build_indexstoredb or
        args.build_pythonkit or
        args.build_sourcekitlsp or
        args.build_tensorflow_swift_apis or
        args.cmake_generator == 'Ninja'
    )
    if ninja_required and toolchain.ninja is None:
        args.build_ninja = True

    # Set the default stdlib-deployment-targets, if none were provided.
    if args.stdlib_deployment_targets is None:
        stdlib_targets = default_stdlib_deployment_targets(args)
        args.stdlib_deployment_targets = [
            target.name for target in stdlib_targets]

    # SwiftPM and XCTest have a dependency on Foundation.
    # On OS X, Foundation is built automatically using xcodebuild.
    # On Linux, we must ensure that it is built manually.
    if ((args.build_swiftpm or args.build_xctest) and
            platform.system() != "Darwin"):
        args.build_foundation = True

    # Foundation has a dependency on libdispatch.
    # On OS X, libdispatch is provided by the OS.
    # On Linux, we must ensure that it is built manually.
    if (args.build_foundation and
            platform.system() != "Darwin"):
        args.build_libdispatch = True

    if args.build_subdir is None:
        args.build_subdir = \
            workspace.compute_build_subdir(args)

    if args.install_destdir is None:
        args.install_destdir = os.path.join(
            SWIFT_BUILD_ROOT, args.build_subdir,
            '{}-{}'.format('toolchain', args.host_target))

    # Add optional stdlib-deployment-targets
    if args.android:
        if args.android_arch == "armv7":
            args.stdlib_deployment_targets.append(
                StdlibDeploymentTarget.Android.armv7.name)
        elif args.android_arch == "aarch64":
            args.stdlib_deployment_targets.append(
                StdlibDeploymentTarget.Android.aarch64.name)

    # Infer platform flags from manually-specified configure targets.
    # This doesn't apply to Darwin platforms, as they are
    # already configured. No building without the platform flag, though.

    android_tgts = [tgt for tgt in args.stdlib_deployment_targets
                    if StdlibDeploymentTarget.Android.contains(tgt)]
    if not args.android and len(android_tgts) > 0:
        # If building natively on an Android host, avoid the NDK
        # cross-compilation configuration.
        if not StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
                                                       .host_target().name):
            args.android = True
        args.build_android = False

    # Include the Darwin supported architectures in the CMake options.
    if args.swift_darwin_supported_archs:
        args.extra_cmake_options.append(
            '-DSWIFT_DARWIN_SUPPORTED_ARCHS:STRING={}'.format(
                args.swift_darwin_supported_archs))

        # Remove unsupported Darwin archs from the standard library
        # deployment targets.
        supported_archs = args.swift_darwin_supported_archs.split(';')
        targets = StdlibDeploymentTarget.get_targets_by_name(
            args.stdlib_deployment_targets)

        args.stdlib_deployment_targets = [
            target.name
            for target in targets
            if (target.platform.is_darwin and
                target.arch in supported_archs)
        ]

    # Filter out any macOS stdlib deployment targets that are not supported
    # by the macOS SDK.
    targets = StdlibDeploymentTarget.get_targets_by_name(
        args.stdlib_deployment_targets)
    args.stdlib_deployment_targets = [
        target.name
        for target in targets
        if (not target.platform.is_darwin or
            target.platform.sdk_supports_architecture(
                target.arch, args.darwin_xcrun_toolchain))
    ]

    # Include the Darwin module-only architectures in the CMake options.
    if args.swift_darwin_module_archs:
        args.extra_cmake_options.append(
            '-DSWIFT_DARWIN_MODULE_ARCHS:STRING={}'.format(
                args.swift_darwin_module_archs))


# -----------------------------------------------------------------------------
# Build Script Impl Wrapping

class BuildScriptInvocation(object):
    """Represent a single build script invocation.
    """

    def __init__(self, toolchain, args):
        self.toolchain = toolchain
        self.args = args

        self.workspace = workspace.Workspace(
            source_root=SWIFT_SOURCE_ROOT,
            build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))

        self.build_libparser_only = args.build_libparser_only

    @property
    def install_all(self):
        return self.args.install_all or self.args.infer_dependencies

    def build_ninja(self):
        if not os.path.exists(self.workspace.source_dir("ninja")):
            fatal_error(
                "can't find source directory for ninja "
                "(tried %s)" % (self.workspace.source_dir("ninja")))

        ninja_build = products.Ninja.new_builder(
            args=self.args,
            toolchain=self.toolchain,
            workspace=self.workspace,
            host=StdlibDeploymentTarget.get_target_for_name(
                self.args.host_target))
        ninja_build.build()
        self.toolchain.ninja = ninja_build.ninja_bin_path

    def convert_to_impl_arguments(self):
        """convert_to_impl_arguments() -> (env, args)

        Convert the invocation to an environment and list of arguments suitable
        for invoking `build-script-impl`.
        """

        # Create local shadows, for convenience.
        args = self.args
        toolchain = self.toolchain

        cmake = CMake(args=args,
                      toolchain=self.toolchain)

        impl_args = [
            "--workspace", self.workspace.source_root,
            "--build-dir", self.workspace.build_root,
            "--install-prefix", args.install_prefix,
            "--host-target", args.host_target,
            "--stdlib-deployment-targets={}".format(
                " ".join(args.stdlib_deployment_targets)),
            "--host-cc", toolchain.cc,
            "--host-cxx", toolchain.cxx,
            "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
            "--darwin-deployment-version-osx=%s" % (
                args.darwin_deployment_version_osx),
            "--darwin-deployment-version-ios=%s" % (
                args.darwin_deployment_version_ios),
            "--darwin-deployment-version-tvos=%s" % (
                args.darwin_deployment_version_tvos),
            "--darwin-deployment-version-watchos=%s" % (
                args.darwin_deployment_version_watchos),
            "--cmake", toolchain.cmake,
            "--cmark-build-type", args.cmark_build_variant,
            "--llvm-build-type", args.llvm_build_variant,
            "--swift-build-type", args.swift_build_variant,
            "--swift-stdlib-build-type", args.swift_stdlib_build_variant,
            "--lldb-build-type", args.lldb_build_variant,
            "--foundation-build-type", args.foundation_build_variant,
            "--libdispatch-build-type", args.libdispatch_build_variant,
            "--libicu-build-type", args.libicu_build_variant,
            "--xctest-build-type", args.build_variant,
            "--llbuild-build-type", args.build_variant,
            "--swift-enable-assertions", str(args.swift_assertions).lower(),
            "--swift-stdlib-enable-assertions", str(
                args.swift_stdlib_assertions).lower(),
            "--swift-analyze-code-coverage", str(
                args.swift_analyze_code_coverage).lower(),
            "--llbuild-enable-assertions", str(
                args.llbuild_assertions).lower(),
            "--lldb-assertions", str(
                args.lldb_assertions).lower(),
            "--cmake-generator", args.cmake_generator,
            "--build-jobs", str(args.build_jobs),
            "--common-cmake-options=%s" % ' '.join(
                pipes.quote(opt) for opt in cmake.common_options()),
            "--build-args=%s" % ' '.join(
                pipes.quote(arg) for arg in cmake.build_args()),
        ]

        # Compute any product specific cmake arguments.
        #
        # NOTE: The sum(list(...)) is b/c compute_product_classes returns a
        # tuple of lists of which the first is the build-script-impl products
        # and the second is the non-build-script-impl-products. It guarantees
        # that when we concatenate these two lists together we get a valid
        # dependency graph.
        for product_class in sum(list(self.compute_product_classes()), []):
            if not product_class.is_build_script_impl_product():
                continue

            product_name = product_class.product_name()
            product_source_name = product_class.product_source_name()
            source_dir = self.workspace.source_dir(product_source_name)

            if not os.path.exists(source_dir):
                fatal_error(
                    "can't find source directory for %s "
                    "(tried %s)" % (product_name, source_dir))

            product = product_class(
                args=args,
                toolchain=self.toolchain,
                source_dir=source_dir,
                # FIXME: This is incorrect since it always assumes the host
                # target I think?
                build_dir=self.workspace.build_dir(
                    args.host_target, product_name))
            cmake_opts = product.cmake_options

            # FIXME: We should be using pipes.quote here but we run into issues
            # with build-script-impl/cmake not being happy with all of the
            # extra "'" in the strings. To fix this easily, we really need to
            # just invoke cmake from build-script directly rather than futzing
            # with build-script-impl. This makes even more sense since there
            # really isn't a security issue here.
            if cmake_opts:
                impl_args += [
                    "--{}-cmake-options={}".format(
                        product_name, ' '.join(cmake_opts))
                ]

        if args.build_stdlib_deployment_targets:
            impl_args += [
                "--build-stdlib-deployment-targets", " ".join(
                    args.build_stdlib_deployment_targets)]
        if args.cross_compile_hosts:
            impl_args += [
                "--cross-compile-hosts", " ".join(args.cross_compile_hosts)]

        if args.test_paths:
            impl_args += ["--test-paths", " ".join(args.test_paths)]

        if toolchain.ninja:
            impl_args += ["--ninja-bin=%s" % toolchain.ninja]
        if args.distcc:
            impl_args += [
                "--distcc",
                "--distcc-pump=%s" % toolchain.distcc_pump
            ]

        # *NOTE* We use normal cmake to pass through tsan/ubsan options. We do
        # NOT pass them to build-script-impl.
        if args.enable_asan:
            impl_args += ["--enable-asan"]
            # If we are on linux, disable leak detection when running ASAN. We
            # have a separate bot that checks for leaks.
            if platform.system() == 'Linux':
                os.environ['ASAN_OPTIONS'] = 'detect_leaks=0'
        if args.enable_ubsan:
            impl_args += ["--enable-ubsan"]

        # If we have lsan, we need to export our suppression list. The actual
        # passing in of the LSAN flag is done via the normal cmake method. We
        # do not pass the flag to build-script-impl.
        if args.enable_lsan:
            supp_file = os.path.join(SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME,
                                     "utils",
                                     "lsan_leaks_suppression_list.txt")
            os.environ['LSAN_OPTIONS'] = 'suppressions={}'.format(supp_file)
        if args.verbose_build:
            impl_args += ["--verbose-build"]
        if args.install_symroot:
            impl_args += [
                "--install-symroot", os.path.abspath(args.install_symroot)
            ]
        if args.install_destdir:
            impl_args += [
                "--install-destdir", os.path.abspath(args.install_destdir)
            ]

        if args.skip_build:
            impl_args += ["--skip-build"]
        if not args.build_benchmarks:
            impl_args += ["--skip-build-benchmarks"]
        # Currently we do not build external benchmarks by default.
        if args.build_external_benchmarks:
            impl_args += ["--skip-build-external-benchmarks=0"]

        # Then add subproject install flags that either skip building them /or/
        # if we are going to build them and install_all is set, we also install
        # them.
        conditional_subproject_configs = [
            (args.build_cmark, "cmark"),
            (args.build_llvm, "llvm"),
            (args.build_swift, "swift"),
            (args.build_foundation, "foundation"),
            (args.build_xctest, "xctest"),
            (args.build_lldb, "lldb"),
            (args.build_llbuild, "llbuild"),
            (args.build_libcxx, "libcxx"),
            (args.build_libdispatch, "libdispatch"),
            (args.build_libicu, "libicu")
        ]
        for (should_build, string_name) in conditional_subproject_configs:
            if not should_build and not self.args.infer_dependencies:
                impl_args += ["--skip-build-{}".format(string_name)]
            elif self.install_all:
                impl_args += ["--install-{}".format(string_name)]

        if args.build_swift_dynamic_stdlib:
            impl_args += ["--build-swift-dynamic-stdlib"]
        if args.build_swift_static_stdlib:
            impl_args += ["--build-swift-static-stdlib"]
        if args.build_swift_stdlib_unittest_extra:
            impl_args += ["--build-swift-stdlib-unittest-extra"]
        if args.build_swift_dynamic_sdk_overlay:
            impl_args += ["--build-swift-dynamic-sdk-overlay"]
        if args.build_swift_static_sdk_overlay:
            impl_args += ["--build-swift-static-sdk-overlay"]

        if not args.build_linux:
            impl_args += ["--skip-build-linux"]
        if not args.build_freebsd:
            impl_args += ["--skip-build-freebsd"]
        if not args.build_cygwin:
            impl_args += ["--skip-build-cygwin"]
        if not args.build_osx:
            impl_args += ["--skip-build-osx"]
        if not args.build_ios_device:
            impl_args += ["--skip-build-ios-device"]
        if not args.build_ios_simulator:
            impl_args += ["--skip-build-ios-simulator"]
        if not args.build_tvos_device:
            impl_args += ["--skip-build-tvos-device"]
        if not args.build_tvos_simulator:
            impl_args += ["--skip-build-tvos-simulator"]
        if not args.build_watchos_device:
            impl_args += ["--skip-build-watchos-device"]
        if not args.build_watchos_simulator:
            impl_args += ["--skip-build-watchos-simulator"]
        if not args.build_android:
            impl_args += ["--skip-build-android"]
        if not args.build_clang_tools_extra:
            impl_args += ["--skip-build-clang-tools-extra"]

        if not args.test and not args.long_test and not args.stress_test:
            impl_args += ["--skip-test-swift"]
        if not args.test:
            impl_args += [
                "--skip-test-cmark",
                "--skip-test-lldb",
                "--skip-test-llbuild",
                "--skip-test-xctest",
                "--skip-test-foundation",
                "--skip-test-libdispatch",
                "--skip-test-libicu",
            ]
        if not args.test_linux:
            impl_args += ["--skip-test-linux"]
        if not args.test_freebsd:
            impl_args += ["--skip-test-freebsd"]
        if not args.test_cygwin:
            impl_args += ["--skip-test-cygwin"]
        if not args.test_osx:
            impl_args += ["--skip-test-osx"]
        if not args.test_ios_host:
            impl_args += ["--skip-test-ios-host"]
        if not args.test_ios_simulator:
            impl_args += ["--skip-test-ios-simulator"]
        if not args.test_ios_32bit_simulator:
            impl_args += ["--skip-test-ios-32bit-simulator"]
        if not args.test_tvos_host:
            impl_args += ["--skip-test-tvos-host"]
        if not args.test_tvos_simulator:
            impl_args += ["--skip-test-tvos-simulator"]
        if not args.test_watchos_host:
            impl_args += ["--skip-test-watchos-host"]
        if not args.test_watchos_simulator:
            impl_args += ["--skip-test-watchos-simulator"]
        if not args.test_android:
            impl_args += ["--skip-test-android"]
        if not args.test_android_host:
            impl_args += ["--skip-test-android-host"]
        if args.build_runtime_with_host_compiler:
            impl_args += ["--build-runtime-with-host-compiler"]
        if args.validation_test:
            impl_args += ["--validation-test"]
        if args.long_test:
            impl_args += ["--long-test"]
        if args.stress_test:
            impl_args += ["--stress-test"]
        if args.skip_local_build:
            impl_args += ["--skip-local-build"]
        if args.only_executable_test:
            impl_args += ["--only-executable-test"]
        if not args.benchmark:
            impl_args += ["--skip-test-benchmarks"]
        if not args.test_optimized:
            impl_args += ["--skip-test-optimized"]
        if not args.test_optimize_for_size:
            impl_args += ["--skip-test-optimize-for-size"]
        if not args.test_optimize_none_with_implicit_dynamic:
            impl_args += ["--skip-test-optimize-none-with-implicit-dynamic"]
        if args.build_libparser_only:
            impl_args += ["--build-libparser-only"]
        if args.android:
            impl_args += [
                "--android-arch", args.android_arch,
                "--android-ndk", args.android_ndk,
                "--android-api-level", args.android_api_level,
                "--android-ndk-gcc-version", args.android_ndk_gcc_version,
                "--android-icu-uc", args.android_icu_uc,
                "--android-icu-uc-include", args.android_icu_uc_include,
                "--android-icu-i18n", args.android_icu_i18n,
                "--android-icu-i18n-include", args.android_icu_i18n_include,
                "--android-icu-data", args.android_icu_data,
            ]
        # If building natively on an Android host, only pass the API level.
        if StdlibDeploymentTarget.Android.contains(StdlibDeploymentTarget
                                                   .host_target().name):
            impl_args += ["--android-api-level", args.android_api_level]
        if args.android_deploy_device_path:
            impl_args += [
                "--android-deploy-device-path",
                args.android_deploy_device_path,
            ]

        if platform.system() == 'Darwin':
            impl_args += [
                "--toolchain-prefix",
                targets.darwin_toolchain_prefix(
                    args.install_prefix),
                "--host-lipo", toolchain.lipo,
            ]

            # Isolate build from the system; Darwin toolchains build against SDKs.
            args.extra_cmake_options.append(
                '-DCMAKE_IGNORE_PATH=/usr/lib;/usr/local/lib;/lib'
            )

        if toolchain.libtool is not None:
            impl_args += [
                "--host-libtool", toolchain.libtool,
            ]

        # If we have extra_swift_args, combine all of them together and then
        # add them as one command.
        if args.extra_swift_args:
            impl_args += [
                "--extra-swift-args=%s" % ';'.join(args.extra_swift_args)
            ]

        # Enable macCatalyst
        if args.maccatalyst:
            (args.extra_cmake_options
             .append('-DSWIFT_ENABLE_MACCATALYST:BOOL=TRUE'))
        if args.maccatalyst_ios_tests:
            impl_args += ["--darwin-test-maccatalyst-ios-like=1"]

        # If we have extra_cmake_options, combine all of them together and then
        # add them as one command.
        if args.extra_cmake_options:
            impl_args += [
                "--extra-cmake-options=%s" % ' '.join(
                    pipes.quote(opt) for opt in args.extra_cmake_options)
            ]

        if args.lto_type is not None:
            impl_args += [
                "--llvm-enable-lto=%s" % args.lto_type,
                "--swift-tools-enable-lto=%s" % args.lto_type
            ]
            if args.llvm_max_parallel_lto_link_jobs is not None:
                impl_args += [
                    "--llvm-num-parallel-lto-link-jobs=%s" %
                    min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
                ]
            if args.swift_tools_max_parallel_lto_link_jobs is not None:
                impl_args += [
                    "--swift-tools-num-parallel-lto-link-jobs=%s" %
                    min(args.swift_tools_max_parallel_lto_link_jobs,
                        args.build_jobs)
                ]

        impl_args += args.build_script_impl_args

        if args.dry_run:
            impl_args += ["--dry-run"]

        if args.clang_profile_instr_use:
            impl_args += [
                "--clang-profile-instr-use=%s" %
                os.path.abspath(args.clang_profile_instr_use)
            ]

        if args.lit_args:
            impl_args += ["--llvm-lit-args=%s" % args.lit_args]

        if args.coverage_db:
            impl_args += [
                "--coverage-db=%s" %
                os.path.abspath(args.coverage_db)
            ]

        if args.llvm_install_components:
            impl_args += [
                "--llvm-install-components=%s" % args.llvm_install_components
            ]

        # Compute the set of host-specific variables, which we pass through to
        # the build script via environment variables.
        host_specific_variables = self.compute_host_specific_variables()
        impl_env = {}
        for (host_target, options) in host_specific_variables.items():
            for (name, value) in options.items():
                # We mangle into an environment variable we can easily evaluate
                # from the `build-script-impl`.
                impl_env["HOST_VARIABLE_{}__{}".format(
                    host_target.replace("-", "_"), name)] = value

        return (impl_env, impl_args)

    def compute_host_specific_variables(self):
        """compute_host_specific_variables(args) -> dict

        Compute the host-specific options, organized as a dictionary keyed by
        host of options.
        """

        args = self.args

        options = {}
        for host_target in [args.host_target] + args.cross_compile_hosts:
            # Compute the host specific configuration.
            try:
                config = HostSpecificConfiguration(host_target, args)
            except argparse.ArgumentError as e:
                exit_rejecting_arguments(six.text_type(e))

            # Convert into `build-script-impl` style variables.
            options[host_target] = {
                "SWIFT_SDKS": " ".join(sorted(
                    config.sdks_to_configure)),
                "SWIFT_STDLIB_TARGETS": " ".join(
                    config.swift_stdlib_build_targets),
                "SWIFT_BENCHMARK_TARGETS": " ".join(
                    config.swift_benchmark_build_targets),
                "SWIFT_RUN_BENCHMARK_TARGETS": " ".join(
                    config.swift_benchmark_run_targets),
                "SWIFT_TEST_TARGETS": " ".join(
                    config.swift_test_run_targets),
            }

        return options

    def compute_product_classes(self):
        """compute_product_classes() -> (list, list)

        Compute the list first of all build-script-impl products and then all
        non-build-script-impl products. It is assumed that concatenating the two
        lists together will result in a valid dependency graph for the
        compilation.

        """

        # FIXME: This is a weird division (returning a list of class objects),
        # but it matches the existing structure of the `build-script-impl`.
        impl_product_classes = []
        if self.args.build_cmark:
            impl_product_classes.append(products.CMark)

        # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
        # Swift still needs a few LLVM targets like tblgen to be built for it to be
        # configured. Instead, handle this in build-script-impl for now.
        impl_product_classes.append(products.LLVM)
        if self.args.build_libcxx:
            impl_product_classes.append(products.LibCXX)
        if self.args.build_libicu:
            impl_product_classes.append(products.LibICU)
        if self.args.build_swift:
            impl_product_classes.append(products.Swift)
        if self.args.build_lldb:
            impl_product_classes.append(products.LLDB)
        if self.args.build_libdispatch:
            impl_product_classes.append(products.LibDispatch)
        if self.args.build_foundation:
            impl_product_classes.append(products.Foundation)
        if self.args.build_xctest:
            impl_product_classes.append(products.XCTest)
        if self.args.build_llbuild:
            impl_product_classes.append(products.LLBuild)
        # Sanity check that all of our impl classes are actually
        # build_script_impl products.
        for prod in impl_product_classes:
            assert(prod.is_build_script_impl_product())

        product_classes = []
        if self.args.build_swiftpm:
            product_classes.append(products.SwiftPM)
        if self.args.build_swift_driver:
            product_classes.append(products.SwiftDriver)
        if self.args.build_swiftsyntax:
            product_classes.append(products.SwiftSyntax)
        if self.args.build_skstresstester:
            product_classes.append(products.SKStressTester)
        if self.args.build_swiftformat:
            product_classes.append(products.SwiftFormat)
        if self.args.build_swiftevolve:
            product_classes.append(products.SwiftEvolve)
        if self.args.build_indexstoredb:
            product_classes.append(products.IndexStoreDB)
        if self.args.build_playgroundsupport:
            product_classes.append(products.PlaygroundSupport)
        if self.args.build_pythonkit:
            product_classes.append(products.PythonKit)
        if self.args.build_tensorflow_swift_apis:
            product_classes.append(products.TensorFlowSwiftAPIs)
        if self.args.build_sourcekitlsp:
            product_classes.append(products.SourceKitLSP)
        if self.args.build_toolchainbenchmarks:
            product_classes.append(products.Benchmarks)
        if self.args.build_swift_inspect:
            product_classes.append(products.SwiftInspect)
        if self.args.tsan_libdispatch_test:
            product_classes.append(products.TSanLibDispatch)
        # Sanity check that all of our non-impl classes are actually
        # not build_script_impl products.
        for prod in product_classes:
            assert(not prod.is_build_script_impl_product())

        # Now that we have our two lists of product_classes, if we are asked to
        # infer dependencies, infer the dependencies now and then re-split the
        # list.
        if self.args.infer_dependencies:
            combined = impl_product_classes + product_classes
            if self.args.verbose_build:
                print("-- Build Graph Inference --")
                print("Initial Product List:")
                for p in combined:
                    print("    {}".format(p.product_name()))

            # Now that we have produced the schedule, resplit. We require our
            # dependencies to respect our build-script-impl property. This means
            # that no build-script-impl products can have dependencies on
            # non-build-script-impl products. Otherwise, it would be unsafe to
            # re-order build-script-impl products in front of non
            # build-script-impl products.
            impl_product_classes = []
            product_classes = []
            is_darwin = platform.system() == 'Darwin'
            final_schedule = build_graph.produce_scheduled_build(combined)[0]
            for p in final_schedule:
                if is_darwin and p.is_nondarwin_only_build_product():
                    continue

                if p.is_build_script_impl_product():
                    impl_product_classes.append(p)
                else:
                    product_classes.append(p)
            if self.args.verbose_build:
                print("Final Build Order:")
                for p in impl_product_classes:
                    print("    {}".format(p.product_name()))
                for p in product_classes:
                    print("    {}".format(p.product_name()))
        return (impl_product_classes, product_classes)

    def execute(self):
        """Execute the invocation with the configured arguments."""

        # Convert to a build-script-impl invocation.
        (self.impl_env, self.impl_args) = self.convert_to_impl_arguments()

        # If using the legacy implementation, delegate all behavior to
        # `build-script-impl`.
        if self.args.legacy_impl:
            # Execute the underlying build script implementation.
            shell.call_without_sleeping([BUILD_SCRIPT_IMPL_PATH] + self.impl_args,
                                        env=self.impl_env, echo=True)
            return

        # Otherwise, we compute and execute the individual actions ourselves.

        # Compute the list of hosts to operate on.
        all_host_names = [
            self.args.host_target] + self.args.cross_compile_hosts
        all_hosts = [StdlibDeploymentTarget.get_target_for_name(name)
                     for name in all_host_names]

        # Compute the list of product classes to operate on.
        #
        # FIXME: This should really be per-host, but the current structure
        # matches that of `build-script-impl`.
        (impl_product_classes, product_classes) = self.compute_product_classes()

        # Execute each "pass".

        # Build...
        for host_target in all_hosts:
            # FIXME: We should only compute these once.
            try:
                config = HostSpecificConfiguration(host_target.name, self.args)
            except argparse.ArgumentError as e:
                exit_rejecting_arguments(six.text_type(e))
            print("Building the standard library for: {}".format(
                " ".join(config.swift_stdlib_build_targets)))
            if config.swift_test_run_targets and (
                    self.args.test or self.args.long_test):
                print("Running Swift tests for: {}".format(
                    " ".join(config.swift_test_run_targets)))
            if config.swift_benchmark_run_targets and self.args.benchmark:
                print("Running Swift benchmarks for: {}".format(
                    " ".join(config.swift_benchmark_run_targets)))

            for product_class in impl_product_classes:
                self._execute_build_action(host_target, product_class)

        # Test...
        for host_target in all_hosts:
            for product_class in impl_product_classes:
                self._execute_test_action(host_target, product_class)

        # Install...
        for host_target in all_hosts:
            for product_class in impl_product_classes:
                self._execute_install_action(host_target, product_class)

        # Non-build-script-impl products...
        # Note: currently only supports building for the host.
        for host_target in [self.args.host_target]:
            for product_class in product_classes:
                if product_class.is_build_script_impl_product():
                    continue
                product_source = product_class.product_source_name()
                product_name = product_class.product_name()
                if product_class.is_swiftpm_unified_build_product():
                    build_dir = self.workspace.swiftpm_unified_build_dir(
                        host_target)
                else:
                    build_dir = self.workspace.build_dir(
                        host_target, product_name)
                product = product_class(
                    args=self.args,
                    toolchain=self.toolchain,
                    source_dir=self.workspace.source_dir(product_source),
                    build_dir=build_dir)
                if product.should_build(host_target):
                    print("--- Building %s ---" % product_name)
                    product.build(host_target)
                if product.should_test(host_target):
                    print("--- Running tests for %s ---" % product_name)
                    product.test(host_target)
                    print("--- Finished tests for %s ---" % product_name)
                if product.should_install(host_target) or \
                   (self.install_all and product.should_build(host_target)):
                    print("--- Installing %s ---" % product_name)
                    product.install(host_target)

        # Extract symbols...
        for host_target in all_hosts:
            self._execute_extract_symbols_action(host_target)

        # Package...
        for host_target in all_hosts:
            self._execute_package_action(host_target)

        # Lipo...
        self._execute_merged_host_lipo_action()

    def _execute_build_action(self, host_target, product_class):
        action_name = "{}-{}-build".format(host_target.name,
                                           product_class.product_name())
        self._execute_action(action_name)

    def _execute_test_action(self, host_target, product_class):
        action_name = "{}-{}-test".format(host_target.name,
                                          product_class.product_name())
        self._execute_action(action_name)

    def _execute_install_action(self, host_target, product_class):
        action_name = "{}-{}-install".format(host_target.name,
                                             product_class.product_name())
        self._execute_action(action_name)

    def _execute_extract_symbols_action(self, host_target):
        action_name = "{}-extractsymbols".format(host_target.name)
        self._execute_action(action_name)

    def _execute_package_action(self, host_target):
        action_name = "{}-package".format(host_target.name)
        self._execute_action(action_name)

    def _execute_merged_host_lipo_action(self):
        self._execute_action("merged-hosts-lipo")

    def _execute_action(self, action_name):
        shell.call_without_sleeping(
            [BUILD_SCRIPT_IMPL_PATH] + self.impl_args +
            ["--only-execute", action_name],
            env=self.impl_env, echo=self.args.verbose_build)


# -----------------------------------------------------------------------------
# Main (preset)

def parse_preset_args():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""Builds Swift using a preset.""")
    parser.add_argument(
        "-n", "--dry-run",
        help="print the commands that would be executed, but do not execute "
             "them",
        action="store_true",
        default=False)
    parser.add_argument(
        "--preset-file",
        help="load presets from the specified file",
        metavar="PATH",
        action="append",
        dest="preset_file_names",
        default=[])
    parser.add_argument(
        "--preset",
        help="use the specified option preset",
        metavar="NAME")
    parser.add_argument(
        "--show-presets",
        help="list all presets and exit",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL)
    parser.add_argument(
        "--distcc",
        help="use distcc",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL,
        default=os.environ.get('USE_DISTCC') == '1')
    parser.add_argument(
        "--cmake-c-launcher",
        help="the absolute path to set CMAKE_C_COMPILER_LAUNCHER",
        metavar="PATH")
    parser.add_argument(
        "--cmake-cxx-launcher",
        help="the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER",
        metavar="PATH")
    parser.add_argument(
        "-j", "--jobs",
        help="the number of parallel build jobs to use",
        type=int,
        dest="build_jobs")
    parser.add_argument(
        "preset_substitutions_raw",
        help="'name=value' pairs that are substituted in the preset",
        nargs="*",
        metavar="SUBSTITUTION")
    parser.add_argument(
        "--expand-invocation", "--expand-build-script-invocation",
        help="Print the expanded build-script invocation generated "
             "by the preset, but do not run the preset",
        action=argparse.actions.StoreTrueAction,
        nargs=argparse.Nargs.OPTIONAL)
    parser.add_argument(
        "--swiftsyntax-install-prefix",
        help="specify the directory to where SwiftSyntax should be installed")
    parser.add_argument(
        "--build-dir",
        help="specify the directory where build artifact should be stored")
    parser.add_argument(
        "--dump-config",
        help="instead of building, write JSON to stdout containing "
             "various values used to build in this configuration",
        action="store_true",
        default=False)
    parser.add_argument(
        "--reconfigure",
        help="Reconfigure all projects as we build",
        action="store_true",
        default=False)

    return parser.parse_args()


def main_preset():
    args = parse_preset_args()

    if len(args.preset_file_names) == 0:
        args.preset_file_names = [
            os.path.join(
                SWIFT_SOURCE_ROOT, SWIFT_REPO_NAME, "utils",
                "build-presets.ini")
        ]

        user_presets_file = os.path.join(HOME, '.swift-build-presets')
        if os.path.isfile(user_presets_file):
            args.preset_file_names.append(user_presets_file)

    preset_parser = presets.PresetParser()

    try:
        preset_parser.read_files(args.preset_file_names)
    except presets.PresetError as e:
        fatal_error(six.text_type(e))

    if args.show_presets:
        for name in sorted(preset_parser.preset_names,
                           key=lambda name: name.lower()):
            print(name)
        return 0

    if not args.preset:
        fatal_error("missing --preset option")

    args.preset_substitutions = {}
    for arg in args.preset_substitutions_raw:
        name, value = arg.split("=", 1)
        args.preset_substitutions[name] = value

    try:
        preset = preset_parser.get_preset(
            args.preset,
            vars=args.preset_substitutions)
    except presets.PresetError as e:
        fatal_error(six.text_type(e))

    preset_args = migration.migrate_swift_sdks(preset.args)

    if args.distcc and (args.cmake_c_launcher or args.cmake_cxx_launcher):
        fatal_error(
            '--distcc can not be used with' +
            ' --cmake-c-launcher or --cmake-cxx-launcher')

    build_script_args = [sys.argv[0]]
    if args.dry_run:
        build_script_args += ["--dry-run"]
    if args.dump_config:
        build_script_args += ["--dump-config"]

    build_script_args += preset_args
    if args.distcc:
        build_script_args += ["--distcc"]
    if args.build_jobs:
        build_script_args += ["--jobs", str(args.build_jobs)]
    if args.swiftsyntax_install_prefix:
        build_script_args += ["--install-swiftsyntax",
                              "--install-destdir",
                              args.swiftsyntax_install_prefix]
    if args.build_dir:
        build_script_args += ["--build-dir", args.build_dir]
    if args.cmake_c_launcher:
        build_script_args += ["--cmake-c-launcher", args.cmake_c_launcher]
    if args.cmake_cxx_launcher:
        build_script_args += ["--cmake-cxx-launcher", args.cmake_cxx_launcher]
    printable_command = shell.quote_command(build_script_args)
    if args.expand_invocation:
        print(printable_command)
        return 0

    if args.reconfigure:
        build_script_args += ["--reconfigure"]

    print_note('using preset "{}", which expands to \n\n{}\n'.format(
        args.preset, printable_command))
    shell.call_without_sleeping(build_script_args)
    return 0


# -----------------------------------------------------------------------------
# Main (normal)

def main_normal():
    parser = driver_arguments.create_argument_parser()

    args = migration.parse_args(parser, sys.argv[1:])

    if args.build_script_impl_args:
        # If we received any impl args, check if `build-script-impl` would
        # accept them or not before any further processing.
        try:
            migration.check_impl_args(BUILD_SCRIPT_IMPL_PATH,
                                      args.build_script_impl_args)
        except ValueError as e:
            exit_rejecting_arguments(e, parser)

        if '--check-args-only' in args.build_script_impl_args:
            return 0

    shell.dry_run = args.dry_run

    # Prepare and validate toolchain
    if args.darwin_xcrun_toolchain is None:
        xcrun_toolchain = os.environ.get('TOOLCHAINS',
                                         defaults.DARWIN_XCRUN_TOOLCHAIN)

        print_note('Using toolchain {}'.format(xcrun_toolchain))
        args.darwin_xcrun_toolchain = xcrun_toolchain

    toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
    os.environ['TOOLCHAINS'] = args.darwin_xcrun_toolchain

    if args.host_cc is not None:
        toolchain.cc = args.host_cc
    if args.host_cxx is not None:
        toolchain.cxx = args.host_cxx
    if args.host_lipo is not None:
        toolchain.lipo = args.host_lipo
    if args.host_libtool is not None:
        toolchain.libtool = args.host_libtool
    if args.cmake is not None:
        toolchain.cmake = args.cmake

    cmake = CMake(args=args, toolchain=toolchain)
    # Check the CMake version is sufficient on Linux and build from source
    # if not.
    cmake_path = cmake.check_cmake_version(SWIFT_SOURCE_ROOT, SWIFT_BUILD_ROOT)
    if cmake_path is not None:
        toolchain.cmake = cmake_path
        args.cmake = cmake_path

    # Preprocess the arguments to apply defaults.
    apply_default_arguments(toolchain, args)

    # Validate the arguments.
    validate_arguments(toolchain, args)

    # Create the build script invocation.
    invocation = BuildScriptInvocation(toolchain, args)

    # Sanitize the runtime environment.
    initialize_runtime_environment()

    # Show SDKs, if requested.
    if args.show_sdks:
        print_xcodebuild_versions()

    if args.dump_config:
        print(JSONDumper().encode(invocation))
        return 0

    # Clean build directory if requested.
    if args.clean:
        clean_delay()
        shell.rmtree(invocation.workspace.build_root)

    # Create build directory.
    shell.makedirs(invocation.workspace.build_root)

    # Build ninja if required, which will update the toolchain.
    if args.build_ninja:
        invocation.build_ninja()

    # Execute the underlying build script implementation.
    invocation.execute()

    if args.symbols_package:
        print('--- Creating symbols package ---')
        print('-- Package file: {} --'.format(args.symbols_package))

        if platform.system() == 'Darwin':
            prefix = targets.darwin_toolchain_prefix(args.install_prefix)
            prefix = os.path.join(args.host_target, prefix.lstrip('/'))
        else:
            prefix = args.install_prefix

        # As a security measure, `tar` normally strips leading '/' from paths
        # it is archiving. To stay safe, we change working directories, then
        # run `tar` without the leading '/' (we remove it ourselves to keep
        # `tar` from emitting a warning).
        with shell.pushd(args.install_symroot):
            tar(source=prefix.lstrip('/'),
                destination=args.symbols_package)

    return 0


# -----------------------------------------------------------------------------

def main():
    if not SWIFT_SOURCE_ROOT:
        fatal_error(
            "could not infer source root directory " +
            "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")

    if not os.path.isdir(SWIFT_SOURCE_ROOT):
        fatal_error(
            "source root directory \'" + SWIFT_SOURCE_ROOT +
            "\' does not exist " +
            "(forgot to set $SWIFT_SOURCE_ROOT environment variable?)")

    # Determine if we are invoked in the preset mode and dispatch accordingly.
    if any([(opt.startswith("--preset") or opt == "--show-presets")
            for opt in sys.argv[1:]]):
        return main_preset()
    else:
        return main_normal()


if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)