Merge pull request #32256 from gottesmm/pr-6a889299976ad0e3dfbc1849e1b7edaf177c53ce

[build-script] Add option --infer to infer dependencies.

Author: gottesmm

test/lit.cfg

@@ -146,7 +146,7 @@ config.test_format = swift_test.SwiftTest(coverage_mode=config.coverage_mode,
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m', '.c',
-                   '.swiftinterface', '.test-sh']
+                   '.swiftinterface', '.test-sh', '.test']
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -162,6 +162,11 @@ config.test_source_root = os.path.dirname(__file__)
 # test_exec_root: The root path where tests should be run.
 swift_obj_root = getattr(config, 'swift_obj_root', None)
 
+# cmake. The path to the cmake executable we used to configure swift.
+assert(config.cmake)
+config.substitutions.append( ('%cmake',  config.cmake) )
+lit_config.note('Using cmake: ' + config.cmake)
+
 # Set llvm_{src,obj}_root for use by others.
 config.llvm_src_root = getattr(config, 'llvm_src_root', None)
 config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)

test/lit.site.cfg.in

@@ -14,6 +14,7 @@ import os
 import platform
 import sys
 
+config.cmake = "@CMAKE_COMMAND@"
 config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"

utils/build-script

@@ -35,6 +35,7 @@ from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
 
 import six
 
+from swift_build_support.swift_build_support import build_graph
 from swift_build_support.swift_build_support import products
 from swift_build_support.swift_build_support import shell
 from swift_build_support.swift_build_support import targets
@@ -375,6 +376,10 @@ class BuildScriptInvocation(object):
 
         self.build_libparser_only = args.build_libparser_only
 
+    @property
+    def install_all(self):
+        return self.args.install_all or self.args.infer_dependencies
+
     def build_ninja(self):
         if not os.path.exists(self.workspace.source_dir("ninja")):
             fatal_error(
@@ -548,20 +553,28 @@ class BuildScriptInvocation(object):
         # Currently we do not build external benchmarks by default.
         if args.build_external_benchmarks:
             impl_args += ["--skip-build-external-benchmarks=0"]
-        if not args.build_foundation:
-            impl_args += ["--skip-build-foundation"]
-        if not args.build_xctest:
-            impl_args += ["--skip-build-xctest"]
-        if not args.build_lldb:
-            impl_args += ["--skip-build-lldb"]
-        if not args.build_llbuild:
-            impl_args += ["--skip-build-llbuild"]
-        if not args.build_libcxx:
-            impl_args += ["--skip-build-libcxx"]
-        if not args.build_libdispatch:
-            impl_args += ["--skip-build-libdispatch"]
-        if not args.build_libicu:
-            impl_args += ["--skip-build-libicu"]
+
+        # Then add subproject install flags that either skip building them /or/
+        # if we are going to build them and install_all is set, we also install
+        # them.
+        conditional_subproject_configs = [
+            (args.build_cmark, "cmark"),
+            (args.build_llvm, "llvm"),
+            (args.build_swift, "swift"),
+            (args.build_foundation, "foundation"),
+            (args.build_xctest, "xctest"),
+            (args.build_lldb, "lldb"),
+            (args.build_llbuild, "llbuild"),
+            (args.build_libcxx, "libcxx"),
+            (args.build_libdispatch, "libdispatch"),
+            (args.build_libicu, "libicu")
+        ]
+        for (should_build, string_name) in conditional_subproject_configs:
+            if not should_build and not self.args.infer_dependencies:
+                impl_args += ["--skip-build-{}".format(string_name)]
+            elif self.install_all:
+                impl_args += ["--install-{}".format(string_name)]
+
         if args.build_swift_dynamic_stdlib:
             impl_args += ["--build-swift-dynamic-stdlib"]
         if args.build_swift_static_stdlib:
@@ -816,13 +829,16 @@ class BuildScriptInvocation(object):
         # FIXME: This is a weird division (returning a list of class objects),
         # but it matches the existing structure of the `build-script-impl`.
         impl_product_classes = []
-        impl_product_classes.append(products.CMark)
-        impl_product_classes.append(products.LLVM)
+        if self.args.build_cmark:
+            impl_product_classes.append(products.CMark)
+        if self.args.build_llvm:
+            impl_product_classes.append(products.LLVM)
         if self.args.build_libcxx:
             impl_product_classes.append(products.LibCXX)
         if self.args.build_libicu:
             impl_product_classes.append(products.LibICU)
-        impl_product_classes.append(products.Swift)
+        if self.args.build_swift:
+            impl_product_classes.append(products.Swift)
         if self.args.build_lldb:
             impl_product_classes.append(products.LLDB)
         if self.args.build_libdispatch:
@@ -868,6 +884,30 @@ class BuildScriptInvocation(object):
         for prod in product_classes:
             assert(not prod.is_build_script_impl_product())
 
+        # Now that we have our two lists of product_classes, if we are asked to
+        # infer dependencies, infer the dependencies now and then re-split the
+        # list.
+        if self.args.infer_dependencies:
+            combined = impl_product_classes + product_classes
+
+            # Now that we have produced the schedule, resplit. We require our
+            # dependencies to respect our build-script-impl property. This means
+            # that no build-script-impl products can have dependencies on
+            # non-build-script-impl products. Otherwise, it would be unsafe to
+            # re-order build-script-impl products in front of non
+            # build-script-impl products.
+            impl_product_classes = []
+            product_classes = []
+            is_darwin = platform.system() == 'Darwin'
+            final_schedule = build_graph.produce_scheduled_build(combined)[0]
+            for p in final_schedule:
+                if is_darwin and p.is_nondarwin_only_build_product():
+                    continue
+
+                if p.is_build_script_impl_product():
+                    impl_product_classes.append(p)
+                else:
+                    product_classes.append(p)
         return (impl_product_classes, product_classes)
 
     def execute(self):
@@ -956,7 +996,8 @@ class BuildScriptInvocation(object):
                     print("--- Running tests for %s ---" % product_name)
                     product.test(host_target)
                     print("--- Finished tests for %s ---" % product_name)
-                if product.should_install(host_target):
+                if product.should_install(host_target) or \
+                   (self.install_all and product.should_build(host_target)):
                     print("--- Installing %s ---" % product_name)
                     product.install(host_target)
 

utils/build_swift/build_swift/driver_arguments.py

@@ -345,6 +345,8 @@ def create_argument_parser():
            help='the path to install debug symbols into')
     option('--install-destdir', store_path,
            help='the path to use as the filesystem root for the installation')
+    option('--install-all', toggle_true,
+           help='Assume all built products should be installed')
 
     option(['-j', '--jobs'], store_int('build_jobs'),
            default=multiprocessing.cpu_count(),
@@ -545,6 +547,9 @@ def create_argument_parser():
     # -------------------------------------------------------------------------
     in_group('Options to select projects')
 
+    option('--infer', store_true('infer_dependencies'),
+           help='Infer any downstream dependencies from enabled projects')
+
     option(['-l', '--lldb'], store_true('build_lldb'),
            help='build LLDB')
 
@@ -1096,6 +1101,16 @@ def create_argument_parser():
 
     # -------------------------------------------------------------------------
     in_group('Build-script-impl arguments (for disambiguation)')
+
+    # We need to represent these options so that we can skip installing them if
+    # the user is running in install-all mode.
+    option('--skip-build-cmark', toggle_false('build_cmark'),
+           help='skip building cmark')
+    option('--skip-build-llvm', toggle_false('build_llvm'),
+           help='skip building llvm')
+    option('--skip-build-swift', toggle_false('build_swift'),
+           help='skip building swift')
+
     # We need to list --skip-test-swift explicitly because otherwise argparse
     # will auto-expand arguments like --skip-test-swift to the only known
     # argument --skip-test-swiftevolve.

utils/build_swift/tests/build_swift/test_presets.py

@@ -39,6 +39,7 @@
     'extra_swift_args': '',
     'install_destdir': '/tmp/install',
     'install_symroot': '/tmp/install/symroot',
+    'install_all': False,
     'install_toolchain_dir': '/tmp/install/toolchain',
     'install_prefix': '/usr',
     'installable_package': '/tmp/install/pkg',

utils/build_swift/tests/expected_options.py

@@ -61,6 +61,9 @@
     'build_cygwin': True,
     'build_external_benchmarks': False,
     'build_foundation': False,
+    'build_cmark': True,
+    'build_swift': True,
+    'build_llvm': True,
     'build_freebsd': True,
     'build_ios': True,
     'build_ios_device': False,
@@ -159,9 +162,11 @@
     'host_test': False,
     'only_executable_test': False,
     'only_non_executable_test': False,
+    'infer_dependencies': False,
     'install_prefix': targets.install_prefix(),
     'install_symroot': None,
     'install_destdir': None,
+    'install_all': False,
     'ios': False,
     'ios_all': False,
     'legacy_impl': False,
@@ -482,6 +487,7 @@ class BuildScriptImplOption(_BaseOption):
     SetTrueOption('-p', dest='build_swiftpm'),
 
     SetTrueOption('--legacy-impl', dest='legacy_impl'),
+    SetTrueOption('--infer', dest='infer_dependencies'),
 
     EnableOption('--android'),
     EnableOption('--build-external-benchmarks'),
@@ -537,6 +543,10 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--watchos'),
     EnableOption('--xctest', dest='build_xctest'),
 
+    DisableOption('--skip-build-cmark', dest='build_cmark'),
+    DisableOption('--skip-build-llvm', dest='build_llvm'),
+    DisableOption('--skip-build-swift', dest='build_swift'),
+
     DisableOption('--skip-build-android', dest='build_android'),
     DisableOption('--skip-build-benchmarks', dest='build_benchmarks'),
     DisableOption('--skip-build-cygwin', dest='build_cygwin'),
@@ -632,6 +642,7 @@ class BuildScriptImplOption(_BaseOption):
     PathOption('--install-prefix'),
     PathOption('--install-symroot'),
     PathOption('--install-destdir'),
+    EnableOption('--install-all'),
     PathOption('--symbols-package'),
     PathOption('--cmake-c-launcher'),
     PathOption('--cmake-cxx-launcher'),

utils/swift_build_support/swift_build_support/cmake.py

@@ -272,7 +272,7 @@ def check_cmake_version(self, source_root, build_root):
             cmake_binary = 'cmake'
 
         installed_ver = self.installed_cmake_version(cmake_binary)
-        if installed_ver > self.cmake_source_version(cmake_source_dir):
+        if installed_ver >= self.cmake_source_version(cmake_source_dir):
             return
         else:
             # Build CMake from source and return the path to the executable.

validation-test/BuildSystem/infer_implies_install_all.test

@@ -0,0 +1,10 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --infer --swiftpm --cmake %cmake 2>&1 | %FileCheck %s
+
+# CHECK: --- Installing cmark ---
+# CHECK: --- Installing llvm ---
+# CHECK: --- Installing swift ---
+# CHECK: --- Installing llbuild ---
+# CHECK: --- Building swiftpm ---
+# CHECK: --- Installing swiftpm ---

validation-test/BuildSystem/install_all.test

@@ -0,0 +1,7 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake 2>&1 | %FileCheck %s
+
+# CHECK: --- Installing cmark ---
+# CHECK: --- Installing llvm ---
+# CHECK: --- Installing swift ---

validation-test/BuildSystem/install_all_linux.test

@@ -0,0 +1,13 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --llbuild --swiftpm --foundation --libdispatch --cmake %cmake 2>&1 | %FileCheck %s
+
+# REQUIRES: OS=linux-gnu
+
+# CHECK-DAG: --- Installing cmark ---
+# CHECK-DAG: --- Installing swift ---
+# CHECK-DAG: --- Installing llvm ---
+# CHECK-DAG: --- Installing llbuild ---
+# CHECK-DAG: --- Installing foundation ---
+# CHECK-DAG: --- Installing libdispatch ---
+# CHECK-DAG: --- Installing swiftpm ---