Merge branch 'main' into tbkka-remoteMirror-MPE-spareBits-v2

Author: tbkka

CMakeLists.txt

@@ -192,6 +192,13 @@ set(SWIFT_TOOLS_ENABLE_LTO OFF CACHE STRING "Build Swift tools with LTO. One
     option only affects the tools that run on the host (the compiler), and has
     no effect on the target libraries (the standard library and the runtime).")
 
+option(SWIFT_TOOLS_LD64_LTO_CODEGEN_ONLY_FOR_SUPPORTING_TARGETS
+    "When building ThinLTO using ld64 on Darwin, controls whether to opt out of
+    LLVM IR optimizations when linking targets that will get
+    little benefit from it (e.g. tools for bootstrapping or
+    debugging Swift)"
+    FALSE)
+
 option(BOOTSTRAPPING_MODE [=[
 How to build the swift compiler modules. Possible values are
     OFF:           build without swift modules

cmake/modules/AddSwift.cmake

@@ -636,7 +636,7 @@ macro(add_swift_lib_subdirectory name)
 endmacro()
 
 function(add_swift_host_tool executable)
-  set(options HAS_SWIFT_MODULES)
+  set(options HAS_SWIFT_MODULES THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY)
   set(single_parameter_options SWIFT_COMPONENT BOOTSTRAPPING)
   set(multiple_parameter_options LLVM_LINK_COMPONENTS)
 
@@ -859,6 +859,19 @@ function(add_swift_host_tool executable)
     endif()
   endif()
 
+  if(ASHT_THINLTO_LD64_ADD_FLTO_CODEGEN_ONLY)
+    string(CONCAT lto_codegen_only_link_options
+      "$<"
+        "$<AND:"
+          "$<BOOL:${LLVM_LINKER_IS_LD64}>,"
+          "$<BOOL:${SWIFT_TOOLS_LD64_LTO_CODEGEN_ONLY_FOR_SUPPORTING_TARGETS}>,"
+          "$<STREQUAL:${SWIFT_TOOLS_ENABLE_LTO},thin>"
+        ">:"
+        "LINKER:-flto-codegen-only"
+      ">")
+    target_link_options(${executable} PRIVATE "${lto_codegen_only_link_options}")
+  endif()
+
   if(NOT ${ASHT_SWIFT_COMPONENT} STREQUAL "no_component")
     add_dependencies(${ASHT_SWIFT_COMPONENT} ${executable})
     swift_install_in_component(TARGETS ${executable}

cmake/modules/SwiftComponents.cmake

@@ -64,11 +64,13 @@
 #   Swift code.
 # * toolchain-tools -- a subset of tools that we will install to the OSS toolchain.
 # * testsuite-tools -- extra tools required to run the Swift testsuite.
+# * parser-lib -- Build the syntax parser library used by SwiftSyntax.
+# * static-mirror-lib -- Build the static mirror library used by SwiftStaticMirror.
 # * toolchain-dev-tools -- install development tools useful in a shared toolchain
 # * llvm-toolchain-dev-tools -- install LLVM development tools useful in a shared toolchain
 # * dev -- headers and libraries required to use Swift compiler as a library.
 set(_SWIFT_DEFINED_COMPONENTS
-  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
+  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;stdlib;stdlib-experimental;sdk-overlay;parser-lib;static-mirror-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
 
 # The default install components include all of the defined components, except
 # for the following exceptions.

cmake/modules/UnixCompileRules.cmake

@@ -11,3 +11,20 @@ set(CMAKE_C_ARCHIVE_FINISH "")
 set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> crs <TARGET> <LINK_FLAGS> <OBJECTS>")
 set(CMAKE_CXX_ARCHIVE_APPEND "<CMAKE_AR> qs <TARGET> <LINK_FLAGS> <OBJECTS>")
 set(CMAKE_CXX_ARCHIVE_FINISH "")
+
+# When archiving LTO-based .o files with ar/ranlib/libtool on Darwin, the tools
+# use libLTO.dylib to inspect the bitcode files. However, by default the
+# "host" libLTO.dylib is loaded, which might be too old and not understand our
+# just-built bitcode format. So let's instead ask ar/ranlib/libtool to use the
+# just-built libLTO.dylib from the toolchain that we're using to build.
+if(APPLE AND SWIFT_NATIVE_CLANG_TOOLS_PATH)
+  set(liblto_path "${SWIFT_NATIVE_CLANG_TOOLS_PATH}/../lib/libLTO.dylib")
+
+  set(CMAKE_C_ARCHIVE_CREATE "${CMAKE_COMMAND} -E env LIBLTO_PATH=${liblto_path} <CMAKE_AR> crs <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_C_ARCHIVE_APPEND "${CMAKE_COMMAND} -E env LIBLTO_PATH=${liblto_path} <CMAKE_AR> qs <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_C_ARCHIVE_FINISH "")
+
+  set(CMAKE_CXX_ARCHIVE_CREATE "${CMAKE_COMMAND} -E env LIBLTO_PATH=${liblto_path} <CMAKE_AR> crs <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_CXX_ARCHIVE_APPEND "${CMAKE_COMMAND} -E env LIBLTO_PATH=${liblto_path} <CMAKE_AR> qs <TARGET> <LINK_FLAGS> <OBJECTS>")
+  set(CMAKE_CXX_ARCHIVE_FINISH "")
+endif()

docs/DebuggingTheCompiler.md

@@ -714,39 +714,31 @@ which causes the miscompile.
 Currently there is no tool to automatically identify the bad optimization, but
 it's quite easy to do this manually:
 
-1. Find the offending optimization with bisecting:
-
-  a. Add the compiler option `-Xllvm -sil-opt-pass-count=<n>`, where `<n>`
-     is the number of optimizations to run.
-
-  b. Bisect: find n where the executable crashes, but does not crash
-     with n-1. First just try n = 10, 100, 1000, 10000, etc. to find
-     an upper bound). Then can either bisect the invocation by hand or
-     place the invocation into a script and use
-     `./llvm-project/llvm/utils/bisect` to automatically bisect
-     based on the scripts error code. Example invocation:
-
-       bisect --start=0 --end=10000 ./invoke_swift_passing_N.sh "%(count)s"
-
-  c. Once one finds `n`, Add another option `-Xllvm -sil-print-pass-name`. The output can be
-     large, so it's best to redirect stderr to a file (`2> output`).
-     In the output search for the last pass before `stage Address Lowering`.
-     It should be the `Run #<n-1>`. This line tells you the name of the bad
-     optimization pass and on which function it run.
-
-2. Get the SIL before and after the bad optimization.
-
-  a. Add the compiler option
-     `-Xllvm -sil-print-function='<function>'`
-     where `<function>` is the function name (including the preceding `$`).
-     For example:
-     `-Xllvm -sil-print-function='$s4test6testityS2iF'`.
-     Again, the output can be large, so it's best to redirect stderr to a file.
-  b. From the output, copy the SIL of the function *before* the bad
-     run into a separate file and the SIL *after* the bad run into a file.
-  c. Compare both SIL files and try to figure out what the optimization pass
-     did wrong. To simplify the comparison, it's sometimes helpful to replace
-     all SIL values (e.g. `%27`) with a constant string (e.g. `%x`).
+1. Add the compiler option `-Xllvm -sil-opt-pass-count=<n>`, where `<n>`
+   is the number of optimizations to run.
+
+2. Bisect: find n where the executable crashes, but does not crash
+   with n-1. First just try n = 10, 100, 1000, 10000, etc. to find
+   an upper bound). Then can either bisect the invocation by hand or
+   place the invocation into a script and use
+   `./llvm-project/llvm/utils/bisect` to automatically bisect
+   based on the scripts error code. Example invocation:
+
+     bisect --start=0 --end=10000 ./invoke_swift_passing_N.sh "%(count)s"
+
+3. Add another option `-Xllvm -sil-print-last`. The output can be
+   large, so it's best to redirect stderr to a file (`2> output`).
+   The output contains the SIL before and after the bad optimization.
+
+4. Copy the two functions from the output into separate files and
+   compare both files. Try to figure out what the optimization pass
+   did wrong. To simplify the comparison, it's sometimes helpful to replace
+   all SIL values (e.g. `%27`) with a constant string (e.g. `%x`).
+
+5. If the bad optimization is SILCombine or SimplifyCFG (which do a lot of
+   transformations in a single run) it's helpful to continue bisecting on
+   the sub-pass number. The option `-Xllvm -sil-opt-pass-count=<n>.<m>`
+   can be used for that, where `m` is the sub-pass number.
 
 ### Using git-bisect in the presence of branch forwarding/feature branches
 

docs/Lexicon.md

@@ -594,6 +594,16 @@ of `Optional<Int>`.) Sugared types preserve information about the form
 and use of the type even though the behavior usually does not change
 (except for things like access control). Contrast with [canonical type](#canonical-type).
 
+## TBD
+
+Text-based dynamic library files (TBDs) are a textual representation of
+the information in a dynamic library / shared library that is required
+by the static linker.
+
+Apple’s SDKs originally used Mach-O Dynamic Library Stubs. Mach-O Dynamic
+Library Stubs are dynamic library files, but with all the text and data
+stripped out.
+
 ## thunk
 
 In the Swift compiler, a synthesized function whose only purpose is to

docs/README.md

@@ -102,6 +102,8 @@ documentation, please create a thread on the Swift forums under the
 - [DependencyAnalysis.md](/docs/DependencyAnalysis.md):
   Describes different kinds of dependencies across files in the same module,
   important for understanding incremental builds.
+- [DifferentiableProgrammingImplementation.md](/docs/DifferentiableProgrammingImplementation.md):
+  Describes how automatic differentiation is implemented in the Swift compiler.
 - C and ObjC interoperability: Clang Importer and PrintAsClang
   - [CToSwiftNameTranslation.md](/docs/CToSwiftNameTranslation.md):
     Describes how C and ObjC entities are imported into Swift

docs/ReferenceGuides/UnderscoredAttributes.md

@@ -39,6 +39,15 @@ Most notably, default argument expressions are implicitly
 `@_alwaysEmitIntoClient`, which means that adding a default argument to a
 function which did not have one previously does not break ABI.
 
+## `@_backDeploy(availabilitySpec ...)`
+
+Causes the body of a function to be emitted into the module interface to be
+available for inlining in clients with deployment targets lower than the formal
+availability of the function. When inlined, the body of the function is
+transformed such that it calls the library's copy of the function if it is
+available at runtime. Otherwise, the copy of the original function body is
+executed.
+
 ## `@_assemblyVision`
 
 Forces emission of assembly vision remarks for a function or method, showing

docs/SIL.rst

@@ -2065,7 +2065,7 @@ called a "forwarding instruction" and any use with such a user instruction a
 "forwarding use". This inference generally occurs upon instruction construction
 and as a result:
 
-* When manipulating forwarding instructions programatically, one must manually
+* When manipulating forwarding instructions programmatically, one must manually
   update their forwarded ownership since most of the time the ownership is
   stored in the instruction itself. Don't worry though because the SIL verifier
   will catch this error for you if you forget to do so!
@@ -2075,7 +2075,7 @@ and as a result:
   parsed operand.
   In some cases the forwarding ownership kind is different from the ownership kind
   of its operand. In such cases, textual SIL represents the forwarding ownership kind
-  explicity.
+  explicitly.
   Eg: ::
 
     %cast = unchecked_ref_cast %val : $Klass to $Optional<Klass>, forwarding: @unowned

include/swift-c/CommonString/CommonString.h

@@ -0,0 +1,35 @@
+//===--- CommonString.h - C API for Swift Dependency Scanning ---*- C -*-===//
+ //
+ // This source file is part of the Swift.org open source project
+ //
+ // Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ // Licensed under Apache License v2.0 with Runtime Library Exception
+ //
+ // See https://swift.org/LICENSE.txt for license information
+ // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+ //
+ //===----------------------------------------------------------------------===//
+
+ #ifndef SWIFT_C_LIB_SWIFT_COMMON_STRING_H
+ #define SWIFT_C_LIB_SWIFT_COMMON_STRING_H
+
+ #include <stdbool.h>
+ #include <stddef.h>
+ #include <stdint.h>
+
+ /**
+  * A character string used to pass around dependency scan result metadata.
+  * Lifetime of the string is strictly tied to the object whose field it
+  * represents. When the owning object is released, string memory is freed.
+  */
+ typedef struct {
+   const void *data;
+   size_t length;
+ } swiftscan_string_ref_t;
+
+ typedef struct {
+   swiftscan_string_ref_t *strings;
+   size_t count;
+ } swiftscan_string_set_t;
+
+ #endif // SWIFT_C_LIB_SWIFT_COMMON_STRING_H

include/swift-c/DependencyScan/BinaryScan.h

@@ -15,15 +15,21 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "DependencyScanMacros.h"
-#include "CommonString.h"
-
 #ifndef SWIFT_C_DEPENDENCY_SCAN_H
 #define SWIFT_C_DEPENDENCY_SCAN_H
 
+#include "DependencyScanMacros.h"
+#include "swift-c/CommonString/CommonString.h"
+
+/// The version constants for the SwiftDependencyScan C API.
+/// SWIFTSCAN_VERSION_MINOR should increase when there are API additions.
+/// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes.
+#define SWIFTSCAN_VERSION_MAJOR 0
+#define SWIFTSCAN_VERSION_MINOR 2
+
 SWIFTSCAN_BEGIN_DECLS
 
-//=== Public Dependency Scanner Data Types -------------------------------===//
+//=== Public Scanner Data Types -------------------------------------------===//
 
 typedef enum {
   // This dependency info encodes two ModuleDependencyKind types:
@@ -345,8 +351,6 @@ swiftscan_scanner_cache_load(swiftscan_scanner_t scanner,
 SWIFTSCAN_PUBLIC void
 swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner);
 
-//=== Experimental compiler invocation operations -------------------------===//
-
 /// An entry point to invoke the compiler via a library call.
 SWIFTSCAN_PUBLIC int invoke_swift_compiler(int argc, const char **argv);