Implement global index store cache

bazelbuild#567

swift and clang write index data based on the status of what resides in
`index-store-path`. When using a transient per-swift-library index, it
writes O( M imports * N libs) indexer data and slows down compilation
significantly: to the tune of 300% slow down and 6GB+ index data in my
testing. Bazel likes to use per `swift_library` data in order to remote
cache them, which needs extra consideration to work with the design of
swift and clang and preserve the performance

This PR does 2 things to make index while building use the original
model of the index-store so we don't have to patch clang and swift for
this yet. When the flag `-Xwrapped-swift-enable-global-index-store` is
set:

1. it writes to a global index cache, `bazel-out/global_index_store`
2. it copies indexstore data (records and units) to where Bazel
   expected them for caching: e.g. the original location with
   `swift.index_while_building`

Note that while this uses a `index-import` binary, it is very different
than how index-import currently works. In the variant this expects, it
is program that can _copy_ index data for specific compiler outputs.
This has the effect fo remote caching the subset that was used in
compilation for this library.

I added a detailed description of this feature in the RFC to add it
there MobileNativeFoundation/index-import#53.  In practice,
transitive imports will be indexed by deps and if they aren't cached
then the fallback is Xcode wil index transitive files if we set those as
deps on the unit files

Author: jerrymarino

swift/internal/compiling.bzl

@@ -46,6 +46,7 @@ load(
     "SWIFT_FEATURE_FULL_DEBUG_INFO",
     "SWIFT_FEATURE_INDEX_WHILE_BUILDING",
     "SWIFT_FEATURE_LAYERING_CHECK",
+    "SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2",
     "SWIFT_FEATURE_MODULE_MAP_HOME_IS_CWD",
     "SWIFT_FEATURE_NO_EMBED_DEBUG_MODULE",
     "SWIFT_FEATURE_NO_GENERATED_MODULE_MAP",
@@ -781,6 +782,7 @@ def compile_action_configs(
             configurators = [_index_while_building_configurator],
             features = [SWIFT_FEATURE_INDEX_WHILE_BUILDING],
         ),
+
         swift_toolchain_config.action_config(
             actions = [swift_action_names.COMPILE],
             configurators = [
@@ -793,6 +795,11 @@ def compile_action_configs(
                 SWIFT_FEATURE_DISABLE_SYSTEM_INDEX,
             ],
         ),
+        swift_toolchain_config.action_config(
+            actions = [swift_action_names.COMPILE],
+            configurators = [_index_while_building_configurator],
+            features = [SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2],
+        ),
 
         # User-defined conditional compilation flags (defined for Swift; those
         # passed directly to ClangImporter are handled above).
@@ -1685,6 +1692,15 @@ def compile(
     else:
         vfsoverlay_file = None
 
+    # For index while building v2 it uses a global index store which is
+    # configured by this flag
+    additional_copts = []
+    if is_feature_enabled(
+        feature_configuration = feature_configuration,
+        feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2,
+    ):
+        additional_copts.append("-Xwrapped-swift=-enable-global-index-store")
+
     prerequisites = struct(
         additional_inputs = additional_inputs,
         bin_dir = bin_dir,
@@ -1700,7 +1716,7 @@ def compile(
         source_files = srcs,
         transitive_modules = transitive_modules,
         transitive_swiftmodules = transitive_swiftmodules,
-        user_compile_flags = copts,
+        user_compile_flags = copts + additional_copts,
         vfsoverlay_file = vfsoverlay_file,
         vfsoverlay_search_path = _SWIFTMODULES_VFS_ROOT,
         # Merge the compile outputs into the prerequisites.
@@ -2027,10 +2043,19 @@ def _declare_compile_outputs(
     # Configure index-while-building if requested. IDEs and other indexing tools
     # can enable this feature on the command line during a build and then access
     # the index store artifacts that are produced.
-    index_while_building = is_feature_enabled(
+    index_while_building_v1 = is_feature_enabled(
         feature_configuration = feature_configuration,
         feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING,
     )
+    index_while_building_v2 = is_feature_enabled(
+        feature_configuration = feature_configuration,
+        feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2,
+    )
+
+    if index_while_building_v1 and index_while_building_v2:
+        fail("can't use both swift.index_while_building and swift.index_while_building_v2")
+
+    index_while_building = (index_while_building_v1 or index_while_building_v2)
     if (
         index_while_building and
         not _index_store_path_overridden(user_compile_flags)

swift/internal/feature_names.bzl

@@ -90,6 +90,10 @@ SWIFT_FEATURE_ENABLE_TESTING = "swift.enable_testing"
 # warnings otherwise.
 SWIFT_FEATURE_FULL_DEBUG_INFO = "swift.full_debug_info"
 
+# Index while building - using a global index store cache
+# https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/
+SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2 = "swift.index_while_building_v2"
+
 # If enabled, the compilation action for a target will produce an index store.
 SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building"
 

tools/worker/BUILD

@@ -18,6 +18,14 @@ config_setting(
     },
 )
 
+# Internal hinge for index while building V2 feature
+config_setting(
+    name = "index_while_building_v2",
+    values = {
+        "features": "swift.index_while_building_v2"
+    },
+)
+
 cc_library(
     name = "compile_with_worker",
     srcs = [
@@ -37,6 +45,18 @@ cc_library(
         "@com_github_nlohmann_json//:json",
         "@com_google_protobuf//:protobuf",
     ],
+    copts = select({
+        ":index_while_building_v2": [
+            "-DINDEX_IMPORT_PATH=\\\"$(rootpath @build_bazel_rules_swift_index_import//:index_import)\\\""
+        ],
+        "//conditions:default": [],
+    }),
+    data = select({
+        ":index_while_building_v2": [
+            "@build_bazel_rules_swift_index_import//:index_import"
+        ],
+        "//conditions:default": [],
+    })
 )
 
 cc_library(

tools/worker/work_processor.cc

@@ -22,9 +22,12 @@
 #include <nlohmann/json.hpp>
 #include <sstream>
 #include <string>
+#include <vector>
 
 #include "tools/common/file_system.h"
 #include "tools/common/path_utils.h"
+#include "tools/common/process.h"
+#include "tools/common/string_utils.h"
 #include "tools/common/temp_file.h"
 #include "tools/worker/output_file_map.h"
 #include "tools/worker/swift_runner.h"
@@ -63,6 +66,11 @@ void WorkProcessor::ProcessWorkRequest(
   bool is_wmo = false;
 
   std::string prev_arg;
+
+  std::string bazel_index_store_path;
+  bool enable_global_index_store = false;
+
+
   for (auto arg : request.arguments()) {
     auto original_arg = arg;
     // Peel off the `-output-file-map` argument, so we can rewrite it if
@@ -76,16 +84,41 @@ void WorkProcessor::ProcessWorkRequest(
       is_wmo = true;
     }
 
+    // Peel off index-store-path, we will conditionally pop on a new one if
+    // it passes -Xwrapped-swift-enable-global-index-store
+    if (arg == "-index-store-path") {
+      arg.clear();
+    } else if (prev_arg == "-index-store-path") {
+      bazel_index_store_path = arg;
+      arg.clear();
+    }
+    if (arg == "-Xwrapped-swift=-enable-global-index-store") {
+      arg.clear();
+      enable_global_index_store = true;
+    }
+
     if (!arg.empty()) {
       params_file_stream << arg << '\n';
     }
 
     prev_arg = original_arg;
   }
 
+  // Material within this index store is namespaced to arcitecture, os, etc
+  auto exec_root = GetCurrentDirectory();
+  auto global_index_store = exec_root + "/bazel-out/global_index_store";
+  if (enable_global_index_store) {
+    processed_args.push_back("-index-store-path");
+    processed_args.push_back(global_index_store);
+  } else if (!bazel_index_store_path.empty()) {
+    processed_args.push_back("-index-store-path");
+    processed_args.push_back(bazel_index_store_path);
+  }
+
   if (!output_file_map_path.empty()) {
+    output_file_map.ReadFromPath(output_file_map_path);
+
     if (!is_wmo) {
-      output_file_map.ReadFromPath(output_file_map_path);
 
       // Rewrite the output file map to use the incremental storage area and
       // pass the compiler the path to the rewritten file.
@@ -129,7 +162,6 @@ void WorkProcessor::ProcessWorkRequest(
   SwiftRunner swift_runner(processed_args, /*force_response_file=*/true);
 
   int exit_code = swift_runner.Run(&stderr_stream, /*stdout_to_stderr=*/true);
-
   if (!is_wmo) {
     // Copy the output files from the incremental storage area back to the
     // locations where Bazel declared the files.
@@ -143,6 +175,43 @@ void WorkProcessor::ProcessWorkRequest(
     }
   }
 
+  // Import global indexes if compilation succeeds
+  if (exit_code != EXIT_FAILURE && enable_global_index_store) {
+    std::vector<std::string> ii_args;
+
+    // The index-import runfile path is pased as a define
+    #if defined(INDEX_IMPORT_PATH)
+      ii_args.push_back(INDEX_IMPORT_PATH);
+    #else
+      // Logical error
+      std::cerr << "Incorrectly compiled work_processor.cc";
+      exit_code = EXIT_FAILURE;
+    #endif
+
+    // Use the output path map to deterine what compilation ouputs to import
+    auto outputs  = output_file_map.incremental_outputs();
+    std::map<std::string, std::string>::iterator it;
+    for (it = outputs.begin(); it != outputs.end(); it++) {
+      // Need the actual output paths of the compiler - not bazel
+      auto output_path =  is_wmo ? it->first : it->second;
+
+      auto file_type = output_path.substr(output_path.find_last_of(".") + 1);
+      if (file_type == "o") {
+        ii_args.push_back("-import-output-file");
+        ii_args.push_back(output_path);
+      }
+    }
+
+    // Copy back from the global index store to bazel's index store
+    ii_args.push_back(global_index_store);
+
+    ii_args.push_back(exec_root + "/" + bazel_index_store_path);
+
+    const std::vector<std::string>& ii_args_ = ii_args;
+    // Dupes the output into the action for failure cases
+    exit_code = RunSubProcess(ii_args_, &stderr_stream, /*stdout_to_stderr=*/true);
+  }
+
   response->set_exit_code(exit_code);
   response->set_output(stderr_stream.str());
 }