Implement 'first pass' of index while building V2 (#217)

* Implement 'first pass' of index while building V2

This concludes the first step if mitigating perf issues and first PR in
the series for index while building - docs/index_while_building.md.

Hinging on the feature, swift.index_while_building_v2, it moves
rules_ios to use a global index and pulls in the swift PR
bazelbuild/rules_swift#567, and index-import PR
MobileNativeFoundation/index-import#53 .

In hopes of mitigating performance problems of processing the global
index in this way it flips the `-incremental` bit in the latest flag.
Longer term we will not be processing a global index, per
docs/index_while_building.md so this is temporary measure.

This PR implements line items from 'Index while building V2 - first
pass' in the roadmap for this feature defined in
docs/index_while_building.md.

Author: jerrymarino

.bazelrc

@@ -20,3 +20,10 @@ build --deleted_packages tests/ios/frameworks/sources-with-prebuilt-binaries
 # Enable dbg compilation mode in this repo, so we can test xcodeproj-built
 # binaries contain debug symbol tables.
 build --compilation_mode=dbg
+
+# Use 'worker' strategy for swift compilation
+build --strategy=SwiftCompile=worker
+
+# This flips index_while_building_v2 - see docs/index_while_building.md for a
+# detailed summary
+build --features swift.index_while_building_v2

BUILD.bazel

@@ -0,0 +1,15 @@
+# Pull buildifer.mac as an http_file, then depend on the file group to make an
+# executable
+load("@build_bazel_rules_swift//swift/internal:feature_names.bzl", "SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2")
+
+sh_binary(
+    name = "buildifier",
+    srcs = ["@buildifier.mac//file"],
+)
+
+config_setting(
+    name = "index_while_building_v2",
+    values = {
+        "features": SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2,
+    },
+)

rules/library.bzl

@@ -582,17 +582,31 @@ def apple_library(name, library_tools = {}, export_private_headers = True, names
         lib_names.append(cpp_libname)
 
     additional_objc_copts.append("-I.")
+    index_while_building_objc_copts = select({
+        "@build_bazel_rules_ios//:index_while_building_v2": [
+            # Note: this won't work work for remote caching yet. It uses a
+            # _different_ global index for objc than so that the BEP grep in
+            # rules_ios picks this up.
+            # Checkout the task roadmap for future improvements:
+            # Docs/index_while_building.md
+            "-index-store-path",
+            "bazel-out/rules_ios_global_index_store.indexstore",
+        ],
+        "//conditions:default": [
+            "-index-store-path",
+            "$(GENDIR)/{package}/rules_ios_objc_library_{libname}.indexstore".format(
+                package = native.package_name(),
+                libname = objc_libname,
+            ),
+        ],
+    })
 
-    additional_objc_copts.extend(("-index-store-path", "$(GENDIR)/{package}/rules_ios_objc_library_{libname}.indexstore".format(
-        package = native.package_name(),
-        libname = objc_libname,
-    )))
     objc_library(
         name = objc_libname,
         srcs = objc_sources + objc_private_hdrs + objc_non_exported_hdrs,
         non_arc_srcs = objc_non_arc_sources,
         hdrs = objc_hdrs,
-        copts = copts_by_build_setting.objc_copts + objc_copts + additional_objc_copts,
+        copts = copts_by_build_setting.objc_copts + objc_copts + additional_objc_copts + index_while_building_objc_copts,
         deps = deps + private_deps + lib_names,
         module_map = module_map,
         sdk_dylibs = sdk_dylibs,

rules/repositories.bzl

@@ -53,13 +53,15 @@ def rules_ios_dependencies():
         sha256 = "1fff3fa1e565111a8f678b4698792101844f57b2e78c5e374431d0ebe97f6b6c",
     )
 
+    # Note: this ref is a cherry-pick of the rules_swift PR
+    # https://github.com/bazelbuild/rules_swift/pull/567
     _maybe(
         github_repo,
         name = "build_bazel_rules_swift",
-        ref = "ed81c15f9b577880c13b987101100cbac03f45c2",
-        project = "bazelbuild",
+        ref = "703165622cf87cabe253bf746a8129f8021ec001",
+        project = "bazel-ios",
         repo = "rules_swift",
-        sha256 = "9527ef2617be16115ed514d442b6d53d8d824054fd97e5b3ab689fb9d978b8ed",
+        sha256 = "cf553875aae12744846b5e484879098e9c9153883febfc4074aa9305765a923f",
     )
 
     _maybe(
@@ -81,9 +83,12 @@ def rules_ios_dependencies():
         sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c",
     )
 
+    # Note: it relies on `index-import` to import indexes. Longer term this
+    # dependency may be added by rules_swift
+    # This release is a build of this PR https://github.com/lyft/index-import/pull/53
     _maybe(
         http_archive,
-        name = "com_github_lyft_index_import",
+        name = "build_bazel_rules_swift_index_import",
         build_file_content = """\
 load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
 
@@ -108,9 +113,9 @@ native_binary(
     visibility = ["//visibility:public"],
 )
 """,
-        canonical_id = "index-import-5.2.1.4",
-        urls = ["https://github.com/lyft/index-import/releases/download/5.2.1.4/index-import.zip"],
-        sha256 = "62f42816baf3b690682b5d6fe543a3c5a4a6ea7499ce1f4e8326c7bd2175989a",
+        canonical_id = "index-import-5.3.2.5",
+        urls = ["https://github.com/bazel-ios/index-import/releases/download/5.3.2.5/index-import.zip"],
+        sha256 = "79e9b2cd3e988155b86668c56d95705e1a4a7c7b6d702ff5ded3a18d1291a39a",
     )
 
     _maybe(

rules/xcodeproj.bzl

@@ -837,7 +837,7 @@ https://www.rubydoc.info/github/CocoaPods/Xcodeproj/Xcodeproj/Constants
         "_workspace_checks": attr.label(executable = False, default = Label("//tools/xcodeproj_shims:IDEWorkspaceChecks.plist"), allow_single_file = ["plist"]),
         "output_processor": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:output-processor.rb"), cfg = "host", allow_single_file = True),
         "_xcodegen": attr.label(executable = True, default = Label("@com_github_yonaskolb_xcodegen//:xcodegen"), cfg = "host"),
-        "index_import": attr.label(executable = True, default = Label("@com_github_lyft_index_import//:index_import"), cfg = "host"),
+        "index_import": attr.label(executable = True, default = Label("@build_bazel_rules_swift_index_import//:index_import"), cfg = "host"),
         "clang_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:clang-stub"), cfg = "host"),
         "ld_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:ld-stub"), cfg = "host"),
         "swiftc_stub": attr.label(executable = True, default = Label("//tools/xcodeproj_shims:swiftc-stub"), cfg = "host"),

tests/ios/xcodeproj/Single-Static-Framework-Project.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/ios/xcodeproj/Test-Imports-App-Project.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/ios/xcodeproj/Test-MultipleConfigs-Project-WithTransitiveFlag.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/ios/xcodeproj/Test-MultipleConfigs-Project.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/ios/xcodeproj/TestWithHostApp.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/macos/xcodeproj/Single-Application-Project-AllTargets.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/macos/xcodeproj/Single-Application-Project-DirectTargetsOnly.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tests/macos/xcodeproj/Test-Target-With-Test-Host-Project.xcodeproj/bazelinstallers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \

tools/xcodeproj_shims/installers/_indexstore.sh

@@ -32,6 +32,7 @@ readonly xcode_external="$BAZEL_WORKSPACE_ROOT/bazel-$(basename "$SRCROOT")/exte
 
 
 $BAZEL_INSTALLERS_DIR/index-import \
+    -incremental \
     -remap "$bazel_module=$xcode_module" \
     -remap "$bazel_swift_object=$xcode_object" \
     -remap "$bazel_objc_object=$xcode_object" \