[7.1.0] Reproducible extension (#21306)

- Allow module extension to declare it is reproducible and opt-out from
the lockfile
- Update bazelrc command from build to common to keep the same
StarlarkSemantics for other commands
- Update fetch option documentation

---------

Co-authored-by: Xùdōng Yáng <[email protected]>

Author: SalmaSamy

.bazelrc

@@ -61,7 +61,7 @@ build --java_language_version=11
 build --tool_java_language_version=11
 
 # Fail if a glob doesn't match anything (https://github.com/bazelbuild/bazel/issues/8195)
-build --incompatible_disallow_empty_glob
+common --incompatible_disallow_empty_glob
 
 # Manually enable cc toolchain resolution before it is flipped. https://github.com/bazelbuild/bazel/issues/7260
 build --incompatible_enable_cc_toolchain_resolution

site/en/external/overview.md

@@ -94,6 +94,18 @@ Normally, Bazel only fetches a repo when it needs something from the repo,
 and the repo hasn't already been fetched. If the repo has already been fetched
 before, Bazel only re-fetches it if its definition has changed.
 
+The `fetch` command can be used to initiate a pre-fetch for a repository,
+target, or all necessary repositories to perform any build. This capability
+enables offline builds using the `--nofetch` option.
+
+The `--fetch` option serves to manage network access. Its default value is true.
+However, when set to false (`--nofetch`), the command will utilize any cached
+version of the dependency, and if none exists, the command will result in
+failure.
+
+See [fetch options](/reference/command-line-reference#fetch-options) for more
+information about controlling fetch.
+
 ### Directory layout {:#directory-layout}
 
 After being fetched, the repo can be found in the subdirectory `external` in the

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java

@@ -121,18 +121,23 @@ public void afterCommand() throws AbruptExitException {
 
     // Add the new resolved extensions
     for (var event : extensionResolutionEventsMap.values()) {
+      LockFileModuleExtension extension = event.getModuleExtension();
+      if (!extension.shouldLockExtesnsion()) {
+        continue;
+      }
+
       var oldExtensionEntries = updatedExtensionMap.get(event.getExtensionId());
       ImmutableMap<ModuleExtensionEvalFactors, LockFileModuleExtension> extensionEntries;
       if (oldExtensionEntries != null) {
         // extension exists, add the new entry to the existing map
         extensionEntries =
             new ImmutableMap.Builder<ModuleExtensionEvalFactors, LockFileModuleExtension>()
                 .putAll(oldExtensionEntries)
-                .put(event.getExtensionFactors(), event.getModuleExtension())
+                .put(event.getExtensionFactors(), extension)
                 .buildKeepingLast();
       } else {
         // new extension
-        extensionEntries = ImmutableMap.of(event.getExtensionFactors(), event.getModuleExtension());
+        extensionEntries = ImmutableMap.of(event.getExtensionFactors(), extension);
       }
       updatedExtensionMap.put(event.getExtensionId(), extensionEntries);
     }
@@ -164,12 +169,13 @@ private boolean shouldKeepExtension(
     // If there is a new event for this extension, compare it with the existing ones
     ModuleExtensionResolutionEvent extEvent = extensionResolutionEventsMap.get(extensionId);
     if (extEvent != null) {
+      boolean doNotLockExtension = !extEvent.getModuleExtension().shouldLockExtesnsion();
       boolean dependencyOnOsChanged =
           lockedExtensionKey.getOs().isEmpty() != extEvent.getExtensionFactors().getOs().isEmpty();
       boolean dependencyOnArchChanged =
           lockedExtensionKey.getArch().isEmpty()
               != extEvent.getExtensionFactors().getArch().isEmpty();
-      if (dependencyOnOsChanged || dependencyOnArchChanged) {
+      if (doNotLockExtension || dependencyOnOsChanged || dependencyOnArchChanged) {
         return false;
       }
     }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java

@@ -54,6 +54,11 @@ public static Builder builder() {
 
   public abstract Builder toBuilder();
 
+  public boolean shouldLockExtesnsion() {
+    return getModuleExtensionMetadata().isEmpty()
+        || !getModuleExtensionMetadata().get().getReproducible();
+  }
+
   /** Builder type for {@link LockFileModuleExtension}. */
   @AutoValue.Builder
   public abstract static class Builder {

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java

@@ -206,11 +206,24 @@ public boolean rootModuleHasNonDevDependency() {
               @ParamType(type = String.class),
               @ParamType(type = NoneType.class)
             }),
+        @Param(
+            name = "reproducible",
+            doc =
+                "States that this module extension ensures complete reproducibility, thereby it "
+                    + "should not be stored in the lockfile.",
+            positional = false,
+            named = true,
+            defaultValue = "False",
+            allowedTypes = {
+              @ParamType(type = Boolean.class),
+            }),
       })
   public ModuleExtensionMetadata extensionMetadata(
-      Object rootModuleDirectDepsUnchecked, Object rootModuleDirectDevDepsUnchecked)
+      Object rootModuleDirectDepsUnchecked,
+      Object rootModuleDirectDevDepsUnchecked,
+      boolean reproducible)
       throws EvalException {
     return ModuleExtensionMetadata.create(
-        rootModuleDirectDepsUnchecked, rootModuleDirectDevDepsUnchecked);
+        rootModuleDirectDepsUnchecked, rootModuleDirectDevDepsUnchecked, reproducible);
   }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java

@@ -63,38 +63,44 @@ public abstract class ModuleExtensionMetadata implements StarlarkValue {
 
   abstract UseAllRepos getUseAllRepos();
 
+  abstract boolean getReproducible();
+
   private static ModuleExtensionMetadata create(
       @Nullable Set<String> explicitRootModuleDirectDeps,
       @Nullable Set<String> explicitRootModuleDirectDevDeps,
-      UseAllRepos useAllRepos) {
+      UseAllRepos useAllRepos,
+      boolean reproducible) {
     return new AutoValue_ModuleExtensionMetadata(
         explicitRootModuleDirectDeps != null
             ? ImmutableSet.copyOf(explicitRootModuleDirectDeps)
             : null,
         explicitRootModuleDirectDevDeps != null
             ? ImmutableSet.copyOf(explicitRootModuleDirectDevDeps)
             : null,
-        useAllRepos);
+        useAllRepos,
+        reproducible);
   }
 
   static ModuleExtensionMetadata create(
-      Object rootModuleDirectDepsUnchecked, Object rootModuleDirectDevDepsUnchecked)
+      Object rootModuleDirectDepsUnchecked,
+      Object rootModuleDirectDevDepsUnchecked,
+      boolean reproducible)
       throws EvalException {
     if (rootModuleDirectDepsUnchecked == Starlark.NONE
         && rootModuleDirectDevDepsUnchecked == Starlark.NONE) {
-      return create(null, null, UseAllRepos.NO);
+      return create(null, null, UseAllRepos.NO, reproducible);
     }
 
     // When root_module_direct_deps = "all", accept both root_module_direct_dev_deps = None and
     // root_module_direct_dev_deps = [], but not root_module_direct_dev_deps = ["some_repo"].
     if (rootModuleDirectDepsUnchecked.equals("all")
         && rootModuleDirectDevDepsUnchecked.equals(StarlarkList.immutableOf())) {
-      return create(null, null, UseAllRepos.REGULAR);
+      return create(null, null, UseAllRepos.REGULAR, reproducible);
     }
 
     if (rootModuleDirectDevDepsUnchecked.equals("all")
         && rootModuleDirectDepsUnchecked.equals(StarlarkList.immutableOf())) {
-      return create(null, null, UseAllRepos.DEV);
+      return create(null, null, UseAllRepos.DEV, reproducible);
     }
 
     if (rootModuleDirectDepsUnchecked.equals("all")
@@ -152,7 +158,11 @@ static ModuleExtensionMetadata create(
       }
     }
 
-    return create(explicitRootModuleDirectDeps, explicitRootModuleDirectDevDeps, UseAllRepos.NO);
+    return create(
+        explicitRootModuleDirectDeps,
+        explicitRootModuleDirectDevDeps,
+        UseAllRepos.NO,
+        reproducible);
   }
 
   public void evaluate(

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java

@@ -152,20 +152,27 @@ public SkyValue compute(SkyKey skyKey, Environment env)
     }
 
     // Check the lockfile first for that module extension
+    LockFileModuleExtension lockedExtension = null;
     LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
     if (!lockfileMode.equals(LockfileMode.OFF)) {
       BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
       if (lockfile == null) {
         return null;
       }
-      try {
-        SingleExtensionEvalValue singleExtensionEvalValue =
-            tryGettingValueFromLockFile(env, extensionId, extension, usagesValue, lockfile);
-        if (singleExtensionEvalValue != null) {
-          return singleExtensionEvalValue;
+      var lockedExtensionMap = lockfile.getModuleExtensions().get(extensionId);
+      lockedExtension =
+          lockedExtensionMap == null ? null : lockedExtensionMap.get(extension.getEvalFactors());
+      if (lockedExtension != null) {
+        try {
+          SingleExtensionEvalValue singleExtensionEvalValue =
+              tryGettingValueFromLockFile(
+                  env, extensionId, extension, usagesValue, lockfile, lockedExtension);
+          if (singleExtensionEvalValue != null) {
+            return singleExtensionEvalValue;
+          }
+        } catch (NeedsSkyframeRestartException e) {
+          return null;
         }
-      } catch (NeedsSkyframeRestartException e) {
-        return null;
       }
     }
 
@@ -182,6 +189,24 @@ public SkyValue compute(SkyKey skyKey, Environment env)
     Optional<ModuleExtensionMetadata> moduleExtensionMetadata =
         moduleExtensionResult.getModuleExtensionMetadata();
 
+    if (lockfileMode.equals(LockfileMode.ERROR)) {
+      boolean extensionShouldHaveBeenLocked =
+          moduleExtensionMetadata.map(metadata -> !metadata.getReproducible()).orElse(true);
+      // If this extension was not found in the lockfile, and after evaluation we found that it is
+      // not reproducible, then error indicating that it was expected to be in the lockfile.
+      if (lockedExtension == null && extensionShouldHaveBeenLocked) {
+        throw new SingleExtensionEvalFunctionException(
+            ExternalDepsException.withMessage(
+                Code.BAD_MODULE,
+                "The module extension '%s'%s does not exist in the lockfile",
+                extensionId,
+                extension.getEvalFactors().isEmpty()
+                    ? ""
+                    : " for platform " + extension.getEvalFactors()),
+            Transience.PERSISTENT);
+      }
+    }
+
     // At this point the extension has been evaluated successfully, but SingleExtensionEvalFunction
     // may still fail if imported repositories were not generated. However, since imports do not
     // influence the evaluation of the extension and the validation also runs when the extension
@@ -220,30 +245,13 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile(
       ModuleExtensionId extensionId,
       RunnableExtension extension,
       SingleExtensionUsagesValue usagesValue,
-      BazelLockFileValue lockfile)
+      BazelLockFileValue lockfile,
+      LockFileModuleExtension lockedExtension)
       throws SingleExtensionEvalFunctionException,
           InterruptedException,
           NeedsSkyframeRestartException {
     LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
 
-    var lockedExtensionMap = lockfile.getModuleExtensions().get(extensionId);
-    LockFileModuleExtension lockedExtension =
-        lockedExtensionMap == null ? null : lockedExtensionMap.get(extension.getEvalFactors());
-    if (lockedExtension == null) {
-      if (lockfileMode.equals(LockfileMode.ERROR)) {
-        throw new SingleExtensionEvalFunctionException(
-            ExternalDepsException.withMessage(
-                Code.BAD_MODULE,
-                "The module extension '%s'%s does not exist in the lockfile",
-                extensionId,
-                extension.getEvalFactors().isEmpty()
-                    ? ""
-                    : " for platform " + extension.getEvalFactors()),
-            Transience.PERSISTENT);
-      }
-      return null;
-    }
-
     ImmutableMap<ModuleKey, ModuleExtensionUsage> lockedExtensionUsages;
     try {
       // TODO(salmasamy) might be nicer to precompute this table when we construct

src/main/java/com/google/devtools/build/lib/pkgcache/PackageOptions.java

@@ -173,12 +173,14 @@ public ParallelismConverter() throws OptionsParsingException {
   public int maxDirectoriesToEagerlyVisitInGlobbing;
 
   @Option(
-    name = "fetch",
-    defaultValue = "true",
-    documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
-    effectTags = {OptionEffectTag.UNKNOWN},
-    help = "Allows the command to fetch external dependencies"
-  )
+      name = "fetch",
+      defaultValue = "true",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help =
+          "Allows the command to fetch external dependencies. If set to false, the command will"
+              + " utilize any cached version of the dependency, and if none exists, the command"
+              + " will result in failure.")
   public boolean fetch;
 
   @Option(