[native_assets_builder] Use file content hashing (#1750)

This PR changes the caching behavior for hooks to be file content hashing instead of last modified timestamps.

Closes: #1593.

In addition to using file hashes, a timestamp is passed in to detect if files were modified during a build. The moment of hashing contents is after a build is finished, but we should consider files changed after the build started to invalidate the build. If this happens, the build succeeds, but the cache is invalidated and a warning is printed to the logger.

The implementation was modeled after the [`FileStore` in flutter_tools](https://github.com/flutter/flutter/blob/1e824af6bd217beb0cc201394c9832316b6150da/packages/flutter_tools/lib/src/build_system/file_store.dart). However, it was adapted to support caching directories and to match the code style in this repository.

Directory caching is defined as follows: the hash of the names of the children. This excludes recursive descendants, and excludes the contents of children. For recursive caching (and glob patterns), the populator of the cache should do the glob/recursion to add all directories and files.

### Testing

* The existing caching tests (such as `pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart`) cover caching behavior.
* Now that last-modified are no longer used, some sleeps have been removed from tests. � 

### Performance

Adding a stopwatch to pkgs/native_assets_builder/test/build_runner/build_runner_caching_test.dart for the second invocation of the hooks so that it is cached. 

* `lastModified` timestamps: 0.028 seconds (pre this PR)
* `package:crypto` `md5`: 0.047 seconds (current PR)
* `package:xxh3` `xxh3`: 0.042 seconds

The implementation does not use parallel system IO for loading files (no `Future.wait`), but does use async I/O to allow flutter_tools to run other `Target`s in parallel.

The (pre and post this PR) implementation is fast enough for a handful of packages with native assets in a `flutter run`. The implementation (pre and post this PR) is not fast enough for hot restart / hot reload with 10+ packages with native assets. So, when exploring support for that, we'll need to revisit the implementation.

### Related issues not addressed in this PR

* #32
  * Changing environment variables should also cause the hook to rerun.
* #1578
  * Hook compilation can potentially be shared. (This requires taking more directory locks due to concurrent and re-entrant invocations.)

Author: dcharkes

pkgs/native_assets_builder/lib/src/build_runner/build_runner.dart

@@ -10,15 +10,14 @@ import 'package:logging/logging.dart';
 import 'package:native_assets_cli/native_assets_cli_internal.dart';
 import 'package:package_config/package_config.dart';
 
+import '../dependencies_hash_file/dependencies_hash_file.dart';
 import '../locking/locking.dart';
 import '../model/build_dry_run_result.dart';
 import '../model/build_result.dart';
 import '../model/hook_result.dart';
 import '../model/link_result.dart';
 import '../package_layout/package_layout.dart';
-import '../utils/file.dart';
 import '../utils/run_process.dart';
-import '../utils/uri.dart';
 import 'build_planner.dart';
 
 typedef DependencyMetadata = Map<String, Metadata>;
@@ -451,6 +450,8 @@ class NativeAssetsBuildRunner {
     return hookResult;
   }
 
+  // TODO(https://github.com/dart-lang/native/issues/32): Rerun hook if
+  // environment variables change.
   Future<HookOutput?> _runHookForPackageCached(
     Hook hook,
     HookConfig config,
@@ -473,7 +474,7 @@ class NativeAssetsBuildRunner {
         final (
           compileSuccess,
           hookKernelFile,
-          hookLastSourceChange,
+          hookHashesFile,
         ) = await _compileHookForPackageCached(
           config.packageName,
           config.outputDirectory,
@@ -488,7 +489,14 @@ class NativeAssetsBuildRunner {
 
         final buildOutputFile =
             File.fromUri(config.outputDirectory.resolve(hook.outputName));
-        if (buildOutputFile.existsSync()) {
+        final dependenciesHashFile = File.fromUri(
+          config.outputDirectory
+              .resolve('../dependencies.dependencies_hash_file.json'),
+        );
+        final dependenciesHashes =
+            DependenciesHashFile(file: dependenciesHashFile);
+        final lastModifiedCutoffTime = DateTime.now();
+        if (buildOutputFile.existsSync() && dependenciesHashFile.existsSync()) {
           late final HookOutput output;
           try {
             output = _readHookOutputFromUri(hook, buildOutputFile);
@@ -503,17 +511,13 @@ ${e.message}
             return null;
           }
 
-          final lastBuilt = output.timestamp.roundDownToSeconds();
-          final dependenciesLastChange =
-              await Dependencies(output.dependencies).lastModified();
-          if (lastBuilt.isAfter(dependenciesLastChange) &&
-              lastBuilt.isAfter(hookLastSourceChange)) {
+          final outdated =
+              (await dependenciesHashes.findOutdatedFileSystemEntity()) != null;
+          if (!outdated) {
             logger.info(
               [
                 'Skipping ${hook.name} for ${config.packageName} in $outDir.',
-                'Last build on $lastBuilt.',
-                'Last dependencies change on $dependenciesLastChange.',
-                'Last hook change on $hookLastSourceChange.',
+                'Last build on ${output.timestamp}.',
               ].join(' '),
             );
             // All build flags go into [outDir]. Therefore we do not have to
@@ -522,7 +526,7 @@ ${e.message}
           }
         }
 
-        return await _runHookForPackage(
+        final result = await _runHookForPackage(
           hook,
           config,
           validator,
@@ -533,6 +537,24 @@ ${e.message}
           hookKernelFile,
           packageLayout,
         );
+        if (result == null) {
+          if (await dependenciesHashFile.exists()) {
+            await dependenciesHashFile.delete();
+          }
+        } else {
+          final modifiedDuringBuild = await dependenciesHashes.hashFiles(
+            [
+              ...result.dependencies,
+              // Also depend on the hook source code.
+              hookHashesFile.uri,
+            ],
+            validBeforeLastModified: lastModifiedCutoffTime,
+          );
+          if (modifiedDuringBuild != null) {
+            logger.severe('File modified during build. Build must be rerun.');
+          }
+        }
+        return result;
       },
     );
   }
@@ -644,7 +666,10 @@ ${e.message}
   /// It does not reuse the cached kernel for different configs due to
   /// reentrancy requirements. For more info see:
   /// https://github.com/dart-lang/native/issues/1319
-  Future<(bool success, File kernelFile, DateTime lastSourceChange)>
+  ///
+  /// TODO(https://github.com/dart-lang/native/issues/1578): Compile only once
+  /// instead of per config. This requires more locking.
+  Future<(bool success, File kernelFile, File cacheFile)>
       _compileHookForPackageCached(
     String packageName,
     Uri outputDirectory,
@@ -659,29 +684,17 @@ ${e.message}
     final depFile = File.fromUri(
       outputDirectory.resolve('../hook.dill.d'),
     );
+    final dependenciesHashFile = File.fromUri(
+      outputDirectory.resolve('../hook.dependencies_hash_file.json'),
+    );
+    final dependenciesHashes = DependenciesHashFile(file: dependenciesHashFile);
+    final lastModifiedCutoffTime = DateTime.now();
     final bool mustCompile;
-    final DateTime sourceLastChange;
-    if (!await depFile.exists()) {
+    if (!await dependenciesHashFile.exists()) {
       mustCompile = true;
-      sourceLastChange = DateTime.now();
     } else {
-      // Format: `path/to/my.dill: path/to/my.dart, path/to/more.dart`
-      final depFileContents = await depFile.readAsString();
-      final dartSourceFiles = depFileContents
-          .trim()
-          .split(' ')
-          .skip(1) // '<kernel file>:'
-          .map((u) => Uri.file(u).fileSystemEntity)
-          .toList();
-      final dartFilesLastChange = await dartSourceFiles.lastModified();
-      final packageConfigLastChange =
-          await packageConfigUri.fileSystemEntity.lastModified();
-      sourceLastChange = packageConfigLastChange.isAfter(dartFilesLastChange)
-          ? packageConfigLastChange
-          : dartFilesLastChange;
-      final kernelLastChange = await kernelFile.lastModified();
-      mustCompile = sourceLastChange == kernelLastChange ||
-          sourceLastChange.isAfter(kernelLastChange);
+      mustCompile =
+          (await dependenciesHashes.findOutdatedFileSystemEntity()) != null;
     }
     final bool success;
     if (!mustCompile) {
@@ -696,8 +709,28 @@ ${e.message}
         kernelFile,
         depFile,
       );
+
+      if (success) {
+        // Format: `path/to/my.dill: path/to/my.dart, path/to/more.dart`
+        final depFileContents = await depFile.readAsString();
+        final dartSources = depFileContents
+            .trim()
+            .split(' ')
+            .skip(1) // '<kernel file>:'
+            .map(Uri.file)
+            .toList();
+        final modifiedDuringBuild = await dependenciesHashes.hashFiles(
+          dartSources,
+          validBeforeLastModified: lastModifiedCutoffTime,
+        );
+        if (modifiedDuringBuild != null) {
+          logger.severe('File modified during build. Build must be rerun.');
+        }
+      } else {
+        await dependenciesHashFile.delete();
+      }
     }
-    return (success, kernelFile, sourceLastChange);
+    return (success, kernelFile, dependenciesHashFile);
   }
 
   Future<bool> _compileHookForPackage(
@@ -859,12 +892,6 @@ ${compileResult.stdout}
   }
 }
 
-extension on DateTime {
-  DateTime roundDownToSeconds() =>
-      DateTime.fromMillisecondsSinceEpoch(millisecondsSinceEpoch -
-          millisecondsSinceEpoch % const Duration(seconds: 1).inMilliseconds);
-}
-
 extension on Uri {
   Uri get parent => File(toFilePath()).parent.uri;
 }