Merge pull request #1917 from tyrielv/tyrielv/hydration-status-4-projection

Replace git ls-tree with projection-based folder count

Author: tyrielv

GVFS/GVFS.Common/GVFSConstants.cs

@@ -134,7 +134,6 @@ public static class GitStatusCache
             {
                 public const string Name = "gitStatusCache";
                 public static readonly string CachePath = Path.Combine(Name, "GitStatusCache.dat");
-                public static readonly string TreeCount = Path.Combine(Name, "TreeCountCache.dat");
             }
 
             public static class HydrationStatus

GVFS/GVFS.Common/GVFSEnlistment.cs

@@ -54,6 +54,11 @@ private GVFSEnlistment(string enlistmentRoot, string gitBinPath, GitAuthenticati
 
         public string GVFSLogsRoot { get; }
 
+        /// <summary>
+        /// Path to the git index file.
+        /// </summary>
+        public string GitIndexPath => Path.Combine(this.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Index);
+
         public string LocalCacheRoot { get; private set; }
 
         public string BlobSizesRoot { get; private set; }

GVFS/GVFS.Common/Git/GitProcess.cs

@@ -682,11 +682,6 @@ public Result MultiPackIndexRepack(string gitObjectDirectory, string batchSize)
             return this.InvokeGitAgainstDotGitFolder($"-c pack.threads=1 -c repack.packKeptObjects=true multi-pack-index repack --object-dir=\"{gitObjectDirectory}\" --batch-size={batchSize} --no-progress");
         }
 
-        public Result GetHeadTreeId()
-        {
-            return this.InvokeGitAgainstDotGitFolder("rev-parse \"HEAD^{tree}\"", usePreCommandHook: false);
-        }
-
         public Process GetGitProcess(string command, string workingDirectory, string dotGitDirectory, bool useReadObjectHook, bool redirectStandardError, string gitObjectsDirectory, bool usePreCommandHook)
         {
             ProcessStartInfo processInfo = new ProcessStartInfo(this.gitBinPath);

GVFS/GVFS.Common/GitStatusCache.cs

@@ -50,6 +50,8 @@ public class GitStatusCache : IDisposable
         private CancellationTokenSource shutdownTokenSource;
         private Task activeHydrationTask;
 
+        private Func<int> projectedFolderCountProvider;
+
         private volatile CacheState cacheState = CacheState.Dirty;
 
         private object cacheFileLock = new object();
@@ -72,6 +74,23 @@ public GitStatusCache(GVFSContext context, TimeSpan backoffTime)
             this.wakeUpThread = new AutoResetEvent(false);
         }
 
+        /// <summary>
+        /// Sets the provider used to get the total projected folder count for hydration
+        /// summary computation. Must be called before <see cref="Initialize"/> for
+        /// hydration summary to function.
+        /// </summary>
+        /// <remarks>
+        /// This is set post-construction because of a circular dependency:
+        /// InProcessMount creates GitStatusCache before FileSystemCallbacks,
+        /// but the provider requires GitIndexProjection, which is created
+        /// inside FileSystemCallbacks. FileSystemCallbacks calls this method
+        /// after GitIndexProjection is available.
+        /// </remarks>
+        public void SetProjectedFolderCountProvider(Func<int> provider)
+        {
+            this.projectedFolderCountProvider = provider;
+        }
+
         public virtual void Initialize()
         {
             this.isInitialized = true;
@@ -183,17 +202,16 @@ public virtual void Dispose()
 
             // Wait for the hydration task to complete before disposing the
             // token source it may still be using.
-            if (this.activeHydrationTask != null)
+            Task hydrationTask = Interlocked.Exchange(ref this.activeHydrationTask, null);
+            if (hydrationTask != null)
             {
                 try
                 {
-                    this.activeHydrationTask.Wait();
+                    hydrationTask.Wait(TimeSpan.FromSeconds(5));
                 }
                 catch (AggregateException)
                 {
                 }
-
-                this.activeHydrationTask = null;
             }
 
             if (this.shutdownTokenSource != null)
@@ -346,11 +364,12 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff)
                 // Run hydration summary in parallel with git status — they are independent
                 // operations and neither should delay the other.
                 Task hydrationTask = Task.Run(() => this.UpdateHydrationSummary());
-                this.activeHydrationTask = hydrationTask;
+                Interlocked.Exchange(ref this.activeHydrationTask, hydrationTask);
 
                 bool rebuildStatusCacheSucceeded = this.TryRebuildStatusCache();
 
                 // Wait for hydration to complete before logging final stats.
+                // Exceptions are observed here to avoid unobserved task exceptions.
                 try
                 {
                     hydrationTask.Wait();
@@ -385,6 +404,11 @@ private void RebuildStatusCacheIfNeeded(bool ignoreBackoff)
 
         private void UpdateHydrationSummary()
         {
+            if (this.projectedFolderCountProvider == null)
+            {
+                return;
+            }
+
             bool enabled = TEST_EnableHydrationSummaryOverride
                 ?? this.context.Repository.LibGit2RepoInvoker.GetConfigBoolOrDefault(GVFSConstants.GitConfig.ShowHydrationStatus, GVFSConstants.GitConfig.ShowHydrationStatusDefault);
             if (!enabled)
@@ -409,7 +433,7 @@ private void UpdateHydrationSummary()
                  * and this is also a convenient place to log telemetry for it.
                  */
                 EnlistmentHydrationSummary hydrationSummary =
-                    EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.context.FileSystem, this.context.Tracer, cancellationToken: this.shutdownTokenSource.Token);
+                    EnlistmentHydrationSummary.CreateSummary(this.context.Enlistment, this.context.FileSystem, this.context.Tracer, this.projectedFolderCountProvider, this.shutdownTokenSource.Token);
                 EventMetadata metadata = new EventMetadata();
                 metadata.Add("Area", EtwArea);
                 if (hydrationSummary.IsValid)

GVFS/GVFS.Common/HealthCalculator/EnlistmentHydrationSummary.cs

@@ -1,10 +1,8 @@
 using GVFS.Common.FileSystem;
-using GVFS.Common.Git;
 using GVFS.Common.Tracing;
 using System;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Threading;
 
 namespace GVFS.Common
@@ -45,6 +43,7 @@ public static EnlistmentHydrationSummary CreateSummary(
             GVFSEnlistment enlistment,
             PhysicalFileSystem fileSystem,
             ITracer tracer,
+            Func<int> projectedFolderCountProvider,
             CancellationToken cancellationToken = default)
         {
             Stopwatch totalStopwatch = Stopwatch.StartNew();
@@ -57,7 +56,6 @@ public static EnlistmentHydrationSummary CreateSummary(
                 phaseStopwatch.Restart();
                 int totalFileCount = GetIndexFileCount(enlistment, fileSystem);
                 long indexReadMs = phaseStopwatch.ElapsedMilliseconds;
-
                 cancellationToken.ThrowIfCancellationRequested();
 
                 EnlistmentPathData pathData = new EnlistmentPathData();
@@ -66,15 +64,13 @@ public static EnlistmentHydrationSummary CreateSummary(
                 phaseStopwatch.Restart();
                 pathData.LoadPlaceholdersFromDatabase(enlistment);
                 long placeholderLoadMs = phaseStopwatch.ElapsedMilliseconds;
-
                 cancellationToken.ThrowIfCancellationRequested();
 
                 phaseStopwatch.Restart();
                 pathData.LoadModifiedPaths(enlistment, tracer);
                 long modifiedPathsLoadMs = phaseStopwatch.ElapsedMilliseconds;
+                cancellationToken.ThrowIfCancellationRequested();
 
-                cancellationToken.ThrowIfCancellationRequested();
-
                 int hydratedFileCount = pathData.ModifiedFilePaths.Count + pathData.PlaceholderFilePaths.Count;
                 int hydratedFolderCount = pathData.ModifiedFolderPaths.Count + pathData.PlaceholderFolderPaths.Count;
 
@@ -100,11 +96,12 @@ public static EnlistmentHydrationSummary CreateSummary(
                     return soFar;
                 }
 
-                /* Getting all the directories is also slow, but not as slow as reading the entire index,
-                 * GetTotalPathCount caches the count so this is only slow occasionally,
-                 * and the GitStatusCache manager also calls this to ensure it is updated frequently. */
+                /* Get the total folder count from the caller-provided function.
+                 * In the mount process, this comes from the in-memory projection (essentially free).
+                 * In gvfs health --status fallback, this parses the git index via GitIndexProjection. */
+                cancellationToken.ThrowIfCancellationRequested();
                 phaseStopwatch.Restart();
-                int totalFolderCount = GetHeadTreeCount(enlistment, fileSystem, tracer);
+                int totalFolderCount = projectedFolderCountProvider();
                 long treeCountMs = phaseStopwatch.ElapsedMilliseconds;
 
                 EmitDurationTelemetry(tracer, totalStopwatch.ElapsedMilliseconds, indexReadMs, placeholderLoadMs, modifiedPathsLoadMs, treeCountMs, earlyExit: false);
@@ -169,7 +166,7 @@ private static void EmitDurationTelemetry(
         /// </summary>
         internal static int GetIndexFileCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem)
         {
-            string indexPath = Path.Combine(enlistment.WorkingDirectoryBackingRoot, GVFSConstants.DotGit.Index);
+            string indexPath = enlistment.GitIndexPath;
             using (var indexFile = fileSystem.OpenFileStream(indexPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, callFlushFileBuffers: false))
             {
                 if (indexFile.Length < 12)
@@ -193,77 +190,5 @@ internal static int GetIndexFileCount(GVFSEnlistment enlistment, PhysicalFileSys
             }
         }
 
-        /// <summary>
-        /// Get the total number of trees in the repo at HEAD.
-        /// </summary>
-        /// <remarks>
-        /// This is used as the denominator in displaying percentage of hydrated
-        /// directories as part of git status pre-command hook.
-        /// It can take several seconds to calculate, so we cache it near the git status cache.
-        /// </remarks>
-        /// <returns>
-        /// The number of subtrees at HEAD, which may be 0.
-        /// Will return 0 if unsuccessful.
-        /// </returns>
-        internal static int GetHeadTreeCount(GVFSEnlistment enlistment, PhysicalFileSystem fileSystem, ITracer tracer)
-        {
-            var gitProcess = enlistment.CreateGitProcess();
-            var headResult = gitProcess.GetHeadTreeId();
-            if (headResult.ExitCodeIsFailure)
-            {
-                tracer.RelatedError($"Failed to get HEAD tree ID: \nOutput: {headResult.Output}\n\nError:{headResult.Errors}");
-                return 0;
-            }
-            var headSha = headResult.Output.Trim();
-            var cacheFile = Path.Combine(
-                enlistment.DotGVFSRoot,
-                GVFSConstants.DotGVFS.GitStatusCache.TreeCount);
-
-            // Load from cache if cache matches current HEAD.
-            if (fileSystem.FileExists(cacheFile))
-            {
-                try
-                {
-                    var lines = fileSystem.ReadLines(cacheFile).ToArray();
-                    if (lines.Length == 2
-                        && lines[0] == headSha
-                        && int.TryParse(lines[1], out int cachedCount))
-                    {
-                        return cachedCount;
-                    }
-                }
-                catch (Exception ex)
-                {
-                    tracer.RelatedWarning($"Failed to read tree count cache file at {cacheFile}: {ex}");
-                    // Ignore errors reading the cache
-                }
-            }
-
-            int totalPathCount = 0;
-            GitProcess.Result folderResult = gitProcess.LsTree(
-                GVFSConstants.DotGit.HeadName,
-                line => totalPathCount++,
-                recursive: true,
-                showDirectories: true);
-
-            if (GitProcess.Result.SuccessCode != folderResult.ExitCode)
-            {
-                tracer.RelatedError($"Failed to get tree count from HEAD: \nOutput: {folderResult.Output}\n\nError:{folderResult.Errors}");
-                return 0;
-            }
-
-            try
-            {
-                fileSystem.CreateDirectory(Path.GetDirectoryName(cacheFile));
-                fileSystem.WriteAllText(cacheFile, $"{headSha}\n{totalPathCount}");
-            }
-            catch (Exception ex)
-            {
-                // Ignore errors writing the cache
-                tracer.RelatedWarning($"Failed to write tree count cache file at {cacheFile}: {ex}");
-            }
-
-            return totalPathCount;
-        }
     }
 }