Merge pull request #588 Allow --files-list option for prefetch

This PR addresses #580 and adds --files-list option to prefetch verb. It also adds a magic file name, to allow passing in both --files-list and --folders-list via stdin.

Author: derrickstolee

GVFS/FastFetch/FastFetchVerb.cs

@@ -105,8 +105,8 @@ public class FastFetchVerb
             Default = "",
             Required = false,
             HelpText = "Sets the path and filename for git.exe if it isn't expected to be on %PATH%.")]
-        public string GitBinPath { get; set; }
-        
+        public string GitBinPath { get; set; }
+
         [Option(
             "folders",
             Required = false,
@@ -171,8 +171,8 @@ private int ExecuteWithExitCode()
             {
                 Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
                 return ExitFailure;
-            }
-            
+            }
+
             this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
             this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
             this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount;
@@ -185,8 +185,8 @@ private int ExecuteWithExitCode()
             {
                 Console.WriteLine("Must be run within a git repo");
                 return ExitFailure;
-            }
-            
+            }
+
             string commitish = this.Commit ?? this.Branch;
             if (string.IsNullOrWhiteSpace(commitish))
             {
@@ -238,11 +238,11 @@ private int ExecuteWithExitCode()
                     tracer.RelatedError(error);
                     Console.WriteLine(error);
                     return ExitFailure;
-                }
-                
+                }
+
                 RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
                 BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig);
-                if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error))
+                if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, readListFromStdIn: false, error: out error))
                 {
                     tracer.RelatedError(error);
                     Console.WriteLine(error);

GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs

@@ -83,70 +83,47 @@ public BlobPrefetcher(
 
         public List<string> FolderList { get; }
 
-        public static bool TryLoadFolderList(Enlistment enlistment, string foldersInput, string folderListFile, List<string> folderListOutput, out string error)
+        public static bool TryLoadFolderList(Enlistment enlistment, string foldersInput, string folderListFile, List<string> folderListOutput, bool readListFromStdIn, out string error)
         {
-            folderListOutput.AddRange(
-                foldersInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
-                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: true)));
-
-            if (!string.IsNullOrWhiteSpace(folderListFile))
-            {
-                if (File.Exists(folderListFile))
-                {
-                    IEnumerable<string> allLines = File.ReadAllLines(folderListFile)
-                        .Select(line => line.Trim())
-                        .Where(line => !string.IsNullOrEmpty(line))
-                        .Where(line => !line.StartsWith(GVFSConstants.GitCommentSign.ToString()))
-                        .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: true));
-
-                    folderListOutput.AddRange(allLines);
-                }
-                else
-                {
-                    error = string.Format("Could not find '{0}' for folder list.", folderListFile);
-                    return false;
-                }
-            }
-
-            folderListOutput.RemoveAll(string.IsNullOrWhiteSpace);
-
-            foreach (string folder in folderListOutput)
-            {
-                if (folder.Contains("*"))
-                {
-                    error = "Wildcards are not supported for folders. Invalid entry: " + folder;
-                    return false;
-                }
-            }
-
-            error = null;
-            return true;
+            return TryLoadFileOrFolderList(
+                    enlistment,
+                    foldersInput,
+                    folderListFile,
+                    isFolder: true,
+                    readListFromStdIn: readListFromStdIn,
+                    output: folderListOutput,
+                    elementValidationFunction: s =>
+                        s.Contains("*") ?
+                            "Wildcards are not supported for folders. Invalid entry: " + s :
+                            null,
+                    error: out error);
         }
 
-        public static bool TryLoadFileList(Enlistment enlistment, string filesInput, List<string> fileListOutput, out string error)
+        public static bool TryLoadFileList(Enlistment enlistment, string filesInput, string filesListFile, List<string> fileListOutput, bool readListFromStdIn, out string error)
         {
-            fileListOutput.AddRange(
-                filesInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
-                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: false)));
-
-            foreach (string file in fileListOutput)
-            {
-                if (file.IndexOf('*', 1) != -1)
+            return TryLoadFileOrFolderList(
+                enlistment,
+                filesInput,
+                filesListFile,
+                readListFromStdIn: readListFromStdIn,
+                isFolder: false,
+                output: fileListOutput,
+                elementValidationFunction: s =>
                 {
-                    error = "Only prefix wildcards are supported. Invalid entry: " + file;
-                    return false;
-                }
+                    if (s.IndexOf('*', 1) != -1)
+                    {
+                        return "Only prefix wildcards are supported. Invalid entry: " + s;
+                    }
 
-                if (file.EndsWith(GVFSConstants.GitPathSeparatorString) ||
-                    file.EndsWith(pathSeparatorString))
-                {
-                    error = "Folders are not allowed in the file list. Invalid entry: " + file;
-                    return false;
-                }
-            }
+                    if (s.EndsWith(GVFSConstants.GitPathSeparatorString) ||
+                        s.EndsWith(pathSeparatorString))
+                    {
+                        return "Folders are not allowed in the file list. Invalid entry: " + s;
+                    }
 
-            error = null;
-            return true;
+                    return null;
+                },
+                error: out error);
         }
 
         public static bool IsNoopPrefetch(
@@ -173,7 +150,7 @@ public static bool IsNoopPrefetch(
 
                 tracer.RelatedEvent(
                     EventLevel.Informational,
-                    "BlobPrefetcher.IsNoopPrefetch", 
+                    "BlobPrefetcher.IsNoopPrefetch",
                     new EventMetadata
                     {
                         { "Last" + PrefetchArgs.CommitId, lastCommitId },
@@ -231,7 +208,7 @@ public virtual void Prefetch(string branchOrCommit, bool isBranch)
             int matchedBlobCount;
             int downloadedBlobCount;
             int hydratedFileCount;
-            
+
             this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out hydratedFileCount);
         }
 
@@ -290,7 +267,7 @@ public void PrefetchWithStats(
                     return;
                 }
             }
-            
+
             BlockingCollection<string> availableBlobs = new BlockingCollection<string>();
 
             ////
@@ -300,7 +277,7 @@ public void PrefetchWithStats(
             //    |           |              |                 |
             //     ------------------------------------------------------> fileHydrator
             ////
-            
+
             // diff
             //  Inputs:
             //      * files/folders
@@ -310,7 +287,7 @@ public void PrefetchWithStats(
             //      * FileAddOperations (property): Repo-relative paths corresponding to those blob ids
             DiffHelper diff = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false);
 
-            // blobFinder 
+            // blobFinder
             //  Inputs:
             //      * requiredBlobs (in param): Blob ids from output of `diff`
             //  Outputs:
@@ -513,9 +490,77 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects)
             }
         }
 
+        private static IEnumerable<string> GetFilesFromVerbParameter(string valueString)
+        {
+            return valueString.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+        }
+
+        private static IEnumerable<string> GetFilesFromFile(string fileName, out string error)
+        {
+            error = null;
+            if (string.IsNullOrWhiteSpace(fileName))
+            {
+                return Enumerable.Empty<string>();
+            }
+
+            if (!File.Exists(fileName))
+            {
+                error = string.Format("Could not find '{0}' list file.", fileName);
+                return Enumerable.Empty<string>();
+            }
+
+            return File.ReadAllLines(fileName)
+                        .Select(line => line.Trim());
+        }
+
+        private static IEnumerable<string> GetFilesFromStdin(bool shouldRead)
+        {
+            if (!shouldRead)
+            {
+                yield break;
+            }
+
+            string line;
+            while ((line = Console.In.ReadLine()) != null)
+            {
+                yield return line.Trim();
+            }
+        }
+
+        private static bool TryLoadFileOrFolderList(Enlistment enlistment, string valueString, string listFileName, bool readListFromStdIn, bool isFolder, List<string> output, Func<string, string> elementValidationFunction, out string error)
+        {
+            output.AddRange(
+                GetFilesFromVerbParameter(valueString)
+                .Union(GetFilesFromFile(listFileName, out string fileReadError))
+                .Union(GetFilesFromStdin(readListFromStdIn))
+                .Where(path => !path.StartsWith(GVFSConstants.GitCommentSign.ToString()))
+                .Where(path => !string.IsNullOrWhiteSpace(path))
+                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: isFolder)));
+
+            if (!string.IsNullOrWhiteSpace(fileReadError))
+            {
+                error = fileReadError;
+                return false;
+            }
+
+            string[] errorArray = output
+                .Select(elementValidationFunction)
+                .Where(s => !string.IsNullOrWhiteSpace(s))
+                .ToArray();
+
+            if (errorArray != null && errorArray.Length > 0)
+            {
+                error = string.Join("\n", errorArray);
+                return false;
+            }
+
+            error = null;
+            return true;
+        }
+
         private static string ToAbsolutePath(Enlistment enlistment, string path, bool isFolder)
         {
-            string absolute = 
+            string absolute =
                 path.StartsWith("*")
                 ? path
                 : Path.Combine(enlistment.WorkingDirectoryRoot, path.Replace(GVFSConstants.GitPathSeparator, Path.DirectorySeparatorChar).TrimStart(Path.DirectorySeparatorChar));

GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs

@@ -3,6 +3,7 @@
 using GVFS.FunctionalTests.Tools;
 using GVFS.Tests.Should;
 using NUnit.Framework;
+using System;
 using System.IO;
 using System.Threading;
 
@@ -93,7 +94,7 @@ public void PrefetchFolderListFromFile()
                     "gvfs/"
                 });
 
-            this.ExpectBlobCount(this.Enlistment.Prefetch("--folders-list " + tempFilePath), 279);
+            this.ExpectBlobCount(this.Enlistment.Prefetch("--folders-list \"" + tempFilePath + "\""), 279);
             File.Delete(tempFilePath);
         }
 
@@ -133,7 +134,7 @@ public void PrefetchCleansUpStalePrefetchLock()
         [Category(Categories.MacTODO.M4)]
         public void PrefetchCleansUpPackDir()
         {
-            string multiPackIndexLockFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem),  MultiPackIndexLock);
+            string multiPackIndexLockFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem), MultiPackIndexLock);
             string oldGitTempFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem), "tmp_midx_XXXX");
 
             this.fileSystem.WriteAllText(multiPackIndexLockFile, this.Enlistment.EnlistmentRoot);
@@ -147,6 +148,58 @@ public void PrefetchCleansUpPackDir()
         }
 
         [TestCase, Order(12)]
+        public void PrefetchFilesFromFileListFile()
+        {
+            string tempFilePath = Path.Combine(Path.GetTempPath(), "temp.file");
+            try
+            {
+                File.WriteAllLines(
+                    tempFilePath,
+                    new[]
+                    {
+                        Path.Combine("GVFS", "GVFS", "Program.cs"),
+                        Path.Combine("GVFS", "GVFS.FunctionalTests", "GVFS.FunctionalTests.csproj")
+                    });
+
+                this.ExpectBlobCount(this.Enlistment.Prefetch($"--files-list \"{tempFilePath}\""), 2);
+            }
+            finally
+            {
+                File.Delete(tempFilePath);
+            }
+        }
+
+        [TestCase, Order(13)]
+        public void PrefetchFilesFromFileListStdIn()
+        {
+            string input = string.Join(
+                Environment.NewLine,
+                new[]
+                {
+                    Path.Combine("GVFS", "GVFS", "packages.config"),
+                    Path.Combine("GVFS", "GVFS.FunctionalTests", "App.config")
+                });
+
+            this.ExpectBlobCount(this.Enlistment.Prefetch("--stdin-files-list", standardInput: input), 2);
+        }
+
+        [TestCase, Order(14)]
+        public void PrefetchFolderListFromStdin()
+        {
+            string input = string.Join(
+                Environment.NewLine,
+                new[]
+                {
+                    "# A comment",
+                    " ",
+                    "gvfs/",
+                    "gvfs/gvfs",
+                    "gvfs/"
+                });
+
+            this.ExpectBlobCount(this.Enlistment.Prefetch("--stdin-folders-list", standardInput: input), 279);
+        }
+
         public void PrefetchPathsWithLsTreeTypeInPath()
         {
             ProcessResult checkoutResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + LsTreeTypeInPathBranchName);

GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs

@@ -197,9 +197,9 @@ public bool TryMountGVFS(out string output)
             return this.gvfsProcess.TryMount(out output);
         }
 
-        public string Prefetch(string args, bool failOnError = true)
+        public string Prefetch(string args, bool failOnError = true, string standardInput = null)
         {
-            return this.gvfsProcess.Prefetch(args, failOnError);
+            return this.gvfsProcess.Prefetch(args, failOnError, standardInput);
         }
 
         public void Repair(bool confirm)