Allow --files-list option for prefetch

GVFS/FastFetch/FastFetchVerb.cs

@@ -105,8 +105,8 @@ public class FastFetchVerb
             Default = "",
             Required = false,
             HelpText = "Sets the path and filename for git.exe if it isn't expected to be on %PATH%.")]
-        public string GitBinPath { get; set; }
-        
+        public string GitBinPath { get; set; }
+
         [Option(
             "folders",
             Required = false,
@@ -171,8 +171,8 @@ private int ExecuteWithExitCode()
             {
                 Console.WriteLine("Cannot use --force-checkout option without --checkout option.");
                 return ExitFailure;
-            }
-            
+            }
+
             this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount;
             this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads);
             this.IndexThreadCount = this.IndexThreadCount > 0 ? this.IndexThreadCount : Environment.ProcessorCount;
@@ -185,8 +185,8 @@ private int ExecuteWithExitCode()
             {
                 Console.WriteLine("Must be run within a git repo");
                 return ExitFailure;
-            }
-            
+            }
+
             string commitish = this.Commit ?? this.Branch;
             if (string.IsNullOrWhiteSpace(commitish))
             {
@@ -238,11 +238,11 @@ private int ExecuteWithExitCode()
                     tracer.RelatedError(error);
                     Console.WriteLine(error);
                     return ExitFailure;
-                }
-                
+                }
+
                 RetryConfig retryConfig = new RetryConfig(this.MaxAttempts, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes));
                 BlobPrefetcher prefetcher = this.GetFolderPrefetcher(tracer, enlistment, cacheServer, retryConfig);
-                if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, out error))
+                if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.FolderList, this.FolderListFile, prefetcher.FolderList, readListFromStdIn: false, error: out error))
                 {
                     tracer.RelatedError(error);
                     Console.WriteLine(error);

GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs

@@ -16,6 +16,7 @@ namespace GVFS.Common.Prefetch
 {
     public class BlobPrefetcher
     {
+        public const string StdInFileName = "stdin";
         protected const string RefsHeadsGitPath = "refs/heads/";
 
         protected readonly Enlistment Enlistment;
@@ -83,70 +84,36 @@ public BlobPrefetcher(
 
         public List<string> FolderList { get; }
 
-        public static bool TryLoadFolderList(Enlistment enlistment, string foldersInput, string folderListFile, List<string> folderListOutput, out string error)
+        public static bool TryLoadFolderList(Enlistment enlistment, string foldersInput, string folderListFile, List<string> folderListOutput, bool readListFromStdIn, out string error)
         {
-            folderListOutput.AddRange(
-                foldersInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
-                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: true)));
-
-            if (!string.IsNullOrWhiteSpace(folderListFile))
-            {
-                if (File.Exists(folderListFile))
-                {
-                    IEnumerable<string> allLines = File.ReadAllLines(folderListFile)
-                        .Select(line => line.Trim())
-                        .Where(line => !string.IsNullOrEmpty(line))
-                        .Where(line => !line.StartsWith(GVFSConstants.GitCommentSign.ToString()))
-                        .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: true));
-
-                    folderListOutput.AddRange(allLines);
-                }
-                else
-                {
-                    error = string.Format("Could not find '{0}' for folder list.", folderListFile);
-                    return false;
-                }
-            }
-
-            folderListOutput.RemoveAll(string.IsNullOrWhiteSpace);
-
-            foreach (string folder in folderListOutput)
-            {
-                if (folder.Contains("*"))
-                {
-                    error = "Wildcards are not supported for folders. Invalid entry: " + folder;
-                    return false;
-                }
-            }
-
-            error = null;
-            return true;
+            return TryLoadFileOrFolderList(enlistment, foldersInput, folderListFile, isFolder: true, readListFromStdIn: readListFromStdIn, output: folderListOutput, elementValidationFunction: s => s.Contains("*") ? "Wildcards are not supported for folders. Invalid entry: " + s : null, error: out error);
         }
 
-        public static bool TryLoadFileList(Enlistment enlistment, string filesInput, List<string> fileListOutput, out string error)
+        public static bool TryLoadFileList(Enlistment enlistment, string filesInput, string filesListFile, List<string> fileListOutput, bool readListFromStdIn, out string error)
         {
-            fileListOutput.AddRange(
-                filesInput.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
-                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: false)));
-
-            foreach (string file in fileListOutput)
-            {
-                if (file.IndexOf('*', 1) != -1)
+            return TryLoadFileOrFolderList(
+                enlistment,
+                filesInput,
+                filesListFile,
+                readListFromStdIn: readListFromStdIn,
+                isFolder: false,
+                output: fileListOutput,
+                elementValidationFunction: s =>
                 {
-                    error = "Only prefix wildcards are supported. Invalid entry: " + file;
-                    return false;
-                }
+                    if (s.IndexOf('*', 1) != -1)
+                    {
+                        return "Only prefix wildcards are supported. Invalid entry: " + s;
+                    }
 
-                if (file.EndsWith(GVFSConstants.GitPathSeparatorString) ||
-                    file.EndsWith(pathSeparatorString))
-                {
-                    error = "Folders are not allowed in the file list. Invalid entry: " + file;
-                    return false;
-                }
-            }
+                    if (s.EndsWith(GVFSConstants.GitPathSeparatorString) ||
+                        s.EndsWith(pathSeparatorString))
+                    {
+                        return "Folders are not allowed in the file list. Invalid entry: " + s;
+                    }
 
-            error = null;
-            return true;
+                    return null;
+                },
+                error: out error);
         }
 
         public static bool IsNoopPrefetch(
@@ -173,7 +140,7 @@ public static bool IsNoopPrefetch(
 
                 tracer.RelatedEvent(
                     EventLevel.Informational,
-                    "BlobPrefetcher.IsNoopPrefetch", 
+                    "BlobPrefetcher.IsNoopPrefetch",
                     new EventMetadata
                     {
                         { "Last" + PrefetchArgs.CommitId, lastCommitId },
@@ -231,7 +198,7 @@ public virtual void Prefetch(string branchOrCommit, bool isBranch)
             int matchedBlobCount;
             int downloadedBlobCount;
             int hydratedFileCount;
-            
+
             this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out hydratedFileCount);
         }
 
@@ -290,7 +257,7 @@ public void PrefetchWithStats(
                     return;
                 }
             }
-            
+
             BlockingCollection<string> availableBlobs = new BlockingCollection<string>();
 
             ////
@@ -300,7 +267,7 @@ public void PrefetchWithStats(
             //    |           |              |                 |
             //     ------------------------------------------------------> fileHydrator
             ////
-            
+
             // diff
             //  Inputs:
             //      * files/folders
@@ -310,7 +277,7 @@ public void PrefetchWithStats(
             //      * FileAddOperations (property): Repo-relative paths corresponding to those blob ids
             DiffHelper diff = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList, includeSymLinks: false);
 
-            // blobFinder 
+            // blobFinder
             //  Inputs:
             //      * requiredBlobs (in param): Blob ids from output of `diff`
             //  Outputs:
@@ -513,9 +480,77 @@ protected void DownloadMissingCommit(string commitSha, GitObjects gitObjects)
             }
         }
 
+        private static IEnumerable<string> GetFilesFromParameter(string valueString)
+        {
+            return valueString.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
+        }
+
+        private static IEnumerable<string> GetFilesFromFile(string fileName, out string error)
+        {
+            error = null;
+            if (string.IsNullOrWhiteSpace(fileName))
+            {
+                return Enumerable.Empty<string>();
+            }
+
+            if (!File.Exists(fileName))
+            {
+                error = string.Format("Could not find '{0}' list file.", fileName);
+                return Enumerable.Empty<string>();
+            }
+
+            return File.ReadAllLines(fileName)
+                        .Select(line => line.Trim());
+        }
+
+        private static IEnumerable<string> GetFilesFromStdin(bool shouldRead)
+        {
+            if (!shouldRead)
+            {
+                yield break;
+            }
+
+            string line;
+            while ((line = Console.In.ReadLine()) != null)
+            {
+                yield return line.Trim();
+            }
+        }
+
+        private static bool TryLoadFileOrFolderList(Enlistment enlistment, string valueString, string listFileName, bool readListFromStdIn, bool isFolder, List<string> output, Func<string, string> elementValidationFunction, out string error)
+        {
+            output.AddRange(
+                GetFilesFromParameter(valueString)
+                .Union(GetFilesFromFile(listFileName, out string fileReadError))
+                .Union(GetFilesFromStdin(readListFromStdIn))
+                .Where(path => !path.StartsWith(GVFSConstants.GitCommentSign.ToString()))
+                .Where(path => !string.IsNullOrWhiteSpace(path))
+                .Select(path => BlobPrefetcher.ToAbsolutePath(enlistment, path, isFolder: isFolder)));
+
+            if (!string.IsNullOrWhiteSpace(fileReadError))
+            {
+                error = fileReadError;
+                return false;
+            }
+
+            string[] errorArray = output
+                .Select(elementValidationFunction)
+                .Where(s => !string.IsNullOrWhiteSpace(s))
+                .ToArray();
+
+            if (errorArray != null && errorArray.Length > 0)
+            {
+                error = string.Join("\n", errorArray);
+                return false;
+            }
+
+            error = null;
+            return true;
+        }
+
         private static string ToAbsolutePath(Enlistment enlistment, string path, bool isFolder)
         {
-            string absolute = 
+            string absolute =
                 path.StartsWith("*")
                 ? path
                 : Path.Combine(enlistment.WorkingDirectoryRoot, path.Replace(GVFSConstants.GitPathSeparator, Path.DirectorySeparatorChar).TrimStart(Path.DirectorySeparatorChar));

GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs

@@ -3,6 +3,7 @@
 using GVFS.FunctionalTests.Tools;
 using GVFS.Tests.Should;
 using NUnit.Framework;
+using System;
 using System.IO;
 using System.Threading;
 
@@ -132,7 +133,7 @@ public void PrefetchCleansUpStalePrefetchLock()
         [Category(Categories.MacTODO.M4)]
         public void PrefetchCleansUpPackDir()
         {
-            string multiPackIndexLockFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem),  MultiPackIndexLock);
+            string multiPackIndexLockFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem), MultiPackIndexLock);
             string oldGitTempFile = Path.Combine(this.Enlistment.GetPackRoot(this.fileSystem), "tmp_midx_XXXX");
 
             this.fileSystem.WriteAllText(multiPackIndexLockFile, this.Enlistment.EnlistmentRoot);
@@ -145,6 +146,41 @@ public void PrefetchCleansUpPackDir()
             multiPackIndexLockFile.ShouldNotExistOnDisk(this.fileSystem);
         }
 
+        [TestCase, Order(12)]
+        public void PrefetchFilesFromFileListFile()
+        {
+            try
+            {
+                File.WriteAllLines(
+                    "fileList",
+                    new[]
+                    {
+                        Path.Combine("GVFS", "GVFS", "Program.cs"),
+                        Path.Combine("GVFS", "GVFS.FunctionalTests", "GVFS.FunctionalTests.csproj")
+                    });
+
+                this.ExpectBlobCount(this.Enlistment.Prefetch("--files-list fileList"), 2);
+            }
+            finally
+            {
+                File.Delete("fileList");
+            }
+        }
+
+        [TestCase, Order(13)]
+        public void PrefetchFilesFromFileListStdIn()
+        {
+            var input = string.Join(
+                Environment.NewLine,
+                new[]
+                {
+                    Path.Combine("GVFS", "GVFS", "packages.config"),
+                    Path.Combine("GVFS", "GVFS.FunctionalTests", "App.config")
+                });
+
+            this.ExpectBlobCount(this.Enlistment.Prefetch("--stdin-files-list", standardInput: input), 2);
+        }
+
         private void ExpectBlobCount(string output, int expectedCount)
         {
             output.ShouldContain("Matched blobs:    " + expectedCount);

GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs

@@ -196,9 +196,9 @@ public bool TryMountGVFS(out string output)
             return this.gvfsProcess.TryMount(out output);
         }
 
-        public string Prefetch(string args, bool failOnError = true)
+        public string Prefetch(string args, bool failOnError = true, string standardInput = null)
         {
-            return this.gvfsProcess.Prefetch(args, failOnError);
+            return this.gvfsProcess.Prefetch(args, failOnError, standardInput);
         }
 
         public void Repair(bool confirm)

GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs

@@ -1,6 +1,6 @@
 using GVFS.Tests.Should;
 using System.Diagnostics;
-
+
 namespace GVFS.FunctionalTests.Tools
 {
     public class GVFSProcess
@@ -42,9 +42,9 @@ public bool TryMount(out string output)
             return this.IsEnlistmentMounted();
         }
 
-        public string Prefetch(string args, bool failOnError)
+        public string Prefetch(string args, bool failOnError, string standardInput = null)
         {
-            return this.CallGVFS("prefetch \"" + this.enlistmentRoot + "\" " + args, failOnError);
+            return this.CallGVFS("prefetch \"" + this.enlistmentRoot + "\" " + args, failOnError, standardInput: standardInput);
         }
 
         public void Repair(bool confirm)
@@ -90,7 +90,7 @@ public string RunServiceVerb(string argument)
             return this.CallGVFS("service " + argument, failOnError: true);
         }
 
-        private string CallGVFS(string args, bool failOnError = false, string trace = null)
+        private string CallGVFS(string args, bool failOnError = false, string trace = null, string standardInput = null)
         {
             ProcessStartInfo processInfo = null;
             processInfo = new ProcessStartInfo(this.pathToGVFS);
@@ -99,6 +99,10 @@ private string CallGVFS(string args, bool failOnError = false, string trace = nu
             processInfo.WindowStyle = ProcessWindowStyle.Hidden;
             processInfo.UseShellExecute = false;
             processInfo.RedirectStandardOutput = true;
+            if (standardInput != null)
+            {
+                processInfo.RedirectStandardInput = true;
+            }
 
             if (trace != null)
             {
@@ -107,6 +111,12 @@ private string CallGVFS(string args, bool failOnError = false, string trace = nu
 
             using (Process process = Process.Start(processInfo))
             {
+                if (standardInput != null)
+                {
+                    process.StandardInput.Write(standardInput);
+                    process.StandardInput.Close();
+                }
+
                 string result = process.StandardOutput.ReadToEnd();
                 process.WaitForExit();