Skip to content

Commit 7669d6c

Browse files
tyrielvtyrielv
authored
Merge pull request #1943 from tyrielv/tyrielv/fix-concurrent-worktree-mount
Improve resiliency of git worktree commands
2 parents 99dcb4a + e876cd7 commit 7669d6c

3 files changed

Lines changed: 405 additions & 90 deletions

File tree

‎GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorktreeTests.cs‎

Lines changed: 285 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,314 @@
1-
using GVFS.FunctionalTests.Tools;
1+
using GVFS.Common;
2+
using GVFS.Common.NamedPipes;
3+
using GVFS.FunctionalTests.Tools;
24
using GVFS.Tests.Should;
35
using NUnit.Framework;
46
using System;
57
using System.Diagnostics;
68
using System.IO;
9+
using System.Linq;
10+
using System.Text;
11+
using System.Threading;
12+
using ProcessResult = GVFS.FunctionalTests.Tools.ProcessResult;
713

814
namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture
915
{
1016
[TestFixture]
1117
[Category(Categories.GitCommands)]
1218
public class WorktreeTests : TestsWithEnlistmentPerFixture
1319
{
14-
private const string WorktreeBranchA = "worktree-test-branch-a";
15-
private const string WorktreeBranchB = "worktree-test-branch-b";
20+
private const int MinWorktreeCount = 4;
1621

1722
[TestCase]
1823
public void ConcurrentWorktreeAddCommitRemove()
1924
{
20-
string worktreePathA = Path.Combine(this.Enlistment.EnlistmentRoot, "test-wt-a-" + Guid.NewGuid().ToString("N").Substring(0, 8));
21-
string worktreePathB = Path.Combine(this.Enlistment.EnlistmentRoot, "test-wt-b-" + Guid.NewGuid().ToString("N").Substring(0, 8));
25+
int count = Math.Max(Environment.ProcessorCount, MinWorktreeCount);
26+
string[] worktreePaths;
27+
string[] branchNames;
28+
29+
// Adaptively scale down if concurrent adds overwhelm the primary
30+
// GVFS mount. CI runners with fewer resources may not handle as
31+
// many concurrent git operations as a developer workstation.
32+
while (true)
33+
{
34+
this.InitWorktreeArrays(count, out worktreePaths, out branchNames);
35+
ProcessResult[] addResults = this.ConcurrentWorktreeAdd(worktreePaths, branchNames, count);
36+
37+
bool overloaded = addResults.Any(r =>
38+
r.ExitCode != 0 &&
39+
r.Errors != null &&
40+
r.Errors.Contains("does not appear to be mounted"));
41+
42+
// Only retry if ALL failures are overload-related. If any
43+
// failure has a different cause, it's a real regression and
44+
// must not be masked by retrying at lower concurrency.
45+
bool hasNonOverloadFailure = addResults.Any(r =>
46+
r.ExitCode != 0 &&
47+
!(r.Errors != null && r.Errors.Contains("does not appear to be mounted")));
48+
49+
if (hasNonOverloadFailure)
50+
{
51+
// Fall through to the assertion loop below which will
52+
// report the specific failure(s).
53+
}
54+
else if (overloaded)
55+
{
56+
this.CleanupAllWorktrees(worktreePaths, branchNames, count);
57+
int reduced = count / 2;
58+
if (reduced < MinWorktreeCount)
59+
{
60+
Assert.Fail(
61+
$"Primary GVFS mount overloaded even at count={count}. " +
62+
$"Cannot reduce below {MinWorktreeCount}.");
63+
}
64+
65+
count = reduced;
66+
continue;
67+
}
68+
69+
// Non-overload failures are real errors
70+
for (int i = 0; i < count; i++)
71+
{
72+
addResults[i].ExitCode.ShouldEqual(0,
73+
$"worktree add [{i}] failed: {addResults[i].Errors}");
74+
}
75+
76+
break;
77+
}
2278

2379
try
2480
{
25-
// 1. Create both worktrees in parallel
26-
ProcessResult addResultA = null;
27-
ProcessResult addResultB = null;
28-
System.Threading.Tasks.Parallel.Invoke(
29-
() => addResultA = GitHelpers.InvokeGitAgainstGVFSRepo(
30-
this.Enlistment.RepoRoot,
31-
$"worktree add -b {WorktreeBranchA} \"{worktreePathA}\""),
32-
() => addResultB = GitHelpers.InvokeGitAgainstGVFSRepo(
33-
this.Enlistment.RepoRoot,
34-
$"worktree add -b {WorktreeBranchB} \"{worktreePathB}\""));
35-
36-
addResultA.ExitCode.ShouldEqual(0, $"worktree add A failed: {addResultA.Errors}");
37-
addResultB.ExitCode.ShouldEqual(0, $"worktree add B failed: {addResultB.Errors}");
38-
39-
// 2. Verify both have projected files
40-
Directory.Exists(worktreePathA).ShouldBeTrue("Worktree A directory should exist");
41-
Directory.Exists(worktreePathB).ShouldBeTrue("Worktree B directory should exist");
42-
File.Exists(Path.Combine(worktreePathA, "Readme.md")).ShouldBeTrue("Readme.md should be projected in A");
43-
File.Exists(Path.Combine(worktreePathB, "Readme.md")).ShouldBeTrue("Readme.md should be projected in B");
44-
45-
// 3. Verify git status is clean in both
46-
ProcessResult statusA = GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathA, "status --porcelain");
47-
ProcessResult statusB = GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathB, "status --porcelain");
48-
statusA.ExitCode.ShouldEqual(0, $"git status A failed: {statusA.Errors}");
49-
statusB.ExitCode.ShouldEqual(0, $"git status B failed: {statusB.Errors}");
50-
statusA.Output.Trim().ShouldBeEmpty("Worktree A should have clean status");
51-
statusB.Output.Trim().ShouldBeEmpty("Worktree B should have clean status");
52-
53-
// 4. Verify worktree list shows all three
81+
// 2. Primary assertion: verify GVFS mount is running for each
82+
// worktree by probing the worktree-specific named pipe.
83+
for (int i = 0; i < count; i++)
84+
{
85+
this.AssertWorktreeMounted(worktreePaths[i], $"worktree [{i}]");
86+
}
87+
88+
// 3. Verify projected files are visible (secondary assertion)
89+
for (int i = 0; i < count; i++)
90+
{
91+
Directory.Exists(worktreePaths[i]).ShouldBeTrue(
92+
$"Worktree [{i}] directory should exist");
93+
File.Exists(Path.Combine(worktreePaths[i], "Readme.md")).ShouldBeTrue(
94+
$"Readme.md should be projected in [{i}]");
95+
}
96+
97+
// 4. Verify git status is clean in each worktree
98+
for (int i = 0; i < count; i++)
99+
{
100+
ProcessResult status = GitHelpers.InvokeGitAgainstGVFSRepo(
101+
worktreePaths[i], "status --porcelain");
102+
status.ExitCode.ShouldEqual(0,
103+
$"git status [{i}] failed: {status.Errors}");
104+
status.Output.Trim().ShouldBeEmpty(
105+
$"Worktree [{i}] should have clean status");
106+
}
107+
108+
// 5. Verify worktree list shows all entries
54109
ProcessResult listResult = GitHelpers.InvokeGitAgainstGVFSRepo(
55110
this.Enlistment.RepoRoot, "worktree list");
56111
listResult.ExitCode.ShouldEqual(0, $"worktree list failed: {listResult.Errors}");
57112
string listOutput = listResult.Output;
58-
Assert.IsTrue(listOutput.Contains(worktreePathA.Replace('\\', '/')),
59-
$"worktree list should contain A. Output: {listOutput}");
60-
Assert.IsTrue(listOutput.Contains(worktreePathB.Replace('\\', '/')),
61-
$"worktree list should contain B. Output: {listOutput}");
62-
63-
// 5. Make commits in both worktrees
64-
File.WriteAllText(Path.Combine(worktreePathA, "from-a.txt"), "created in worktree A");
65-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathA, "add from-a.txt")
66-
.ExitCode.ShouldEqual(0);
67-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathA, "commit -m \"commit from A\"")
68-
.ExitCode.ShouldEqual(0);
69-
70-
File.WriteAllText(Path.Combine(worktreePathB, "from-b.txt"), "created in worktree B");
71-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathB, "add from-b.txt")
72-
.ExitCode.ShouldEqual(0);
73-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathB, "commit -m \"commit from B\"")
74-
.ExitCode.ShouldEqual(0);
75-
76-
// 6. Verify commits are visible from all worktrees (shared objects)
77-
GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, $"log -1 --format=%s {WorktreeBranchA}")
78-
.Output.ShouldContain(expectedSubstrings: new[] { "commit from A" });
79-
GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, $"log -1 --format=%s {WorktreeBranchB}")
80-
.Output.ShouldContain(expectedSubstrings: new[] { "commit from B" });
81-
82-
// A can see B's commit and vice versa
83-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathA, $"log -1 --format=%s {WorktreeBranchB}")
84-
.Output.ShouldContain(expectedSubstrings: new[] { "commit from B" });
85-
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePathB, $"log -1 --format=%s {WorktreeBranchA}")
86-
.Output.ShouldContain(expectedSubstrings: new[] { "commit from A" });
87-
88-
// 7. Remove both in parallel
89-
ProcessResult removeA = null;
90-
ProcessResult removeB = null;
91-
System.Threading.Tasks.Parallel.Invoke(
92-
() => removeA = GitHelpers.InvokeGitAgainstGVFSRepo(
93-
this.Enlistment.RepoRoot,
94-
$"worktree remove --force \"{worktreePathA}\""),
95-
() => removeB = GitHelpers.InvokeGitAgainstGVFSRepo(
96-
this.Enlistment.RepoRoot,
97-
$"worktree remove --force \"{worktreePathB}\""));
98-
99-
removeA.ExitCode.ShouldEqual(0, $"worktree remove A failed: {removeA.Errors}");
100-
removeB.ExitCode.ShouldEqual(0, $"worktree remove B failed: {removeB.Errors}");
101-
102-
// 8. Verify cleanup
103-
Directory.Exists(worktreePathA).ShouldBeFalse("Worktree A directory should be deleted");
104-
Directory.Exists(worktreePathB).ShouldBeFalse("Worktree B directory should be deleted");
113+
for (int i = 0; i < count; i++)
114+
{
115+
Assert.IsTrue(
116+
listOutput.Contains(worktreePaths[i].Replace('\\', '/')),
117+
$"worktree list should contain [{i}]. Output: {listOutput}");
118+
}
119+
120+
// 6. Make commits in all worktrees
121+
for (int i = 0; i < count; i++)
122+
{
123+
File.WriteAllText(
124+
Path.Combine(worktreePaths[i], $"from-{i}.txt"),
125+
$"created in worktree {i}");
126+
GitHelpers.InvokeGitAgainstGVFSRepo(worktreePaths[i], $"add from-{i}.txt")
127+
.ExitCode.ShouldEqual(0);
128+
GitHelpers.InvokeGitAgainstGVFSRepo(
129+
worktreePaths[i], $"commit -m \"commit from {i}\"")
130+
.ExitCode.ShouldEqual(0);
131+
}
132+
133+
// 7. Verify commits are visible from main repo
134+
for (int i = 0; i < count; i++)
135+
{
136+
GitHelpers.InvokeGitAgainstGVFSRepo(
137+
this.Enlistment.RepoRoot, $"log -1 --format=%s {branchNames[i]}")
138+
.Output.ShouldContain(expectedSubstrings: new[] { $"commit from {i}" });
139+
}
140+
141+
// 8. Verify cross-worktree commit visibility (shared objects)
142+
for (int i = 0; i < count; i++)
143+
{
144+
int other = (i + 1) % count;
145+
GitHelpers.InvokeGitAgainstGVFSRepo(
146+
worktreePaths[i], $"log -1 --format=%s {branchNames[other]}")
147+
.Output.ShouldContain(expectedSubstrings: new[] { $"commit from {other}" });
148+
}
149+
150+
// 9. Remove all worktrees in parallel
151+
ProcessResult[] removeResults = new ProcessResult[count];
152+
using (CountdownEvent barrier = new CountdownEvent(count))
153+
{
154+
Thread[] threads = new Thread[count];
155+
for (int i = 0; i < count; i++)
156+
{
157+
int idx = i;
158+
threads[idx] = new Thread(() =>
159+
{
160+
barrier.Signal();
161+
barrier.Wait();
162+
removeResults[idx] = GitHelpers.InvokeGitAgainstGVFSRepo(
163+
this.Enlistment.RepoRoot,
164+
$"worktree remove --force \"{worktreePaths[idx]}\"");
165+
});
166+
threads[idx].Start();
167+
}
168+
169+
foreach (Thread t in threads)
170+
{
171+
t.Join();
172+
}
173+
}
174+
175+
for (int i = 0; i < count; i++)
176+
{
177+
removeResults[i].ExitCode.ShouldEqual(0,
178+
$"worktree remove [{i}] failed: {removeResults[i].Errors}");
179+
}
180+
181+
// 10. Verify cleanup
182+
for (int i = 0; i < count; i++)
183+
{
184+
Directory.Exists(worktreePaths[i]).ShouldBeFalse(
185+
$"Worktree [{i}] directory should be deleted");
186+
}
105187
}
106188
finally
107189
{
108-
this.ForceCleanupWorktree(worktreePathA, WorktreeBranchA);
109-
this.ForceCleanupWorktree(worktreePathB, WorktreeBranchB);
190+
this.CleanupAllWorktrees(worktreePaths, branchNames, count);
191+
}
192+
}
193+
194+
private void InitWorktreeArrays(int count, out string[] paths, out string[] branches)
195+
{
196+
paths = new string[count];
197+
branches = new string[count];
198+
for (int i = 0; i < count; i++)
199+
{
200+
string suffix = Guid.NewGuid().ToString("N").Substring(0, 8);
201+
paths[i] = Path.Combine(this.Enlistment.EnlistmentRoot, $"test-wt-{i}-{suffix}");
202+
branches[i] = $"worktree-test-branch-{i}-{suffix}";
203+
}
204+
}
205+
206+
private ProcessResult[] ConcurrentWorktreeAdd(string[] paths, string[] branches, int count)
207+
{
208+
ProcessResult[] results = new ProcessResult[count];
209+
using (CountdownEvent barrier = new CountdownEvent(count))
210+
{
211+
Thread[] threads = new Thread[count];
212+
for (int i = 0; i < count; i++)
213+
{
214+
int idx = i;
215+
threads[idx] = new Thread(() =>
216+
{
217+
barrier.Signal();
218+
barrier.Wait();
219+
results[idx] = GitHelpers.InvokeGitAgainstGVFSRepo(
220+
this.Enlistment.RepoRoot,
221+
$"worktree add -b {branches[idx]} \"{paths[idx]}\"");
222+
});
223+
threads[idx].Start();
224+
}
225+
226+
foreach (Thread t in threads)
227+
{
228+
t.Join();
229+
}
230+
}
231+
232+
return results;
233+
}
234+
235+
/// <summary>
236+
/// Asserts that the GVFS mount for a worktree is running by probing
237+
/// the worktree-specific named pipe. This is the definitive signal
238+
/// that ProjFS projection is active — much stronger than File.Exists
239+
/// which depends on projection timing.
240+
/// </summary>
241+
private void AssertWorktreeMounted(string worktreePath, string label)
242+
{
243+
string basePipeName = GVFSPlatform.Instance.GetNamedPipeName(
244+
this.Enlistment.EnlistmentRoot);
245+
string suffix = GVFSEnlistment.GetWorktreePipeSuffix(worktreePath);
246+
247+
Assert.IsNotNull(suffix,
248+
$"Could not determine pipe suffix for {label} at {worktreePath}. " +
249+
$"The worktree .git file may be missing or malformed.");
250+
251+
string pipeName = basePipeName + suffix;
252+
253+
using (NamedPipeClient client = new NamedPipeClient(pipeName))
254+
{
255+
if (!client.Connect(10000))
256+
{
257+
string diagnostics = this.CaptureWorktreeDiagnostics(worktreePath);
258+
Assert.Fail(
259+
$"GVFS mount is NOT running for {label}.\n" +
260+
$"Path: {worktreePath}\n" +
261+
$"Pipe: {pipeName}\n" +
262+
$"This indicates the post-hook 'gvfs mount' failed silently.\n" +
263+
$"Diagnostics:\n{diagnostics}");
264+
}
265+
}
266+
}
267+
268+
private string CaptureWorktreeDiagnostics(string worktreePath)
269+
{
270+
StringBuilder sb = new StringBuilder();
271+
272+
sb.AppendLine($" Directory exists: {Directory.Exists(worktreePath)}");
273+
if (Directory.Exists(worktreePath))
274+
{
275+
string dotGit = Path.Combine(worktreePath, ".git");
276+
sb.AppendLine($" .git file exists: {File.Exists(dotGit)}");
277+
if (File.Exists(dotGit))
278+
{
279+
try
280+
{
281+
sb.AppendLine($" .git contents: {File.ReadAllText(dotGit).Trim()}");
282+
}
283+
catch (Exception ex)
284+
{
285+
sb.AppendLine($" .git read failed: {ex.Message}");
286+
}
287+
}
288+
289+
try
290+
{
291+
string[] entries = Directory.GetFileSystemEntries(worktreePath);
292+
sb.AppendLine($" Directory listing ({entries.Length} entries):");
293+
foreach (string entry in entries)
294+
{
295+
sb.AppendLine($" {Path.GetFileName(entry)}");
296+
}
297+
}
298+
catch (Exception ex)
299+
{
300+
sb.AppendLine($" Directory listing failed: {ex.Message}");
301+
}
302+
}
303+
304+
return sb.ToString();
305+
}
306+
307+
private void CleanupAllWorktrees(string[] paths, string[] branches, int count)
308+
{
309+
for (int i = 0; i < count; i++)
310+
{
311+
this.ForceCleanupWorktree(paths[i], branches[i]);
110312
}
111313
}
112314

0 commit comments

Comments
 (0)