Fixed: flapping clean-nfs-cache/TestCompare

packages/orchestrator/cmd/clean-nfs-cache/compare_test.go

@@ -32,22 +32,22 @@
 
 	printSummary := func(start time.Time, atimes []time.Duration, deletedBytes uint64) {
 		mean, sd := standardDeviation(atimes)
-		t.Logf("Cleaned %d out of %d bytes in %v; file age %v (%v)",
+		t.Logf("Cleaned %d (target %d) bytes in %v; file age %v (%v)",
 			deletedBytes, targetBytesToDelete, time.Since(start), mean.Round(time.Hour), sd.Round(time.Minute))
 	}
 
-	for _, nScan := range []int{1, 4, 16, 1024} {
-		for _, nDel := range []int{1, 2, 8, 1024} {
-			for _, nStat := range []int{1, 4, 16, 1024} {
+	for _, nScan := range []int{1, 4, 16, 32} {
+		for _, nDel := range []int{1, 2, 4, 16} {
+			for _, nStat := range []int{1, 4, 16, 32} {
 				t.Run(fmt.Sprintf("Scan%v-Del%v-Stat%v", nScan, nDel, nStat), func(t *testing.T) {
-					t.Parallel()
-
 					path := t.TempDir()
 					ex.CreateTestDir(path, NDirs, NFiles, testFileSize)
 					t.Cleanup(func() {
 						os.RemoveAll(path)
 					})
 					start := time.Now()
+					// log, _ := logger.NewDevelopmentLogger()
+					log := logger.NewNopLogger()
 					c := ex.NewCleaner(ex.Options{
 						Path:                path,
 						DeleteN:             NFiles / 100,
@@ -58,8 +58,7 @@
 						MaxConcurrentDelete: nDel,
 						TargetBytesToDelete: targetBytesToDelete,
 						MaxErrorRetries:     10,
-					}, logger.NewNopLogger())
-
+					}, log)
 					err := c.Clean(ctx)
 					require.NoError(t, err)
 					require.GreaterOrEqual(t, c.DeletedBytes.Load(), targetBytesToDelete)

packages/orchestrator/cmd/clean-nfs-cache/ex/clean.go

@@ -197,6 +197,13 @@ func (c *Cleaner) Clean(ctx context.Context) error {
 	runtime.ReadMemStats(&baseMem)
 	batchNumber := 0
 
+	// All scanner workers would immediately try to scan the root directory,
+	// so we scan it once here to populate the initial directory tree.
+	_, err := c.scanDir(ctx, []*Dir{c.root})
+	if err != nil {
+		return fmt.Errorf("initial scan of root directory failed: %w", err)
+	}
+
 	for range c.MaxConcurrentStat {
 		running.Add(1)
 		go c.Statter(ctx, running)

packages/orchestrator/cmd/clean-nfs-cache/ex/scan.go

@@ -37,7 +37,9 @@ func (c *Cleaner) Scanner(ctx context.Context, candidateCh chan<- *Candidate, er
 
 			default:
 				if !errors.Is(err, ErrNoFiles) {
-					c.Info(ctx, "error during scanning", zap.Error(err))
+					c.Info(ctx, "error during scanning",
+						zap.Int("continousCount", continuousErrors),
+						zap.Error(err))
 				}
 				continuousErrors++
 				if continuousErrors >= c.MaxErrorRetries {
@@ -94,12 +96,12 @@ func (c *Cleaner) findCandidate(ctx context.Context, path []*Dir) (*Candidate, e
 	}
 }
 
-func (d *Dir) randomSubdirOrOldestFile() (cadidate *File, randomSubdir *Dir, err error) {
+func (d *Dir) randomSubdirOrOldestFile() (randomCandidate *File, randomSubdir *Dir, err error) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 
 	if len(d.Files) == 0 && len(d.Dirs) == 0 {
-		return nil, nil, ErrNoFiles
+		return nil, nil, fmt.Errorf("no candidates found in %s: %w", d.Name, ErrNoFiles)
 	}
 	itemCount := len(d.Dirs) + len(d.Files)
 	i := rand.Intn(itemCount)
@@ -129,7 +131,7 @@ func (c *Cleaner) scanDir(ctx context.Context, path []*Dir) (out *Dir, err error
 	case dirStateScanning:
 		d.mu.Unlock()
 
-		return nil, ErrBusy
+		return nil, fmt.Errorf("%w: directory %s is busy being scanned", ErrBusy, c.abs(path, ""))
 
 	default:
 		// continue