darwin/arm64: use the number of perf cores as the default number of workers

Starting around macOS v15 the parallel performance of the APFS file
system appears slightly improved and now the ideal number of fastwalk
workers is generally the number of performance cores currently
available.

Therefore, this commit changes the default number of workers on
darwin/arm64 to be the value of sysctl("hw.perflevel0.physicalcpu")
which is the number of currently available performance cores (link to
xnu source below).

https://github.com/apple-oss-distributions/xnu/blob/43a90889846e00bfb5cf1d255cdc0a701a1e05a4/bsd/sys/sysctl.h#L1244

Author: charlievieth

fastwalk.go

@@ -77,18 +77,23 @@ func DefaultNumWorkers() int {
 	if numCPU < 4 {
 		return 4
 	}
+	// User manually set GOMAXPROCS - respect it.
+	if numCPU != runtime.NumCPU() {
+		return min(numCPU, 32)
+	}
 	// Darwin IO performance on APFS can slow with increased parallelism.
-	// Depending on CPU, stat(2) performance is best around 4-10 workers
-	// and file IO is best around 4 workers. More workers only benefit CPU
-	// intensive tasks.
-	//
-	// NB(Charlie): As of macOS 15, the parallel performance of readdir_r(3)
-	// and stat(2) calls has improved and is now generally the number of
-	// performance cores (on ARM Macs).
+	// For Intel CPUs (and maybe older arm64 CPUs) performance is best
+	// around 4-10 workers and file IO is best around 4 workers. More workers
+	// only benefit CPU intensive tasks.
 	//
-	// TODO: Consider using the value of sysctl("hw.perflevel0.physicalcpu").
-	// TODO: Find someone with a Mac Studio to test higher core counts.
+	// As of macOS 15 (on ARM Macs), the parallel performance of readdir_r(3)
+	// and stat(2) calls has improved and the ideal number of workers is now
+	// generally the number of performance cores.
 	if runtime.GOOS == "darwin" {
+		if n := darwinNumPerfCores(); n > 0 {
+			return n
+		}
+		// This is primarily for Intel CPUs.
 		switch {
 		case numCPU <= 8:
 			return 4
@@ -98,10 +103,7 @@ func DefaultNumWorkers() int {
 			return 10
 		}
 	}
-	if numCPU > 32 {
-		return 32
-	}
-	return numCPU
+	return min(numCPU, 32)
 }
 
 // DefaultToSlash returns true if this is a Go program compiled for Windows

fastwalk_test.go

@@ -10,11 +10,13 @@ import (
 	"io/fs"
 	"math"
 	"os"
+	"os/exec"
 	"os/user"
 	"path/filepath"
 	"reflect"
 	"regexp"
 	"runtime"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -1343,6 +1345,83 @@ func TestSkipAll(t *testing.T) {
 	}
 }
 
+func TestDefaultNumWorkers(t *testing.T) {
+	if os.Getenv("FASTWALK_TEST_COOKIE") == "123456" {
+		fmt.Printf("## DEFAULT_NUM_WORKERS: %d\n", fastwalk.DefaultNumWorkers())
+		return
+	}
+
+	exe, err := os.Executable()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	re := regexp.MustCompile(`## DEFAULT_NUM_WORKERS:\s+(\d+)`)
+	parseOut := func(t *testing.T, out []byte) int {
+		a := re.FindSubmatch(out)
+		if len(a) != 2 {
+			t.Fatal(`Sub-test output does not contain: "## DEFAULT_NUM_WORKERS"`)
+		}
+		n, err := strconv.Atoi(string(a[1]))
+		if err != nil {
+			t.Fatal(err)
+		}
+		return n
+	}
+
+	runTest := func(t *testing.T, maxProcs, want int) {
+		cmd := exec.Command(exe, "-test.run", "^TestDefaultNumWorkers$")
+		cmd.Env = cmd.Environ()
+		cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool {
+			return strings.HasPrefix(s, "GOMAXPROCS=") || strings.HasPrefix(s, "GOFLAGS=")
+		})
+		cmd.Env = append(cmd.Env, "FASTWALK_TEST_COOKIE=123456")
+		if maxProcs > 0 {
+			cmd.Env = append(cmd.Env, fmt.Sprintf("GOMAXPROCS=%d", maxProcs))
+		}
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("test failed: %s", bytes.TrimSpace(out))
+		}
+		got := parseOut(t, out)
+		if got != want {
+			t.Fatalf("got: %d want: %d", got, want)
+		}
+	}
+
+	t.Run("GOMAXPROCS", func(t *testing.T) {
+		for _, procs := range []int{2, 4, 8, 20, 64} {
+			t.Run(strconv.Itoa(procs), func(t *testing.T) {
+				want := max(4, min(procs, 32))
+				runTest(t, procs, want)
+			})
+		}
+	})
+
+	t.Run("Darwin_ARM64", func(t *testing.T) {
+		if runtime.GOOS != "darwin" && runtime.GOARCH != "arm64" {
+			t.Skip("test only supported on darwin/arm64")
+		}
+		// Not all platforms have syscall.SysctlUint32 so shell out to
+		// "sysctl". This is ugly but saves us from having to duplicate
+		// a large portion of this code.
+		out, err := exec.Command("sysctl", "hw.perflevel0.physicalcpu").CombinedOutput()
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, val, ok := strings.Cut(strings.TrimSpace(string(out)), " ")
+		if !ok {
+			t.Fatalf("Invalid sysctl output: %q", out)
+		}
+		want, err := strconv.Atoi(val)
+		if err != nil {
+			t.Fatal(err)
+		}
+		want = max(want, 4) // 4 is the minimum
+		runTest(t, -1, int(want))
+	})
+}
+
 func BenchmarkSortModeString(b *testing.B) {
 	var s string
 	for i := 0; i < b.N; i++ {

numcpu_darwin_arm64.go

@@ -0,0 +1,25 @@
+//go:build darwin && arm64
+
+package fastwalk
+
+import "syscall"
+
+// darwinNumPerfCores returns the number of performance cores currently
+// available on macOS/arm64.
+func darwinNumPerfCores() int {
+	// "hw.physicalcpu" is the number of physical processors available in
+	// the current power management mode.
+	//
+	// https://github.com/apple-oss-distributions/xnu/blob/43a90889846e00bfb5cf1d255cdc0a701a1e05a4/bsd/sys/sysctl.h#L1244
+	//
+	// NB: We do not cache this value since it could change with the power
+	// management mode.
+	//
+	// TODO: Find someone with a Mac Studio or MacPro to see if this still
+	// holds true when the CPU is basically two "fused" ones.
+	n, err := syscall.SysctlUint32("hw.perflevel0.physicalcpu")
+	if err != nil {
+		return -1
+	}
+	return int(n)
+}

numcpu_darwin_arm64_test.go

@@ -0,0 +1,11 @@
+package fastwalk
+
+import "testing"
+
+func TestDarwinNumPerfCores(t *testing.T) {
+	n := darwinNumPerfCores()
+	// Test that n is reasonable
+	if !(0 < n && n < 4096) {
+		t.Fatalf("expected a value between 0..4096 got: %d", n)
+	}
+}

numcpu_portable.go

@@ -0,0 +1,7 @@
+//go:build !(darwin && arm64)
+
+package fastwalk
+
+func darwinNumPerfCores() int {
+	return -1
+}