add background build system with tool version pinning

Author: leighmcculloch

main.go

@@ -122,6 +122,7 @@ Configuration is loaded from (in order, merged):
 	rootCmd.Flags().Bool("force-build", false, "Force rebuild of container image")
 	rootCmd.Flags().Bool("no-cache", false, "Disable build cache (implies --force-build)")
 	rootCmd.Flags().BoolP("verbose", "v", false, "Show detailed output instead of progress bar")
+	rootCmd.Flags().String("tool-version", "", "Pin a specific tool version (forces synchronous build)")
 
 	// Define command groups (order here determines display order in --help)
 	rootCmd.AddGroup(
@@ -147,6 +148,7 @@ Configuration is loaded from (in order, merged):
 		toolCmd.Flags().Bool("no-cache", false, "Disable build cache (implies --force-build)")
 		toolCmd.Flags().BoolP("verbose", "v", false, "Show detailed output instead of progress bar")
 		toolCmd.Flags().String("entrypoint", "", "Run a custom command instead of the tool (e.g. /bin/bash)")
+		toolCmd.Flags().String("tool-version", "", "Pin a specific tool version (forces synchronous build)")
 		rootCmd.AddCommand(toolCmd)
 	}
 
@@ -282,6 +284,18 @@ Use --local or --global to skip the prompt.`,
 	shellCmd.Flags().String("backend", "", "Backend to use: docker, container (default: both)")
 	rootCmd.AddCommand(shellCmd)
 
+	// Hidden __build subcommand — used by background build launcher.
+	buildCmd := &cobra.Command{
+		Use:    "__build",
+		Hidden: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir, _ := cmd.Flags().GetString("dir")
+			return runBackgroundBuild(dir, stderr)
+		},
+	}
+	buildCmd.Flags().String("dir", "", "Build state directory")
+	rootCmd.AddCommand(buildCmd)
+
 	rootCmd.Version = version
 	rootCmd.SetVersionTemplate("silo version {{.Version}}\n")
 
@@ -346,16 +360,20 @@ func runSilo(cmd *cobra.Command, args []string, stdout, stderr io.Writer) error
 	// Get verbose flag
 	verbose, _ := cmd.Flags().GetBool("verbose")
 
+	// Get tool-version flag
+	toolVersion, _ := cmd.Flags().GetString("tool-version")
+
 	// Run the tool
 	return run.Tool(run.Options{
-		ToolDef:    *toolDef,
-		Config:     cfg,
-		Dockerfile: Dockerfile(supportedTools),
-		ForceBuild: forceBuild,
-		NoCache:    noCache,
-		Verbose:    verbose,
-		Stdout:     stdout,
-		Stderr:     stderr,
+		ToolDef:     *toolDef,
+		Config:      cfg,
+		Dockerfile:  Dockerfile(supportedTools),
+		ForceBuild:  forceBuild,
+		NoCache:     noCache,
+		Verbose:     verbose,
+		ToolVersion: toolVersion,
+		Stdout:      stdout,
+		Stderr:      stderr,
 	})
 }
 
@@ -389,18 +407,22 @@ func runTool(cmd *cobra.Command, toolDef tools.Tool, args []string, stdout, stde
 	// Get entrypoint flag
 	entrypoint, _ := cmd.Flags().GetString("entrypoint")
 
+	// Get tool-version flag
+	toolVersion, _ := cmd.Flags().GetString("tool-version")
+
 	// Run the tool
 	return run.Tool(run.Options{
-		ToolDef:    toolDef,
-		ToolArgs:   toolArgs,
-		Config:     cfg,
-		Dockerfile: Dockerfile(supportedTools),
-		ForceBuild: forceBuild,
-		NoCache:    noCache,
-		Verbose:    verbose,
-		Entrypoint: entrypoint,
-		Stdout:     stdout,
-		Stderr:     stderr,
+		ToolDef:     toolDef,
+		ToolArgs:    toolArgs,
+		Config:      cfg,
+		Dockerfile:  Dockerfile(supportedTools),
+		ForceBuild:  forceBuild,
+		NoCache:     noCache,
+		Verbose:     verbose,
+		Entrypoint:  entrypoint,
+		ToolVersion: toolVersion,
+		Stdout:      stdout,
+		Stderr:      stderr,
 	})
 }
 
@@ -911,3 +933,62 @@ func formatMemoryUsage(bytes uint64, isRunning bool) string {
 	}
 	return humanize.IBytes(bytes)
 }
+
+// runBackgroundBuild is the entry point for the hidden `silo __build` command.
+// It reads all build parameters from the manifest in the build state directory,
+// acquires a build lock, builds the image, and updates state.
+func runBackgroundBuild(dir string, stderr io.Writer) error {
+	if dir == "" {
+		return fmt.Errorf("__build: --dir is required")
+	}
+
+	// Read build manifest.
+	imageTag, tool, backendType, dockerfile, buildArgs, err := run.ReadBuildManifest(dir)
+	if err != nil {
+		return fmt.Errorf("__build: %w", err)
+	}
+
+	// Acquire build lock — exit cleanly if another build is already running.
+	lock, err := run.TryLock(imageTag)
+	if err != nil {
+		return fmt.Errorf("__build: lock: %w", err)
+	}
+	if lock == nil {
+		// Another process is already building this image.
+		return nil
+	}
+	defer lock.Unlock()
+
+	// Create backend.
+	backendClient, err := run.CreateBackend(backendType, stderr, false)
+	if err != nil {
+		lock.WriteStatus("failed")
+		return fmt.Errorf("__build: backend: %w", err)
+	}
+	defer backendClient.Close()
+
+	// Build image.
+	fmt.Fprintf(stderr, "==> Background build: %s (%s)\n", tool, imageTag)
+	ctx := context.Background()
+	_, err = backendClient.Build(ctx, backend.BuildOptions{
+		Dockerfile: dockerfile,
+		Target:     tool,
+		Tag:        imageTag,
+		BuildArgs:  buildArgs,
+		OnProgress: func(msg string) {
+			fmt.Fprint(stderr, msg)
+		},
+	})
+	if err != nil {
+		lock.WriteStatus("failed")
+		return fmt.Errorf("__build: build: %w", err)
+	}
+
+	lock.WriteStatus("done")
+	fmt.Fprintf(stderr, "==> Background build complete: %s\n", imageTag)
+
+	// Update last-image so next run picks up the new image.
+	run.SaveLastImage(tool, imageTag)
+
+	return nil
+}

run/bgbuild.go

@@ -0,0 +1,109 @@
+package run
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/leighmcculloch/silo/config"
+)
+
+// BackgroundBuildOptions contains the parameters needed to launch a
+// detached background build process.
+type BackgroundBuildOptions struct {
+	ImageTag   string
+	Dockerfile string
+	BuildArgs  map[string]string
+	Backend    string
+	Tool       string
+}
+
+// buildManifest is the JSON structure written to the build state directory.
+// The __build command reads this single file to get all build parameters.
+type buildManifest struct {
+	ImageTag   string            `json:"image_tag"`
+	Tool       string            `json:"tool"`
+	Backend    string            `json:"backend,omitempty"`
+	BuildArgs  map[string]string `json:"build_args"`
+	Dockerfile string            `json:"dockerfile"`
+}
+
+// ReadBuildManifest reads the build manifest from a state directory.
+func ReadBuildManifest(dir string) (imageTag, tool, backend, dockerfile string, buildArgs map[string]string, err error) {
+	data, err := os.ReadFile(filepath.Join(dir, "build.json"))
+	if err != nil {
+		return "", "", "", "", nil, fmt.Errorf("read manifest: %w", err)
+	}
+	var m buildManifest
+	if err := json.Unmarshal(data, &m); err != nil {
+		return "", "", "", "", nil, fmt.Errorf("parse manifest: %w", err)
+	}
+	if m.ImageTag == "" || m.Tool == "" {
+		return "", "", "", "", nil, fmt.Errorf("manifest missing required fields (image_tag, tool)")
+	}
+	return m.ImageTag, m.Tool, m.Backend, m.Dockerfile, m.BuildArgs, nil
+}
+
+// LaunchBackgroundBuild writes the build inputs to the build state directory
+// and spawns a detached `silo __build` subprocess that will survive if the
+// parent session exits.
+func LaunchBackgroundBuild(opts BackgroundBuildOptions) error {
+	dir := buildDir(opts.ImageTag)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("bgbuild: mkdir: %w", err)
+	}
+
+	// Write all build parameters to a single manifest file.
+	manifest := buildManifest{
+		ImageTag:   opts.ImageTag,
+		Tool:       opts.Tool,
+		Backend:    opts.Backend,
+		BuildArgs:  opts.BuildArgs,
+		Dockerfile: opts.Dockerfile,
+	}
+	manifestJSON, err := json.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("bgbuild: marshal manifest: %w", err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "build.json"), manifestJSON, 0o644); err != nil {
+		return fmt.Errorf("bgbuild: write manifest: %w", err)
+	}
+
+	// Find the silo binary (same binary that is currently running).
+	selfPath, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("bgbuild: find executable: %w", err)
+	}
+
+	// Open log file in the shared logs directory so it appears in `silo logs`.
+	logDir := filepath.Join(config.XDGStateHomeDir(), "silo", "logs")
+	_ = os.MkdirAll(logDir, 0o755)
+	logName := fmt.Sprintf("%s-build-%s.log", time.Now().Format("20060102-150405"), opts.Tool)
+	logPath := filepath.Join(logDir, logName)
+	logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return fmt.Errorf("bgbuild: open log: %w", err)
+	}
+
+	cmd := exec.Command(selfPath, "__build", "--dir", dir)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setsid: true, // detach from parent session
+	}
+
+	if err := cmd.Start(); err != nil {
+		logFile.Close()
+		return fmt.Errorf("bgbuild: start: %w", err)
+	}
+
+	// Release the file and process handles — the child is fully detached.
+	logFile.Close()
+	_ = cmd.Process.Release()
+
+	return nil
+}

run/buildlock.go

@@ -0,0 +1,115 @@
+package run
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"github.com/leighmcculloch/silo/config"
+)
+
+// BuildLock holds a filesystem-based advisory lock for a build.
+type BuildLock struct {
+	file *os.File
+	dir  string
+}
+
+// buildDir returns the state directory for a given image tag build.
+func buildDir(imageTag string) string {
+	return filepath.Join(config.XDGStateHomeDir(), "silo", "builds", imageTag)
+}
+
+// TryLock attempts to acquire an exclusive non-blocking flock for the given
+// image tag. Returns a *BuildLock on success. Returns nil, nil if another
+// process already holds the lock (i.e. a build is in progress).
+func TryLock(imageTag string) (*BuildLock, error) {
+	dir := buildDir(imageTag)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, fmt.Errorf("buildlock: mkdir: %w", err)
+	}
+
+	lockPath := filepath.Join(dir, "lock")
+	f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("buildlock: open: %w", err)
+	}
+
+	err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+	if err != nil {
+		f.Close()
+		// EWOULDBLOCK means another process holds the lock.
+		if err == syscall.EWOULDBLOCK {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("buildlock: flock: %w", err)
+	}
+
+	// Write PID for observability.
+	_ = f.Truncate(0)
+	_, _ = f.Seek(0, 0)
+	_, _ = fmt.Fprintf(f, "%d\n", os.Getpid())
+
+	// Write status file.
+	_ = os.WriteFile(filepath.Join(dir, "status"), []byte("building\n"), 0o644)
+
+	return &BuildLock{file: f, dir: dir}, nil
+}
+
+// Unlock releases the flock and removes the build directory.
+func (bl *BuildLock) Unlock() {
+	if bl.file != nil {
+		_ = syscall.Flock(int(bl.file.Fd()), syscall.LOCK_UN)
+		bl.file.Close()
+	}
+	_ = os.RemoveAll(bl.dir)
+}
+
+// WriteStatus writes a status string to the build directory.
+func (bl *BuildLock) WriteStatus(status string) {
+	_ = os.WriteFile(filepath.Join(bl.dir, "status"), []byte(status+"\n"), 0o644)
+}
+
+// Dir returns the build directory path.
+func (bl *BuildLock) Dir() string {
+	return bl.dir
+}
+
+// IsBuilding checks whether a build for the given image tag is currently in
+// progress. It attempts a non-blocking flock: if it fails, a live process
+// holds the lock. If it succeeds, the old lock was stale (process died) and
+// we clean it up and return false.
+func IsBuilding(imageTag string) bool {
+	dir := buildDir(imageTag)
+	lockPath := filepath.Join(dir, "lock")
+
+	f, err := os.OpenFile(lockPath, os.O_RDWR, 0o644)
+	if err != nil {
+		// No lock file means no build.
+		return false
+	}
+	defer f.Close()
+
+	err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+	if err != nil {
+		// Could not acquire → another process holds it → building.
+		return true
+	}
+
+	// We acquired the lock → stale. Release and clean up.
+	_ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+	f.Close()
+	_ = os.RemoveAll(dir)
+	return false
+}
+
+// ReadBuildPID reads the PID from a build lock directory, or 0 if unavailable.
+func ReadBuildPID(imageTag string) int {
+	data, err := os.ReadFile(filepath.Join(buildDir(imageTag), "lock"))
+	if err != nil {
+		return 0
+	}
+	pid, _ := strconv.Atoi(string(data[:len(data)-1])) // trim newline
+	return pid
+}