Support template creation in local dev

.gitignore

@@ -13,4 +13,5 @@ tests/periodic-test/build-template/e2b.toml
 .air
 go.work.sum
 .infisical.json
-.vscode/mise-tools
+.vscode/mise-tools
+/packages/fc-kernels

DEV-LOCAL.md

@@ -3,12 +3,18 @@
 > Note: Linux is required for developing on bare metal. This is a work in progress. Not everything will function as expected.
 
 1. `sudo modprobe nbd nbds_max=64`
-2. `make local-infra`: runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
-3. `cd packages/db && make migrate-local` initialize the database
-4. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development 
-5. `cd packages/api && make run-local` run the api locally 
-6. `cd packages/orchestrator && make build-debug && sudo make run-local` run the orchestrator and template-manager locally.
-7. `cd packages/client-proxy && make run-local` run the client-proxy locally.
+2. `echo always | sudo tee /sys/kernel/mm/transparent_hugepage/enabled` enable huge pages
+3. `sudo sysctl -w vm.nr_hugepages=2048` enable huge pages
+4. `make download-public-kernels` download linux kernels 
+5. `make local-infra` runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
+6. `cd packages/db && make migrate-local` initialize the database
+7. `cd packages/envd && make build-debug` create the envd that will be embedded in templates
+8. `cd packages/fc-versions && make build` build the firecracker versions
+9. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development 
+10. `cd packages/api && make run-local` run the api locally 
+11. `cd packages/orchestrator && make run-local` run the orchestrator and template-manager locally.
+12. `cd packages/client-proxy && make run-local` run the client-proxy locally.
+13. `cd packages/shared/script && make local-build-base-template` instructs orchestrator to create the 'base' template
 
 # Services
 - grafana: http://localhost:53000)

Makefile

@@ -97,6 +97,11 @@ copy-public-builds:
 	gsutil cp -r gs://e2b-prod-public-builds/kernels/* gs://$(GCP_PROJECT_ID)-fc-kernels/
 	gsutil cp -r gs://e2b-prod-public-builds/firecrackers/* gs://$(GCP_PROJECT_ID)-fc-versions/
 
+.PHONY: download-public-kernels
+download-public-kernels:
+	mkdir -p ./packages/fc-kernels
+	gsutil cp -r gs://e2b-prod-public-builds/kernels/* ./packages/fc-kernels/
+
 .PHONY: generate
 generate: generate/api generate/orchestrator generate/client-proxy generate/envd generate/db generate/shared generate-tests generate-mocks
 generate/%:

packages/api/.env.local

@@ -4,7 +4,5 @@ OTEL_COLLECTOR_GRPC_ENDPOINT=localhost:4317
 POSTGRES_CONNECTION_STRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
 REDIS_URL=localhost:6379
 SANDBOX_ACCESS_TOKEN_HASH_SEED=--sandbox-access-token-hash-seed--
-SD_ORCHESTRATOR_PROVIDER=STATIC
-SD_ORCHESTRATOR_STATIC=127.0.0.1
-SD_EDGE_PROVIDER=STATIC
-SD_EDGE_STATIC=127.0.0.1
+LOCAL_CLUSTER_ENDPOINT=localhost:3001
+LOCAL_CLUSTER_TOKEN=--edge-secret--

packages/api/internal/auth/middleware.go

@@ -83,8 +83,6 @@ func (a *commonAuthenticator[T]) Authenticate(ctx context.Context, input *openap
 		return fmt.Errorf("%s %w", a.errorMessage, err)
 	}
 
-	telemetry.ReportEvent(ctx, "api key extracted")
-
 	// If the API key is valid, we will get a result back
 	result, validationError := a.validationFunction(ctx, headerKey)
 	if validationError != nil {

packages/orchestrator/.env.local

@@ -1,8 +1,14 @@
 ARTIFACTS_REGISTRY_PROVIDER=Local
 ENVIRONMENT=local
+FIRECRACKER_VERSIONS_DIR=../fc-versions/builds
+HOST_ENVD_PATH=../envd/bin/envd
+HOST_KERNELS_DIR=../fc-kernels
+LOCAL_TEMPLATE_STORAGE_BASE_PATH=./tmp/local-template-storage
+SANDBOX_CACHE_DIR=./tmp/sandbox-cache-dir
 ORCHESTRATOR_BASE_PATH=./tmp/
 ORCHESTRATOR_LOCK_PATH=./tmp/.lock
 ORCHESTRATOR_SERVICES=orchestrator,template-manager
 OTEL_COLLECTOR_GRPC_ENDPOINT=localhost:4317
 REDIS_URL=localhost:6379
+SNAPSHOT_CACHE_DIR=./tmp/snapshot-cache
 STORAGE_PROVIDER=Local

packages/orchestrator/Makefile

@@ -57,8 +57,10 @@ endef
 
 .PHONY: run-local
 run-local:
+	make build-debug
 	$(call setup_local_env)
-	NODE_ID=$$(hostname) ./bin/orchestrator --port 3000
+	mkdir -p $(ORCHESTRATOR_BASE_PATH)/build-templates
+	NODE_ID=$$(hostname) sudo -E ./bin/orchestrator --port 3000
 
 .PHONY: upload/clean-nfs-cache
 upload/clean-nfs-cache:

packages/orchestrator/benchmark_test.go

@@ -13,6 +13,7 @@
 	"testing"
 	"time"
 
+	"github.com/e2b-dev/infra/packages/shared/pkg/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel"
@@ -21,6 +22,7 @@
 	"go.uber.org/zap"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/paths"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/proxy"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox"
 	blockmetrics "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block/metrics"
@@ -37,7 +39,6 @@
 	sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
-	"github.com/e2b-dev/infra/packages/shared/pkg/utils"
 )
 
 var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator")
@@ -53,8 +54,8 @@
 		baseImage           = "e2bdev/base"
 		kernelVersion       = "vmlinux-6.1.102"
 		fcVersion           = "v1.10.1_1fcdaec08"
-		templateID          = "fcb33d09-3141-42c4-8d3b-c2df411681db"
-		buildID             = "ba6aae36-74f7-487a-b6f7-74fd7c94e479"
+		templateID          = "fcb33d09-3141-42c4-8d3b-c2df411681dc"
+		buildID             = "ba6aae36-74f7-487a-b6f7-74fd7c94e478"
 		useHugePages        = false
 		allowInternetAccess = true
 		templateVersion     = "v2.0.0"
@@ -98,15 +99,15 @@
 
 	// hacks, these should go away
 	b.Setenv("ARTIFACTS_REGISTRY_PROVIDER", "Local")
-	b.Setenv("USE_LOCAL_NAMESPACE_STORAGE", "true")
-	b.Setenv("STORAGE_PROVIDER", "Local")
-	b.Setenv("ORCHESTRATOR_BASE_PATH", tempDir)
-	b.Setenv("HOST_ENVD_PATH", abs(filepath.Join("..", "envd", "bin", "envd")))
 	b.Setenv("FIRECRACKER_VERSIONS_DIR", abs(filepath.Join("..", "fc-versions", "builds")))
+	b.Setenv("HOST_ENVD_PATH", abs(filepath.Join("..", "envd", "bin", "envd")))
 	b.Setenv("HOST_KERNELS_DIR", abs(kernelsDir))
+	b.Setenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH", abs(filepath.Join(persistenceDir, "templates")))
+	b.Setenv("ORCHESTRATOR_BASE_PATH", tempDir)
 	b.Setenv("SANDBOX_DIR", abs(sandboxDir))
 	b.Setenv("SNAPSHOT_CACHE_DIR", abs(filepath.Join(tempDir, "snapshot-cache")))
-	b.Setenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH", abs(filepath.Join(persistenceDir, "templates")))
+	b.Setenv("STORAGE_PROVIDER", "Local")
+	b.Setenv("USE_LOCAL_NAMESPACE_STORAGE", "true")
 
 	config, err := cfg.Parse()
 	require.NoError(b, err)
@@ -231,6 +232,7 @@
 
 	builder := build.NewBuilder(
 		logger,
+		config.BuilderConfig,
 		featureFlags,
 		sandboxFactory,
 		persistenceTemplate,
@@ -259,7 +261,7 @@
 			HugePages:  sandboxConfig.HugePages,
 		}
 
-		metadata := storage.TemplateFiles{
+		metadata := paths.TemplateFiles{
 			BuildID:            buildID,
 			KernelVersion:      kernelVersion,
 			FirecrackerVersion: fcVersion,
@@ -288,7 +290,7 @@
 	}
 
 	for b.Loop() {
-		tc.testOneItem(b, buildID, kernelVersion, fcVersion)
+		tc.testOneItem(b, config.BuilderConfig, buildID, kernelVersion, fcVersion)
 	}
 }
 
@@ -312,7 +314,7 @@
 	runtime        sandbox.RuntimeMetadata
 }
 
-func (tc *testContainer) testOneItem(b *testing.B, buildID, kernelVersion, fcVersion string) {
+func (tc *testContainer) testOneItem(b *testing.B, config cfg.BuilderConfig, buildID, kernelVersion, fcVersion string) {
 	b.Helper()
 
 	ctx, span := tracer.Start(b.Context(), "testOneItem")
@@ -341,11 +343,12 @@
 	meta, err := sbx.Template.Metadata()
 	require.NoError(b, err)
 
-	templateMetadata := meta.SameVersionTemplate(storage.TemplateFiles{
-		BuildID:            buildID,
-		KernelVersion:      kernelVersion,
-		FirecrackerVersion: fcVersion,
-	})
+	templateMetadata := meta.SameVersionTemplate(paths.NewWithVersions(
+		config,
+		buildID,
+		kernelVersion,
+		fcVersion,
+	))
 	snap, err := sbx.Pause(ctx, templateMetadata)
 	require.NoError(b, err)
 	require.NotNil(b, snap)

packages/orchestrator/cmd/build-template/main.go

@@ -14,6 +14,7 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/paths"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/proxy"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox"
 	blockmetrics "github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/block/metrics"
@@ -191,6 +192,7 @@ func buildTemplate(
 
 	builder := build.NewBuilder(
 		logger,
+		c.BuilderConfig,
 		featureFlags,
 		sandboxFactory,
 		persistenceTemplate,
@@ -221,11 +223,7 @@ func buildTemplate(
 		HugePages:  true,
 	}
 
-	metadata := storage.TemplateFiles{
-		BuildID:            buildID,
-		KernelVersion:      kernelVersion,
-		FirecrackerVersion: fcVersion,
-	}
+	metadata := paths.NewWithVersions(c.BuilderConfig, buildID, kernelVersion, fcVersion)
 	_, err = builder.Build(ctx, metadata, template, logger.Core())
 	if err != nil {
 		return fmt.Errorf("error building template: %w", err)

packages/orchestrator/cmd/inspect-data/main.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/paths"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 )
 
@@ -18,10 +20,13 @@ func main() {
 
 	flag.Parse()
 
-	template := storage.TemplateFiles{
-		BuildID: *buildId,
+	config, err := cfg.ParseBuilder()
+	if err != nil {
+		log.Fatal(err)
 	}
 
+	template := paths.New(config, *buildId)
+
 	var storagePath string
 	var blockSize int64
 

packages/orchestrator/cmd/inspect-header/main.go

@@ -7,6 +7,8 @@ import (
 	"log"
 	"unsafe"
 
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/paths"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage/header"
 )
@@ -17,10 +19,13 @@ func main() {
 
 	flag.Parse()
 
-	template := storage.TemplateFiles{
-		BuildID: *buildId,
+	config, err := cfg.ParseBuilder()
+	if err != nil {
+		log.Fatal(err)
 	}
 
+	template := paths.New(config, *buildId)
+
 	var storagePath string
 
 	switch *kind {

packages/orchestrator/cmd/simulate-headers-merge/main.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/google/uuid"
 
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
+	"github.com/e2b-dev/infra/packages/orchestrator/internal/paths"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage/header"
 )
@@ -21,13 +23,13 @@ func main() {
 
 	flag.Parse()
 
-	baseTemplate := storage.TemplateFiles{
-		BuildID: *baseBuildId,
+	config, err := cfg.ParseBuilder()
+	if err != nil {
+		log.Fatal(err)
 	}
 
-	diffTemplate := storage.TemplateFiles{
-		BuildID: *diffBuildId,
-	}
+	baseTemplate := paths.New(config, *baseBuildId)
+	diffTemplate := paths.New(config, *diffBuildId)
 
 	var baseStoragePath string
 	var diffStoragePath string