Support template creation in local dev

.gitignore

@@ -13,4 +13,5 @@ tests/periodic-test/build-template/e2b.toml
 .air
 go.work.sum
 .infisical.json
-.vscode/mise-tools
+.vscode/mise-tools
+/packages/fc-kernels

DEV-LOCAL.md

@@ -3,14 +3,35 @@
 > Note: Linux is required for developing on bare metal. This is a work in progress. Not everything will function as expected.
 
 1. `sudo modprobe nbd nbds_max=64`
-2. `make local-infra`: runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
-3. `cd packages/db && make migrate-local` initialize the database
-4. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development 
-5. `cd packages/api && make run-local` run the api locally 
-6. `cd packages/orchestrator && make build-debug && sudo make run-local` run the orchestrator and template-manager locally.
-7. `cd packages/client-proxy && make run-local` run the client-proxy locally.
+2. `sudo sysctl -w vm.nr_hugepages=2048` enable huge pages
+3. `make download-public-kernels` download linux kernels 
+4. `make local-infra` runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
+5. `cd packages/db && make migrate-local` initialize the database
+6. `cd packages/envd && make build-debug` create the envd that will be embedded in templates
+7. `cd packages/fc-versions && make build` build the firecracker versions
+8. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development 
+9. `cd packages/api && make run-local` run the api locally 
+10. `cd packages/orchestrator && make run-local` run the orchestrator and template-manager locally.
+11. `cd packages/client-proxy && make run-local` run the client-proxy locally.
+12. `cd packages/shared/script && make local-build-base-template` instructs orchestrator to create the 'base' template
 
 # Services
-- grafana: http://localhost:53000)
+- grafana: http://localhost:53000
 - postgres: postgres:[email protected]:5432
-- clickhouse: clickhouse:[email protected]:9000
+- clickhouse (http): http://localhost:8123
+- clickhouse (native): clickhouse:[email protected]:9000
+- redis: localhost:6379
+- otel collector (grpc): localhost:4317
+- otel collector (http): localhost:4318
+- vector: localhost:30006
+- e2b api: http://localhost:3000
+- e2b client proxy: http://localhost:3002
+- e2b orchestrator: http://localhost:5008
+
+# Client configuration
+```dotenv
+E2B_API_KEY=e2b_53ae1fed82754c17ad8077fbc8bcdd90
+E2B_ACCESS_TOKEN=sk_e2b_89215020937a4c989cde33d7bc647715
+E2B_API_URL=http://localhost:3000
+E2B_ENVD_API_URL=http://localhost:3002
+```

Makefile

@@ -97,6 +97,11 @@ copy-public-builds:
 	gsutil cp -r gs://e2b-prod-public-builds/kernels/* gs://$(GCP_PROJECT_ID)-fc-kernels/
 	gsutil cp -r gs://e2b-prod-public-builds/firecrackers/* gs://$(GCP_PROJECT_ID)-fc-versions/
 
+.PHONY: download-public-kernels
+download-public-kernels:
+	mkdir -p ./packages/fc-kernels
+	gsutil cp -r gs://e2b-prod-public-builds/kernels/* ./packages/fc-kernels/
+
 .PHONY: generate
 generate: generate/api generate/orchestrator generate/client-proxy generate/envd generate/db generate/shared generate-tests generate-mocks
 generate/%:

packages/api/.env.local

@@ -1,10 +1,9 @@
 DNS_PORT=9953
 ENVIRONMENT=local
+LOGS_COLLECTOR_ADDRESS=http://localhost:30006
 OTEL_COLLECTOR_GRPC_ENDPOINT=localhost:4317
 POSTGRES_CONNECTION_STRING=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable
 REDIS_URL=localhost:6379
 SANDBOX_ACCESS_TOKEN_HASH_SEED=--sandbox-access-token-hash-seed--
-SD_ORCHESTRATOR_PROVIDER=STATIC
-SD_ORCHESTRATOR_STATIC=127.0.0.1
-SD_EDGE_PROVIDER=STATIC
-SD_EDGE_STATIC=127.0.0.1
+LOCAL_CLUSTER_ENDPOINT=localhost:3001
+LOCAL_CLUSTER_TOKEN=--edge-secret--

packages/client-proxy/.env.local

@@ -1,5 +1,7 @@
 EDGE_SECRET=--edge-secret--
 EDGE_URL=http://localhost:3000
+ENVIRONMENT=local
+LOGS_COLLECTOR_ADDRESS=http://localhost:30006
 LOKI_URL=http://localhost:3100
 NODE_IP=127.0.0.1
 REDIS_URL=localhost:6379

packages/local-dev/docker-compose.yaml

@@ -30,6 +30,12 @@ services:
     command: -config.file=/etc/loki/local-config.yaml
     ports:
       - "3100:3100"
+    healthcheck:
+      test: wget --quiet --tries=1 --output-document=- http://localhost:3100/ready | grep -q -w ready || exit 1
+      start_period: 20s
+      interval: 10s
+      timeout: 1s
+      retries: 12  # try for 2 minutes
 
   memcached:
     image: memcached:1.6.38
@@ -73,7 +79,7 @@ services:
 
   tempo:
     image: grafana/tempo:2.8.2
-    command: ["-config.file=/etc/tempo.yaml"]
+    command: [ "-config.file=/etc/tempo.yaml" ]
     volumes:
       - ./tempo.yaml:/etc/tempo.yaml
       - tempo:/var/tempo
@@ -95,7 +101,20 @@ services:
     volumes:
       - tempo:/var/tempo
 
+  vector:
+    image: timberio/vector:0.34.X-alpine
+    depends_on:
+      - loki
+    environment:
+      VECTOR_CONFIG: "/etc/vector.toml"
+      VECTOR_LOG: "warn"
+    ports:
+      - "30006:30006"
+    volumes:
+      - ./vector.toml:/etc/vector.toml
+
+
 volumes:
-  clickhouse: {}
-  postgres: {}
-  tempo: {}
+  clickhouse: { }
+  postgres: { }
+  tempo: { }

packages/local-dev/vector.toml

@@ -0,0 +1,112 @@
+data_dir = "alloc/data/vector/"
+
+[api]
+enabled = true
+address = "0.0.0.0:44313"
+
+[sources.http_server]
+type = "http_server"
+address = "0.0.0.0:30006"
+encoding = "ndjson"
+path_key = "_path"
+
+[transforms.add_source_http_server]
+type = "remap"
+inputs = ["http_server"]
+source = """
+del(."_path")
+.sandboxID = .instanceID
+.timestamp = parse_timestamp(.timestamp, format: "%+") ?? now()
+# Normalize keys
+if exists(.sandbox_id) {
+  .sandboxID = .sandbox_id
+}
+if exists(.build_id) {
+  .buildID = .build_id
+}
+if exists(.env_id) {
+  .envID = .env_id
+}
+if exists(.team_id) {
+  .teamID = .team_id
+}
+if exists(."template.id") {
+  .templateID = ."template.id"
+  del(."template.id")
+}
+if exists(."sandbox.id") {
+  .sandboxID = ."sandbox.id"
+  del(."sandbox.id")
+}
+if exists(."build.id") {
+  .buildID = ."build.id"
+  del(."build.id")
+}
+if exists(."env.id") {
+  .envID = ."env.id"
+  del(."env.id")
+}
+if exists(."team.id") {
+  .teamID = ."team.id"
+  del(."team.id")
+}
+
+# Apply defaults if not already set
+if !exists(.envID) {
+  .envID = "unknown"
+}
+if !exists(.category) {
+  .category = "default"
+}
+if !exists(.teamID) {
+  .teamID = "unknown"
+}
+if !exists(.sandboxID) {
+  .sandboxID = "unknown"
+}
+if !exists(.buildID) {
+  .buildID = "unknown"
+}
+if !exists(.service) {
+  .service = "envd"
+}
+"""
+
+[transforms.internal_routing]
+type = "route"
+inputs = [ "add_source_http_server" ]
+
+[transforms.internal_routing.route]
+internal = '.internal == true'
+
+[transforms.remove_internal]
+type = "remap"
+inputs = [ "internal_routing._unmatched" ]
+source = '''
+del(.internal)
+'''
+
+# Enable debugging of logs to the console
+# [sinks.console_loki]
+# type = "console"
+# inputs = ["remove_internal"]
+# encoding.codec = "json"
+
+[sinks.local_loki_logs]
+type = "loki"
+inputs = [ "remove_internal" ]
+endpoint = "http://loki:3100"
+encoding.codec = "json"
+# This is recommended behavior for Loki 2.4.0 and newer and is default in Vector 0.39.0 and newer
+# https://vector.dev/docs/reference/configuration/sinks/loki/#out_of_order_action
+# https://vector.dev/releases/0.39.0/
+out_of_order_action = "accept"
+
+[sinks.local_loki_logs.labels]
+source = "logs-collector"
+service = "{{ service }}"
+teamID = "{{ teamID }}"
+envID = "{{ envID }}"
+buildID = "{{ buildID }}"
+sandboxID = "{{ sandboxID }}"
+category = "{{ category }}"

packages/orchestrator/.env.local

@@ -1,8 +1,15 @@
 ARTIFACTS_REGISTRY_PROVIDER=Local
 ENVIRONMENT=local
+FIRECRACKER_VERSIONS_DIR=../fc-versions/builds
+HOST_ENVD_PATH=../envd/bin/envd
+HOST_KERNELS_DIR=../fc-kernels
+LOCAL_TEMPLATE_STORAGE_BASE_PATH=./tmp/local-template-storage
+LOGS_COLLECTOR_ADDRESS=http://localhost:30006
 ORCHESTRATOR_BASE_PATH=./tmp/
 ORCHESTRATOR_LOCK_PATH=./tmp/.lock
 ORCHESTRATOR_SERVICES=orchestrator,template-manager
 OTEL_COLLECTOR_GRPC_ENDPOINT=localhost:4317
 REDIS_URL=localhost:6379
+SANDBOX_CACHE_DIR=./tmp/sandbox-cache-dir
+SNAPSHOT_CACHE_DIR=./tmp/snapshot-cache
 STORAGE_PROVIDER=Local

packages/orchestrator/Makefile

@@ -58,7 +58,7 @@ endef
 .PHONY: run-local
 run-local:
 	$(call setup_local_env)
-	NODE_ID=$$(hostname) ./bin/orchestrator --port 3000
+	NODE_ID=$$(hostname) ./bin/orchestrator
 
 .PHONY: upload/clean-nfs-cache
 upload/clean-nfs-cache:

packages/orchestrator/benchmark_test.go

@@ -98,15 +98,15 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
 
 	// hacks, these should go away
 	b.Setenv("ARTIFACTS_REGISTRY_PROVIDER", "Local")
-	b.Setenv("USE_LOCAL_NAMESPACE_STORAGE", "true")
-	b.Setenv("STORAGE_PROVIDER", "Local")
-	b.Setenv("ORCHESTRATOR_BASE_PATH", tempDir)
-	b.Setenv("HOST_ENVD_PATH", abs(filepath.Join("..", "envd", "bin", "envd")))
 	b.Setenv("FIRECRACKER_VERSIONS_DIR", abs(filepath.Join("..", "fc-versions", "builds")))
+	b.Setenv("HOST_ENVD_PATH", abs(filepath.Join("..", "envd", "bin", "envd")))
 	b.Setenv("HOST_KERNELS_DIR", abs(kernelsDir))
+	b.Setenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH", abs(filepath.Join(persistenceDir, "templates")))
+	b.Setenv("ORCHESTRATOR_BASE_PATH", tempDir)
 	b.Setenv("SANDBOX_DIR", abs(sandboxDir))
 	b.Setenv("SNAPSHOT_CACHE_DIR", abs(filepath.Join(tempDir, "snapshot-cache")))
-	b.Setenv("LOCAL_TEMPLATE_STORAGE_BASE_PATH", abs(filepath.Join(persistenceDir, "templates")))
+	b.Setenv("STORAGE_PROVIDER", "Local")
+	b.Setenv("USE_LOCAL_NAMESPACE_STORAGE", "true")
 
 	config, err := cfg.Parse()
 	require.NoError(b, err)
@@ -230,6 +230,7 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
 	require.NoError(b, err)
 
 	builder := build.NewBuilder(
+		config.BuilderConfig,
 		logger,
 		featureFlags,
 		sandboxFactory,

packages/orchestrator/cmd/build-template/main.go

@@ -47,12 +47,17 @@ func main() {
 	fcVersion := flag.String("firecracker", "", "firecracker version")
 	flag.Parse()
 
+	builderConfig, err := cfg.ParseBuilder()
+	if err != nil {
+		log.Fatalf("error parsing builder config: %v", err)
+	}
+
 	networkConfig, err := network.ParseConfig()
 	if err != nil {
-		log.Fatalf("error parsing config: %v", err)
+		log.Fatalf("error parsing network config: %v", err)
 	}
 
-	err = buildTemplate(ctx, *kernelVersion, *fcVersion, *templateID, *buildID, networkConfig)
+	err = buildTemplate(ctx, *kernelVersion, *fcVersion, *templateID, *buildID, builderConfig, networkConfig)
 	if err != nil {
 		log.Fatalf("error building template: %v", err)
 	}
@@ -64,6 +69,7 @@ func buildTemplate(
 	fcVersion,
 	templateID,
 	buildID string,
+	builderConfig cfg.BuilderConfig,
 	networkConfig network.Config,
 ) error {
 	ctx, cancel := context.WithTimeout(parentCtx, time.Minute*5)
@@ -190,6 +196,7 @@ func buildTemplate(
 	sandboxFactory := sandbox.NewFactory(c.BuilderConfig, networkPool, devicePool, featureFlags)
 
 	builder := build.NewBuilder(
+		builderConfig,
 		logger,
 		featureFlags,
 		sandboxFactory,