Skip to content

Commit 1c34d0d

Browse files
dobracdobrac
committed
feat: add sandbox egress firewall for ip addresses
1 parent 8014191 commit 1c34d0d

File tree

10 files changed

+411
-228
lines changed

10 files changed

+411
-228
lines changed

‎packages/orchestrator/benchmark_test.go‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
3434
"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
3535
featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags"
36+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
3637
"github.com/e2b-dev/infra/packages/shared/pkg/limit"
3738
sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox"
3839
"github.com/e2b-dev/infra/packages/shared/pkg/storage"
@@ -49,17 +50,18 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
4950

5051
// test configuration
5152
const (
52-
testType = onlyStart
53-
baseImage = "e2bdev/base"
54-
kernelVersion = "vmlinux-6.1.102"
55-
fcVersion = "v1.10.1_1fcdaec08"
56-
templateID = "fcb33d09-3141-42c4-8d3b-c2df411681db"
57-
buildID = "ba6aae36-74f7-487a-b6f7-74fd7c94e479"
58-
useHugePages = false
59-
allowInternetAccess = true
60-
templateVersion = "v2.0.0"
53+
testType = onlyStart
54+
baseImage = "e2bdev/base"
55+
kernelVersion = "vmlinux-6.1.102"
56+
fcVersion = "v1.10.1_1fcdaec08"
57+
templateID = "fcb33d09-3141-42c4-8d3b-c2df411681db"
58+
buildID = "ba6aae36-74f7-487a-b6f7-74fd7c94e479"
59+
useHugePages = false
60+
templateVersion = "v2.0.0"
6161
)
6262

63+
firewall := &orchestrator.SandboxFirewallConfig{}
64+
6365
// cache paths, to speed up test runs. these paths aren't wiped between tests
6466
persistenceDir := filepath.Join(os.TempDir(), "e2b-orchestrator-benchmark")
6567
kernelsDir := filepath.Join(persistenceDir, "kernels")
@@ -182,12 +184,12 @@ func BenchmarkBaseImageLaunch(b *testing.B) {
182184

183185
accessToken := "access-token"
184186
sandboxConfig := sandbox.Config{
185-
BaseTemplateID: templateID,
186-
Vcpu: 2,
187-
RamMB: 512,
188-
TotalDiskSizeMB: 2 * 1024,
189-
HugePages: useHugePages,
190-
AllowInternetAccess: ptr(allowInternetAccess),
187+
BaseTemplateID: templateID,
188+
Vcpu: 2,
189+
RamMB: 512,
190+
TotalDiskSizeMB: 2 * 1024,
191+
HugePages: useHugePages,
192+
Firewall: firewall,
191193
Envd: sandbox.EnvdMetadata{
192194
Vars: map[string]string{"HELLO": "WORLD"},
193195
AccessToken: &accessToken,

‎packages/orchestrator/internal/sandbox/network/pool.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"go.opentelemetry.io/otel/metric"
1313
"go.uber.org/zap"
1414

15+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
1516
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
1617
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
1718
)
@@ -127,7 +128,7 @@ func (p *Pool) Populate(ctx context.Context) {
127128
}
128129
}
129130

130-
func (p *Pool) Get(ctx context.Context, allowInternet bool) (*Slot, error) {
131+
func (p *Pool) Get(ctx context.Context, firewall *orchestrator.SandboxFirewallConfig) (*Slot, error) {
131132
var slot *Slot
132133

133134
select {
@@ -154,7 +155,7 @@ func (p *Pool) Get(ctx context.Context, allowInternet bool) (*Slot, error) {
154155
}
155156
}
156157

157-
err := slot.ConfigureInternet(ctx, allowInternet)
158+
err := slot.ConfigureInternet(ctx, firewall)
158159
if err != nil {
159160
return nil, fmt.Errorf("error setting slot internet access: %w", err)
160161
}

‎packages/orchestrator/internal/sandbox/network/slot.go‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
netutils "k8s.io/utils/net"
1717

1818
"github.com/e2b-dev/infra/packages/shared/pkg/env"
19+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
1920
)
2021

2122
var tracer = otel.Tracer("github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/network")
@@ -248,14 +249,13 @@ func (s *Slot) CloseFirewall() error {
248249
return nil
249250
}
250251

251-
func (s *Slot) ConfigureInternet(ctx context.Context, allowInternet bool) (e error) {
252+
func (s *Slot) ConfigureInternet(ctx context.Context, firewall *orchestrator.SandboxFirewallConfig) (e error) {
252253
_, span := tracer.Start(ctx, "slot-internet-configure", trace.WithAttributes(
253254
attribute.String("namespace_id", s.NamespaceID()),
254-
attribute.Bool("allow_internet", allowInternet),
255255
))
256256
defer span.End()
257257

258-
if allowInternet {
258+
if firewall == nil || len(firewall.GetEgress().GetAllowedCidrs()) == 0 && len(firewall.GetEgress().GetBlockedCidrs()) == 0 {
259259
// Internet access is allowed by default.
260260
return nil
261261
}
@@ -269,9 +269,18 @@ func (s *Slot) ConfigureInternet(ctx context.Context, allowInternet bool) (e err
269269
defer n.Close()
270270

271271
err = n.Do(func(_ ns.NetNS) error {
272-
err = s.Firewall.AddBlockedIP("0.0.0.0/0")
273-
if err != nil {
274-
return fmt.Errorf("error setting firewall rules: %w", err)
272+
for _, cidr := range firewall.GetEgress().GetAllowedCidrs() {
273+
err = s.Firewall.AddAllowedIP(cidr)
274+
if err != nil {
275+
return fmt.Errorf("error setting firewall rules: %w", err)
276+
}
277+
}
278+
279+
for _, cidr := range firewall.GetEgress().GetBlockedCidrs() {
280+
err = s.Firewall.AddBlockedIP(cidr)
281+
if err != nil {
282+
return fmt.Errorf("error setting firewall rules: %w", err)
283+
}
275284
}
276285

277286
return nil

‎packages/orchestrator/internal/sandbox/sandbox.go‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ type Config struct {
6161
TotalDiskSizeMB int64
6262
HugePages bool
6363

64-
AllowInternetAccess *bool
64+
Firewall *orchestrator.SandboxFirewallConfig
6565

6666
Envd EnvdMetadata
6767
}
@@ -185,13 +185,7 @@ func (f *Factory) CreateSandbox(
185185
}
186186
}()
187187

188-
// TODO: Temporarily set this based on global config, should be removed later (it should be passed as a parameter in build)
189-
allowInternet := f.config.AllowSandboxInternet
190-
if config.AllowInternetAccess != nil {
191-
allowInternet = *config.AllowInternetAccess
192-
}
193-
194-
ipsCh := getNetworkSlotAsync(ctx, f.networkPool, cleanup, allowInternet)
188+
ipsCh := getNetworkSlotAsync(ctx, f.networkPool, cleanup, config.Firewall)
195189
defer func() {
196190
// Ensure the slot is received from chan so the slot is cleaned up properly in cleanup
197191
<-ipsCh
@@ -373,14 +367,7 @@ func (f *Factory) ResumeSandbox(
373367
}
374368
}()
375369

376-
// TODO: Temporarily set this based on global config, should be removed later
377-
// (it should be passed as a non nil parameter from API)
378-
allowInternet := f.config.AllowSandboxInternet
379-
if config.AllowInternetAccess != nil {
380-
allowInternet = *config.AllowInternetAccess
381-
}
382-
383-
ipsCh := getNetworkSlotAsync(ctx, f.networkPool, cleanup, allowInternet)
370+
ipsCh := getNetworkSlotAsync(ctx, f.networkPool, cleanup, config.Firewall)
384371
defer func() {
385372
// Ensure the slot is received from chan before ResumeSandbox returns so the slot is cleaned up properly in cleanup
386373
<-ipsCh
@@ -915,7 +902,7 @@ func getNetworkSlotAsync(
915902
ctx context.Context,
916903
networkPool *network.Pool,
917904
cleanup *Cleanup,
918-
allowInternet bool,
905+
firewall *orchestrator.SandboxFirewallConfig,
919906
) chan networkSlotRes {
920907
ctx, span := tracer.Start(ctx, "get-network-slot")
921908
defer span.End()
@@ -925,7 +912,7 @@ func getNetworkSlotAsync(
925912
go func() {
926913
defer close(r)
927914

928-
ips, err := networkPool.Get(ctx, allowInternet)
915+
ips, err := networkPool.Get(ctx, firewall)
929916
if err != nil {
930917
r <- networkSlotRes{nil, fmt.Errorf("failed to get network slot: %w", err)}
931918

‎packages/orchestrator/internal/server/main.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"go.uber.org/zap"
99
"golang.org/x/sync/semaphore"
1010

11+
"github.com/e2b-dev/infra/packages/orchestrator/internal/cfg"
1112
"github.com/e2b-dev/infra/packages/orchestrator/internal/events"
1213
"github.com/e2b-dev/infra/packages/orchestrator/internal/proxy"
1314
"github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox"
@@ -25,6 +26,7 @@ import (
2526
type Server struct {
2627
orchestrator.UnimplementedSandboxServiceServer
2728

29+
config cfg.Config
2830
sandboxFactory *sandbox.Factory
2931
info *service.ServiceInfo
3032
sandboxes *sandbox.Map
@@ -40,6 +42,7 @@ type Server struct {
4042
}
4143

4244
type ServiceConfig struct {
45+
Config cfg.Config
4346
Tel *telemetry.Client
4447
NetworkPool *network.Pool
4548
DevicePool *nbd.DevicePool
@@ -55,6 +58,7 @@ type ServiceConfig struct {
5558

5659
func New(cfg ServiceConfig) *Server {
5760
server := &Server{
61+
config: cfg.Config,
5862
sandboxFactory: cfg.SandboxFactory,
5963
info: cfg.Info,
6064
proxy: cfg.Proxy,

‎packages/orchestrator/internal/server/sandboxes.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,19 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
9999
return nil, fmt.Errorf("failed to get template snapshot data: %w", err)
100100
}
101101

102+
firewall := req.GetSandbox().GetFirewall()
103+
104+
// TODO: Temporarily set this based on global config, should be removed later
105+
// (it should be passed as a non nil parameter from API)
106+
allowInternet := s.config.AllowSandboxInternet
107+
if req.GetSandbox().AllowInternetAccess != nil {
108+
allowInternet = req.GetSandbox().GetAllowInternetAccess()
109+
}
110+
if !allowInternet {
111+
firewall.Egress = firewall.GetEgress()
112+
firewall.Egress.BlockedCidrs = []string{"0.0.0.0/0"}
113+
}
114+
102115
sbx, err := s.sandboxFactory.ResumeSandbox(
103116
ctx,
104117
template,
@@ -110,7 +123,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
110123
TotalDiskSizeMB: req.GetSandbox().GetTotalDiskSizeMb(),
111124
HugePages: req.GetSandbox().GetHugePages(),
112125

113-
AllowInternetAccess: req.GetSandbox().AllowInternetAccess,
126+
Firewall: req.GetSandbox().GetFirewall(),
114127

115128
Envd: sandbox.EnvdMetadata{
116129
Version: req.GetSandbox().GetEnvdVersion(),

‎packages/orchestrator/internal/template/build/phases/base/builder.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
artifactsregistry "github.com/e2b-dev/infra/packages/shared/pkg/artifacts-registry"
3232
"github.com/e2b-dev/infra/packages/shared/pkg/dockerhub"
3333
featureflags "github.com/e2b-dev/infra/packages/shared/pkg/feature-flags"
34+
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
3435
"github.com/e2b-dev/infra/packages/shared/pkg/id"
3536
"github.com/e2b-dev/infra/packages/shared/pkg/storage"
3637
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
@@ -204,15 +205,13 @@ func (bb *BaseBuilder) buildLayerFromOCI(
204205
return metadata.Template{}, fmt.Errorf("error creating provision rootfs: %w", err)
205206
}
206207

207-
// Allow sandbox internet access during provisioning
208-
allowInternetAccess := true
209-
210208
baseSbxConfig := sandbox.Config{
211209
Vcpu: bb.Config.VCpuCount,
212210
RamMB: bb.Config.MemoryMB,
213211
HugePages: bb.Config.HugePages,
214212

215-
AllowInternetAccess: &allowInternetAccess,
213+
// Allow sandbox internet access during provisioning
214+
Firewall: &orchestrator.SandboxFirewallConfig{},
216215

217216
Envd: sandbox.EnvdMetadata{
218217
Version: bb.EnvdVersion,

‎packages/orchestrator/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ func run(config cfg.Config) (success bool) {
349349
sandboxFactory := sandbox.NewFactory(config.BuilderConfig, networkPool, devicePool, featureFlags)
350350

351351
orchestratorService := server.New(server.ServiceConfig{
352+
Config: config,
352353
SandboxFactory: sandboxFactory,
353354
Tel: tel,
354355
NetworkPool: networkPool,

‎packages/orchestrator/orchestrator.proto‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,20 @@ message SandboxConfig {
99
// Data required for creating a new sandbox.
1010
string template_id = 1;
1111
string build_id = 2;
12-
12+
1313
string kernel_version = 3;
1414
string firecracker_version = 4;
15-
15+
1616
bool huge_pages = 5;
17-
17+
1818
string sandbox_id = 6;
1919
map<string, string> env_vars = 7;
20-
20+
2121
// Metadata about the sandbox.
2222
map<string, string> metadata = 8;
2323
optional string alias = 9;
2424
string envd_version = 10;
25-
25+
2626
int64 vcpu = 11;
2727
int64 ram_mb = 12;
2828

@@ -44,6 +44,17 @@ message SandboxConfig {
4444
// This is optional only for backwards compatibility.
4545
// After migration, the optional keyword can be removed.
4646
optional bool allow_internet_access = 21;
47+
48+
optional SandboxFirewallConfig firewall = 22;
49+
}
50+
51+
message SandboxFirewallConfig {
52+
optional SandboxFirewallEgressConfig egress = 1;
53+
}
54+
55+
message SandboxFirewallEgressConfig {
56+
repeated string allowed_cidrs = 1;
57+
repeated string blocked_cidrs = 2;
4758
}
4859

4960
message SandboxCreateRequest {

0 commit comments

Comments
 (0)