soft fail when normalization fix is not applied

Author: dobrac

packages/orchestrator/internal/sandbox/sandbox.go

@@ -835,7 +835,11 @@ func pauseProcessMemory(
 
 	err = header.ValidateMappings(memfileHeader.Mapping, memfileHeader.Metadata.Size, memfileHeader.Metadata.BlockSize)
 	if err != nil {
-		return nil, nil, fmt.Errorf("invalid memfile header mappings: %w", err)
+		if memfileHeader.IsNormalizeFixApplied() {
+			return nil, nil, fmt.Errorf("invalid memfile header mappings: %w", err)
+		}
+
+		zap.L().Warn("memfile header mappings are invalid, but normalize fix is not applied", zap.Error(err))
 	}
 
 	return memfileDiff, memfileHeader, nil
@@ -896,7 +900,11 @@ func pauseProcessRootfs(
 
 	err = header.ValidateMappings(rootfsHeader.Mapping, rootfsHeader.Metadata.Size, rootfsHeader.Metadata.BlockSize)
 	if err != nil {
-		return nil, nil, fmt.Errorf("invalid rootfs header mappings: %w", err)
+		if rootfsHeader.IsNormalizeFixApplied() {
+			return nil, nil, fmt.Errorf("invalid rootfs header mappings: %w", err)
+		}
+
+		zap.L().Warn("rootfs header mappings are invalid, but normalize fix is not applied", zap.Error(err))
 	}
 
 	return rootfsDiff, rootfsHeader, nil

packages/shared/pkg/storage/header/header.go

@@ -5,8 +5,11 @@ import (
 
 	"github.com/bits-and-blooms/bitset"
 	"github.com/google/uuid"
+	"go.uber.org/zap"
 )
 
+const NormalizeFixVersion = 3
+
 type Header struct {
 	Metadata    *Metadata
 	blockStarts *bitset.BitSet
@@ -49,6 +52,12 @@ func NewHeader(metadata *Metadata, mapping []*BuildMap) (*Header, error) {
 	}, nil
 }
 
+// IsNormalizeFixApplied is a helper method to soft fail for older versions of the header where fix for normalization was not applied.
+// This should be removed in the future.
+func (t *Header) IsNormalizeFixApplied() bool {
+	return t.Metadata.Version >= NormalizeFixVersion
+}
+
 func (t *Header) GetShiftedMapping(offset int64) (mappedOffset int64, mappedLength int64, buildID *uuid.UUID, err error) {
 	mapping, shift, err := t.getMapping(offset)
 	if err != nil {
@@ -60,18 +69,39 @@ func (t *Header) GetShiftedMapping(offset int64) (mappedOffset int64, mappedLeng
 	buildID = &mapping.BuildId
 
 	if mappedLength < 0 {
-		return 0, 0, nil, fmt.Errorf("mapped length for offset %d is negative: %d", offset, mappedLength)
+		if t.IsNormalizeFixApplied() {
+			return 0, 0, nil, fmt.Errorf("mapped length for offset %d is negative: %d", offset, mappedLength)
+		}
+
+		zap.L().Warn("mapped length is negative, but normalize fix is not applied",
+			zap.Int64("offset", offset),
+			zap.Int64("mappedLength", mappedLength),
+		)
 	}
 
 	return mappedOffset, mappedLength, buildID, nil
 }
 
 func (t *Header) getMapping(offset int64) (*BuildMap, int64, error) {
 	if offset < 0 || offset >= int64(t.Metadata.Size) {
-		return nil, 0, fmt.Errorf("offset %d is out of bounds (size: %d)", offset, t.Metadata.Size)
+		if t.IsNormalizeFixApplied() {
+			return nil, 0, fmt.Errorf("offset %d is out of bounds (size: %d)", offset, t.Metadata.Size)
+		}
+
+		zap.L().Warn("offset is out of bounds, but normalize fix is not applied",
+			zap.Int64("offset", offset),
+			zap.Int64("size", int64(t.Metadata.Size)),
+		)
 	}
 	if offset%int64(t.Metadata.BlockSize) != 0 {
-		return nil, 0, fmt.Errorf("offset %d is not aligned to block size %d", offset, t.Metadata.BlockSize)
+		if t.IsNormalizeFixApplied() {
+			return nil, 0, fmt.Errorf("offset %d is not aligned to block size %d", offset, t.Metadata.BlockSize)
+		}
+
+		zap.L().Warn("offset is not aligned to block size, but normalize fix is not applied",
+			zap.Int64("offset", offset),
+			zap.Int64("blockSize", int64(t.Metadata.BlockSize)),
+		)
 	}
 
 	block := BlockIdx(offset, int64(t.Metadata.BlockSize))
@@ -90,8 +120,17 @@ func (t *Header) getMapping(offset int64) (*BuildMap, int64, error) {
 
 	// Verify that the offset falls within this mapping's range
 	if shift >= int64(mapping.Length) {
-		return nil, 0, fmt.Errorf("offset %d (block %d) is beyond the end of mapping at offset %d (ends at %d)",
-			offset, block, mapping.Offset, mapping.Offset+mapping.Length)
+		if t.IsNormalizeFixApplied() {
+			return nil, 0, fmt.Errorf("offset %d (block %d) is beyond the end of mapping at offset %d (ends at %d)",
+				offset, block, mapping.Offset, mapping.Offset+mapping.Length)
+		}
+
+		zap.L().Warn("offset is beyond the end of mapping, but normalize fix is not applied",
+			zap.Int64("offset", offset),
+			zap.Int64("block", block),
+			zap.Uint64("mappingOffset", mapping.Offset),
+			zap.Uint64("mappingEnd", mapping.Offset+mapping.Length),
+		)
 	}
 
 	return mapping, shift, nil

packages/shared/pkg/storage/header/serialization.go

@@ -13,7 +13,7 @@ import (
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 )
 
-const metadataVersion = 2
+const metadataVersion = 3
 
 type Metadata struct {
 	Version    uint64