Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,6 @@ spec:
command:
- /manager
env:
{{- range .Values.env }}
{{- if eq .name "DYN_OPERATOR_ALLOW_GMS_SNAPSHOT" }}
{{- fail "DYN_OPERATOR_ALLOW_GMS_SNAPSHOT is reserved; use featureGates.gmsSnapshot instead" }}
{{- end }}
{{- end }}
{{- if .Values.featureGates.gmsSnapshot }}
- name: DYN_OPERATOR_ALLOW_GMS_SNAPSHOT
value: "1"
{{- end }}
{{- with .Values.env }}
{{- toYaml . | nindent 8 }}
{{- end }}
Expand Down
6 changes: 0 additions & 6 deletions deploy/helm/charts/platform/components/operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,7 @@
# the cluster-wide operator.
upgradeCRD: true

featureGates:
# -- Temporary internal gate for GMS + Snapshot.
gmsSnapshot: false

# Environment variables to pass to operator Deployment.
#
# Prefer featureGates.gmsSnapshot for the temporary GMS + Snapshot gate.
env: []

# -- DEVELOPMENT AND TESTING ONLY: Namespace-restricted mode is not supported for production. Use cluster-wide mode for production deployments.
Expand Down
10 changes: 1 addition & 9 deletions deploy/operator/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ func registerControllers(
}
}

if runtimeConfig.Gate.Enabled(features.GMSSnapshot) {
if runtimeConfig.Gate.Enabled(features.Checkpoint) {
if err := controller.SetupGMSPodReplacement(mgr, setupOptions); err != nil {
return err
}
Expand Down Expand Up @@ -703,14 +703,6 @@ func registerWebhookHandlers(
setupLog.Info("POD_SERVICE_ACCOUNT/POD_NAMESPACE not set; operator SA self-identification disabled")
}

// Temporary internal gate for GMS + Snapshot.
if gate.Enabled(features.GMSSnapshot) {
setupLog.Info(
"INTERNAL OVERRIDE: GMS + Snapshot admission rule disabled via env var; do NOT enable in production",
"envVar", features.GMSSnapshotEnvVar,
)
}

if err := webhooksetup.Setup(mgr, webhooksetup.Options{
Config: operatorCfg,
RuntimeConfig: runtimeConfig,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,6 @@ spec:
dtype: "bfloat16"
maxModelLen: 2048

# GMS + Snapshot is temporarily disabled because of known GPU driver
# restore issues. Keep this false.
gpuMemoryService:
enabled: false

# Job configuration for checkpoint creation
job:
activeDeadlineSeconds: 3600
Expand Down
63 changes: 18 additions & 45 deletions deploy/operator/internal/checkpoint/checkpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/features"
gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms"
snapshotprotocol "github.com/ai-dynamo/dynamo/deploy/snapshot/protocol"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -433,7 +432,7 @@ func TestCreateOrGetAutoCheckpointDoesNotReuseDifferentCheckpointWithSameLegacyH
},
}

ckpt, err := CreateOrGetAutoCheckpoint(ctx, c, testNamespace, testHash, identity, corev1.PodTemplateSpec{}, "", "", nil, nil, features.Defaults())
ckpt, err := CreateOrGetAutoCheckpoint(ctx, c, testNamespace, testHash, identity, corev1.PodTemplateSpec{}, "", "", nil, nil)
require.NoError(t, err)
assert.Equal(t, "checkpoint-"+testHash, ckpt.Name)

Expand All @@ -447,7 +446,7 @@ func TestCreateOrGetAutoCheckpointSetsDefaultArtifactVersion(t *testing.T) {
s := testScheme()
c := fake.NewClientBuilder().WithScheme(s).Build()

ckpt, err := CreateOrGetAutoCheckpoint(ctx, c, testNamespace, testHash, testIdentity(), corev1.PodTemplateSpec{}, "", "", nil, nil, features.Defaults())
ckpt, err := CreateOrGetAutoCheckpoint(ctx, c, testNamespace, testHash, testIdentity(), corev1.PodTemplateSpec{}, "", "", nil, nil)
require.NoError(t, err)
require.NotNil(t, ckpt.Annotations)
assert.Equal(t, snapshotprotocol.DefaultCheckpointArtifactVersion, ckpt.Annotations[snapshotprotocol.CheckpointArtifactVersionAnnotation])
Expand All @@ -461,12 +460,12 @@ func TestCreateOrGetAutoCheckpointSetsDefaultArtifactVersion(t *testing.T) {
assert.True(t, commonController.ContainsFinalizer(stored))
}

func TestCreateOrGetAutoCheckpointRejectsGMSSnapshotWhenGateDisabled(t *testing.T) {
func TestCreateOrGetAutoCheckpointAcceptsGMSCheckpoint(t *testing.T) {
ctx := context.Background()
s := testScheme()
c := fake.NewClientBuilder().WithScheme(s).Build()

_, err := CreateOrGetAutoCheckpoint(
ckpt, err := CreateOrGetAutoCheckpoint(
ctx,
c,
testNamespace,
Expand All @@ -477,10 +476,10 @@ func TestCreateOrGetAutoCheckpointRejectsGMSSnapshotWhenGateDisabled(t *testing.
"",
&nvidiacomv1alpha1.GPUMemoryServiceSpec{Enabled: true},
nil,
features.Gates{},
)
require.Error(t, err)
assert.Contains(t, err.Error(), "GMS + Snapshot is temporarily disabled")
require.NoError(t, err)
require.NotNil(t, ckpt.Spec.GPUMemoryService)
assert.True(t, ckpt.Spec.GPUMemoryService.Enabled)
}

func TestCreateOrGetAutoCheckpointRetainStoresDeletionPolicy(t *testing.T) {
Expand All @@ -506,7 +505,6 @@ func TestCreateOrGetAutoCheckpointRetainStoresDeletionPolicy(t *testing.T) {
nvidiacomv1alpha1.CheckpointDeletionPolicyRetain,
nil,
owner,
features.Defaults(),
)
require.NoError(t, err)

Expand Down Expand Up @@ -552,7 +550,6 @@ func TestCreateOrGetAutoCheckpointUpdatesExistingDeletionPolicyAndFinalizer(t *t
nvidiacomv1alpha1.CheckpointDeletionPolicyDelete,
nil,
owner,
features.Defaults(),
)
require.NoError(t, err)
assert.Equal(t, string(nvidiacomv1alpha1.CheckpointDeletionPolicyDelete), ckpt.Annotations[consts.CheckpointDeletionPolicyAnnotation])
Expand Down Expand Up @@ -769,7 +766,7 @@ func TestResolveCheckpointForService(t *testing.T) {
t.Run("nil or disabled config returns disabled", func(t *testing.T) {
c := fake.NewClientBuilder().WithScheme(s).Build()
for _, cfg := range []*nvidiacomv1alpha1.ServiceCheckpointConfig{nil, {Enabled: false}} {
info, err := ResolveCheckpointForService(ctx, c, testNamespace, cfg, features.Defaults())
info, err := ResolveCheckpointForService(ctx, c, testNamespace, cfg)
require.NoError(t, err)
assert.False(t, info.Enabled)
}
Expand All @@ -780,15 +777,15 @@ func TestResolveCheckpointForService(t *testing.T) {
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true,
Mode: nvidiacomv1alpha1.CheckpointModeManual,
}, features.Defaults())
})
require.NoError(t, err)
assert.True(t, info.Enabled)
assert.False(t, info.Exists)
})

t.Run("config without ref or identity resolves enabled without error", func(t *testing.T) {
c := fake.NewClientBuilder().WithScheme(s).Build()
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{Enabled: true}, features.Defaults())
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{Enabled: true})
require.NoError(t, err)
assert.True(t, info.Enabled)
assert.False(t, info.Exists)
Expand All @@ -813,7 +810,7 @@ func TestResolveCheckpointForService(t *testing.T) {

info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, CheckpointRef: &ref,
}, features.Gates{GMSSnapshot: true})
})
require.NoError(t, err)
assert.True(t, info.Exists)
assert.True(t, info.Ready)
Expand All @@ -823,30 +820,6 @@ func TestResolveCheckpointForService(t *testing.T) {
assert.True(t, info.GPUMemoryService.Enabled)
})

t.Run("checkpointRef rejects GMS checkpoint when gate is disabled", func(t *testing.T) {
hash, err := ComputeIdentityHash(testIdentity())
require.NoError(t, err)
ckpt := &nvidiacomv1alpha1.DynamoCheckpoint{
ObjectMeta: metav1.ObjectMeta{Name: hash, Namespace: testNamespace},
Spec: nvidiacomv1alpha1.DynamoCheckpointSpec{
Identity: testIdentity(),
GPUMemoryService: &nvidiacomv1alpha1.GPUMemoryServiceSpec{Enabled: true},
},
Status: nvidiacomv1alpha1.DynamoCheckpointStatus{
Phase: nvidiacomv1alpha1.DynamoCheckpointPhaseReady,
IdentityHash: hash,
},
}
c := fake.NewClientBuilder().WithScheme(s).WithObjects(ckpt).WithStatusSubresource(ckpt).Build()
ref := hash

_, err = ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, CheckpointRef: &ref,
}, features.Gates{})
require.Error(t, err)
assert.Contains(t, err.Error(), "GMS + Snapshot is temporarily disabled")
})

t.Run("checkpointRef resolves not-ready CR", func(t *testing.T) {
hash, err := ComputeIdentityHash(testIdentity())
require.NoError(t, err)
Expand All @@ -860,7 +833,7 @@ func TestResolveCheckpointForService(t *testing.T) {

info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, CheckpointRef: &ref,
}, features.Defaults())
})
require.NoError(t, err)
assert.True(t, info.Exists)
assert.False(t, info.Ready)
Expand All @@ -871,7 +844,7 @@ func TestResolveCheckpointForService(t *testing.T) {
ref := "nonexistent"
_, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, CheckpointRef: &ref,
}, features.Defaults())
})
assert.ErrorContains(t, err, "nonexistent")
})

Expand All @@ -890,7 +863,7 @@ func TestResolveCheckpointForService(t *testing.T) {

info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, CheckpointRef: &ref,
}, features.Defaults())
})
require.NoError(t, err)
assert.Equal(t, "not-the-hash", info.CheckpointName)
assert.Equal(t, hash, info.Hash)
Expand All @@ -913,7 +886,7 @@ func TestResolveCheckpointForService(t *testing.T) {

info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, Identity: &identity,
}, features.Defaults())
})
require.NoError(t, err)
assert.True(t, info.Exists)
assert.True(t, info.Ready)
Expand All @@ -938,7 +911,7 @@ func TestResolveCheckpointForService(t *testing.T) {

info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, Identity: &identity,
}, features.Defaults())
})
require.NoError(t, err)
assert.True(t, info.Exists)
assert.False(t, info.Ready)
Expand All @@ -950,7 +923,7 @@ func TestResolveCheckpointForService(t *testing.T) {
identity := testIdentity()
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{
Enabled: true, Identity: &identity,
}, features.Defaults())
})
require.NoError(t, err)
assert.False(t, info.Exists)
assert.False(t, info.Ready)
Expand All @@ -959,7 +932,7 @@ func TestResolveCheckpointForService(t *testing.T) {

t.Run("enabled without ref or identity waits for auto-created checkpoint", func(t *testing.T) {
c := fake.NewClientBuilder().WithScheme(s).Build()
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{Enabled: true}, features.Defaults())
info, err := ResolveCheckpointForService(ctx, c, testNamespace, &nvidiacomv1alpha1.ServiceCheckpointConfig{Enabled: true})
require.NoError(t, err)
assert.True(t, info.Enabled)
assert.False(t, info.Exists)
Expand Down
37 changes: 37 additions & 0 deletions deploy/operator/internal/checkpoint/compatibility.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package checkpoint

import (
"errors"

nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
)

const (
checkpointInterPodCompatibilityMessage = "Snapshot with gpuMemoryService.mode=InterPod is unsupported"
checkpointFailoverCompatibilityMessage = "Snapshot with active/passive failover is temporarily unsupported"
)

// ValidateCheckpointCompatibility returns unsupported checkpoint combinations
// in stable policy order.
func ValidateCheckpointCompatibility(experimental *nvidiacomv1beta1.ExperimentalSpec) []error {
if experimental == nil ||
experimental.Checkpoint == nil || !experimental.Checkpoint.Enabled {
return nil
}

var violations []error
if experimental.GPUMemoryService != nil &&
experimental.GPUMemoryService.Mode == nvidiacomv1beta1.GMSModeInterPod {
violations = append(violations, errors.New(checkpointInterPodCompatibilityMessage))
}
if experimental.Failover != nil {
violations = append(violations, errors.New(checkpointFailoverCompatibilityMessage))
}

return violations
}
68 changes: 68 additions & 0 deletions deploy/operator/internal/checkpoint/compatibility_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package checkpoint

import (
"testing"

nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
"github.com/stretchr/testify/assert"
)

func TestValidateCheckpointCompatibility(t *testing.T) {
tests := []struct {
name string
experimental *nvidiacomv1beta1.ExperimentalSpec
wantErrs []string
}{
{name: "no experimental features"},
{
name: "no checkpoint configuration",
experimental: &nvidiacomv1beta1.ExperimentalSpec{
GPUMemoryService: &nvidiacomv1beta1.GPUMemoryServiceSpec{Mode: nvidiacomv1beta1.GMSModeInterPod},
Failover: &nvidiacomv1beta1.FailoverSpec{},
},
},
{
name: "disabled checkpoint ignores incompatible settings",
experimental: &nvidiacomv1beta1.ExperimentalSpec{
Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{},
GPUMemoryService: &nvidiacomv1beta1.GPUMemoryServiceSpec{Mode: nvidiacomv1beta1.GMSModeInterPod},
Failover: &nvidiacomv1beta1.FailoverSpec{},
},
},
{
name: "enabled checkpoint with intra-pod GMS",
experimental: &nvidiacomv1beta1.ExperimentalSpec{
Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true},
GPUMemoryService: &nvidiacomv1beta1.GPUMemoryServiceSpec{Mode: nvidiacomv1beta1.GMSModeIntraPod},
},
},
{
name: "enabled checkpoint with inter-pod GMS and failover",
experimental: &nvidiacomv1beta1.ExperimentalSpec{
Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true},
GPUMemoryService: &nvidiacomv1beta1.GPUMemoryServiceSpec{Mode: nvidiacomv1beta1.GMSModeInterPod},
Failover: &nvidiacomv1beta1.FailoverSpec{},
},
wantErrs: []string{
checkpointInterPodCompatibilityMessage,
checkpointFailoverCompatibilityMessage,
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
violations := ValidateCheckpointCompatibility(tt.experimental)
var gotErrs []string
for _, violation := range violations {
gotErrs = append(gotErrs, violation.Error())
}
assert.Equal(t, tt.wantErrs, gotErrs)
})
}
}
Loading
Loading