Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ type DynamoComponentDeploymentSharedSpec struct {
// main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
// semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
// not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
// It does not change the image or rendered Pod, and changing only this field does not trigger a rollout.
// It does not change the image. Setting or changing an override that resolves to version 1.5.0 or
// later may trigger a rollout. Keep it consistent with the image's runtime version.
// +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$`
// +optional
RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ type DynamoComponentDeploymentSharedSpec struct {
// main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
// no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
// parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
// example "1.4.0". It does not change the image or rendered Pod, and changing only this field does
// not trigger a rollout.
// example "1.4.0". It does not change the image. Setting or changing an override that resolves to
// version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version.
// +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$`
// +optional
RuntimeVersionOverride string `json:"runtimeVersionOverride,omitempty"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11454,7 +11454,8 @@ spec:
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image or rendered Pod, and changing only this field does not trigger a rollout.
It does not change the image. Setting or changing an override that resolves to version 1.5.0 or
later may trigger a rollout. Keep it consistent with the image's runtime version.
pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$
type: string
scalingAdapter:
Expand Down Expand Up @@ -20439,8 +20440,8 @@ spec:
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image or rendered Pod, and changing only this field does
not trigger a rollout.
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version.
pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$
type: string
scalingAdapter:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11754,7 +11754,8 @@ spec:
main image. DGD admission requires it when spec.extraPodSpec.mainContainer.image has no parseable
semantic-version tag; controller-generated DCDs may omit it. Set it also when the parsed tag is
not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for example "1.4.0".
It does not change the image or rendered Pod, and changing only this field does not trigger a rollout.
It does not change the image. Setting or changing an override that resolves to version 1.5.0 or
later may trigger a rollout. Keep it consistent with the image's runtime version.
pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$
type: string
scalingAdapter:
Expand Down Expand Up @@ -20897,8 +20898,8 @@ spec:
main image. DGD admission requires it when spec.podTemplate.spec.containers[name=main].image has
no parseable semantic-version tag; controller-generated DCDs may omit it. Set it also when the
parsed tag is not the Dynamo runtime version. Use the canonical MAJOR.MINOR.PATCH value, for
example "1.4.0". It does not change the image or rendered Pod, and changing only this field does
not trigger a rollout.
example "1.4.0". It does not change the image. Setting or changing an override that resolves to
version 1.5.0 or later may trigger a rollout. Keep it consistent with the image's runtime version.
pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$
type: string
scalingAdapter:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,63 @@ func TestShouldTriggerRollingUpdate_IgnoresReplicaChanges(t *testing.T) {
assert.False(t, trigger)
}

func TestShouldTriggerRollingUpdate_UsesResolvedRuntimeVersion(t *testing.T) {
t.Log("define rollout decisions from current and desired runtime versions")
tests := []struct {
name string
image string
currentOverride string
desiredOverride string
wantTrigger bool
}{
{
name: "triggers when the resolved override changes",
image: "registry.example/dynamo:custom",
currentOverride: "1.5.0",
desiredOverride: "1.5.1",
wantTrigger: true,
},
{
name: "does not trigger for an equivalent image-derived override",
image: "registry.example/dynamo:1.5.0",
desiredOverride: "1.5.0",
wantTrigger: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Log("create a worker DGD at the current runtime version")
dgd := createTestDGD("test-dgd", map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
"worker": {ComponentType: consts.ComponentTypeWorker},
})
dgd.Spec.Components[0].RuntimeVersionOverride = tt.currentOverride
dgd.Spec.Components[0].PodTemplate = &corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: consts.MainContainerName,
Image: tt.image,
}},
},
}

t.Log("record the current v2 worker hash")
dgd.Annotations = map[string]string{
consts.AnnotationCurrentWorkerHashV2: betaDGDWorkersSpecHash(t, dgd),
}

t.Log("apply the desired runtime override")
dgd.Spec.Components[0].RuntimeVersionOverride = tt.desiredOverride

t.Log("evaluate whether the desired runtime requires rollout")
r := createTestReconcilerWithStatus(dgd)
trigger, err := r.shouldTriggerRollingUpdate(dgd)
require.NoError(t, err)
assert.Equal(t, tt.wantTrigger, trigger)
})
}
}

func TestCanonicalWorkerHashLifecycle_FirstDeploySpecChangeAndCompletion(t *testing.T) {
dgd := createTestDGD("test-dgd", map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{
"worker": {
Expand Down
62 changes: 55 additions & 7 deletions deploy/operator/internal/dynamo/hash.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,14 @@ import (

"github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
"github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion"
)

const dgdWorkerHashPlaceholderValue = "worker-hash-placeholder"

// Preserve hashes for runtimes released before version hashing.
var minimumHashedRuntimeVersion = runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}
Comment thread
tmonty12 marked this conversation as resolved.
Resolved

// ComputeLegacyAlphaDGDWorkersSpecHash returns the v1alpha1 worker hash that a
// pre-v1beta1 controller would compute for the DGD's current spec. Conversion
// must preserve every v1alpha1 hash input shape this depends on.
Expand Down Expand Up @@ -60,9 +64,10 @@ func ComputeDGDWorkersSpecHash(dgd *v1beta1.DynamoGraphDeployment) (string, erro
}

type workerTemplate struct {
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Spec v1beta1.DynamoComponentDeploymentSpec `json:"spec"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
RuntimeVersion string `json:"runtimeVersion,omitempty"`
Spec v1beta1.DynamoComponentDeploymentSpec `json:"spec"`
}

workerDCDs := make(map[string]workerTemplate, len(dcds))
Expand All @@ -76,9 +81,10 @@ func ComputeDGDWorkersSpecHash(dgd *v1beta1.DynamoGraphDeployment) (string, erro
return "", fmt.Errorf("duplicate generated worker DCD component name %q", componentName)
}
workerDCDs[componentName] = workerTemplate{
Labels: GetDCDKubeLabels(dcd),
Annotations: GetDCDKubeAnnotations(dcd),
Spec: workerHashSpec(dcd),
Labels: GetDCDKubeLabels(dcd),
Annotations: GetDCDKubeAnnotations(dcd),
RuntimeVersion: resolvedRuntimeVersionForHash(&dcd.Spec.DynamoComponentDeploymentSharedSpec),
Spec: workerHashSpec(dcd),
}
}
}
Expand All @@ -101,8 +107,50 @@ func workerHashSpec(dcd *v1beta1.DynamoComponentDeployment) v1beta1.DynamoCompon
spec.MinAvailable = nil
spec.ScalingAdapter = nil

// RuntimeVersionOverride has no rendered Pod effect yet.
// Hash the resolved version separately so equivalent image-derived and
// explicit versions produce the same worker hash.
spec.RuntimeVersionOverride = ""

return *spec
}

// resolvedRuntimeVersionForHash returns the canonical runtime version included
// in the v2 worker hash. The hash identifies a worker generation: changing it
// creates a new generation and triggers a managed rollout. The hash should
// change exactly when a downstream rendered workload PodSpec changes.
// Computing that directly would require refactoring this path to construct the
// full component set and render the final Deployment or Grove PodSpec.
//
// An alternative considered was resolving the runtime version, evaluating all
// runtime feature gates, and hashing those decisions. That would be more
// precise: upgrading from 1.5.0 to 1.6.0 would preserve the hash when both
// versions enable the same gates and render the same PodSpec. Hashing the
// canonical resolved version is simpler and also makes equivalent image-derived
// and explicit versions hash identically.
//
// Runtime versions should normally change together with the image, which already
// changes the hash. This value matters primarily when runtimeVersionOverride is
// corrected without changing the image. If that correction changes downstream
// version-gated rendering, the worker hash changes and triggers the required
// rollout.
//
// Versions before 1.5.0 are omitted. runtimeVersionOverride was introduced in
// 1.4.0, but runtime-version feature-gated rendering begins in 1.5.0. Hashing
// 1.4.x versions would change existing worker hashes, triggering a rollout even
// though their rendered PodSpecs remain unchanged.
func resolvedRuntimeVersionForHash(component *v1beta1.DynamoComponentDeploymentSharedSpec) string {
if component == nil {
return ""
}

image := ""
if main := GetMainContainer(component); main != nil {
image = main.Image
}
version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride)
if err != nil || version.Compare(minimumHashedRuntimeVersion) < 0 {
return ""
}

return version.String()
}
84 changes: 75 additions & 9 deletions deploy/operator/internal/dynamo/hash_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,28 @@ func mustComputeBetaDGDWorkersSpecHash(t testing.TB, dgd *v1beta1.DynamoGraphDep
return hash
}

func betaDGDWithRuntimeVersion(t testing.TB, image, override string) *v1beta1.DynamoGraphDeployment {
t.Helper()

// Convert the standard worker fixture to the current API version.
dgd := betaDGD(t, baseDGD(map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"worker": {ComponentType: commonconsts.ComponentTypeWorker},
}))

// Configure the inputs used to resolve the worker runtime version.
dgd.Spec.Components[0].RuntimeVersionOverride = override
dgd.Spec.Components[0].PodTemplate = &corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Name: commonconsts.MainContainerName,
Image: image,
}},
},
}

return dgd
}

func TestComputeBetaDGDWorkersSpecHash_Deterministic(t *testing.T) {
dgd := baseDGD(map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"prefill": {ComponentType: commonconsts.ComponentTypePrefill, Replicas: ptr.To(int32(2))},
Expand Down Expand Up @@ -296,16 +318,60 @@ func TestComputeBetaDGDWorkersSpecHash_IgnoresNonRolloutFields(t *testing.T) {
assert.Equal(t, baseHash, mustComputeBetaDGDWorkersSpecHash(t, betaDGD(t, disabledScalingAdapter)))
}

func TestComputeBetaDGDWorkersSpecHash_IgnoresRuntimeVersionOverride(t *testing.T) {
base := betaDGD(t, baseDGD(map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
"worker": {ComponentType: commonconsts.ComponentTypeWorker},
}))
baseHash := mustComputeBetaDGDWorkersSpecHash(t, base)

withOverride := base.DeepCopy()
withOverride.Spec.Components[0].RuntimeVersionOverride = "1.4.0"
func TestComputeBetaDGDWorkersSpecHash_UsesResolvedRuntimeVersion(t *testing.T) {
t.Log("define resolved-runtime-version hash comparisons")
tests := []struct {
name string
leftImage string
leftOverride string
rightImage string
rightOverride string
wantEqual bool
}{
{
name: "omits resolved versions before 1.5",
leftImage: "registry.example/runtime:custom",
rightImage: "registry.example/runtime:custom",
rightOverride: "1.4.9",
wantEqual: true,
},
{
name: "canonicalizes equivalent implicit and explicit versions",
leftImage: "nvcr.io/nvidia/ai-dynamo/runtime:v1.5.0-cuda13",
rightImage: "nvcr.io/nvidia/ai-dynamo/runtime:v1.5.0-cuda13",
rightOverride: "1.5.0",
wantEqual: true,
},
{
name: "changes at the 1.5 boundary",
leftImage: "registry.example/runtime:custom",
leftOverride: "1.4.9",
rightImage: "registry.example/runtime:custom",
rightOverride: "1.5.0",
},
{
name: "tracks override changes after 1.5",
leftImage: "registry.example/runtime:custom",
leftOverride: "1.5.0",
rightImage: "registry.example/runtime:custom",
rightOverride: "1.5.1",
},
}

assert.Equal(t, baseHash, mustComputeBetaDGDWorkersSpecHash(t, withOverride))
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Log("compute worker hashes for the compared runtime versions")
left := mustComputeBetaDGDWorkersSpecHash(t, betaDGDWithRuntimeVersion(t, tt.leftImage, tt.leftOverride))
right := mustComputeBetaDGDWorkersSpecHash(t, betaDGDWithRuntimeVersion(t, tt.rightImage, tt.rightOverride))

t.Log("assert whether the resolved versions share a worker generation")
if tt.wantEqual {
assert.Equal(t, left, right)
} else {
assert.NotEqual(t, left, right)
}
})
}
}

func TestComputeBetaDGDWorkersSpecHash_TracksPreservedAlphaResourceMetadata(t *testing.T) {
Expand Down
9 changes: 9 additions & 0 deletions deploy/operator/internal/runtimeversion/runtimeversion.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,15 @@ func ParseImageVersion(image string) (Version, error) {
return fromSemver(version), nil
}

// Resolve returns the override or the version derived from the image tag.
func Resolve(image, override string) (Version, error) {
if override != "" {
return Parse(override)
}

return ParseImageVersion(image)
}

func fromSemver(version *semver.Version) Version {
return Version{Major: version.Major(), Minor: version.Minor(), Patch: version.Patch()}
}
Expand Down
Loading
Loading