Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"fmt"

nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -36,7 +37,10 @@ type componentRestartProgressResolver struct {
reader client.Reader
}

const dcdNotFoundReason = "resource not found"
const (
dcdNotFoundReason = "resource not found"
dcdRestartNotObservedReason = "DynamoComponentDeployment has not observed the requested restart"
)

func newComponentRestartProgressResolver(reader client.Reader) *componentRestartProgressResolver {
return &componentRestartProgressResolver{reader: reader}
Expand Down Expand Up @@ -64,9 +68,14 @@ func (r *componentRestartProgressResolver) checkComponentFullyUpdated(
dgd *nvidiacomv1beta1.DynamoGraphDeployment,
componentName string,
) (bool, string) {
// Restart progress is only resolved while a restart ID is requested.
restartID := ""
if dgd.Spec.Restart != nil {
restartID = dgd.Spec.Restart.ID
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
}
if currentWorkerHashes(dgd).empty() {
resourceName := dynamo.GetDCDResourceName(dgd, componentName, "")
return checkDCDReady(ctx, r.reader, resourceName, dgd.Namespace)
return checkDCDReady(ctx, r.reader, resourceName, dgd.Namespace, restartID)
}

hashes, err := desiredWorkerHashes(dgd)
Expand All @@ -77,7 +86,7 @@ func (r *componentRestartProgressResolver) checkComponentFullyUpdated(
var lastReason string
for _, hash := range activeWorkerHashCandidates(dgd, hashes) {
resourceName := dynamo.GetDCDResourceName(dgd, componentName, hash)
ready, reason := checkDCDReady(ctx, r.reader, resourceName, dgd.Namespace)
ready, reason := checkDCDReady(ctx, r.reader, resourceName, dgd.Namespace, restartID)
if ready || reason != dcdNotFoundReason {
return ready, reason
}
Expand All @@ -91,6 +100,7 @@ func checkDCDReady(
reader client.Reader,
resourceName string,
namespace string,
restartID string,
) (bool, string) {
logger := log.FromContext(ctx)
dcd := &nvidiacomv1beta1.DynamoComponentDeployment{}
Expand Down Expand Up @@ -118,6 +128,15 @@ func checkDCDReady(
return false, fmt.Sprintf("spec not yet processed: generation=%d, observedGeneration=%d", dcd.Generation, dcd.Status.ObservedGeneration)
}

// A ready DCD from the previous revision must not satisfy a new restart,
// for example when the first write of the restarted DCD failed.
if !componentRestartApplied(dcd, restartID) {
logger.V(1).Info("DynamoComponentDeployment has not observed the requested restart",
"resourceName", resourceName,
"restartID", restartID)
return false, dcdRestartNotObservedReason
}

for _, condition := range dcd.Status.Conditions {
if condition.Type != nvidiacomv1beta1.DynamoComponentDeploymentConditionTypeAvailable {
continue
Expand All @@ -136,3 +155,22 @@ func checkDCDReady(
logger.V(1).Info("DynamoComponentDeployment missing Available condition", "resourceName", resourceName)
return false, "Available condition not found"
}

// componentRestartApplied requires the requested restart on every effective
// pod template of the DCD. dcd is non-nil and is never mutated.
func componentRestartApplied(dcd *nvidiacomv1beta1.DynamoComponentDeployment, restartID string) bool {
if restartID == "" {
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
return false
}
// Check the shared template and every role template the DCD renders.
templates := dynamo.ComponentPodTemplates(&dcd.Spec.DynamoComponentDeploymentSharedSpec)
if len(templates) == 0 {
return false
}
for _, template := range templates {
if template.Annotations[consts.RestartAnnotation] != restartID {
return false
}
}
return true
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package controller
import (
"context"

"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo"
grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -65,6 +66,12 @@ func (r *groveRestartProgressResolver) Resolve(
return inProgress
}

// Restart progress is only resolved while a restart ID is requested.
restartID := ""
if req.DGD.Spec.Restart != nil {
restartID = req.DGD.Spec.Restart.ID
}

updatedInProgress := make([]string, 0, len(inProgress))
for _, componentName := range inProgress {
component := req.DGD.GetComponentByName(componentName)
Expand All @@ -74,6 +81,18 @@ func (r *groveRestartProgressResolver) Resolve(
}
resourceName := dynamo.GroveComponentResourceName(pcsName, componentName)

// A ready clique from the previous revision must not satisfy a new
// restart, for example when the first PodCliqueSet write failed.
if !groveRestartApplied(pcs, componentName, restartID) {
logger.V(1).Info(
"component has not observed the requested restart",
"componentName", componentName,
"restartID", restartID,
)
updatedInProgress = append(updatedInProgress, componentName)
continue
}

var (
isReady bool
reason string
Expand Down Expand Up @@ -111,3 +130,23 @@ func (r *groveRestartProgressResolver) Resolve(

return updatedInProgress
}

// groveRestartApplied requires restartID on every clique rendered for
// componentName in the observed PodCliqueSet spec. pcs is non-nil, never mutated.
func groveRestartApplied(pcs *grovev1alpha1.PodCliqueSet, componentName string, restartID string) bool {
if restartID == "" {
return false
}
// Select cliques the way groveRestartAnnotations maps them to components.
found := false
for _, clique := range pcs.Spec.Template.Cliques {
if clique == nil || clique.Labels[consts.KubeLabelDynamoComponent] != componentName {
continue
}
if clique.Annotations[consts.RestartAnnotation] != restartID {
return false
}
found = true
}
return found
}
22 changes: 22 additions & 0 deletions deploy/operator/internal/controller/dgd_lpx_handoff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ func TestGroveComponentSelectionDoesNotMutateDGD(t *testing.T) {
Containers: []corev1.Container{{Name: "main", Image: "frontend"}},
}},
})
source.Spec.Restart = &v1beta1.Restart{ID: "restart-1"}
}
before := source.DeepCopy()
req := groveReconcileRequest{DGD: source, IsDelegated: (*v1beta1.DynamoComponentDeploymentSharedSpec).ManagedByExternalController}
Expand All @@ -112,6 +113,13 @@ func TestGroveComponentSelectionDoesNotMutateDGD(t *testing.T) {
}
pcs := &grovev1alpha1.PodCliqueSet{
Name: pcsName, Namespace: source.Namespace, Generation: 1,
Spec: grovev1alpha1.PodCliqueSetSpec{Template: grovev1alpha1.PodCliqueSetTemplateSpec{
Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{{
Name: "ordinary-frontend",
Labels: map[string]string{consts.KubeLabelDynamoComponent: "ordinary-frontend"},
Annotations: map[string]string{consts.RestartAnnotation: "restart-1"},
}},
}},
Status: grovev1alpha1.PodCliqueSetStatus{ObservedGeneration: new(int64(1)), Conditions: []metav1.Condition{{
Type: groveconstants.ConditionTopologyLevelsUnavailable, Status: metav1.ConditionTrue,
Reason: groveconstants.ConditionReasonClusterTopologyNotFound, Message: "missing ordinary topology",
Expand Down Expand Up @@ -672,6 +680,20 @@ func TestSpecDecodeRestartRollsTheSharedChildOnce(t *testing.T) {
groveReq := groveReconcileRequest{DGD: source, IsDelegated: (*v1beta1.DynamoComponentDeploymentSharedSpec).ManagedByExternalController}
pcs := &grovev1alpha1.PodCliqueSet{
Name: dynamo.PCSNameForDGD(groveReq.DGD, groveReq.IsDelegated), Namespace: source.Namespace, Generation: 2,
Spec: grovev1alpha1.PodCliqueSetSpec{Template: grovev1alpha1.PodCliqueSetTemplateSpec{
Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{
{
Name: "frontend",
Labels: map[string]string{consts.KubeLabelDynamoComponent: "frontend"},
Annotations: map[string]string{consts.RestartAnnotation: "pair-restart"},
},
{
Name: "prefill",
Labels: map[string]string{consts.KubeLabelDynamoComponent: "prefill"},
Annotations: map[string]string{consts.RestartAnnotation: "pair-restart"},
},
},
}},
Status: grovev1alpha1.PodCliqueSetStatus{ObservedGeneration: new(int64(1))},
}
require.NoError(t, kube.Create(t.Context(), pcs))
Expand Down
Loading
Loading