Add maximum lifetime constraint for a workspace

Author: Prince Rachit Sinha

chart/templates/ws-manager-configmap.yaml

@@ -82,6 +82,7 @@ data:
                 "headlessWorkspace": "60m",
                 "initialization": "30m",
                 "regularWorkspace": "30m",
+                "maxLifetime": "2160m",
                 "startup": "60m",
                 "contentFinalization": "60m",
                 "stopping": "60m",

components/installer/pkg/components/ws-manager/configmap.go

@@ -86,6 +86,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				HeadlessWorkspace:   util.Duration(1 * time.Hour),
 				Initialization:      util.Duration(30 * time.Minute),
 				RegularWorkspace:    util.Duration(30 * time.Minute),
+				MaxLifetime:         util.Duration(2160 * time.Minute),
 				TotalStartup:        util.Duration(1 * time.Hour),
 				ContentFinalization: util.Duration(1 * time.Hour),
 				Stopping:            util.Duration(1 * time.Hour),

components/ws-manager-api/go/config/config.go

@@ -122,6 +122,8 @@ type WorkspaceTimeoutConfiguration struct {
 	Initialization util.Duration `json:"initialization"`
 	// RegularWorkspace is the time a regular workspace can be without activity before it's shutdown
 	RegularWorkspace util.Duration `json:"regularWorkspace"`
+	// MaxLifetime is the maximum lifetime of a regular workspace
+	MaxLifetime util.Duration `json:"maxLifetime"`
 	// HeadlessWorkspace is the maximum runtime a headless workspace can have (including startup)
 	HeadlessWorkspace util.Duration `json:"headlessWorkspace"`
 	// AfterClose is the time a workspace lives after it has been marked closed
@@ -188,6 +190,7 @@ func (c *Configuration) Validate() error {
 		validation.Field(&c.Timeouts.HeadlessWorkspace, validation.Required),
 		validation.Field(&c.Timeouts.Initialization, validation.Required),
 		validation.Field(&c.Timeouts.RegularWorkspace, validation.Required),
+		validation.Field(&c.Timeouts.MaxLifetime, validation.Required),
 		validation.Field(&c.Timeouts.TotalStartup, validation.Required),
 		validation.Field(&c.Timeouts.ContentFinalization, validation.Required),
 		validation.Field(&c.Timeouts.Stopping, validation.Required),

components/ws-manager/config-schema.json

@@ -173,6 +173,7 @@
         "startup",
         "initialization",
         "regularWorkspace",
+        "maxLifetime",
         "headlessWorkspace",
         "afterClose",
         "stopping",
@@ -194,6 +195,9 @@
         "regularWorkspace": {
           "type": "string"
         },
+        "maxLifetime": {
+          "type": "string"
+        },
         "startup": {
           "type": "string"
         },

components/ws-manager/example-config.json

@@ -41,6 +41,7 @@
       "headlessWorkspace": "60m",
       "initialization": "30m",
       "regularWorkspace": "30m",
+      "maxLifetime": "2160m",
       "startup": "30m"
     },
     "eventTraceLog": "-",

components/ws-manager/pkg/manager/integration_test.go

@@ -87,6 +87,7 @@ func forIntegrationTestGetManager(t *testing.T) *Manager {
 			Initialization:      util.Duration(30 * time.Minute),
 			TotalStartup:        util.Duration(45 * time.Minute),
 			RegularWorkspace:    util.Duration(60 * time.Minute),
+			MaxLifetime:         util.Duration(2160 * time.Minute),
 			HeadlessWorkspace:   util.Duration(90 * time.Minute),
 			Stopping:            util.Duration(60 * time.Minute),
 			ContentFinalization: util.Duration(15 * time.Minute),

components/ws-manager/pkg/manager/monitor.go

@@ -976,7 +976,7 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb
 	}
 }
 
-// markTimedoutWorkspaces finds workspaces which haven't been active recently and marks them as timed out
+// markTimedoutWorkspaces finds workspaces which can be timeout due to inactivity or max lifetime allowed
 func (m *Monitor) markTimedoutWorkspaces(ctx context.Context) (err error) {
 	span, ctx := tracing.FromContext(ctx, "markTimedoutWorkspaces")
 	defer tracing.FinishSpan(span, &err)

components/ws-manager/pkg/manager/status.go

@@ -667,6 +667,7 @@ const (
 	activityPullingImages      activity = "pulling images"
 	activityRunningHeadless    activity = "running the headless workspace"
 	activityNone               activity = "period of inactivity"
+	activityMaxLifetime        activity = "maximum lifetime"
 	activityClosed             activity = "after being closed"
 	activityInterrupted        activity = "workspace interruption"
 	activityStopping           activity = "stopping"
@@ -718,22 +719,32 @@ func (m *Manager) isWorkspaceTimedOut(wso workspaceObjects) (reason string, err
 	case api.WorkspacePhase_RUNNING:
 		timeout := m.Config.Timeouts.RegularWorkspace
 		activity := activityNone
+		if ctv, ok := wso.Pod.Annotations[customTimeoutAnnotation]; ok {
+			if ct, err := time.ParseDuration(ctv); err == nil {
+				timeout = util.Duration(ct)
+			} else {
+				log.WithError(err).WithField("customTimeout", ctv).WithFields(wsk8s.GetOWIFromObject(&wso.Pod.ObjectMeta)).Warn("pod had custom timeout annotation set, but could not parse its value. Defaulting to ws-manager config.")
+			}
+		}
 		if wso.IsWorkspaceHeadless() {
 			timeout = m.Config.Timeouts.HeadlessWorkspace
 			lastActivity = &start
 			activity = activityRunningHeadless
+			return decide(*lastActivity, timeout, activity)
 		} else if lastActivity == nil {
 			// the workspace is up and running, but the user has never produced any activity
 			return decide(start, m.Config.Timeouts.TotalStartup, activityNone)
 		} else if isClosed {
 			return decide(*lastActivity, m.Config.Timeouts.AfterClose, activityClosed)
 		}
-		if ctv, ok := wso.Pod.Annotations[customTimeoutAnnotation]; ok {
-			if ct, err := time.ParseDuration(ctv); err == nil {
-				timeout = util.Duration(ct)
-			} else {
-				log.WithError(err).WithField("customTimeout", ctv).WithFields(wsk8s.GetOWIFromObject(&wso.Pod.ObjectMeta)).Warn("pod had custom timeout annotation set, but could not parse its value. Defaulting to ws-manager config.")
-			}
+		// check if workspace can be inactivity timed out
+		// if not then check if the workspace has reached max lifetime
+		timedOut, err := decide(*lastActivity, timeout, activity)
+		if err != nil {
+			return timedOut, err
+		} else {
+			timeout = m.Config.Timeouts.MaxLifetime
+			activity = activityMaxLifetime
 		}
 		return decide(*lastActivity, timeout, activity)
 

components/ws-manager/pkg/manager/status_test.go

@@ -48,6 +48,7 @@ func TestIsWorkspaceTimedout(t *testing.T) {
 						Initialization:      util.Duration(30 * time.Minute),
 						TotalStartup:        util.Duration(45 * time.Minute),
 						RegularWorkspace:    util.Duration(60 * time.Minute),
+						MaxLifetime:         util.Duration(2160 * time.Minute),
 						HeadlessWorkspace:   util.Duration(90 * time.Minute),
 						Stopping:            util.Duration(60 * time.Minute),
 						ContentFinalization: util.Duration(55 * time.Minute),

components/ws-manager/pkg/manager/testing_test.go

@@ -56,6 +56,7 @@ func forTestingOnlyManagerConfig() config.Configuration {
 			Initialization:      util.Duration(30 * time.Minute),
 			TotalStartup:        util.Duration(45 * time.Minute),
 			RegularWorkspace:    util.Duration(60 * time.Minute),
+			MaxLifetime:         util.Duration(2160 * time.Minute),
 			HeadlessWorkspace:   util.Duration(90 * time.Minute),
 			Stopping:            util.Duration(60 * time.Minute),
 			ContentFinalization: util.Duration(55 * time.Minute),