Add maximum lifetime constraint for a workspace

Author: Prince Rachit Sinha

chart/templates/ws-manager-configmap.yaml

@@ -82,6 +82,7 @@ data:
                 "headlessWorkspace": "60m",
                 "initialization": "30m",
                 "regularWorkspace": "30m",
+                "maxLifetime": "2160m",
                 "startup": "60m",
                 "contentFinalization": "60m",
                 "stopping": "60m",

components/installer/pkg/components/ws-manager/configmap.go

@@ -86,6 +86,7 @@ func configmap(ctx *common.RenderContext) ([]runtime.Object, error) {
 				HeadlessWorkspace:   util.Duration(1 * time.Hour),
 				Initialization:      util.Duration(30 * time.Minute),
 				RegularWorkspace:    util.Duration(30 * time.Minute),
+				MaxLifetime:         util.Duration(2160 * time.Minute),
 				TotalStartup:        util.Duration(1 * time.Hour),
 				ContentFinalization: util.Duration(1 * time.Hour),
 				Stopping:            util.Duration(1 * time.Hour),

components/ws-manager-api/go/config/config.go

@@ -122,6 +122,8 @@ type WorkspaceTimeoutConfiguration struct {
 	Initialization util.Duration `json:"initialization"`
 	// RegularWorkspace is the time a regular workspace can be without activity before it's shutdown
 	RegularWorkspace util.Duration `json:"regularWorkspace"`
+	// MaxLifetime is the maximum lifetime of a regular workspace
+	MaxLifetime util.Duration `json:"maxLifetime"`
 	// HeadlessWorkspace is the maximum runtime a headless workspace can have (including startup)
 	HeadlessWorkspace util.Duration `json:"headlessWorkspace"`
 	// AfterClose is the time a workspace lives after it has been marked closed
@@ -188,6 +190,7 @@ func (c *Configuration) Validate() error {
 		validation.Field(&c.Timeouts.HeadlessWorkspace, validation.Required),
 		validation.Field(&c.Timeouts.Initialization, validation.Required),
 		validation.Field(&c.Timeouts.RegularWorkspace, validation.Required),
+		validation.Field(&c.Timeouts.MaxLifetime, validation.Required),
 		validation.Field(&c.Timeouts.TotalStartup, validation.Required),
 		validation.Field(&c.Timeouts.ContentFinalization, validation.Required),
 		validation.Field(&c.Timeouts.Stopping, validation.Required),

components/ws-manager/config-schema.json

@@ -173,6 +173,7 @@
         "startup",
         "initialization",
         "regularWorkspace",
+        "maxLifetime",
         "headlessWorkspace",
         "afterClose",
         "stopping",
@@ -194,6 +195,9 @@
         "regularWorkspace": {
           "type": "string"
         },
+        "maxLifetime": {
+          "type": "string"
+        },
         "startup": {
           "type": "string"
         },

components/ws-manager/example-config.json

@@ -41,6 +41,7 @@
       "headlessWorkspace": "60m",
       "initialization": "30m",
       "regularWorkspace": "30m",
+      "maxLifetime": "2160m",
       "startup": "30m"
     },
     "eventTraceLog": "-",

components/ws-manager/pkg/manager/integration_test.go

@@ -87,6 +87,7 @@ func forIntegrationTestGetManager(t *testing.T) *Manager {
 			Initialization:      util.Duration(30 * time.Minute),
 			TotalStartup:        util.Duration(45 * time.Minute),
 			RegularWorkspace:    util.Duration(60 * time.Minute),
+			MaxLifetime:         util.Duration(2160 * time.Minute),
 			HeadlessWorkspace:   util.Duration(90 * time.Minute),
 			Stopping:            util.Duration(60 * time.Minute),
 			ContentFinalization: util.Duration(15 * time.Minute),

components/ws-manager/pkg/manager/monitor.go

@@ -976,7 +976,7 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb
 	}
 }
 
-// markTimedoutWorkspaces finds workspaces which haven't been active recently and marks them as timed out
+// markTimedoutWorkspaces finds workspaces which can be timeout due to inactivity or max lifetime allowed
 func (m *Monitor) markTimedoutWorkspaces(ctx context.Context) (err error) {
 	span, ctx := tracing.FromContext(ctx, "markTimedoutWorkspaces")
 	defer tracing.FinishSpan(span, &err)

components/ws-manager/pkg/manager/status.go

@@ -667,6 +667,7 @@ const (
 	activityPullingImages      activity = "pulling images"
 	activityRunningHeadless    activity = "running the headless workspace"
 	activityNone               activity = "period of inactivity"
+	activityMaxLifetime        activity = "maximum lifetime"
 	activityClosed             activity = "after being closed"
 	activityInterrupted        activity = "workspace interruption"
 	activityStopping           activity = "stopping"
@@ -716,8 +717,15 @@ func (m *Manager) isWorkspaceTimedOut(wso workspaceObjects) (reason string, err
 		return decide(start, m.Config.Timeouts.TotalStartup, activity)
 
 	case api.WorkspacePhase_RUNNING:
-		timeout := m.Config.Timeouts.RegularWorkspace
-		activity := activityNone
+		// First check is always for the max lifetime
+		timeout := m.Config.Timeouts.MaxLifetime
+		activity := activityMaxLifetime
+		if msg, err := decide(start, timeout, activity); msg != "" {
+			return msg, err
+		}
+
+		timeout = m.Config.Timeouts.RegularWorkspace
+		activity = activityNone
 		if wso.IsWorkspaceHeadless() {
 			timeout = m.Config.Timeouts.HeadlessWorkspace
 			lastActivity = &start

components/ws-manager/pkg/manager/status_test.go

@@ -48,6 +48,7 @@ func TestIsWorkspaceTimedout(t *testing.T) {
 						Initialization:      util.Duration(30 * time.Minute),
 						TotalStartup:        util.Duration(45 * time.Minute),
 						RegularWorkspace:    util.Duration(60 * time.Minute),
+						MaxLifetime:         util.Duration(2160 * time.Minute),
 						HeadlessWorkspace:   util.Duration(90 * time.Minute),
 						Stopping:            util.Duration(60 * time.Minute),
 						ContentFinalization: util.Duration(55 * time.Minute),

components/ws-manager/pkg/manager/testdata/timeout_regular_maxLifetime.golden

@@ -0,0 +1,3 @@
+{
+    "reason": "workspace timed out after maximum lifetime (36h10m) took longer than 36h00m"
+}