Implement support for Volume components

controllers/controller/component/component_controller.go

@@ -17,6 +17,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/devfile/devworkspace-operator/pkg/library/lifecycle"
+
 	"github.com/devfile/devworkspace-operator/pkg/config"
 
 	"github.com/devfile/devworkspace-operator/pkg/adaptor"
@@ -36,7 +38,6 @@ import (
 	controllerv1alpha1 "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
 
 	devworkspace "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
-	"github.com/devfile/devworkspace-operator/pkg/library"
 )
 
 var configMapDiffOpts = cmp.Options{
@@ -80,7 +81,7 @@ func (r *ComponentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 		return reconcile.Result{}, nil
 	}
 
-	initContainers, mainComponents, err := library.GetInitContainers(devworkspace.DevWorkspaceTemplateSpecContent{
+	initContainers, mainComponents, err := lifecycle.GetInitContainers(devworkspace.DevWorkspaceTemplateSpecContent{
 		Components: instance.Spec.Components,
 		Commands:   instance.Spec.Commands,
 		Events:     instance.Spec.Events,

controllers/workspace/devworkspace_controller.go

@@ -19,26 +19,29 @@ import (
 	"strings"
 	"time"
 
-	batchv1 "k8s.io/api/batch/v1"
+	containerlib "github.com/devfile/devworkspace-operator/pkg/library/container"
+	shimlib "github.com/devfile/devworkspace-operator/pkg/library/shim"
+	storagelib "github.com/devfile/devworkspace-operator/pkg/library/storage"
 
 	controllerv1alpha1 "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
+	"github.com/devfile/devworkspace-operator/controllers/workspace/provision"
+	"github.com/devfile/devworkspace-operator/controllers/workspace/restapis"
 	"github.com/devfile/devworkspace-operator/pkg/common"
 	"github.com/devfile/devworkspace-operator/pkg/config"
 	"github.com/devfile/devworkspace-operator/pkg/timing"
-	appsv1 "k8s.io/api/apps/v1"
-	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
+	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	devworkspace "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
-	"github.com/devfile/devworkspace-operator/controllers/workspace/provision"
-	"github.com/devfile/devworkspace-operator/controllers/workspace/restapis"
 )
 
 type currentStatus struct {
@@ -167,24 +170,32 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 		return reconcile.Result{}, nil
 	}
 
-	// Step one: Create components, and wait for their states to be ready.
 	timing.SetTime(workspace, timing.ComponentsCreated)
-	componentsStatus := provision.SyncComponentsToCluster(workspace, clusterAPI)
-	if !componentsStatus.Continue {
-		if componentsStatus.FailStartup {
-			reqLogger.Info("DevWorkspace start failed")
-			reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
-			if componentsStatus.Message != "" {
-				reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = componentsStatus.Message
-			} else {
-				reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = "Could not find plugins for devworkspace"
-			}
-		} else {
-			reqLogger.Info("Waiting on components to be ready")
-		}
-		return reconcile.Result{Requeue: componentsStatus.Requeue}, componentsStatus.Err
+	// TODO#185 : Move away from using devfile 1.0 constructs; only work on flattened devfiles until
+	// TODO#185 : plugins is figured out.
+	// TODO#185 : Implement defaulting container component for Web Terminals for compatibility
+	devfilePodAdditions, err := containerlib.GetKubeContainersFromDevfile(workspace.Spec.Template)
+	if err != nil {
+		reqLogger.Info("DevWorkspace start failed")
+		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
+		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error processing devfile: %s", err)
+		return reconcile.Result{}, nil
+	}
+	err = storagelib.RewriteContainerVolumeMounts(workspace.Status.WorkspaceId, devfilePodAdditions, workspace.Spec.Template)
+	if err != nil {
+		reqLogger.Info("DevWorkspace start failed")
+		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
+		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error processing devfile volumes: %s", err)
+		return reconcile.Result{}, nil
+	}
+	shimlib.FillDefaultEnvVars(devfilePodAdditions, workspace.Spec.Template)
+	componentDescriptions, err := shimlib.GetComponentDescriptionsFromPodAdditions(devfilePodAdditions, workspace.Spec.Template)
+	if err != nil {
+		reqLogger.Info("DevWorkspace start failed")
+		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
+		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error processing devfile for Theia: %s", err)
+		return reconcile.Result{}, nil
 	}
-	componentDescriptions := componentsStatus.ComponentDescriptions
 	reconcileStatus.Conditions[devworkspace.WorkspaceReady] = ""
 	timing.SetTime(workspace, timing.ComponentsReady)
 
@@ -201,9 +212,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 		componentDescriptions = append([]controllerv1alpha1.ComponentDescription{cheRestApisComponent}, componentDescriptions...)
 	}
 
-	pvcStatus := provision.SyncPVC(workspace, componentDescriptions, r.Client, reqLogger)
-	if pvcStatus.Err != nil || !pvcStatus.Continue {
-		return reconcile.Result{Requeue: true}, pvcStatus.Err
+	if storagelib.NeedsStorage(workspace.Spec.Template) {
+		pvcStatus := provision.SyncPVC(workspace, r.Client, reqLogger)
+		if pvcStatus.Err != nil || !pvcStatus.Continue {
+			return reconcile.Result{Requeue: true}, pvcStatus.Err
+		}
 	}
 
 	rbacStatus := provision.SyncRBAC(workspace, r.Client, reqLogger)
@@ -273,7 +286,7 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 
 	// Step six: Create deployment and wait for it to be ready
 	timing.SetTime(workspace, timing.DeploymentCreated)
-	deploymentStatus := provision.SyncDeploymentToCluster(workspace, podAdditions, componentDescriptions, serviceAcctName, clusterAPI)
+	deploymentStatus := provision.SyncDeploymentToCluster(workspace, podAdditions, serviceAcctName, clusterAPI)
 	if !deploymentStatus.Continue {
 		if deploymentStatus.FailStartup {
 			reqLogger.Info("Workspace start failed")

controllers/workspace/finalize.go

@@ -18,6 +18,8 @@ import (
 	"path"
 	"time"
 
+	storagelib "github.com/devfile/devworkspace-operator/pkg/library/storage"
+
 	"github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
 	"github.com/devfile/devworkspace-operator/controllers/workspace/provision"
 	"github.com/devfile/devworkspace-operator/internal/images"
@@ -223,10 +225,8 @@ func (r *DevWorkspaceReconciler) getClusterCleanupJob(ctx context.Context, works
 	return clusterJob, nil
 }
 
-func isFinalizerNecessary(_ *v1alpha2.DevWorkspace) bool {
-	// TODO: Implement checking whether persistent storage is used (once other choices are possible)
-	// Note this could interfere with cloud-shell until this TODO is resolved.
-	return true
+func isFinalizerNecessary(workspace *v1alpha2.DevWorkspace) bool {
+	return storagelib.NeedsStorage(workspace.Spec.Template)
 }
 
 func hasFinalizer(workspace *v1alpha2.DevWorkspace) bool {

controllers/workspace/provision/deployment.go

@@ -59,13 +59,12 @@ var deploymentDiffOpts = cmp.Options{
 func SyncDeploymentToCluster(
 	workspace *devworkspace.DevWorkspace,
 	podAdditions []v1alpha1.PodAdditions,
-	components []v1alpha1.ComponentDescription,
 	saName string,
 	clusterAPI ClusterAPI) DeploymentProvisioningStatus {
 
 	// [design] we have to pass components and routing pod additions separately because we need mountsources from each
 	// component.
-	specDeployment, err := getSpecDeployment(workspace, podAdditions, components, saName, clusterAPI.Scheme)
+	specDeployment, err := getSpecDeployment(workspace, podAdditions, saName, clusterAPI.Scheme)
 	if err != nil {
 		return DeploymentProvisioningStatus{
 			ProvisioningStatus: ProvisioningStatus{
@@ -160,7 +159,6 @@ func checkDeploymentStatus(deployment *appsv1.Deployment) (ready bool) {
 func getSpecDeployment(
 	workspace *devworkspace.DevWorkspace,
 	podAdditionsList []v1alpha1.PodAdditions,
-	components []v1alpha1.ComponentDescription,
 	saName string,
 	scheme *runtime.Scheme) (*appsv1.Deployment, error) {
 	replicas := int32(1)
@@ -225,8 +223,7 @@ func getSpecDeployment(
 		},
 	}
 
-	if IsPVCRequired(components) {
-		deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, getPersistentVolumeClaim())
+	if needsPVCWorkaround(podAdditions) {
 		// Kubernetes creates directories in a PVC to support subpaths such that only the leaf directory has g+rwx permissions.
 		// This means that mounting the subpath e.g. <workspace-id>/plugins will result in the <workspace-id> directory being
 		// created with 755 permissions, requiring the root UID to remove it.
@@ -345,19 +342,6 @@ func mergePodAdditions(toMerge []v1alpha1.PodAdditions) (*v1alpha1.PodAdditions,
 	return podAdditions, nil
 }
 
-func getPersistentVolumeClaim() corev1.Volume {
-	var workspaceClaim = corev1.PersistentVolumeClaimVolumeSource{
-		ClaimName: config.ControllerCfg.GetWorkspacePVCName(),
-	}
-	pvcVolume := corev1.Volume{
-		Name: config.ControllerCfg.GetWorkspacePVCName(),
-		VolumeSource: corev1.VolumeSource{
-			PersistentVolumeClaim: &workspaceClaim,
-		},
-	}
-	return pvcVolume
-}
-
 func getWorkspaceSubpathVolumeMount(workspaceId string) corev1.VolumeMount {
 	volumeName := config.ControllerCfg.GetWorkspacePVCName()
 
@@ -369,3 +353,13 @@ func getWorkspaceSubpathVolumeMount(workspaceId string) corev1.VolumeMount {
 
 	return workspaceVolumeMount
 }
+
+func needsPVCWorkaround(podAdditions *v1alpha1.PodAdditions) bool {
+	commonPVCName := config.ControllerCfg.GetWorkspacePVCName()
+	for _, vol := range podAdditions.Volumes {
+		if vol.Name == commonPVCName {
+			return true
+		}
+	}
+	return false
+}

controllers/workspace/provision/pvc.go

@@ -14,7 +14,6 @@ package provision
 
 import (
 	devworkspace "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
-	"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
 	"github.com/devfile/devworkspace-operator/pkg/config"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
@@ -23,11 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func SyncPVC(workspace *devworkspace.DevWorkspace, components []v1alpha1.ComponentDescription, client client.Client, reqLogger logr.Logger) ProvisioningStatus {
-	if !IsPVCRequired(components) {
-		return ProvisioningStatus{Continue: true}
-	}
-
+func SyncPVC(workspace *devworkspace.DevWorkspace, client client.Client, reqLogger logr.Logger) ProvisioningStatus {
 	pvc, err := generatePVC(workspace)
 	if err != nil {
 		return ProvisioningStatus{Err: err}
@@ -61,26 +56,3 @@ func generatePVC(workspace *devworkspace.DevWorkspace) (*corev1.PersistentVolume
 		},
 	}, nil
 }
-
-// IsPVCRequired checks to see if we need a PVC for the given devfile.
-// If there is any Containers with VolumeMounts that have the same name as the workspace PVC name then we need a PVC
-func IsPVCRequired(components []v1alpha1.ComponentDescription) bool {
-	volumeName := config.ControllerCfg.GetWorkspacePVCName()
-	for _, comp := range components {
-		for _, cont := range comp.PodAdditions.Containers {
-			for _, vm := range cont.VolumeMounts {
-				if vm.Name == volumeName {
-					return true
-				}
-			}
-		}
-		for _, cont := range comp.PodAdditions.InitContainers {
-			for _, vm := range cont.VolumeMounts {
-				if vm.Name == volumeName {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}

pkg/config/config.go

@@ -251,6 +251,10 @@ func WatchControllerConfig(mgr manager.Manager) error {
 	return err
 }
 
+func SetupConfigForTesting(cm *corev1.ConfigMap) {
+	ControllerCfg.update(cm)
+}
+
 func buildDefaultConfigMap(cm *corev1.ConfigMap) {
 	cm.Name = ConfigMapReference.Name
 	cm.Namespace = ConfigMapReference.Namespace

pkg/library/constants/constants.go

@@ -0,0 +1,20 @@
+//
+// Copyright (c) 2019-2020 Red Hat, Inc.
+// This program and the accompanying materials are made
+// available under the terms of the Eclipse Public License 2.0
+// which is available at https://www.eclipse.org/legal/epl-2.0/
+//
+// SPDX-License-Identifier: EPL-2.0
+//
+// Contributors:
+//   Red Hat, Inc. - initial API and implementation
+//
+
+// Package constants contains constants related to the devfile API spec (e.g. reserved env vars)
+package constants
+
+const (
+	ProjectsRootEnvVar   = "PROJECTS_ROOT"
+	ProjectsSourceEnvVar = "PROJECTS_SOURCE"
+	ProjectsVolumeName   = "projects"
+)

pkg/library/container/container.go

@@ -0,0 +1,76 @@
+//
+// Copyright (c) 2019-2020 Red Hat, Inc.
+// This program and the accompanying materials are made
+// available under the terms of the Eclipse Public License 2.0
+// which is available at https://www.eclipse.org/legal/epl-2.0/
+//
+// SPDX-License-Identifier: EPL-2.0
+//
+// Contributors:
+//   Red Hat, Inc. - initial API and implementation
+//
+
+// Package container contains library functions for converting DevWorkspace Container components to Kubernetes
+// components
+//
+// TODO:
+// - Devfile API spec is unclear on how mountSources should be handled -- mountPath is assumed to be /projects
+//   and volume name is assumed to be "projects"
+//   see issues:
+//     - https://github.com/devfile/api/issues/290
+//     - https://github.com/devfile/api/issues/291
+package container
+
+import (
+	"fmt"
+
+	devworkspace "github.com/devfile/api/pkg/apis/workspaces/v1alpha2"
+	"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
+	"github.com/devfile/devworkspace-operator/pkg/library"
+	"github.com/devfile/devworkspace-operator/pkg/library/lifecycle"
+)
+
+// GetKubeContainersFromDevfile converts container components in a DevWorkspace into Kubernetes containers.
+// If a DevWorkspace container is an init container (i.e. is bound to a preStart event), it will be returned as an
+// init container.
+//
+// This function also provisions volume mounts on containers as follows:
+// - Container component's volume mounts are provisioned with the mount path and name specified in the devworkspace
+// However, no Volumes are added to the returned PodAdditions at this stage; the volumeMounts above are expected to be
+// rewritten as Volumes are added to PodAdditions, in order to support e.g. using one PVC to hold all volumes
+//
+// Note: Requires DevWorkspace to be flattened (i.e. the DevWorkspace contains no Parent or Components of type Plugin)
+func GetKubeContainersFromDevfile(workspace devworkspace.DevWorkspaceTemplateSpec) (*v1alpha1.PodAdditions, error) {
+	if !library.DevWorkspaceIsFlattened(workspace) {
+		return nil, fmt.Errorf("devfile is not flattened")
+	}
+	podAdditions := &v1alpha1.PodAdditions{}
+
+	initContainers, mainComponents, err := lifecycle.GetInitContainers(workspace.DevWorkspaceTemplateSpecContent)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, component := range mainComponents {
+		if component.Container == nil {
+			continue
+		}
+		k8sContainer, err := convertContainerToK8s(component)
+		if err != nil {
+			return nil, err
+		}
+		handleMountSources(k8sContainer, component.Container)
+		podAdditions.Containers = append(podAdditions.Containers, *k8sContainer)
+	}
+
+	for _, container := range initContainers {
+		k8sContainer, err := convertContainerToK8s(container)
+		if err != nil {
+			return nil, err
+		}
+		handleMountSources(k8sContainer, container.Container)
+		podAdditions.InitContainers = append(podAdditions.InitContainers, *k8sContainer)
+	}
+
+	return podAdditions, nil
+}