operator: API to configure kubelet directory path

Not all clusters use /var/lib/kubelet as state directory for kubelet
(kubernetes-csi/csi-driver-host-path#71). There is no way for the operator
to auto-detect this path. Hence, added a new field to deployment spec that
could be used to provide the kubelet path to driver deployment.

FIXES: intel#668

Author: avalluri

docs/install.md

@@ -844,6 +844,7 @@ The current API for PMEM-CSI `Deployment` resources is:
 | nodeSelector | string map | [Labels to use for selecting Nodes](../docs/install.md#run-pmem-csi-on-kubernetes) on which PMEM-CSI driver should run. | `{ "storage": "pmem" }`|
 | pmemPercentage | integer | Percentage of PMEM space to be used by the driver on each node. This is only valid for a driver deployed in `lvm` mode. This field can be modified, but by that time the old value may have been used already. Reducing the percentage is not supported. | 100 |
 | labels | string map | Additional labels for all objects created by the operator. Can be modified after the initial creation, but removed labels will not be removed from existing objects because the operator cannot know which labels it needs to remove and which it has to leave in place. |
+| kubeletDir | string | Kubelet's root directory path | /var/lib/kubelet |
 
 <sup>1</sup> To use the same container image as default driver image
 the operator pod must set with below environment variables with

pkg/apis/pmemcsi/v1alpha1/deployment_types.go

@@ -91,6 +91,8 @@ type DeploymentSpec struct {
 	PMEMPercentage uint16 `json:"pmemPercentage,omitempty"`
 	// Labels contains additional labels for all objects created by the operator.
 	Labels map[string]string `json:"labels,omitempty"`
+	// KubeletDir kubelet's root directory path
+	KubeletDir string `json:"kubeletDir,omitempty"`
 }
 
 // DeploymentStatus defines the observed state of Deployment
@@ -163,6 +165,8 @@ const (
 	DefaultDeviceMode = DeviceModeLVM
 	// DefaultPMEMPercentage PMEM space to reserve for the driver
 	DefaultPMEMPercentage = 100
+	// DefaultKubeletDir default kubelet's path
+	DefaultKubeletDir = "/var/lib/kubelet"
 )
 
 var (
@@ -204,6 +208,7 @@ const (
 	RegistryKey
 	NodeControllerCertificate
 	NodeControllerKey
+	KubeletDir
 )
 
 func (c DeploymentChange) String() string {
@@ -224,6 +229,7 @@ func (c DeploymentChange) String() string {
 		RegistryKey:               "registryKey",
 		NodeControllerCertificate: "nodeControllerCert",
 		NodeControllerKey:         "nodeControllerKey",
+		KubeletDir:                "kubeletDir",
 	}[c]
 }
 
@@ -293,6 +299,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error {
 		d.Spec.PMEMPercentage = DefaultPMEMPercentage
 	}
 
+	if d.Spec.KubeletDir == "" {
+		d.Spec.KubeletDir = DefaultKubeletDir
+	}
+
 	return nil
 }
 
@@ -357,6 +367,9 @@ func (d *Deployment) Compare(other *Deployment) map[DeploymentChange]struct{} {
 	if bytes.Compare(d.Spec.NodeControllerPrivateKey, other.Spec.NodeControllerPrivateKey) != 0 {
 		changes[NodeControllerKey] = struct{}{}
 	}
+	if d.Spec.KubeletDir != other.Spec.KubeletDir {
+		changes[KubeletDir] = struct{}{}
+	}
 
 	return changes
 }
@@ -460,7 +473,12 @@ func GetDeploymentCRDSchema() *apiextensions.JSONSchemaProps {
 								Type: "string",
 							},
 						},
-					}},
+					},
+					"kubeletDir": apiextensions.JSONSchemaProps{
+						Type:        "string",
+						Description: "Kubelet root directory path",
+					},
+				},
 			},
 			"status": apiextensions.JSONSchemaProps{
 				Type:        "object",

pkg/deployments/load.go

@@ -47,6 +47,11 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo
 		*yaml = bytes.ReplaceAll(*yaml, []byte("path: /var/lib/pmem-csi.intel.com"), []byte("path: /var/lib/"+deployment.Name))
 		*yaml = bytes.ReplaceAll(*yaml, []byte("mountPath: /var/lib/pmem-csi.intel.com"), []byte("mountPath: /var/lib/"+deployment.Name))
 
+		// Update kubelet path
+		if deployment.Spec.KubeletDir != api.DefaultKubeletDir {
+			*yaml = bytes.ReplaceAll(*yaml, []byte("/var/lib/kubelet"), []byte(deployment.Spec.KubeletDir))
+		}
+
 		// This assumes that all namespaced objects actually have "namespace: default".
 		*yaml = bytes.ReplaceAll(*yaml, []byte("namespace: default"), []byte("namespace: "+namespace))
 

pkg/pmem-csi-operator/controller/deployment/controller_driver.go

@@ -105,6 +105,8 @@ func (d *PmemCSIDriver) reconcileDeploymentChanges(r *ReconcileDeployment, chang
 			updateAll = true
 		case api.CACertificate, api.RegistryCertificate, api.NodeControllerCertificate:
 			updateSecrets = true
+		case api.KubeletDir:
+			updateNodeDriver = true
 		}
 
 		if err != nil {
@@ -717,7 +719,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "registration-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/plugins_registry/",
+									Path: d.Spec.KubeletDir + "/plugins_registry/",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -726,7 +728,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "mountpoint-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/plugins/kubernetes.io/csi",
+									Path: d.Spec.KubeletDir + "/plugins/kubernetes.io/csi",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -735,7 +737,7 @@ func (d *PmemCSIDriver) getNodeDaemonSet() *appsv1.DaemonSet {
 							Name: "pods-dir",
 							VolumeSource: corev1.VolumeSource{
 								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/var/lib/kubelet/pods",
+									Path: d.Spec.KubeletDir + "/pods",
 									Type: &directoryOrCreate,
 								},
 							},
@@ -908,12 +910,12 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container {
 		VolumeMounts: []corev1.VolumeMount{
 			{
 				Name:             "mountpoint-dir",
-				MountPath:        "/var/lib/kubelet/plugins/kubernetes.io/csi",
+				MountPath:        d.Spec.KubeletDir + "/plugins/kubernetes.io/csi",
 				MountPropagation: &bidirectional,
 			},
 			{
 				Name:             "pods-dir",
-				MountPath:        "/var/lib/kubelet/pods",
+				MountPath:        d.Spec.KubeletDir + "/pods",
 				MountPropagation: &bidirectional,
 			},
 			{

pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go

@@ -45,6 +45,7 @@ type pmemDeployment struct {
 	controllerCPU, controllerMemory                     string
 	nodeCPU, nodeMemory                                 string
 	caCert, regCert, regKey, ncCert, ncKey              []byte
+	kubeletDir                                          string
 }
 
 func getDeployment(d *pmemDeployment) *api.Deployment {
@@ -91,6 +92,9 @@ func getDeployment(d *pmemDeployment) *api.Deployment {
 	spec.RegistryPrivateKey = d.regKey
 	spec.NodeControllerCert = d.ncCert
 	spec.NodeControllerPrivateKey = d.ncKey
+	if d.kubeletDir != "" {
+		spec.KubeletDir = d.kubeletDir
+	}
 
 	return dep
 }
@@ -236,6 +240,7 @@ func TestDeploymentController(t *testing.T) {
 				controllerMemory: "300Mi",
 				nodeCPU:          "1000m",
 				nodeMemory:       "500Mi",
+				kubeletDir:       "/some/directory",
 			}
 
 			dep := getDeployment(d)

pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go

@@ -77,6 +77,9 @@ func UpdateTests() []UpdateTest {
 			}
 			d.Spec.Labels["foo"] = "bar"
 		},
+		"kubeletDir": func(d *api.Deployment) {
+			d.Spec.KubeletDir = "/foo/bar"
+		},
 	}
 
 	full := api.Deployment{