fix: handle the case that a data disk is in an invalid state and make sure the finalizer does not hang

Signed-off-by: Dr. Carsten Leue <carsten.leue@de.ibm.com>

Author: CarstenLeue

onprem/cr.go

@@ -47,6 +47,13 @@ type DataDiskCustomResourceSpec struct {
 	TargetSelector *metav1.LabelSelector `json:"targetSelector"`
 }
 
+type DataDiskStatus struct {
+	// description of the data disk status
+	Description string `json:"description"`
+	// the status flag
+	Status int `json:"status"`
+}
+
 type OnPremCustomResource struct {
 	metav1.TypeMeta `json:",inline"`
 	// Standard object's metadata.
@@ -71,6 +78,9 @@ type DataDiskCustomResource struct {
 	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
 	// +optional
 	Spec DataDiskCustomResourceSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
+
+	// status of this custom resource
+	Status DataDiskStatus `json:"status,omitempty"`
 }
 
 type OnPremCustomResourceOptions struct {

onprem/datadisk.go

@@ -212,11 +212,17 @@ func DataDisksFromRelated(data map[string]any) ([]*DataDiskCustomResource, error
 			// decode each disk
 			for _, dataDisk := range dataDisks {
 				// transcode to the expected format
-				cfg, err := common.Transcode[*DataDiskCustomResource](dataDisk)
+				disk, err := common.Transcode[*DataDiskCustomResource](dataDisk)
 				if err != nil {
 					return nil, err
 				}
-				result = append(result, cfg)
+				// validate the status of the data disk
+				if disk.Status.Status == 1 {
+					result = append(result, disk)
+				} else {
+					// disk is not in a valid status
+					log.Printf("Data Disk [%s] is not in ready state, ignoring, cause: [%s]", disk.Name, disk.Status.Description)
+				}
 			}
 		}
 	}

onprem/domain.go

@@ -194,7 +194,9 @@ func shutDownDomain(client *LivirtClient) func(domain *libvirt.Domain) error {
 		// check if the domain is running
 		state, _, err := conn.DomainGetState(*domain, 0)
 		if err != nil {
-			return err
+			// if we cannot get the domain state, assume it's gone
+			log.Printf("Unable to get the domain state for domain [%s], err: [%v]", domain.Name, err)
+			return nil
 		}
 		if libvirt.DomainState(state) != libvirt.DomainRunning {
 			return nil
@@ -211,7 +213,9 @@ func shutDownDomain(client *LivirtClient) func(domain *libvirt.Domain) error {
 			state, reason, err := conn.DomainGetState(*domain, 0)
 			log.Printf("Domain State [%d], reason [%d]", state, reason)
 			if err != nil {
-				return err
+				// if we cannot get the domain state, assume it's gone
+				log.Printf("Unable to get the domain state for domain [%s], err: [%v]", domain.Name, err)
+				return nil
 			}
 			// check for states that depict a shutdown system
 			switch libvirt.DomainState(state) {
@@ -267,10 +271,14 @@ func DeleteDomainByName(client *LivirtClient) func(name string) error {
 	delDomain := deleteDomain(client)
 
 	return func(name string) error {
+		// log this
+		log.Printf("Deleting domain by name [%s] ...", name)
 		// locate the domain
 		domain, err := conn.DomainLookupByName(name)
 		// TODO check for domain does not exist
 		if err != nil {
+			// log this fact
+			log.Printf("Domain [%s] cannot be located, assuming it's been deleted, cause: [%v]", name, err)
 			return nil
 		}
 		// delete

onprem/volume.go

@@ -61,6 +61,10 @@ func getStorageVolByNameXMLDesc(conn *libvirt.Libvirt) func(pool libvirt.Storage
 	}
 }
 
+func GetStorageVolXMLDesc(client *LivirtClient) func(vol *libvirt.StorageVol) (*libvirtxml.StorageVolume, error) {
+	return getStorageVolXMLDesc(client.LibVirt)
+}
+
 func getStorageVolXMLDesc(conn *libvirt.Libvirt) func(vol *libvirt.StorageVol) (*libvirtxml.StorageVolume, error) {
 	return func(vol *libvirt.StorageVol) (*libvirtxml.StorageVolume, error) {
 		// try to get more info

server/datadisk/actions.go

@@ -15,27 +15,63 @@
 package datadisk
 
 import (
+	"log"
+
 	"github.com/ibm-hyper-protect/k8s-operator-hpcr/onprem"
 	"github.com/ibm-hyper-protect/k8s-operator-hpcr/server/common"
+	C "github.com/ibm-hyper-protect/terraform-provider-hpcr/contract"
+	"libvirt.org/go/libvirtxml"
 )
 
+// createDataDiskReadyAction create the action
+func createDataDiskReadyAction(disk *libvirtxml.StorageVolume) common.Action {
+
+	return func() (*common.ResourceStatus, error) {
+		// metadata to attach
+		metadata := C.RawMap{
+			"Name": disk.Name,
+		}
+		// marshal the disk info into metadata
+		diskStrg, err := onprem.XMLMarshall(disk)
+		if err == nil {
+			metadata["diskXML"] = diskStrg
+		} else {
+			log.Printf("Unable to marshal the disk XML, cause: [%v]", err)
+		}
+		return &common.ResourceStatus{
+			Status:      common.Ready,
+			Description: diskStrg,
+			Error:       nil,
+			Metadata:    metadata,
+		}, nil
+	}
+}
+
 // CreateSyncAction synchronizes the state of the resource and determines what to do next
 func CreateSyncAction(client *onprem.LivirtClient, opt *onprem.DataDiskOptions) common.Action {
 	// checks for the validity of the data disk
 	isDataDiskValid := onprem.IsDataDiskValid(client)
-	_, ok := isDataDiskValid(opt)
+	diskXML, ok := isDataDiskValid(opt)
 	if ok {
 		// ready
-		return common.CreateReadyAction()
+		return createDataDiskReadyAction(diskXML)
 	}
 	// create a disk (will resize if required)
 	diskSync := onprem.CreateDataDiskSync(client)
-	_, err := diskSync(opt)
+	disk, err := diskSync(opt)
 	if err != nil {
+		log.Printf("Unable to create data disk [%s], cause: [%v]", opt.Name, err)
+		return common.CreateErrorAction(err)
+	}
+	// try to get the XML description
+	getDiskXML := onprem.GetStorageVolXMLDesc(client)
+	diskXML, err = getDiskXML(disk)
+	if err != nil {
+		log.Printf("Unable to get disk XML [%s], cause: [%v]", opt.Name, err)
 		return common.CreateErrorAction(err)
 	}
 	// ready
-	return common.CreateReadyAction()
+	return createDataDiskReadyAction(diskXML)
 }
 
 func CreateFinalizeAction(client *onprem.LivirtClient, opt *onprem.DataDiskOptions) common.Action {

server/onprem/actions.go

@@ -116,6 +116,7 @@ func CreateSyncAction(client *onprem.LivirtClient, opt *onprem.InstanceOptions)
 	instSync := onprem.CreateInstanceSync(client)
 	result, err := instSync(opt)
 	if err != nil {
+		log.Printf("Unable to create the VSI [%s], cause: [%v]", opt.Name, err)
 		return common.CreateErrorAction(err)
 	}
 	// log the result
@@ -135,6 +136,7 @@ func CreateFinalizeAction(client *onprem.LivirtClient, opt *onprem.InstanceOptio
 	deleteSync := onprem.DeleteInstanceSync(client)
 	err := deleteSync(opt.StoragePool, opt.Name)
 	if err != nil {
+		log.Printf("Unable to delete the VSI [%s], cause: [%v]", opt.Name, err)
 		return common.CreateErrorAction(err)
 	}
 	// done

server/onprem/onprem.go

@@ -24,6 +24,7 @@ import (
 	C "github.com/ibm-hyper-protect/k8s-operator-hpcr/common"
 	"github.com/ibm-hyper-protect/k8s-operator-hpcr/onprem"
 	"github.com/ibm-hyper-protect/k8s-operator-hpcr/server/common"
+	A "github.com/ibm-hyper-protect/terraform-provider-hpcr/fp/array"
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -67,7 +68,15 @@ func syncOnPrem(req map[string]any) common.Action {
 		return common.CreateErrorAction(err)
 	}
 
-	log.Printf("DataDisks: %v", dataDisks)
+	// dump the attached data disks
+	if A.IsNonEmpty(dataDisks) {
+		// extract names
+		dataDiskNames := A.MonadMap(dataDisks, func(disk *onprem.DataDiskCustomResource) string {
+			return disk.Name
+		})
+		// log the disks
+		log.Printf("DataDisks: %v", dataDiskNames)
+	}
 
 	// attach data disks
 	opt.DataDisks = onprem.DataDiskCustomResourcesToAttachedDataDisks(dataDisks)
@@ -76,6 +85,7 @@ func syncOnPrem(req map[string]any) common.Action {
 	return CreateSyncAction(client, opt)
 }
 
+// finalizeOnPrem deletes a VSI
 func finalizeOnPrem(req map[string]any) common.Action {
 	if !lock.TryLock() {
 		return common.CreateStatusAction(common.Waiting)

tooling/data/busybox.yml

@@ -0,0 +1,19 @@
+kind: HyperProtectContainerRuntimeOnPrem
+apiVersion: hpse.ibm.com/v1
+metadata:
+  name: busybox
+  labels:
+    app: hpcr
+spec:
+  contract: |
+    env: hyper-protect-basic.kdp9q+bIqUWhHBQ+0Bt+lCVLb65UhaWBhfgznuLCVR3RzwCqCoGlRMcZJ7/DleW+/c3fiPxD92LuK9fXPtuM1K4DWOg+SCn4xR0Pvw0ACXvs8+gcrHMm3DHgL4pd8cLygAAiLjYQqjuvyNTyBrxHMWs3X/qasenKqYKd0SeJ0UIb+SECAefIjrFHGg2yIxYkSq/39P2Ssc2wfyYmHIY4CyGPViidL6dSASkdAPTBjk4T2k5gCvE9xZjSiPr2uO+9RCpbpZAzctpPouamCDCrve50pAruxY4Ypn2wDzstRsdhydu0ZFisaXxU5oBhYCcNzDWrGZSAly8BdRNMbSpQBaxjQ2vNc/DGVVOaNwoTcsYyvW3zCc6SIeQ88gW8sjHYeHLnvMWedYQHIyoPnPFQlz1xWJcJsFndGr7OYFges0iyIr7u2jVgVcXbOkQZcdgCVhpBVf40E4uNHhYc+dWdzTnfpRlSXOuMy/Oy1wAPB6Z+rsklm2T189OOlLkWj+9KCcr6BuiNp7o38OwSAnbstXDUZziS+gV+w7LJCuOt2VFzJJ+huHgs8nyUmXVFhcfFvwlZZuSS5PEHAJYZloumnXcJ3ptgQJY8Y+fBkv3xea+Ztjmg9MF/Tarky4FvEG2DEsK7D5deLhUOxPqDPeMz+DGjYaOo7qgn1DyQh9046K8=.U2FsdGVkX1+Br9ZWBatGTfZ9JAQf8OeBQlVKFda/T1kFG058Pw3Q6Gn20vQ/8Y8MwVZfiMLuOsVVjX/x8ERnfwvUE5GLK3XElBA6SxZWfXShKm28I1WXyAI1JHAbXg/607g5KFyHRbrCFOK0He7ZPoDV0cGqnnR6dgrfCEYpkWL6A103kGFckmjASBKEv2k/DHqpTTHDuWZMmKiqF8uQhbG9az5ziiKSEKoLt0iw4Lnh9zowkGThvNy+ul3vshqIkKTQeS590J4jr1iZPQyeHcK3IlplRBoGKMOSakitjVbWK+GLYmkDlAX8KR1D5X4psE82yETTGk0rzAseKSHvOYpVlLlM0ClPkNHLEwpm81zfKRynP9m+FqThTWE6z8yXod4Lxa/kToX6fJjhgFJrGRB/HZeqD3armdU2xAWi3RdJGv3tb0CHeB8P0VF47aR46eRBCBVxu0mbdGS8g7g4YAWvdXiHx4KxnLDEZw830cL2ovCo0AV7hLjJiFjd4w7CpXIRfUIRs0pLwPYPAwlvndDiDsjCNYv84ZJtUP78muKM6Y4RCEGWrXENKFsEITDoXhHi2FnYko62oeH++LnRkgMO/ggN6PnTBH82zh2K2z9tXOI2J3yAm8oYZbd4lodbqFWgC5SUaa1kHxE1pMkwjlM9Ic+zCYezhNogHGtCKXlESsbRsXQz2hPJANUkGjID2Frq+c97zIhzaSojVbuCOPTRXO6b3Wa1I+EzZ3kyUWD5TdHK6eNmULCC5pKR8V9mzPq6bBdkEv/TpNPjkt47S6CjtuSvR5AqOiZjtNQ/nVK2jy8BPPsT4hYMOYs0s9Z0g35HwRbFKKdvtTLeho7JH676ohTvGV/6b8wHjUGCl9/A2jQTz1Yk8Npgd7z0R4TWHqChEiWfJyaE2hnhgCi09U/g/m//SZXOx8x/RDf9w1uA4L6tKYx7c64BA5YM7YgU1GPvhkFvDSrP1rXIE3qmJ+K8VMWovq5qhMg1Jw3OQRxp6tjA5bbXdaurBkKcjd6+JZDkiPpoAsaYGxGIehs43DvKtBObOjmilQLQuHVKrvalFZtTvEkAQ2ESweLafdoLcSc/D/gMip8grnVOxccM4s1FpY2bnbfVvD0s4YEIJb/DHNQLgZD++kmwx18efgqUpQs7v3c/NeLYOElVLJlwe1bvmOg7H54isBBEjkKVdC5CNCJsBAglTZVuQ/W3ZJL9udBn8iI4LHBdf5hSwjM6NYFnY3QIpdo2/JvO0KorIW/od5XuKUjtVXMgZbMkPjtdm15wRL2C+Lhakto2FSb6pd+t0jxAkgxwEO9EKOrtDShDVqq9c0zMQGo7bsube+TjBWFdtKqI4/dSq6mr44dwEA==
+    envWorkloadSignature: 8SxEIJI2fDiON+01D11c7DJEmOyf0h/XNC1pOv5iJIS9pLe8Kw6BW2bAw8LiGL5Tj7HxWsiknGTbiuKwVWi0KcT4G8C9994oyFPNLzSOA7jOT0XA6xj98kHO/Q9iD6ThWeCJAJiGWkqfStd4/U86rxbh6cYee7dlSlv3IbhAu9a+0ziFZQlYG1uPLDcKabZt8M4Tb3gDNjIVlgY8g9fSgNQk7UquhexVkvk4NU+oEjQVGCDZbYRukGoNNODd0+LMtQIDnkjhLGZY54E7XvN1iS9xMQj3an9/g9rCnpdpmEcv/03p/Fz6Lm5VvYLX3poSigN1TgGQv4hn3CsPdlDTkRXDXVkreHQygnXTv9CuvLv6SYhr9vhdPOiDMrjMDYKjiBDSf2aGVqSYuN+Z8kWVjzF9WmDyLQrT82e+UEVwM4hJxCKYdBQtoB4CyjDuDaqWvb0/MEqFEWFwSnuaNgDu9NyFyJDlTW7QEapPuEHLP7VI3rf9AoYSrNgr+bVj8lOCFf5/ros00WXOBgAApQRoiishusFhPTSIgFOcBFGHzy9Pr704hmtX+VNXvvEtuCD1bJ3Ub2BFWm3AD/UVu+qhwBV/M/pmeEVuhLHoWxc8Jq7OVlrZghjmh9Oo+OA1jrymlj9tB4aIC3o21Jh6vvWT2W5/5qVqHGfFffY440B0Q9w=
+    workload: hyper-protect-basic.WQGY6mI2eBig9IxBoERb4/Q8F0nnuGq3bskYxNphbuiu/ticd0Z0HorZMF8cjbg7poROiYcni2dLJmzDtWAJe0rv/WFEnOBfJ/PsHwu218SwkKoc4RxkkiawbU/6+RVGKSYAuAMJoOdixeLbBSLSduZ+Go5DGd6PnleVc4mD83ET0itP4A4nNUE71Aw7fi3+/Mx/u4SPjCXEMaKaT5cLJWhNsNllLoz9c6Mjpj5rToBaOAV+qurhEOtHqhZ4Ki2z8RSM4jXctUTfHZmrVzs/a0aWT2Scw5rNtjE1/B71os5sTx3SJxXVNXOqkR31fUjpwD+Y/lmdfuX2E9ePLNqPGGzkkXZSQtnmn13PvvIY3JoD3h1zYgwxjkopyDwdEqpq90o3QT5Cq8Gf5tV0WOhtJZFHoDMJdU0urIxyf1YFkj8Hij0Sdpw/sXAyeCsQIBXOz0LBmCXsxvC6VZine/6JpgB7KokX8ueKq7tztq4Kz8elKf44GHmg8P9Jpc7RN1FzzZ/16rAafWKsJzPj1TNfvI0uYRdKHXdXqE9gaOYhXJmL4j6ARflVxoUiys7z1Pxt3VBw9FWpxRDlJxT/dcTK8SPBBHXBAsf4T67K/OSTkKlVVbYyy61aNIFPH6ZvXN74hGO/vc3UAh+dGioP6TnlXJ0P+9TaiJycbTvv1+pZ/vg=.U2FsdGVkX18EYcoHOF2mGE7tS3M81nYMuE6xIUYvj2LOOk9vNHsBhYuCs8J0lTLSTVjZ4uAle3rl1+Xnr0BIXtIHydeAylGS+0zQD/ukMdt+6Q4iP/LcYtNxyr5Pl1s8BZlE5fzJXNPQzHFRpYzrVO1zBZCtxbB4XI6ks0WxbvNpr6uD0KyWv5cBWbiqR4Y41YWd2UJc4eqmbVTrb0s7x9w3IgZu3YDA0+ab+1+WseM4P+dlwV92DUIFA9kXGc/cuboBemNwuVPhm1IbE/tSveuN0bJbVGmQ/oCs0u9Ae5n9vlqVb8q15r6GeoK0BwBw6zsIJVYiLEEWUffEmiwgxEfU54jcEi6wjZMwB/F5di3V4otusCrB5WtYb1OOhIt1uCi01Kz2kJ+SLBzoQdpxF4x4w4bUjytD6hpw2evZVUbj1jYrWf4HxiSZSAWXUkZodKzFjPVAifpUAD6Q9+ET6OfsvSY0iAD6dya+ufYyoK1btl4nCz4wu9mE2Vn++IV5pDF+7GcjOvCQjzw/X8ueOg2h1hG59GgWj6PIPMLex8Y+qlw5JW+GoqKcSg/CSruvx+EUQzZ/wpG5/IlSPi316xM/BAQq22i4Ii+n+KXtOdZIrdeG93KXAYT12JpVchq7tNEj0r9iIqs+dEXWMg1TSvR3SHZQpPIsYy9V1za2MChIk/JsGrG+hjKdLAAQZ/y4TFXHn9Gz/QKnW8gSx42ROk2fRaUV0wx7CIf2TLYsDed7pUjNFDPjczHynEmWBILZHAoqprt4XefQ+LoLn5Gux+X4NDjHbEOSIE9I+yffZnHewnnIwa+dkO5R3zqvT8eY
+  imageURL: http://hpcr-qcow2-image.default:8080/hpcr.qcow2
+  storagePool: images
+  targetSelector:
+    matchLabels:
+      config: onpremsample
+  diskSelector: 
+    matchLabels:
+      app: hpcr

tooling/data/webhook.yaml

@@ -0,0 +1,38 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8s-operator-hpcr
+  labels:
+    hpcr: pod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: k8s-operator-hpcr
+  template:
+    metadata:
+      labels:
+        app: k8s-operator-hpcr
+    spec:
+      containers:
+      - name: controller
+        image: ghcr.io/ibm-hyper-protect/k8s-operator-hpcr:latest
+        imagePullPolicy: Never
+        resources:
+          limits:
+            memory: 512Mi
+            cpu: "1"
+          requests:
+            memory: 256Mi
+            cpu: "0.2"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: k8s-operator-hpcr
+spec:
+  selector:
+    app: k8s-operator-hpcr
+  ports:
+  - port: 8080