OLS-1775: Reconciling when ca cert deleted for postgres pod.

cmd/main.go

@@ -244,6 +244,12 @@ func main() {
 						controller.TelemetryPullSecretNamespace: {},
 					},
 				},
+				&corev1.ConfigMap{}: {
+					Namespaces: map[string]cache.Config{
+						namespace:          {},
+						"openshift-config": {},
+					},
+				},
 			},
 		},
 	})

internal/controller/constants.go

@@ -244,6 +244,10 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt'
 	PostgresConfigHashStateCacheKey = "olspostgresconfig-hash"
 	// #nosec G101
 	PostgresSecretHashStateCacheKey = "olspostgressecret-hash"
+	// PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificates
+	PostgresCAHashKey = "hash/postgres-ca"
+	// PostgresCAHashStateCacheKey is the key of the hash value of the OLS Postgres CA certificates in state cache
+	PostgresCAHashStateCacheKey = "olspostgresca-hash"
 	// OperatorNetworkPolicyName is the name of the network policy for the operator
 	OperatorNetworkPolicyName = "lightspeed-operator"
 	// OperatorMetricsPort is the port number of the operator metrics endpoint

internal/controller/ols_app_postgres_assets.go

@@ -201,6 +201,15 @@ func (r *OLSConfigReconciler) generatePostgresDeployment(cr *olsv1alpha1.OLSConf
 							},
 							VolumeMounts: volumeMounts,
 							Resources:    *databaseResources,
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									Exec: &corev1.ExecAction{
+										Command: []string{"pg_isready", "-U", PostgresDefaultUser, "-d", PostgresDefaultDbName},
+									},
+								},
+								InitialDelaySeconds: 10,
+								PeriodSeconds:       5,
+							},
 							Env: []corev1.EnvVar{
 								{
 									Name:  "POSTGRESQL_USER",
@@ -256,16 +265,18 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis
 	// Validate deployment annotations.
 	if existingDeployment.Annotations == nil ||
 		existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] ||
-		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] {
-		updateDeploymentAnnotations(existingDeployment, map[string]string{
+		existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] ||
+		(r.stateCache[PostgresCAHashStateCacheKey] != "" && existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey]) {
+		ann := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+		}
+		if r.stateCache[PostgresCAHashStateCacheKey] != "" {
+			ann[PostgresCAHashKey] = r.stateCache[PostgresCAHashStateCacheKey]
+		}
+		updateDeploymentAnnotations(existingDeployment, ann)
 		// update the deployment template annotation triggers the rolling update
-		updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{
-			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+		updateDeploymentTemplateAnnotations(existingDeployment, ann)
 
 		if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil {
 			return err
@@ -293,7 +304,7 @@ func (r *OLSConfigReconciler) generatePostgresService(cr *olsv1alpha1.OLSConfig)
 			Namespace: r.Options.Namespace,
 			Labels:    generatePostgresSelectorLabels(),
 			Annotations: map[string]string{
-				ServingCertSecretAnnotationKey: PostgresCertsSecretName,
+				"service.beta.openshift.io/serving-cert-secret-name": PostgresCertsSecretName,
 			},
 		},
 		Spec: corev1.ServiceSpec{

internal/controller/ols_app_postgres_reconciliator.go

@@ -70,14 +70,15 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c
 	existingDeployment := &appsv1.Deployment{}
 	err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment)
 	if err != nil && errors.IsNotFound(err) {
-		updateDeploymentAnnotations(desiredDeployment, map[string]string{
+		ann := map[string]string{
 			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
 			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
-		updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{
-			PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey],
-			PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey],
-		})
+		}
+		if r.stateCache[PostgresCAHashStateCacheKey] != "" {
+			ann[PostgresCAHashKey] = r.stateCache[PostgresCAHashStateCacheKey]
+		}
+		updateDeploymentAnnotations(desiredDeployment, ann)
+		updateDeploymentTemplateAnnotations(desiredDeployment, ann)
 		r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name)
 		err = r.Create(ctx, desiredDeployment)
 		if err != nil {

internal/controller/ols_app_server_reconciliator.go

@@ -206,6 +206,67 @@ func (r *OLSConfigReconciler) reconcileProxyCAConfigMap(ctx context.Context, cr
 	return nil
 }
 
+func (r *OLSConfigReconciler) reconcilePostgresCASecret(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
+	// Read OpenShift service CA configmap
+	openshiftCA := &corev1.ConfigMap{}
+	err := r.Client.Get(ctx, client.ObjectKey{Name: "openshift-service-ca.crt", Namespace: "openshift-config"}, openshiftCA)
+	if err != nil {
+		r.logger.Info("OpenShift service CA configmap not found, skipping", "configmap", "openshift-service-ca.crt")
+		// Continue with just the secret
+	}
+
+	// Read PostgreSQL CA secret
+	secret := &corev1.Secret{}
+	err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, secret)
+	if err != nil {
+		// If the CA certificates secret is not present yet, set a special "deleted" hash to trigger deployment update
+		if errors.IsNotFound(err) {
+			deletedHash := "DELETED-" + fmt.Sprintf("%d", time.Now().Unix())
+			r.stateCache[PostgresCAHashStateCacheKey] = deletedHash
+			return nil
+		}
+		return fmt.Errorf("failed to get postgres CA certificates secret %s: %w", PostgresCertsSecretName, err)
+	}
+
+	// Calculate hash combining both OpenShift service CA and PostgreSQL CA data
+	hashData := ""
+
+	// Add OpenShift service CA data if available
+	if openshiftCA.Name != "" {
+		for key, value := range openshiftCA.Data {
+			hashData += fmt.Sprintf("openshift-%s:%s:", key, value)
+		}
+		hashData += fmt.Sprintf("openshift-version:%s:", openshiftCA.GetResourceVersion())
+	}
+
+	// Add PostgreSQL CA secret data
+	caCertData, exists := secret.Data["service-ca.crt"]
+	if exists {
+		hashData += fmt.Sprintf("postgres-service-ca:%s:", string(caCertData))
+	}
+	hashData += fmt.Sprintf("postgres-version:%s", secret.GetResourceVersion())
+
+	computedHash, err := hashBytes([]byte(hashData))
+	if err != nil {
+		return fmt.Errorf("failed to generate postgres CA cert hash: %w", err)
+	}
+
+	// Store the computed hash in state cache for comparison
+	oldHash := r.stateCache[PostgresCAHashStateCacheKey]
+	r.stateCache[PostgresCAHashStateCacheKey] = computedHash
+
+	// Debug logging to check for idempotency issues
+	r.logger.Info("CA hash debug", "existing", oldHash, "computed", computedHash, "openshift-version", openshiftCA.GetResourceVersion(), "postgres-version", secret.GetResourceVersion())
+
+	if oldHash == computedHash {
+		r.logger.Info("Postgres CA reconciliation skipped - hash unchanged", "hash", computedHash)
+		return nil
+	}
+
+	r.logger.Info("Postgres CA hash changed, will trigger deployment update", "oldHash", oldHash, "newHash", computedHash)
+	return nil
+}
+
 func (r *OLSConfigReconciler) reconcileServiceAccount(ctx context.Context, cr *olsv1alpha1.OLSConfig) error {
 	sa, err := r.generateServiceAccount(cr)
 	if err != nil {

internal/controller/olsconfig_controller.go

@@ -151,7 +151,7 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		r.logger.Error(err, "Failed to get olsconfig")
 		return ctrl.Result{RequeueAfter: 1 * time.Second}, err
 	}
-	r.logger.Info("reconciliation starts", "olsconfig generation", olsconfig.Generation)
+	r.logger.Info("reconciliation starts", "olsconfig generation", olsconfig.Generation, "triggered by", req.NamespacedName)
 
 	err = r.reconcileConsoleUI(ctx, olsconfig)
 	if err != nil {
@@ -162,6 +162,21 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	// Update status condition for Console Plugin
 	r.updateStatusCondition(ctx, olsconfig, typeConsolePluginReady, true, "All components are successfully deployed", nil)
 
+	// Reconcile postgres CA certificate first to ensure hash is updated before postgres deployment reconciliation
+	// Only do this if we have a CA certificate to reconcile
+	if r.shouldReconcilePostgresCA(ctx) {
+		r.logger.Info("Starting postgres CA certificates secret reconciliation")
+		err = r.reconcilePostgresCASecret(ctx, olsconfig)
+		if err != nil {
+			r.logger.Error(err, "Failed to reconcile postgres CA certificates secret")
+			r.updateStatusCondition(ctx, olsconfig, typeCRReconciled, false, "Failed", err)
+			return ctrl.Result{RequeueAfter: 1 * time.Second}, err
+		}
+		r.logger.Info("Completed postgres CA certificates secret reconciliation")
+	} else {
+		r.logger.Info("Skipping postgres CA certificates secret reconciliation")
+	}
+
 	err = r.reconcilePostgresServer(ctx, olsconfig)
 	if err != nil {
 		r.logger.Error(err, "Failed to reconcile ols postgres")
@@ -200,6 +215,13 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	return ctrl.Result{RequeueAfter: r.Options.ReconcileInterval}, nil
 }
 
+// shouldReconcilePostgresCA checks if we should reconcile the postgres CA certificate
+func (r *OLSConfigReconciler) shouldReconcilePostgresCA(ctx context.Context) bool {
+	secret := &corev1.Secret{}
+	err := r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, secret)
+	return err == nil
+}
+
 // updateStatusCondition updates the status condition of the OLSConfig Custom Resource instance.
 // TODO: Should we support Unknown status and ObservedGeneration?
 // TODO: conditionType must be metav1.Condition?
@@ -248,7 +270,6 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.PersistentVolumeClaim{}).
 		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)).
-		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)).
 		Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(configMapWatcherFilter)).
 		Owns(&consolev1.ConsolePlugin{}).
 		Owns(&monv1.ServiceMonitor{}).

internal/controller/resource_watchers.go

@@ -10,19 +10,56 @@ import (
 )
 
 func secretWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
-	annotations := obj.GetAnnotations()
-	if annotations == nil {
-		return nil
+	// Try each specific watcher in order
+	if requests := postgresCASecretWatcher(ctx, obj); len(requests) > 0 {
+		return requests
+	}
+
+	if requests := telemetryPullSecretWatcher(ctx, obj); len(requests) > 0 {
+		return requests
+	}
+
+	if requests := annotatedSecretWatcher(ctx, obj); len(requests) > 0 {
+		return requests
 	}
-	crName, exist := annotations[WatcherAnnotationKey]
-	if !exist {
-		return nil
+
+	return nil
+}
+
+func postgresCASecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request {
+	if obj.GetName() == PostgresCertsSecretName && obj.GetNamespace() == "openshift-lightspeed" {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
+	}
+	return nil
+}
+
+func telemetryPullSecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request {
+	if obj.GetNamespace() == TelemetryPullSecretNamespace && obj.GetName() == TelemetryPullSecretName {
+		return []reconcile.Request{
+			{NamespacedName: types.NamespacedName{
+				Name: OLSConfigName,
+			}},
+		}
 	}
-	return []reconcile.Request{
-		{NamespacedName: types.NamespacedName{
-			Name: crName,
-		}},
+	return nil
+}
+
+func annotatedSecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request {
+	annotations := obj.GetAnnotations()
+	if annotations != nil {
+		if crName, exist := annotations[WatcherAnnotationKey]; exist {
+			return []reconcile.Request{
+				{NamespacedName: types.NamespacedName{
+					Name: crName,
+				}},
+			}
+		}
 	}
+	return nil
 }
 
 func annotateSecretWatcher(secret *corev1.Secret) {
@@ -34,31 +71,24 @@ func annotateSecretWatcher(secret *corev1.Secret) {
 	secret.SetAnnotations(annotations)
 }
 
-func telemetryPullSecretWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
-	if obj.GetNamespace() != TelemetryPullSecretNamespace || obj.GetName() != TelemetryPullSecretName {
-		return nil
-	}
-	return []reconcile.Request{
-		{NamespacedName: types.NamespacedName{
-			Name: OLSConfigName,
-		}},
+func configMapWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
+	// Check for OpenShift service CA configmap first
+	if obj.GetName() == "openshift-service-ca.crt" && obj.GetNamespace() == "openshift-config" {
+		return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: OLSConfigName}}}
 	}
-}
 
-func configMapWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request {
+	// Check for annotated configmaps
 	annotations := obj.GetAnnotations()
-	if annotations == nil {
-		return nil
-	}
-	crName, exist := annotations[WatcherAnnotationKey]
-	if !exist {
-		return nil
-	}
-	return []reconcile.Request{
-		{NamespacedName: types.NamespacedName{
-			Name: crName,
-		}},
+	if annotations != nil {
+		if crName, exist := annotations[WatcherAnnotationKey]; exist {
+			return []reconcile.Request{
+				{NamespacedName: types.NamespacedName{
+					Name: crName,
+				}},
+			}
+		}
 	}
+	return nil
 }
 
 func annotateConfigMapWatcher(cm *corev1.ConfigMap) {

test/e2e/client.go

@@ -620,3 +620,24 @@ func (c *Client) CreatePVC(name, storageClassName string, volumeSize resource.Qu
 		}
 	}, nil
 }
+
+func (c *Client) ExecInPod(podName, namespace, containerName string, command []string) (string, error) {
+	ctx, cancel := context.WithTimeout(c.ctx, c.timeout)
+	defer cancel()
+
+	// Build kubectl exec command
+	args := []string{"exec", "-n", namespace, podName}
+	if containerName != "" {
+		args = append(args, "-c", containerName)
+	}
+	args = append(args, "--")
+	args = append(args, command...)
+
+	cmd := exec.CommandContext(ctx, "kubectl", args...)
+	if c.kubeconfigPath != "" {
+		cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", c.kubeconfigPath))
+	}
+
+	output, err := cmd.CombinedOutput()
+	return string(output), err
+}