diff --git a/cmd/main.go b/cmd/main.go index 9bb1a977f..e0b1d8935 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -244,6 +244,12 @@ func main() { controller.TelemetryPullSecretNamespace: {}, }, }, + &corev1.ConfigMap{}: { + Namespaces: map[string]cache.Config{ + namespace: {}, + "openshift-config": {}, + }, + }, }, }, }) diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 5274d4983..fb7b08746 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -244,6 +244,10 @@ ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt' PostgresConfigHashStateCacheKey = "olspostgresconfig-hash" // #nosec G101 PostgresSecretHashStateCacheKey = "olspostgressecret-hash" + // PostgresCAHashKey is the key of the hash value of the OLS Postgres CA certificates + PostgresCAHashKey = "hash/postgres-ca" + // PostgresCAHashStateCacheKey is the key of the hash value of the OLS Postgres CA certificates in state cache + PostgresCAHashStateCacheKey = "olspostgresca-hash" // OperatorNetworkPolicyName is the name of the network policy for the operator OperatorNetworkPolicyName = "lightspeed-operator" // OperatorMetricsPort is the port number of the operator metrics endpoint diff --git a/internal/controller/ols_app_postgres_assets.go b/internal/controller/ols_app_postgres_assets.go index ee6157722..a2a78272f 100644 --- a/internal/controller/ols_app_postgres_assets.go +++ b/internal/controller/ols_app_postgres_assets.go @@ -201,6 +201,15 @@ func (r *OLSConfigReconciler) generatePostgresDeployment(cr *olsv1alpha1.OLSConf }, VolumeMounts: volumeMounts, Resources: *databaseResources, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{"pg_isready", "-U", PostgresDefaultUser, "-d", PostgresDefaultDbName}, + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 5, + }, Env: []corev1.EnvVar{ { Name: "POSTGRESQL_USER", @@ -256,16 +265,18 @@ func (r *OLSConfigReconciler) updatePostgresDeployment(ctx context.Context, exis // Validate deployment annotations. if existingDeployment.Annotations == nil || existingDeployment.Annotations[PostgresConfigHashKey] != r.stateCache[PostgresConfigHashStateCacheKey] || - existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] { - updateDeploymentAnnotations(existingDeployment, map[string]string{ + existingDeployment.Annotations[PostgresSecretHashKey] != r.stateCache[PostgresSecretHashStateCacheKey] || + (r.stateCache[PostgresCAHashStateCacheKey] != "" && existingDeployment.Annotations[PostgresCAHashKey] != r.stateCache[PostgresCAHashStateCacheKey]) { + ann := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + } + if r.stateCache[PostgresCAHashStateCacheKey] != "" { + ann[PostgresCAHashKey] = r.stateCache[PostgresCAHashStateCacheKey] + } + updateDeploymentAnnotations(existingDeployment, ann) // update the deployment template annotation triggers the rolling update - updateDeploymentTemplateAnnotations(existingDeployment, map[string]string{ - PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + updateDeploymentTemplateAnnotations(existingDeployment, ann) if _, err := setDeploymentContainerEnvs(existingDeployment, desiredDeployment.Spec.Template.Spec.Containers[0].Env, PostgresDeploymentName); err != nil { return err @@ -293,7 +304,7 @@ func (r *OLSConfigReconciler) generatePostgresService(cr *olsv1alpha1.OLSConfig) Namespace: r.Options.Namespace, Labels: generatePostgresSelectorLabels(), Annotations: map[string]string{ - ServingCertSecretAnnotationKey: PostgresCertsSecretName, + "service.beta.openshift.io/serving-cert-secret-name": PostgresCertsSecretName, }, }, Spec: corev1.ServiceSpec{ diff --git a/internal/controller/ols_app_postgres_reconciliator.go b/internal/controller/ols_app_postgres_reconciliator.go index 3706b5e80..a236ec155 100644 --- a/internal/controller/ols_app_postgres_reconciliator.go +++ b/internal/controller/ols_app_postgres_reconciliator.go @@ -70,14 +70,15 @@ func (r *OLSConfigReconciler) reconcilePostgresDeployment(ctx context.Context, c existingDeployment := &appsv1.Deployment{} err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresDeploymentName, Namespace: r.Options.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { - updateDeploymentAnnotations(desiredDeployment, map[string]string{ + ann := map[string]string{ PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) - updateDeploymentTemplateAnnotations(desiredDeployment, map[string]string{ - PostgresConfigHashKey: r.stateCache[PostgresConfigHashStateCacheKey], - PostgresSecretHashKey: r.stateCache[PostgresSecretHashStateCacheKey], - }) + } + if r.stateCache[PostgresCAHashStateCacheKey] != "" { + ann[PostgresCAHashKey] = r.stateCache[PostgresCAHashStateCacheKey] + } + updateDeploymentAnnotations(desiredDeployment, ann) + updateDeploymentTemplateAnnotations(desiredDeployment, ann) r.logger.Info("creating a new OLS postgres deployment", "deployment", desiredDeployment.Name) err = r.Create(ctx, desiredDeployment) if err != nil { diff --git a/internal/controller/ols_app_server_reconciliator.go b/internal/controller/ols_app_server_reconciliator.go index aa5fb9178..c8f32b813 100644 --- a/internal/controller/ols_app_server_reconciliator.go +++ b/internal/controller/ols_app_server_reconciliator.go @@ -206,6 +206,67 @@ func (r *OLSConfigReconciler) reconcileProxyCAConfigMap(ctx context.Context, cr return nil } +func (r *OLSConfigReconciler) reconcilePostgresCASecret(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { + // Read OpenShift service CA configmap + openshiftCA := &corev1.ConfigMap{} + err := r.Client.Get(ctx, client.ObjectKey{Name: "openshift-service-ca.crt", Namespace: "openshift-config"}, openshiftCA) + if err != nil { + r.logger.Info("OpenShift service CA configmap not found, skipping", "configmap", "openshift-service-ca.crt") + // Continue with just the secret + } + + // Read PostgreSQL CA secret + secret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, secret) + if err != nil { + // If the CA certificates secret is not present yet, set a special "deleted" hash to trigger deployment update + if errors.IsNotFound(err) { + deletedHash := "DELETED-" + fmt.Sprintf("%d", time.Now().Unix()) + r.stateCache[PostgresCAHashStateCacheKey] = deletedHash + return nil + } + return fmt.Errorf("failed to get postgres CA certificates secret %s: %w", PostgresCertsSecretName, err) + } + + // Calculate hash combining both OpenShift service CA and PostgreSQL CA data + hashData := "" + + // Add OpenShift service CA data if available + if openshiftCA.Name != "" { + for key, value := range openshiftCA.Data { + hashData += fmt.Sprintf("openshift-%s:%s:", key, value) + } + hashData += fmt.Sprintf("openshift-version:%s:", openshiftCA.GetResourceVersion()) + } + + // Add PostgreSQL CA secret data + caCertData, exists := secret.Data["service-ca.crt"] + if exists { + hashData += fmt.Sprintf("postgres-service-ca:%s:", string(caCertData)) + } + hashData += fmt.Sprintf("postgres-version:%s", secret.GetResourceVersion()) + + computedHash, err := hashBytes([]byte(hashData)) + if err != nil { + return fmt.Errorf("failed to generate postgres CA cert hash: %w", err) + } + + // Store the computed hash in state cache for comparison + oldHash := r.stateCache[PostgresCAHashStateCacheKey] + r.stateCache[PostgresCAHashStateCacheKey] = computedHash + + // Debug logging to check for idempotency issues + r.logger.Info("CA hash debug", "existing", oldHash, "computed", computedHash, "openshift-version", openshiftCA.GetResourceVersion(), "postgres-version", secret.GetResourceVersion()) + + if oldHash == computedHash { + r.logger.Info("Postgres CA reconciliation skipped - hash unchanged", "hash", computedHash) + return nil + } + + r.logger.Info("Postgres CA hash changed, will trigger deployment update", "oldHash", oldHash, "newHash", computedHash) + return nil +} + func (r *OLSConfigReconciler) reconcileServiceAccount(ctx context.Context, cr *olsv1alpha1.OLSConfig) error { sa, err := r.generateServiceAccount(cr) if err != nil { diff --git a/internal/controller/olsconfig_controller.go b/internal/controller/olsconfig_controller.go index 54c12faf4..19b9b098a 100644 --- a/internal/controller/olsconfig_controller.go +++ b/internal/controller/olsconfig_controller.go @@ -151,7 +151,7 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( r.logger.Error(err, "Failed to get olsconfig") return ctrl.Result{RequeueAfter: 1 * time.Second}, err } - r.logger.Info("reconciliation starts", "olsconfig generation", olsconfig.Generation) + r.logger.Info("reconciliation starts", "olsconfig generation", olsconfig.Generation, "triggered by", req.NamespacedName) err = r.reconcileConsoleUI(ctx, olsconfig) if err != nil { @@ -162,6 +162,21 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Update status condition for Console Plugin r.updateStatusCondition(ctx, olsconfig, typeConsolePluginReady, true, "All components are successfully deployed", nil) + // Reconcile postgres CA certificate first to ensure hash is updated before postgres deployment reconciliation + // Only do this if we have a CA certificate to reconcile + if r.shouldReconcilePostgresCA(ctx) { + r.logger.Info("Starting postgres CA certificates secret reconciliation") + err = r.reconcilePostgresCASecret(ctx, olsconfig) + if err != nil { + r.logger.Error(err, "Failed to reconcile postgres CA certificates secret") + r.updateStatusCondition(ctx, olsconfig, typeCRReconciled, false, "Failed", err) + return ctrl.Result{RequeueAfter: 1 * time.Second}, err + } + r.logger.Info("Completed postgres CA certificates secret reconciliation") + } else { + r.logger.Info("Skipping postgres CA certificates secret reconciliation") + } + err = r.reconcilePostgresServer(ctx, olsconfig) if err != nil { r.logger.Error(err, "Failed to reconcile ols postgres") @@ -200,6 +215,13 @@ func (r *OLSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{RequeueAfter: r.Options.ReconcileInterval}, nil } +// shouldReconcilePostgresCA checks if we should reconcile the postgres CA certificate +func (r *OLSConfigReconciler) shouldReconcilePostgresCA(ctx context.Context) bool { + secret := &corev1.Secret{} + err := r.Client.Get(ctx, client.ObjectKey{Name: PostgresCertsSecretName, Namespace: r.Options.Namespace}, secret) + return err == nil +} + // updateStatusCondition updates the status condition of the OLSConfig Custom Resource instance. // TODO: Should we support Unknown status and ObservedGeneration? // TODO: conditionType must be metav1.Condition? @@ -248,7 +270,6 @@ func (r *OLSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Owns(&corev1.PersistentVolumeClaim{}). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(secretWatcherFilter)). - Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(telemetryPullSecretWatcherFilter)). Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(configMapWatcherFilter)). Owns(&consolev1.ConsolePlugin{}). Owns(&monv1.ServiceMonitor{}). diff --git a/internal/controller/resource_watchers.go b/internal/controller/resource_watchers.go index c54747c12..8c720231b 100644 --- a/internal/controller/resource_watchers.go +++ b/internal/controller/resource_watchers.go @@ -10,19 +10,56 @@ import ( ) func secretWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { - annotations := obj.GetAnnotations() - if annotations == nil { - return nil + // Try each specific watcher in order + if requests := postgresCASecretWatcher(ctx, obj); len(requests) > 0 { + return requests + } + + if requests := telemetryPullSecretWatcher(ctx, obj); len(requests) > 0 { + return requests + } + + if requests := annotatedSecretWatcher(ctx, obj); len(requests) > 0 { + return requests } - crName, exist := annotations[WatcherAnnotationKey] - if !exist { - return nil + + return nil +} + +func postgresCASecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request { + if obj.GetName() == PostgresCertsSecretName && obj.GetNamespace() == "openshift-lightspeed" { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } + } + return nil +} + +func telemetryPullSecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request { + if obj.GetNamespace() == TelemetryPullSecretNamespace && obj.GetName() == TelemetryPullSecretName { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: OLSConfigName, + }}, + } } - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: crName, - }}, + return nil +} + +func annotatedSecretWatcher(ctx context.Context, obj client.Object) []reconcile.Request { + annotations := obj.GetAnnotations() + if annotations != nil { + if crName, exist := annotations[WatcherAnnotationKey]; exist { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: crName, + }}, + } + } } + return nil } func annotateSecretWatcher(secret *corev1.Secret) { @@ -34,31 +71,24 @@ func annotateSecretWatcher(secret *corev1.Secret) { secret.SetAnnotations(annotations) } -func telemetryPullSecretWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { - if obj.GetNamespace() != TelemetryPullSecretNamespace || obj.GetName() != TelemetryPullSecretName { - return nil - } - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: OLSConfigName, - }}, +func configMapWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { + // Check for OpenShift service CA configmap first + if obj.GetName() == "openshift-service-ca.crt" && obj.GetNamespace() == "openshift-config" { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: OLSConfigName}}} } -} -func configMapWatcherFilter(ctx context.Context, obj client.Object) []reconcile.Request { + // Check for annotated configmaps annotations := obj.GetAnnotations() - if annotations == nil { - return nil - } - crName, exist := annotations[WatcherAnnotationKey] - if !exist { - return nil - } - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: crName, - }}, + if annotations != nil { + if crName, exist := annotations[WatcherAnnotationKey]; exist { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: crName, + }}, + } + } } + return nil } func annotateConfigMapWatcher(cm *corev1.ConfigMap) { diff --git a/test/e2e/client.go b/test/e2e/client.go index 1a063b279..d9e889237 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -620,3 +620,24 @@ func (c *Client) CreatePVC(name, storageClassName string, volumeSize resource.Qu } }, nil } + +func (c *Client) ExecInPod(podName, namespace, containerName string, command []string) (string, error) { + ctx, cancel := context.WithTimeout(c.ctx, c.timeout) + defer cancel() + + // Build kubectl exec command + args := []string{"exec", "-n", namespace, podName} + if containerName != "" { + args = append(args, "-c", containerName) + } + args = append(args, "--") + args = append(args, command...) + + cmd := exec.CommandContext(ctx, "kubectl", args...) + if c.kubeconfigPath != "" { + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", c.kubeconfigPath)) + } + + output, err := cmd.CombinedOutput() + return string(output), err +}