Check if CRLs are downloaded when determining ready status

This fixes OCPBUGS-29894

Author: rfredette

pkg/cmd/infra/router/template.go

@@ -633,6 +633,7 @@ func (o *TemplateRouterOptions) Run(stopCh <-chan struct{}) error {
 		if err != nil {
 			return err
 		}
+		checkCRLs := metrics.CRLsUpdated()
 		checkController := metrics.ControllerLive()
 		liveChecks := []healthz.HealthChecker{checkController}
 		if !(isTrue(env("ROUTER_BIND_PORTS_BEFORE_SYNC", ""))) {
@@ -687,7 +688,7 @@ func (o *TemplateRouterOptions) Run(stopCh <-chan struct{}) error {
 				Name:            o.RouterName,
 			},
 			LiveChecks:  liveChecks,
-			ReadyChecks: []healthz.HealthChecker{checkBackend, checkSync, metrics.ProcessRunning(stopCh)},
+			ReadyChecks: []healthz.HealthChecker{checkBackend, checkSync, metrics.ProcessRunning(stopCh), checkCRLs},
 		}
 
 		if tlsConfig, err := makeTLSConfig(30 * time.Second); err != nil {

pkg/router/crl/crl.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	logf "github.com/openshift/router/log"
@@ -66,6 +67,9 @@ var (
 	CRLFilename = filepath.Join(mtlsLatestSymlink, crlBasename)
 	// CABundleFilename is the fully qualified path to the currently in use CA bundle.
 	CABundleFilename = filepath.Join(mtlsLatestSymlink, caBundleBasename)
+	// crlsUpdated is true when all CRLs have been successfully updated, and false when there are missing CRLs.
+	crlsUpdated = false
+	crlsMutex   = sync.Mutex{}
 )
 
 // authorityKeyIdentifier is a certificate's authority key identifier.
@@ -143,19 +147,24 @@ func ManageCRLs(caBundleFilename string, caUpdateChannel <-chan struct{}, update
 			log.Error(err, "failed to parse CA bundle", "CA bundle filename", caBundleFilename)
 			nextUpdate = time.Now().Add(errorBackoffTime)
 		}
+		if !shouldHaveCRLs {
+			SetCRLsUpdated(true)
+		}
 		for {
 			updated := false
 			if nextUpdate.IsZero() {
 				log.V(4).Info("no nextUpdate. only watching for CA updates")
 				select {
 				case <-caUpdateChannel:
+					SetCRLsUpdated(false)
 					caUpdated = true
 				}
 			} else {
 				log.V(4).Info("nextUpdate is at " + nextUpdate.Format(time.RFC3339))
 				select {
 				case <-time.After(time.Until(nextUpdate)):
 				case <-caUpdateChannel:
+					SetCRLsUpdated(false)
 					caUpdated = true
 				}
 			}
@@ -175,8 +184,9 @@ func ManageCRLs(caBundleFilename string, caUpdateChannel <-chan struct{}, update
 				nextUpdate = time.Now().Add(errorBackoffTime)
 				continue
 			}
-			// After successfully updating the CRL file, reset caUpdated
+			// After successfully updating the CRL file, reset caUpdated and mark CRLs as updated
 			caUpdated = false
+			SetCRLsUpdated(true)
 			if updated {
 				updateCallback(shouldHaveCRLs)
 			}
@@ -506,3 +516,15 @@ func makeStagingDirectory() (string, error) {
 	}
 	return stagingDirName, nil
 }
+
+func GetCRLsUpdated() bool {
+	crlsMutex.Lock()
+	defer crlsMutex.Unlock()
+	return crlsUpdated
+}
+
+func SetCRLsUpdated(value bool) {
+	crlsMutex.Lock()
+	defer crlsMutex.Unlock()
+	crlsUpdated = value
+}

pkg/router/metrics/health.go

@@ -12,6 +12,7 @@ import (
 
 	"k8s.io/apiserver/pkg/server/healthz"
 
+	"github.com/openshift/router/pkg/router/crl"
 	"github.com/openshift/router/pkg/router/metrics/probehttp"
 	templateplugin "github.com/openshift/router/pkg/router/template"
 )
@@ -75,6 +76,15 @@ func ControllerLive() healthz.HealthChecker {
 
 }
 
+func CRLsUpdated() healthz.HealthChecker {
+	return healthz.NamedCheck("crls-updated", func(r *http.Request) error {
+		if !crl.GetCRLsUpdated() {
+			return fmt.Errorf("missing CRLs")
+		}
+		return nil
+	})
+}
+
 // ProxyProtocolHTTPBackendAvailable returns a healthz check that verifies a backend supporting
 // the HAProxy PROXY protocol responds to a GET to the provided URL with 2xx or 3xx response.
 func ProxyProtocolHTTPBackendAvailable(u *url.URL) healthz.HealthChecker {

pkg/router/template/router.go

@@ -480,6 +480,8 @@ func (r *templateRouter) watchMutualTLSCert() error {
 			log.V(0).Error(err, "failed to establish watch on mTLS certificate directory")
 			return nil
 		}
+	} else {
+		crl.SetCRLsUpdated(true)
 	}
 	return nil
 }