Add support for startup probes

Author: ramperher

designs/component-config.md

@@ -1,7 +1,7 @@
 # ComponentConfig Controller Runtime Support
 Author: @christopherhein
 
-Last Updated on: 03/02/2020
+Last Updated on: 31/01/2024
 
 ## Table of Contents
 
@@ -91,6 +91,7 @@ type ManagerConfiguration interface {
 
 	GetReadinessEndpointName() string
 	GetLivenessEndpointName() string
+	GetStartupEndpointName() string
 
 	GetPort() int
 	GetHost() string
@@ -161,6 +162,7 @@ type ControllerManagerConfigurationHealth struct {
 
 	ReadinessEndpointName string `json:"readinessEndpointName,omitempty"`
 	LivenessEndpointName  string `json:"livenessEndpointName,omitempty"`
+	StartupEndpointName  string `json:"startupEndpointName,omitempty"`
 }
 ```
 

designs/move-cluster-specific-code-out-of-manager.md

@@ -116,6 +116,9 @@ type Manager interface {
 	// AddReadyzCheck allows you to add Readyz checker
 	AddReadyzCheck(name string, check healthz.Checker) error
 
+	// AddStartzCheck allows you to add Startz checker
+	AddStartzCheck(name string, check healthz.Checker) error
+
 	// Start starts all registered Controllers and blocks until the Stop channel is closed.
 	// Returns an error if there is an error starting any controller.
 	// If LeaderElection is used, the binary must be exited immediately after this returns,

pkg/config/v1alpha1/types.go

@@ -135,6 +135,10 @@ type ControllerHealth struct {
 	// LivenessEndpointName, defaults to "healthz"
 	// +optional
 	LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
+
+	// StartupEndpointName, defaults to "startz"
+	// +optional
+	StartupEndpointName string `json:"startupEndpointName,omitempty"`
 }
 
 // ControllerWebhook defines the webhook server for the controller.

pkg/healthz/doc.go

@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package healthz contains helpers from supporting liveness and readiness endpoints.
-// (often referred to as healthz and readyz, respectively).
+// Package healthz contains helpers from supporting liveness, readiness and startup endpoints.
+// (often referred to as healthz, readyz and startz, respectively).
 //
 // This package draws heavily from the apiserver's healthz package
 // ( https://github.com/kubernetes/apiserver/tree/master/pkg/server/healthz )

pkg/manager/internal.go

@@ -56,6 +56,7 @@ const (
 
 	defaultReadinessEndpoint = "/readyz"
 	defaultLivenessEndpoint  = "/healthz"
+	defaultStartupEndpoint   = "/startz"
 )
 
 var _ Runnable = &controllerManager{}
@@ -94,12 +95,18 @@ type controllerManager struct {
 	// Liveness probe endpoint name
 	livenessEndpointName string
 
+	// Startup probe endpoint name
+	startupEndpointName string
+
 	// Readyz probe handler
 	readyzHandler *healthz.Handler
 
 	// Healthz probe handler
 	healthzHandler *healthz.Handler
 
+	// Startz probe handler
+	startzHandler *healthz.Handler
+
 	// pprofListener is used to serve pprof
 	pprofListener net.Listener
 
@@ -213,6 +220,23 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker)
 	return nil
 }
 
+// AddStartzCheck allows you to add Startz checker.
+func (cm *controllerManager) AddStartzCheck(name string, check healthz.Checker) error {
+	cm.Lock()
+	defer cm.Unlock()
+
+	if cm.started {
+		return fmt.Errorf("unable to add new checker because healthz endpoint has already been created")
+	}
+
+	if cm.startzHandler == nil {
+		cm.startzHandler = &healthz.Handler{Checks: map[string]healthz.Checker{}}
+	}
+
+	cm.startzHandler.Checks[name] = check
+	return nil
+}
+
 func (cm *controllerManager) GetHTTPClient() *http.Client {
 	return cm.cluster.GetHTTPClient()
 }
@@ -283,6 +307,11 @@ func (cm *controllerManager) addHealthProbeServer() error {
 		// Append '/' suffix to handle subpaths
 		mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler))
 	}
+	if cm.startzHandler != nil {
+		mux.Handle(cm.startupEndpointName, http.StripPrefix(cm.startupEndpointName, cm.startzHandler))
+		// Append '/' suffix to handle subpaths
+		mux.Handle(cm.startupEndpointName+"/", http.StripPrefix(cm.startupEndpointName, cm.startzHandler))
+	}
 
 	return cm.add(&server{
 		Kind:     "health probe",

pkg/manager/internal/integration/manager_test.go

@@ -158,6 +158,7 @@ var _ = Describe("manger.Manager Start", func() {
 		// Configure health probes.
 		Expect(mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed())
 		Expect(mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed())
+		Expect(mgr.AddStartzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed())
 
 		// Set up Driver reconciler (using v2).
 		driverReconciler := &DriverReconciler{

pkg/manager/manager.go

@@ -73,6 +73,9 @@ type Manager interface {
 	// AddReadyzCheck allows you to add Readyz checker
 	AddReadyzCheck(name string, check healthz.Checker) error
 
+	// AddStartzCheck allows you to add Startz checker
+	AddStartzCheck(name string, check healthz.Checker) error
+
 	// Start starts all registered Controllers and blocks until the context is cancelled.
 	// Returns an error if there is an error starting any controller.
 	//
@@ -228,6 +231,9 @@ type Options struct {
 	// Liveness probe endpoint name, defaults to "healthz"
 	LivenessEndpointName string
 
+	// Startup probe endpoint name, defaults to "healthz"
+	StartupEndpointName string
+
 	// PprofBindAddress is the TCP address that the controller should bind to
 	// for serving pprof.
 	// It can be set to "" or "0" to disable the pprof serving.
@@ -430,6 +436,7 @@ func New(config *rest.Config, options Options) (Manager, error) {
 		healthProbeListener:           healthProbeListener,
 		readinessEndpointName:         options.ReadinessEndpointName,
 		livenessEndpointName:          options.LivenessEndpointName,
+		startupEndpointName:           options.StartupEndpointName,
 		pprofListener:                 pprofListener,
 		gracefulShutdownTimeout:       *options.GracefulShutdownTimeout,
 		internalProceduresStop:        make(chan struct{}),
@@ -480,6 +487,10 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options,
 		o.LivenessEndpointName = newObj.Health.LivenessEndpointName
 	}
 
+	if o.StartupEndpointName == "" && newObj.Health.StartupEndpointName != "" {
+		o.StartupEndpointName = newObj.Health.StartupEndpointName
+	}
+
 	if o.WebhookServer == nil {
 		port := 0
 		if newObj.Webhook.Port != nil {
@@ -640,6 +651,10 @@ func setOptionsDefaults(options Options) Options {
 		options.LivenessEndpointName = defaultLivenessEndpoint
 	}
 
+	if options.StartupEndpointName == "" {
+		options.StartupEndpointName = defaultStartupEndpoint
+	}
+
 	if options.newHealthProbeListener == nil {
 		options.newHealthProbeListener = defaultHealthProbeListener
 	}

pkg/manager/manager_test.go

@@ -145,6 +145,7 @@ var _ = Describe("manger.Manager", func() {
 						HealthProbeBindAddress: "6060",
 						ReadinessEndpointName:  "/readyz",
 						LivenessEndpointName:   "/livez",
+						StartupEndpointName:    "/startz",
 					},
 					Webhook: v1alpha1.ControllerWebhook{
 						Port:    &port,
@@ -170,6 +171,7 @@ var _ = Describe("manger.Manager", func() {
 			Expect(m.HealthProbeBindAddress).To(Equal("6060"))
 			Expect(m.ReadinessEndpointName).To(Equal("/readyz"))
 			Expect(m.LivenessEndpointName).To(Equal("/livez"))
+			Expect(m.StartupEndpointName).To(Equal("/startz"))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Port).To(Equal(port))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Host).To(Equal("localhost"))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.CertDir).To(Equal("/certs"))
@@ -201,6 +203,7 @@ var _ = Describe("manger.Manager", func() {
 						HealthProbeBindAddress: "6060",
 						ReadinessEndpointName:  "/readyz",
 						LivenessEndpointName:   "/livez",
+						StartupEndpointName:    "/startz",
 					},
 					Webhook: v1alpha1.ControllerWebhook{
 						Port:    &port,
@@ -229,6 +232,7 @@ var _ = Describe("manger.Manager", func() {
 				HealthProbeBindAddress:     "5000",
 				ReadinessEndpointName:      "/readiness",
 				LivenessEndpointName:       "/liveness",
+				StartupEndpointName:        "/startup",
 				WebhookServer: webhook.NewServer(webhook.Options{
 					Port:    8080,
 					Host:    "example.com",
@@ -251,6 +255,7 @@ var _ = Describe("manger.Manager", func() {
 			Expect(m.HealthProbeBindAddress).To(Equal("5000"))
 			Expect(m.ReadinessEndpointName).To(Equal("/readiness"))
 			Expect(m.LivenessEndpointName).To(Equal("/liveness"))
+			Expect(m.StartupEndpointName).To(Equal("/startup"))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Port).To(Equal(8080))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Host).To(Equal("example.com"))
 			Expect(m.WebhookServer.(*webhook.DefaultServer).Options.CertDir).To(Equal("/pki"))
@@ -1399,6 +1404,11 @@ var _ = Describe("manger.Manager", func() {
 	})
 
 	Context("should start serving health probes", func() {
+
+		const (
+			namedCheck = "check"
+		)
+
 		var listener net.Listener
 		var opts Options
 
@@ -1452,7 +1462,6 @@ var _ = Describe("manger.Manager", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			res := fmt.Errorf("not ready yet")
-			namedCheck := "check"
 			err = m.AddReadyzCheck(namedCheck, func(_ *http.Request) error { return res })
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1507,7 +1516,6 @@ var _ = Describe("manger.Manager", func() {
 			Expect(err).NotTo(HaveOccurred())
 
 			res := fmt.Errorf("not alive")
-			namedCheck := "check"
 			err = m.AddHealthzCheck(namedCheck, func(_ *http.Request) error { return res })
 			Expect(err).NotTo(HaveOccurred())
 
@@ -1547,14 +1555,68 @@ var _ = Describe("manger.Manager", func() {
 			defer resp.Body.Close()
 			Expect(resp.StatusCode).To(Equal(http.StatusOK))
 
-			// Check readiness path for individual check
+			// Check liveness path for individual check
 			livenessEndpoint = fmt.Sprint("http://", listener.Addr().String(), path.Join(defaultLivenessEndpoint, namedCheck))
 			res = nil
 			resp, err = http.Get(livenessEndpoint)
 			Expect(err).NotTo(HaveOccurred())
 			defer resp.Body.Close()
 			Expect(resp.StatusCode).To(Equal(http.StatusOK))
 		})
+
+		It("should serve startup endpoint", func() {
+			opts.HealthProbeBindAddress = ":0"
+			m, err := New(cfg, opts)
+			Expect(err).NotTo(HaveOccurred())
+
+			res := fmt.Errorf("not alive")
+			err = m.AddStartzCheck(namedCheck, func(_ *http.Request) error { return res })
+			Expect(err).NotTo(HaveOccurred())
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			go func() {
+				defer GinkgoRecover()
+				Expect(m.Start(ctx)).NotTo(HaveOccurred())
+			}()
+			<-m.Elected()
+
+			startupEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultStartupEndpoint)
+
+			// Controller is not ready
+			resp, err := http.Get(startupEndpoint)
+			Expect(err).NotTo(HaveOccurred())
+			defer resp.Body.Close()
+			Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError))
+
+			// Controller is ready
+			res = nil
+			resp, err = http.Get(startupEndpoint)
+			Expect(err).NotTo(HaveOccurred())
+			defer resp.Body.Close()
+			Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+			// Check startup path without trailing slash without redirect
+			startupEndpoint = fmt.Sprint("http://", listener.Addr().String(), defaultStartupEndpoint)
+			res = nil
+			httpClient := http.Client{
+				CheckRedirect: func(req *http.Request, via []*http.Request) error {
+					return http.ErrUseLastResponse // Do not follow redirect
+				},
+			}
+			resp, err = httpClient.Get(startupEndpoint)
+			Expect(err).NotTo(HaveOccurred())
+			defer resp.Body.Close()
+			Expect(resp.StatusCode).To(Equal(http.StatusOK))
+
+			// Check startup path for individual check
+			startupEndpoint = fmt.Sprint("http://", listener.Addr().String(), path.Join(defaultStartupEndpoint, namedCheck))
+			res = nil
+			resp, err = http.Get(startupEndpoint)
+			Expect(err).NotTo(HaveOccurred())
+			defer resp.Body.Close()
+			Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		})
 	})
 
 	Context("should start serving pprof", func() {