[baseserver] Expose gRPC prometheus metrics

Author: easyCZ

components/common-go/baseserver/options.go

@@ -34,12 +34,13 @@ type config struct {
 
 func defaultConfig() *config {
 	return &config{
-		logger:        log.New(),
-		hostname:      "localhost",
-		httpPort:      9000,
-		grpcPort:      9001,
-		closeTimeout:  5 * time.Second,
-		healthHandler: healthcheck.NewHandler(),
+		logger:          log.New(),
+		hostname:        "localhost",
+		httpPort:        9000,
+		grpcPort:        9001,
+		closeTimeout:    5 * time.Second,
+		healthHandler:   healthcheck.NewHandler(),
+		metricsRegistry: prometheus.NewRegistry(),
 	}
 }
 

components/common-go/baseserver/server.go

@@ -10,6 +10,8 @@ import (
 	gitpod_grpc "github.com/gitpod-io/gitpod/common-go/grpc"
 	"github.com/gitpod-io/gitpod/common-go/pprof"
 	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -165,6 +167,10 @@ func (s *Server) GRPC() *grpc.Server {
 	return s.grpc
 }
 
+func (s *Server) MetricsRegistry() *prometheus.Registry {
+	return s.cfg.metricsRegistry
+}
+
 func (s *Server) close(ctx context.Context) error {
 	if s.listening == nil {
 		return fmt.Errorf("server is not running, invalid close operation")
@@ -221,14 +227,9 @@ func (s *Server) newHTTPMux() *http.ServeMux {
 	mux.HandleFunc("/live", s.cfg.healthHandler.LiveEndpoint)
 	s.Logger().WithField("protocol", "http").Debug("Serving liveliness handler on /live")
 
-	// Metrics endpoint
-	metricsHandler := promhttp.Handler()
-	if s.cfg.metricsRegistry != nil {
-		metricsHandler = promhttp.InstrumentMetricHandler(
-			s.cfg.metricsRegistry, promhttp.HandlerFor(s.cfg.metricsRegistry, promhttp.HandlerOpts{}),
-		)
-	}
-	mux.Handle("/metrics", metricsHandler)
+	mux.Handle("/metrics", promhttp.InstrumentMetricHandler(
+		s.cfg.metricsRegistry, promhttp.HandlerFor(s.cfg.metricsRegistry, promhttp.HandlerOpts{}),
+	))
 	s.Logger().WithField("protocol", "http").Debug("Serving metrics on /metrics")
 
 	mux.Handle(pprof.Path, pprof.Handler())
@@ -240,11 +241,19 @@ func (s *Server) newHTTPMux() *http.ServeMux {
 func (s *Server) initializeGRPC() error {
 	gitpod_grpc.SetupLogging()
 
+	grpcMetrics := grpc_prometheus.NewServerMetrics()
+	grpcMetrics.EnableHandlingTimeHistogram()
+	if err := s.MetricsRegistry().Register(grpcMetrics); err != nil {
+		return fmt.Errorf("failed to register grpc metrics: %w", err)
+	}
+
 	unary := []grpc.UnaryServerInterceptor{
 		grpc_logrus.UnaryServerInterceptor(s.Logger()),
+		grpcMetrics.UnaryServerInterceptor(),
 	}
 	stream := []grpc.StreamServerInterceptor{
 		grpc_logrus.StreamServerInterceptor(s.Logger()),
+		grpcMetrics.StreamServerInterceptor(),
 	}
 
 	s.grpc = grpc.NewServer(gitpod_grpc.ServerOptionsWithInterceptors(stream, unary)...)

components/common-go/baseserver/server_test.go

@@ -5,11 +5,15 @@
 package baseserver_test
 
 import (
+	"context"
 	"fmt"
 	"github.com/gitpod-io/gitpod/common-go/baseserver"
 	"github.com/gitpod-io/gitpod/common-go/pprof"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/health/grpc_health_v1"
 	"net/http"
 	"testing"
 )
@@ -77,3 +81,48 @@ func TestServer_ServesPprof(t *testing.T) {
 	require.NoError(t, err)
 	require.Equalf(t, http.StatusOK, resp.StatusCode, "must serve pprof on %s", pprof.Path)
 }
+
+func TestServer_Metrics_gRPC(t *testing.T) {
+	ctx := context.Background()
+	srv := baseserver.NewForTests(t)
+
+	// At this point, there must be metrics registry available for use
+	require.NotNil(t, srv.MetricsRegistry())
+
+	// To actually get gRPC metrics, we need to invoke an RPC, let's use a built-in health service as a mock
+	grpc_health_v1.RegisterHealthServer(srv.GRPC(), &HealthService{})
+
+	// Let's start our server up
+	baseserver.StartServerForTests(t, srv)
+
+	// We need a client to be able to invoke the RPC, let's construct one
+	conn, err := grpc.DialContext(ctx, srv.GRPCAddress(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	client := grpc_health_v1.NewHealthClient(conn)
+
+	// Invoke the RPC
+	_, err = client.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
+	require.NoError(t, err)
+
+	// Finally, we can assert that some metrics were produced.
+	registry := srv.MetricsRegistry()
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+
+	// We expect at least the following. It's not the full set, but a good baseline to sanity check.
+	expected := []string{"grpc_server_handled_total", "grpc_server_handling_seconds", "grpc_server_started_total"}
+	var actual []string
+	for _, metric := range metrics {
+		actual = append(actual, *metric.Name)
+	}
+
+	require.Subset(t, actual, expected)
+}
+
+type HealthService struct {
+	grpc_health_v1.UnimplementedHealthServer
+}
+
+func (h *HealthService) Check(_ context.Context, _ *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) {
+	return &grpc_health_v1.HealthCheckResponse{Status: grpc_health_v1.HealthCheckResponse_SERVING}, nil
+}