[FSSDK-9705] feat: add OpenTelemetry tracing

cmd/optimizely/main.go

@@ -18,6 +18,8 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
+	"fmt"
 	"os"
 	"os/signal"
 	"runtime"
@@ -44,6 +46,14 @@ import (
 	// Initiate the loading of the odpCache plugins
 	_ "github.com/optimizely/agent/plugins/odpcache/all"
 	"github.com/optimizely/go-sdk/pkg/logging"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
 )
 
 // Version holds the admin version
@@ -119,6 +129,95 @@ func initLogging(conf config.LogConfig) {
 	}
 }
 
+func getStdOutTraceProvider(conf config.TracingExporterConfig) (*sdktrace.TracerProvider, error) {
+	f, err := os.Create(conf.Services.StdOut.Filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the trace file, error: %s", err.Error())
+	}
+
+	exp, err := stdouttrace.New(
+		stdouttrace.WithPrettyPrint(),
+		stdouttrace.WithWriter(f),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the collector exporter, error: %s", err.Error())
+	}
+
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(conf.ServiceName),
+			semconv.DeploymentEnvironmentKey.String(conf.Env),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the otel resource, error: %s", err.Error())
+	}
+
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(res),
+	), nil
+}
+
+func getOTELTraceClient(conf config.TracingExporterConfig) (otlptrace.Client, error) {
+	switch conf.Services.Remote.Protocal {
+	case config.TracingRemoteProtocalHTTP:
+		return otlptracehttp.NewClient(
+			otlptracehttp.WithInsecure(),
+			otlptracehttp.WithEndpoint(conf.Services.Remote.Endpoint),
+		), nil
+	case config.TracingRemoteProtocalGRPC:
+		return otlptracegrpc.NewClient(
+			otlptracegrpc.WithInsecure(),
+			otlptracegrpc.WithEndpoint(conf.Services.Remote.Endpoint),
+		), nil
+	default:
+		return nil, errors.New("unknown remote tracing protocal")
+	}
+}
+
+func getRemoteTraceProvider(conf config.TracingExporterConfig) (*sdktrace.TracerProvider, error) {
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(conf.ServiceName),
+			semconv.DeploymentEnvironmentKey.String(conf.Env),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the otel resource, error: %s", err.Error())
+	}
+
+	traceClient, err := getOTELTraceClient(conf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the remote trace client, error: %s", err.Error())
+	}
+
+	traceExporter, err := otlptrace.New(context.Background(), traceClient)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create the remote trace exporter, error: %s", err.Error())
+	}
+
+	bsp := sdktrace.NewBatchSpanProcessor(traceExporter)
+	return sdktrace.NewTracerProvider(
+		sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(conf.Services.Remote.SampleRate))),
+		sdktrace.WithResource(res),
+		sdktrace.WithSpanProcessor(bsp),
+	), nil
+}
+
+func initTracing(conf config.TracingExporterConfig) (*sdktrace.TracerProvider, error) {
+	switch conf.Default {
+	case config.TracingServiceTypeRemote:
+		return getRemoteTraceProvider(conf)
+	case config.TracingServiceTypeStdOut:
+		return getStdOutTraceProvider(conf)
+	default:
+		return nil, errors.New("unknown tracing service type")
+	}
+}
+
 func setRuntimeEnvironment(conf config.RuntimeConfig) {
 	if conf.BlockProfileRate != 0 {
 		log.Warn().Msgf("Setting non-zero blockProfileRate is NOT recommended for production")
@@ -140,6 +239,22 @@ func main() {
 	conf := loadConfig(v)
 	initLogging(conf.Log)
 
+	if conf.Tracing.Enabled {
+		tp, err := initTracing(conf.Tracing.Exporter)
+		if err != nil {
+			log.Panic().Err(err).Msg("Unable to initialize tracing")
+		}
+		defer func() {
+			if err := tp.Shutdown(context.Background()); err != nil {
+				log.Error().Err(err).Msg("Failed to shutdown tracing")
+			}
+		}()
+		otel.SetTracerProvider(tp)
+		log.Info().Msg(fmt.Sprintf("Tracing enabled with service %q", conf.Tracing.Exporter.Default))
+	} else {
+		log.Info().Msg("Tracing disabled")
+	}
+
 	conf.LogConfigWarnings()
 
 	setRuntimeEnvironment(conf.Runtime)

cmd/optimizely/main_test.go

@@ -418,3 +418,92 @@ func TestLoggingWithIncludeSdkKey(t *testing.T) {
 	})
 	assert.False(t, optimizely.ShouldIncludeSDKKey)
 }
+
+func Test_initTracing(t *testing.T) {
+	type args struct {
+		conf config.TracingExporterConfig
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "should return error when exporter type is not supported",
+			args: args{
+				conf: config.TracingExporterConfig{
+					Default: "unsupported",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "should return no error stdout tracing exporter",
+			args: args{
+				conf: config.TracingExporterConfig{
+					Default: "stdout",
+					Services: config.TracingServiceConfig{
+						StdOut: config.TracingStdOutConfig{
+							Filename: "trace.out",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with http protocal",
+			args: args{
+				conf: config.TracingExporterConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocal: "http",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with grpc protocal",
+			args: args{
+				conf: config.TracingExporterConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocal: "grpc",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "should return no error for remote tracing exporter with invalid protocal",
+			args: args{
+				conf: config.TracingExporterConfig{
+					Default: "remote",
+					Services: config.TracingServiceConfig{
+						Remote: config.TracingRemoteConfig{
+							Endpoint: "localhost:1234",
+							Protocal: "udp/invalid",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := initTracing(tt.args.conf)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("initTracing() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+		})
+	}
+}

config.yaml

@@ -26,6 +26,41 @@ log:
     ## to set whether or not the SDK key is included in the logging output.
     includeSdkKey: true
 
+##
+## tracing: tracing configuration
+##
+tracing:
+    ## bydefault tracing is disabled
+    ## to enable tracing set enabled to true
+    enabled: false
+    ## tracing exporter configuration
+    exporter:
+        ## bydefault stdout exporter is enabled
+        ## to enable remote exporter set default as "remote"
+        default: "stdout"
+        ## tracing service name
+        serviceName: "optimizely-agent"
+        ## tracing environment name
+        ## example: for production environment env can be set as "prod"
+        env: "dev"
+        ## tracing service configuration
+        services:
+            ## stdout exporter configuration
+            stdout:
+                ## for stdout tracing data is saved in the specified file
+                filename: "trace.out"
+            ## remote exporter configuration
+            remote:
+                ## remote collector endpoint
+                endpoint: "localhost:4317"
+                ## supported protocals are "http" and "grpc"
+                protocal: "grpc"
+                ## "sampleRate" refers to the rate at which traces are collected and recorded.
+                ## sampleRate >= 1 will always sample.
+                ## sampleRate < 0 are treated as zero i.e. never sample.
+                sampleRate: 1.0
+
+
 ##
 ## http server configuration
 ##

config/config.go

@@ -71,6 +71,9 @@ func NewDefaultConfig() *AgentConfig {
 			IncludeSDKKey: true,
 			Level:         "info",
 		},
+		Tracing: TracingConfig{
+			Enabled: false,
+		},
 		Client: ClientConfig{
 			PollingInterval:     1 * time.Minute,
 			BatchSize:           10,
@@ -139,6 +142,7 @@ type AgentConfig struct {
 	Admin   AdminConfig   `json:"admin"`
 	API     APIConfig     `json:"api"`
 	Log     LogConfig     `json:"log"`
+	Tracing TracingConfig `json:"tracing"`
 	Client  ClientConfig  `json:"client"`
 	Runtime RuntimeConfig `json:"runtime"`
 	Server  ServerConfig  `json:"server"`
@@ -201,6 +205,47 @@ type LogConfig struct {
 	Level         string `json:"level"`
 }
 
+type TracingConfig struct {
+	Enabled  bool                  `json:"enabled"`
+	Exporter TracingExporterConfig `json:"exporter"`
+}
+
+type TracingServiceType string
+
+const (
+	TracingServiceTypeStdOut TracingServiceType = "stdout"
+	TracingServiceTypeRemote TracingServiceType = "remote"
+)
+
+type TracingRemoteProtocal string
+
+const (
+	TracingRemoteProtocalGRPC TracingRemoteProtocal = "grpc"
+	TracingRemoteProtocalHTTP TracingRemoteProtocal = "http"
+)
+
+type TracingExporterConfig struct {
+	Default     TracingServiceType   `json:"default"`
+	ServiceName string               `json:"serviceName"`
+	Env         string               `json:"env"`
+	Services    TracingServiceConfig `json:"services"`
+}
+
+type TracingServiceConfig struct {
+	StdOut TracingStdOutConfig `json:"stdout"`
+	Remote TracingRemoteConfig `json:"remote"`
+}
+
+type TracingStdOutConfig struct {
+	Filename string `json:"filename"`
+}
+
+type TracingRemoteConfig struct {
+	Endpoint   string                `json:"endpoint"`
+	Protocal   TracingRemoteProtocal `json:"protocal"`
+	SampleRate float64               `json:"sampleRate"`
+}
+
 // PluginConfigs defines the generic mapping of middleware plugins
 type PluginConfigs map[string]interface{}
 

go.mod

@@ -18,20 +18,37 @@ require (
 	github.com/rakyll/statik v0.1.7
 	github.com/rs/zerolog v1.29.0
 	github.com/spf13/viper v1.15.0
-	github.com/stretchr/testify v1.8.2
-	golang.org/x/crypto v0.6.0
-	golang.org/x/sync v0.1.0
+	github.com/stretchr/testify v1.8.4
+	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0
+	go.opentelemetry.io/otel/sdk v1.19.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/sync v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.30.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	golang.org/x/net v0.12.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )
 
 require (
@@ -62,8 +79,8 @@ require (
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/twmb/murmur3 v1.1.6 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )