feat: implement comprehensive OpenTelemetry integration for MCP proxies

- Add complete OpenTelemetry instrumentation with OTLP and Prometheus exporters
- Implement HTTP middleware with MCP protocol-aware tracing and metrics
- Add CLI flags for telemetry configuration (--otel-endpoint, --otel-metrics-port, etc.)
- Expose /metrics endpoints on both HTTPSSEProxy and TransparentProxy
- Include comprehensive observability documentation
- Support dual export architecture (OTLP push + Prometheus pull)
- Add tool-specific metrics and argument sanitization for security
- Integrate seamlessly with existing middleware chain

Implements: #597
Fixes: #474

Author: JAORMX

cmd/thv/app/proxy.go

@@ -213,7 +213,7 @@ func proxyCmdFunc(cmd *cobra.Command, args []string) error {
 		port, proxyTargetURI)
 
 	// Create the transparent proxy with middlewares
-	proxy := transparent.NewTransparentProxy(proxyHost, port, serverName, proxyTargetURI, middlewares...)
+	proxy := transparent.NewTransparentProxy(proxyHost, port, serverName, proxyTargetURI, nil, middlewares...)
 	if err := proxy.Start(ctx); err != nil {
 		return fmt.Errorf("failed to start proxy: %v", err)
 	}

cmd/thv/app/run.go

@@ -86,6 +86,14 @@ var (
 	runK8sPodPatch       string
 	runCACertPath        string
 	runVerifyImage       string
+
+	// OpenTelemetry flags
+	runOtelEndpoint     string
+	runOtelServiceName  string
+	runOtelSamplingRate float64
+	runOtelHeaders      []string
+	runOtelInsecure     bool
+	runOtelMetricsPort  int
 )
 
 func init() {
@@ -170,6 +178,20 @@ func init() {
 
 	// Add OIDC validation flags
 	AddOIDCFlags(runCmd)
+
+	// Add OpenTelemetry flags
+	runCmd.Flags().StringVar(&runOtelEndpoint, "otel-endpoint", "",
+		"OpenTelemetry OTLP endpoint URL (e.g., https://api.honeycomb.io)")
+	runCmd.Flags().StringVar(&runOtelServiceName, "otel-service-name", "",
+		"OpenTelemetry service name (defaults to toolhive-mcp-proxy)")
+	runCmd.Flags().Float64Var(&runOtelSamplingRate, "otel-sampling-rate", 0.1,
+		"OpenTelemetry trace sampling rate (0.0-1.0)")
+	runCmd.Flags().StringArrayVar(&runOtelHeaders, "otel-headers", nil,
+		"OpenTelemetry OTLP headers in key=value format (e.g., x-honeycomb-team=your-api-key)")
+	runCmd.Flags().BoolVar(&runOtelInsecure, "otel-insecure", false,
+		"Disable TLS verification for OpenTelemetry endpoint")
+	runCmd.Flags().IntVar(&runOtelMetricsPort, "otel-metrics-port", 0,
+		"Port to expose Prometheus-style /metrics endpoint (0 disables)")
 }
 
 func runCmdFunc(cmd *cobra.Command, args []string) error {
@@ -224,6 +246,12 @@ func runCmdFunc(cmd *cobra.Command, args []string) error {
 		oidcAudience,
 		oidcJwksURL,
 		oidcClientID,
+		runOtelEndpoint,
+		runOtelServiceName,
+		runOtelSamplingRate,
+		runOtelHeaders,
+		runOtelInsecure,
+		runOtelMetricsPort,
 	)
 
 	// Set the Kubernetes pod template patch if provided