Instrument Docker Compose setups to enable Application Observability (#228)

Author: ppcano

README.md

@@ -173,9 +173,19 @@ Now, you can log in to [Grafana Cloud](https://grafana.com/products/cloud/) and
 
 To find the labels applied to the telemetry data, refer to [cloud.alloy](./alloy/cloud.alloy) and [docker-compose-cloud.yaml](./docker-compose-cloud.yaml) .
 
-### Monitor frontend with Grafana Cloud Frontend Observability 
+### Monitor QuickPizza with Grafana Cloud Application and Frontend Observability
 
-To enable [Grafana Cloud Frontend Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/) for QuickPizza:
+The Docker Compose setup is fully instrumented out of the box, so you can jump right into Grafana Cloud Observability apps and start observing the inner workings of the QuickPizza service components.
+
+To enable [Grafana Cloud Application Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/application-observability/) for QuickPizza:
+
+1. In your Grafana Cloud instance, navigate to **Observability > Application**.
+2. Click on **Enable metrics generation** to enable the usage of Application Observability. 
+3. Interact with the QuickPizza app to generate traffic. After a few minutes, the QuickPizza components will be automatically discovered and displayed in the UI.
+
+![Application Observability](./docs/images/grafana-cloud-application-application-observability-quickpizza.png)
+
+To enable [Grafana Cloud Frontend Observability](https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/):
 
 1. In Grafana Cloud, create a new Frontend Observability application and set the domain to `http://localhost:3333`.
 2. Copy the application's Faro web URL.

alloy/cloud.alloy

@@ -9,17 +9,30 @@ otelcol.receiver.otlp "default" {
   }
 
   output {
-    traces  = [otelcol.processor.batch.default.input]
+    traces  = [
+      otelcol.processor.batch.default.input,
+      otelcol.connector.host_info.default.input,
+    ]
+  }
+}
+
+otelcol.connector.host_info "default" {
+  host_identifiers = ["container.name", "container.id", "service.name"]
+  metrics_flush_interval = "10s"
+
+  output {
+    metrics = [otelcol.exporter.prometheus.otlp_metrics.input]
   }
 }
+otelcol.exporter.prometheus "otlp_metrics" {
+  forward_to = [
+    grafana_cloud.stack.receivers.metrics,
+  ]
+}
 
 otelcol.processor.batch "default" {
   output {
-    metrics = []
-    logs = []
-    traces = [
-      grafana_cloud.stack.receivers.traces,
-    ]
+    traces = [grafana_cloud.stack.receivers.traces]
   }
 }
 
@@ -33,6 +46,8 @@ prometheus.scrape "default" {
       "__address__" = env("QUICKPIZZA_HOST"),
       "job" = "quickpizza",
       "instance" = "local",
+      "service_name" = "quickpizza",
+      "service_namespace" = "quickpizza",
     },
   ]
 }
@@ -65,6 +80,10 @@ loki.source.docker "default" {
   host       = "unix:///var/run/docker.sock"
   targets    = discovery.relabel.quickpizza.output
   forward_to = [grafana_cloud.stack.receivers.logs]
+  labels = {
+    service_name = "quickpizza",
+    service_namespace = "quickpizza",
+  }
 }
 
 import.git "grafana_cloud" {

alloy/local-tempo.yaml

@@ -0,0 +1,57 @@
+stream_over_http_enabled: true
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+      duration_slo: 5s
+      throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "tempo:4317"
+        http:
+          endpoint: "tempo:4318"
+
+ingester:
+  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /var/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local # backend configuration to use
+    wal:
+      path: /var/tempo/wal # where to store the wal locally
+    local:
+      path: /var/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics, local-blocks] # enables metrics generator
+      generate_native_histograms: both

alloy/local.alloy

@@ -25,6 +25,7 @@ otelcol.processor.batch "default" {
 
 // Scrape Metrics
 prometheus.scrape "default" {
+  scrape_interval = "60s"
   forward_to = [
     prometheus.remote_write.default.receiver,
   ]
@@ -33,6 +34,8 @@ prometheus.scrape "default" {
       "__address__" = env("QUICKPIZZA_HOST"),
       "job" = "quickpizza",
       "instance" = "local",
+      "service_name" = "quickpizza",
+      "service_namespace" = "quickpizza",
     },
   ]
 }
@@ -86,6 +89,10 @@ loki.source.docker "default" {
   host       = "unix:///var/run/docker.sock"
   targets    = discovery.relabel.quickpizza.output
   forward_to = [loki.write.local.receiver]
+  labels = {
+    service_name = "quickpizza",
+    service_namespace = "quickpizza",
+  }
 }
 
 loki.write "local" {

docker-compose-cloud.yaml

@@ -9,19 +9,18 @@ services:
       QUICKPIZZA_OTLP_ENDPOINT: http://alloy:4318
       QUICKPIZZA_TRUST_CLIENT_TRACEID: 1
       QUICKPIZZA_PYROSCOPE_ENDPOINT: http://alloy:9999
+      # Set to 1 to use component name as OpenTelemetry service name
+      # QUICKPIZZA_OTEL_SERVICE_NAME_LEGACY: 1
       # must be set with an .env file
       QUICKPIZZA_CONF_FARO_URL: "${QUICKPIZZA_CONF_FARO_URL}"
       # must be set with an .env file
       QUICKPIZZA_CONF_FARO_APP_NAME: "${QUICKPIZZA_CONF_FARO_APP_NAME}"
-
+      # OTEL service.instance.id label. Default: local
+      # QUICKPIZZA_OTEL_SERVICE_INSTANCE_ID: local
       # Namespace label in Faro. Default: quickpizza
       # QUICKPIZZA_CONF_FARO_APP_NAMESPACE: quickpizza
       # Enable logging. Possible values: error, warn, debug. Default: info
       # QUICKPIZZA_LOG_LEVEL: debug
-      # Service name label in Pyroscope. Default: quickpizza
-      # QUICKPIZZA_PYROSCOPE_NAME: quickpizza
-      # Namespace label in Pyroscope. Default: quickpizza
-      # QUICKPIZZA_PYROSCOPE_NAMESPACE: quickpizza
 
   alloy:
     image: grafana/alloy:v1.9.1

docker-compose-local.yaml

@@ -11,12 +11,6 @@ services:
       QUICKPIZZA_PYROSCOPE_ENDPOINT: http://alloy:9999
       # Enable logging. Possible values: error, warn, debug. Default: info
       # QUICKPIZZA_LOG_LEVEL: debug
-      # Service name label in Pyroscope. Default: quickpizza
-      # QUICKPIZZA_PYROSCOPE_NAME: quickpizza
-      # Namespace label in Pyroscope. Default: quickpizza
-      # QUICKPIZZA_PYROSCOPE_NAMESPACE: quickpizza
-      # Database configuration (default: in-memory SQLite)
-      # QUICKPIZZA_DB: postgres://user:password@postgres:5432/quickpizza?sslmode=disable
 
   loki:
     image: grafana/loki:3.4.3
@@ -27,6 +21,7 @@ services:
     image: prom/prometheus:v3.2.1
     command:
       - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
       - --enable-feature=native-histograms
       - --config.file=/etc/prometheus/prometheus.yml
     ports:
@@ -36,22 +31,21 @@ services:
     image: grafana/pyroscope:1.14.0
     ports:
       - "4040:4040"
-
+  
   tempo:
     image: grafana/tempo:2.8.1
-    command:
-      - "-storage.trace.backend=local" # tell tempo where to permanently put traces
-      - "-storage.trace.local.path=/tmp/tempo/traces"
-      - "-storage.trace.wal.path=/tmp/tempo/wal" # tell tempo where to store the wal
-      - "-auth.enabled=false" # disables the requirement for the X-Scope-OrgID header
-      - "-server.http-listen-port=3200"
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./alloy/local-tempo.yaml:/etc/tempo.yaml
     ports:
       - "3200:3200"
       - "4317:4317"
       - "4318:4318"
+    depends_on:
+      - prometheus
 
   grafana:
-    image: grafana/grafana:${GRAFANA_VERSION:-12.0.2}
+    image: grafana/grafana:${GRAFANA_VERSION:-12.1.0}
     ports:
       - "3000:3000"
     environment:

docs/images/grafana-cloud-application-application-observability-quickpizza.png

@@ -14,6 +14,7 @@ datasources:
     jsonData:
       tlsAuth: false
       tlsAuthWithCACert: false
+      timeInterval: 60s # Scrape interval (should match Prometheus configuration, defined in local.alloy)
     editable: false
   - name: Tempo
     type: tempo

grafana/datasources/datasource.yaml

@@ -95,13 +95,24 @@ func (t *TraceInstaller) Install(r chi.Router, serviceName string, extraOpts ...
 		}
 	}
 
+	serviceNamespace, ok := os.LookupEnv("QUICKPIZZA_OTEL_SERVICE_NAMESPACE")
+	if !ok {
+		serviceNamespace = "quickpizza"
+	}
+	serviceInstanceID, ok := os.LookupEnv("QUICKPIZZA_OTEL_SERVICE_INSTANCE_ID")
+	if !ok {
+		serviceInstanceID = "local"
+	}
+
 	// We discard the error here as it cannot possibly take place with the parameters we use.
 	res, _ := resource.Merge(
 		resource.Default(),
 		resource.NewWithAttributes(
 			semconv.SchemaURL,
 			semconv.ServiceName(serviceNameAttrValue),
 			attribute.KeyValue{Key: "service.component", Value: attribute.StringValue(serviceName)},
+			attribute.KeyValue{Key: "service.namespace", Value: attribute.StringValue(serviceNamespace)},
+			attribute.KeyValue{Key: "service.instance.id", Value: attribute.StringValue(serviceInstanceID)},
 		),
 	)