Merge pull request #1608 from cortexproject/ruler-notifier-span

Add a tracing span for ruler calling Alertmanager

Author: bboreham

pkg/ruler/ruler.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/go-kit/kit/log/level"
-	opentracing "github.com/opentracing/opentracing-go"
+	ot "github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/prometheus/config"
@@ -280,13 +280,18 @@ func (r *Ruler) getOrCreateNotifier(userID string) (*notifier.Manager, error) {
 	n = newRulerNotifier(&notifier.Options{
 		QueueCapacity: r.cfg.NotificationQueueCapacity,
 		Do: func(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
-			// Note: The passed-in context comes from the Prometheus rule group code
+			// Note: The passed-in context comes from the Prometheus notifier
 			// and does *not* contain the userID. So it needs to be added to the context
 			// here before using the context to inject the userID into the HTTP request.
 			ctx = user.InjectOrgID(ctx, userID)
 			if err := user.InjectOrgIDIntoHTTPRequest(ctx, req); err != nil {
 				return nil, err
 			}
+			// Jaeger complains the passed-in context has an invalid span ID, so start a new root span
+			sp := ot.GlobalTracer().StartSpan("notify", ot.Tag{Key: "organization", Value: userID})
+			defer sp.Finish()
+			ctx = ot.ContextWithSpan(ctx, sp)
+			ot.GlobalTracer().Inject(sp.Context(), ot.HTTPHeaders, ot.HTTPHeadersCarrier(req.Header))
 			return ctxhttp.Do(ctx, client, req)
 		},
 	}, util.Logger)
@@ -313,7 +318,7 @@ func (r *Ruler) Evaluate(userID string, item *workItem) {
 	level.Debug(logger).Log("msg", "evaluating rules...", "num_rules", len(item.group.Rules()))
 	ctx, cancelTimeout := context.WithTimeout(ctx, r.cfg.GroupTimeout)
 	instrument.CollectedRequest(ctx, "Evaluate", evalDuration, nil, func(ctx native_ctx.Context) error {
-		if span := opentracing.SpanFromContext(ctx); span != nil {
+		if span := ot.SpanFromContext(ctx); span != nil {
 			span.SetTag("instance", userID)
 			span.SetTag("groupName", item.groupName)
 		}