Reduce super-noisy traces (#16507)

* [messagebus] Stop reporting messagebus spans

* [server] Don't trace individual updates to clients

Author: geropl

components/gitpod-messagebus/src/messagebus.ts

@@ -10,7 +10,7 @@ import { Disposable } from "@gitpod/gitpod-protocol";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { MessagebusConfiguration } from "./config";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
-import { globalTracer, FORMAT_HTTP_HEADERS, childOf } from "opentracing";
+import { globalTracer, FORMAT_HTTP_HEADERS } from "opentracing";
 import { CancellationToken } from "vscode-jsonrpc/lib/cancellation";
 
 export type WorkspaceSubtopic = "updates" | "log" | "credit" | "headless-log" | "ports";
@@ -540,22 +540,11 @@ export abstract class AbstractTopicListener<T> implements MessagebusListener {
         }
 
         if (msg) {
-            const spanCtx = globalTracer().extract(FORMAT_HTTP_HEADERS, message.properties.headers);
-            let span;
-            if (!!spanCtx && !!spanCtx.toTraceId()) {
-                span = globalTracer().startSpan(`/messagebus/${this.exchangeName}`, {
-                    references: [childOf(spanCtx!)],
-                });
-            }
-
+            // gpl: We decided against tracing here because of the low signal/noise ratio (see history)
             try {
-                this.listener({ span }, msg, message.fields.routingKey);
+                this.listener({}, msg, message.fields.routingKey);
             } catch (e) {
                 log.error("Error while executing message handler", e, { message });
-            } finally {
-                if (span) {
-                    span.finish();
-                }
             }
         }
     }

components/server/src/workspace/gitpod-server-impl.ts

@@ -161,7 +161,7 @@ import { ProjectsService } from "../projects/projects-service";
 import { LocalMessageBroker } from "../messaging/local-message-broker";
 import { IDEOptions } from "@gitpod/gitpod-protocol/lib/ide-protocol";
 import { PartialProject } from "@gitpod/gitpod-protocol/lib/teams-projects-protocol";
-import { ClientMetadata, traceClientMetadata } from "../websocket/websocket-connection-manager";
+import { ClientMetadata } from "../websocket/websocket-connection-manager";
 import { ConfigurationService } from "../config/configuration-service";
 import { EnvVarWithValue, ProjectEnvVar } from "@gitpod/gitpod-protocol/lib/protocol";
 import { InstallationAdminSettings, TelemetryData } from "@gitpod/gitpod-protocol";
@@ -334,16 +334,8 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
     }
 
     protected forwardInstanceUpdateToClient(ctx: TraceContext, instance: WorkspaceInstance) {
-        TraceContext.withSpan(
-            "forwardInstanceUpdateToClient",
-            (ctx) => {
-                traceClientMetadata(ctx, this.clientMetadata);
-                TraceContext.setJsonRPCMetadata(ctx, "onInstanceUpdate");
-
-                this.client?.onInstanceUpdate(this.censorInstance(instance));
-            },
-            ctx,
-        );
+        // gpl: We decided against tracing updates here, because it create far too much noise (cmp. history)
+        this.client?.onInstanceUpdate(this.censorInstance(instance));
     }
 
     setClient(ctx: TraceContext, client: GitpodApiClient | undefined): void {

components/server/src/workspace/messagebus-integration.ts

@@ -18,7 +18,6 @@ import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { HeadlessWorkspaceEvent, HeadlessWorkspaceEventType } from "@gitpod/gitpod-protocol/lib/headless-workspace-log";
 import { Channel, Message } from "amqplib";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
-import * as opentracing from "opentracing";
 import { CancellationTokenSource } from "vscode-ws-jsonrpc";
 import { increaseMessagebusTopicReads } from "../prometheus-metrics";
 import { CreditAlert } from "@gitpod/gitpod-protocol/lib/accounting-protocol";
@@ -108,15 +107,7 @@ export class PrebuildUpdatableQueueListener implements MessagebusListener {
             this.channel.ack(message);
         }
 
-        const spanCtx = opentracing.globalTracer().extract(opentracing.FORMAT_HTTP_HEADERS, message.properties.headers);
-        let span;
-        if (!!spanCtx && !!spanCtx.toTraceId()) {
-            span = opentracing
-                .globalTracer()
-                .startSpan(`/messagebus/${MessageBusHelperImpl.PREBUILD_UPDATABLE_QUEUE}`, {
-                    references: [opentracing.childOf(spanCtx!)],
-                });
-        }
+        // gpl: We decided against tracing here because of the low signal/noise ratio (see history)
 
         let msg: any | undefined;
         try {
@@ -129,13 +120,10 @@ export class PrebuildUpdatableQueueListener implements MessagebusListener {
 
         if (msg) {
             try {
-                this.callback({ span }, msg);
+                this.callback({}, msg);
             } catch (e) {
                 log.error("Error while executing message handler", e, { message });
             } finally {
-                if (span) {
-                    span.finish();
-                }
             }
         }
     }