jb: push backend memory metrics to prometheus

akosyakov · roboquat · commit faa6b3096c44 · 2022-05-18T13:06:32.000+05:30
diff --git a/components/ide/jetbrains/backend-plugin/build.gradle.kts b/components/ide/jetbrains/backend-plugin/build.gradle.kts
@@ -39,6 +39,7 @@ dependencies {
             type = "jar"
         }
     }
+    implementation("io.prometheus:simpleclient_pushgateway:0.15.0")
     compileOnly("javax.websocket:javax.websocket-api:1.1")
     detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.18.1")
     testImplementation(kotlin("test"))
diff --git a/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt b/components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt
@@ -27,6 +27,9 @@ import io.grpc.ManagedChannel
 import io.grpc.ManagedChannelBuilder
 import io.grpc.stub.ClientCallStreamObserver
 import io.grpc.stub.ClientResponseObserver
+import io.prometheus.client.CollectorRegistry
+import io.prometheus.client.Gauge
+import io.prometheus.client.exporter.PushGateway
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.future.await
@@ -52,13 +55,48 @@ class GitpodManager : Disposable {
     }
 
     val devMode = System.getenv("JB_DEV").toBoolean()
+    private val backendKind = System.getenv("JETBRAINS_GITPOD_BACKEND_KIND") ?: "unknown"
+    private val backendQualifier = System.getenv("JETBRAINS_BACKEND_QUALIFIER") ?: "unknown"
 
     private val lifetime = Lifetime.Eternal.createNested()
 
     override fun dispose() {
         lifetime.terminate()
     }
 
+    init {
+        val monitoringJob =  GlobalScope.launch {
+            if (application.isHeadlessEnvironment) {
+                return@launch
+            }
+            val pg = PushGateway("localhost:22999")
+            val registry = CollectorRegistry()
+            val allocatedGauge = Gauge.build()
+                    .name("gitpod_jb_backend_memory_max_bytes")
+                    .help("Total allocated memory of JB backend in bytes.")
+                    .labelNames("product", "qualifier")
+                    .register(registry)
+            val usedGauge = Gauge.build()
+                    .name("gitpod_jb_backend_memory_used_bytes")
+                    .help("Used memory of JB backend in bytes.")
+                    .labelNames("product", "qualifier")
+                    .register(registry)
+            while(isActive) {
+                val totalMemory = Runtime.getRuntime().totalMemory()
+                allocatedGauge.labels(backendKind, backendQualifier).set(totalMemory.toDouble())
+                val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())
+                usedGauge.labels(backendKind, backendQualifier).set(usedMemory.toDouble())
+                try {
+                    pg.push(registry, "jb_backend")
+                } catch (t: Throwable) {
+                    thisLogger().error("gitpod: failed to push monitoring metrics:", t)
+                }
+                delay(5000)
+            }
+        }
+        lifetime.onTerminationOrNow { monitoringJob.cancel() }
+    }
+
     init {
         GlobalScope.launch {
             if (application.isHeadlessEnvironment) {
@@ -269,4 +307,4 @@ class GitpodManager : Disposable {
             serverJob.cancel()
         }
     }
-}
+}
diff --git a/components/ide/jetbrains/image/status/main.go b/components/ide/jetbrains/image/status/main.go
@@ -88,7 +88,7 @@ func main() {
 	if err != nil {
 		log.WithError(err).Error("failed to configure backend Xmx")
 	}
-	go run(wsInfo)
+	go run(wsInfo, alias)
 
 	http.HandleFunc("/joinLink", func(w http.ResponseWriter, r *http.Request) {
 		backendPort := r.URL.Query().Get("backendPort")
@@ -224,11 +224,12 @@ func resolveWorkspaceInfo(ctx context.Context) (*supervisor.ContentStatusRespons
 	return nil, nil, errors.New("failed with attempt 10 times")
 }
 
-func run(wsInfo *supervisor.WorkspaceInfoResponse) {
+func run(wsInfo *supervisor.WorkspaceInfoResponse, alias string) {
 	var args []string
 	args = append(args, "run")
 	args = append(args, wsInfo.GetCheckoutLocation())
 	cmd := remoteDevServerCmd(args)
+	cmd.Env = append(cmd.Env, "JETBRAINS_GITPOD_BACKEND_KIND="+alias)
 	workspaceUrl, err := url.Parse(wsInfo.WorkspaceUrl)
 	if err == nil {
 		cmd.Env = append(cmd.Env, "JETBRAINS_GITPOD_WORKSPACE_HOST="+workspaceUrl.Hostname())
diff --git a/dev/ide/profile-workspace.js b/dev/ide/profile-workspace.js
@@ -6,6 +6,7 @@
 
 const { execSync } = require("child_process");
 const { promises } = require("fs");
+const fetch = require("node-fetch");
 const path = require("path");
 
 /**
@@ -26,13 +27,41 @@ const q90 = (arr) => {
 
 (async () => {
     let wsPodName;
+    let creationTimestamp;
     while (!wsPodName) {
-        wsPodName = execSync("kubectl get pod -l component=workspace -o=custom-columns=:metadata.name", {
-            encoding: "utf8",
-        }).trim();
+        const segments = execSync(
+            "kubectl get pod -l component=workspace -o=custom-columns=:metadata.name,:metadata.creationTimestamp",
+            {
+                encoding: "utf8",
+            },
+        )
+            .trim()
+            .split(/\s+/);
+        wsPodName = segments[0];
+        creationTimestamp = new Date(segments[1]);
         await new Promise((r) => setTimeout(r, 1000));
     }
     console.log(wsPodName);
+    console.log(creationTimestamp);
+
+    const query = async (prefix) => {
+        const age = ((new Date().getTime() - creationTimestamp.getTime()) / 1000).toFixed(0);
+        const response = await fetch.default(
+            encodeURI(`http://localhost:9090/api/v1/query?query=${prefix}{pod="${wsPodName}"}[${age}s])/(1024*1024)`),
+            {
+                method: "GET",
+            },
+        );
+        if (!response.ok) {
+            console.error(`${prefix}: ${response.statusText} (${response.status})`);
+            return "N/A";
+        }
+        /**
+         * @type {{data: {result: { value: [number, string] }[]}}}
+         */
+        const body = await response.json();
+        return Number(body.data.result[0].value[1]).toFixed(2);
+    };
 
     const perfLogPath = path.resolve(__dirname, "perf.log");
     console.log(perfLogPath);
@@ -65,6 +94,7 @@ const q90 = (arr) => {
         } catch (e) {
             console.error(e);
         }
+
         if (top) {
             cores.push(top.cpu.used);
             sumCores += top.cpu.used;
@@ -77,13 +107,32 @@ const q90 = (arr) => {
             avgMemory = sumMemory / measurements;
             maxMemory = Math.max(maxMemory, mem);
 
+            const [avgMax, maxMax, q90Max, avgUsed, maxUsed, q90user] = (
+                await Promise.allSettled([
+                    query("avg_over_time(gitpod_jb_backend_memory_max_bytes"),
+                    query("max_over_time(gitpod_jb_backend_memory_max_bytes"),
+                    query("quantile_over_time(0.9, gitpod_jb_backend_memory_max_bytes"),
+                    query("avg_over_time(gitpod_jb_backend_memory_used_bytes"),
+                    query("max_over_time(gitpod_jb_backend_memory_used_bytes"),
+                    query("quantile_over_time(0.9, gitpod_jb_backend_memory_used_bytes"),
+                ])
+            ).map((v) => {
+                if (v.status === "fulfilled") {
+                    return v.value;
+                }
+                console.error(v.reason);
+                return "N/A";
+            });
+
             await promises.appendFile(
                 perfLogPath,
                 `${((new Date().getTime() - start) / 1000).toFixed(2)}s, cpu(m) ${top.cpu.used.toFixed(
                     2,
                 )}/${avgCores.toFixed(2)}/${maxCores.toFixed(2)}/${q90(cores).toFixed(2)}, memory(Mi) ${mem.toFixed(
                     2,
-                )}/${avgMemory.toFixed(2)}/${maxMemory.toFixed(2)}/${q90(mems).toFixed(2)}\n`,
+                )}/${avgMemory.toFixed(2)}/${maxMemory.toFixed(2)}/${q90(mems).toFixed(
+                    2,
+                )}, allocated(M) ${avgMax}/${maxMax}/${q90Max}, used(M) ${avgUsed}/${maxUsed}/${q90user}\n`,
             );
         }
         await new Promise((r) => setTimeout(r, 1000));

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ dependencies {`
`39`	`39`	`type = "jar"`
`40`	`40`	`}`
`41`	`41`	`}`
	`42`	`+ implementation("io.prometheus:simpleclient_pushgateway:0.15.0")`
`42`	`43`	`compileOnly("javax.websocket:javax.websocket-api:1.1")`
`43`	`44`	`detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.18.1")`
`44`	`45`	`testImplementation(kotlin("test"))`