jb: push backend memory metrics to prometheus

Author: akosyakov

components/ide/jetbrains/backend-plugin/build.gradle.kts

@@ -39,6 +39,7 @@ dependencies {
             type = "jar"
         }
     }
+    implementation("io.prometheus:simpleclient_pushgateway:0.15.0")
     compileOnly("javax.websocket:javax.websocket-api:1.1")
     detektPlugins("io.gitlab.arturbosch.detekt:detekt-formatting:1.18.1")
     testImplementation(kotlin("test"))

components/ide/jetbrains/backend-plugin/src/main/kotlin/io/gitpod/jetbrains/remote/GitpodManager.kt

@@ -19,7 +19,6 @@ import git4idea.config.GitVcsApplicationSettings
 import io.gitpod.gitpodprotocol.api.GitpodClient
 import io.gitpod.gitpodprotocol.api.GitpodServerLauncher
 import io.gitpod.jetbrains.remote.services.HeartbeatService
-import io.gitpod.jetbrains.remote.utils.LatencyRecord
 import io.gitpod.jetbrains.remote.utils.Retrier.retry
 import io.gitpod.supervisor.api.*
 import io.gitpod.supervisor.api.Info.WorkspaceInfoResponse
@@ -28,6 +27,9 @@ import io.grpc.ManagedChannel
 import io.grpc.ManagedChannelBuilder
 import io.grpc.stub.ClientCallStreamObserver
 import io.grpc.stub.ClientResponseObserver
+import io.prometheus.client.CollectorRegistry
+import io.prometheus.client.Histogram
+import io.prometheus.client.exporter.PushGateway
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.future.await
@@ -39,8 +41,6 @@ import java.net.URI
 import java.net.http.HttpClient
 import java.net.http.HttpRequest
 import java.net.http.HttpResponse
-import java.nio.file.Files
-import java.nio.file.Paths
 import java.time.Duration
 import java.util.concurrent.CancellationException
 import java.util.concurrent.CompletableFuture
@@ -56,40 +56,46 @@ class GitpodManager : Disposable {
     }
 
     val devMode = System.getenv("JB_DEV").toBoolean()
+    private val backendKind = System.getenv("JETBRAINS_GITPOD_BACKEND_KIND") ?: "unknown"
 
     private val lifetime = Lifetime.Eternal.createNested()
 
     override fun dispose() {
         lifetime.terminate()
     }
 
-    private val memProfileMode = System.getenv("JB_MEM_PROFILE").toBoolean()
-
     init {
-        val memProfileJob = GlobalScope.launch {
-            if (application.isHeadlessEnvironment || !memProfileMode) {
+        val monitoringJob =  GlobalScope.launch {
+            if (application.isHeadlessEnvironment) {
                 return@launch
             }
-            Files.newBufferedWriter(Paths.get("/tmp/jb_mem_profile.log")).use { writer ->
-                writer.write("")
-                writer.flush()
-                val allocatedRecord = LatencyRecord()
-                val usedRecord = LatencyRecord()
-                while (isActive) {
-                    val totalMemory = Runtime.getRuntime().totalMemory() / (1024 * 1024)
-                    val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024)
-                    allocatedRecord.update(totalMemory)
-                    usedRecord.update(usedMemory)
-                    writer.write("allocated - current ${totalMemory}M, avg ${allocatedRecord.averageLatency}M, max ${allocatedRecord.maxLatency}M, 90% percentile ${allocatedRecord.percentile(
-                            90)}M\n")
-                    writer.write("used - current ${usedMemory}M,avg ${usedRecord.averageLatency}M, max ${usedRecord.maxLatency}M, 90% percentile ${usedRecord.percentile(
-                            90)}M\n")
-                    writer.flush()
-                    delay(1000)
+            val pg = PushGateway("localhost:22999")
+            val registry = CollectorRegistry()
+            val labels = mapOf("kind" to backendKind)
+            val allocatedHistogram = Histogram.build()
+                    .name("jb_backend_allocated_memory_megabytes")
+                    .help("Allocated memory of JB backend in megabytes.")
+                    .buckets(512.0, 1024.0, 2048.0, 4096.0, 8192.0)
+                    .register(registry)
+            val usedHistogram = Histogram.build()
+                    .name("jb_backend_used_memory_megabytes")
+                    .help("Used memory of JB backend in megabytes.")
+                    .buckets(512.0, 1024.0, 2048.0, 4096.0, 8192.0)
+                    .register(registry)
+            while(isActive) {
+                val totalMemory = Runtime.getRuntime().totalMemory() / (1024 * 1024)
+                val usedMemory = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / (1024 * 1024)
+                allocatedHistogram.observeWithExemplar(totalMemory.toDouble(), labels)
+                usedHistogram.observeWithExemplar(usedMemory.toDouble(), labels)
+                try {
+                    pg.push(registry, "jb_backend")
+                } catch (t: Throwable) {
+                    thisLogger().error("gitpod: failed to push monitoring metrics:", t)
                 }
+                delay(5000)
             }
         }
-        lifetime.onTerminationOrNow { memProfileJob.cancel() }
+        lifetime.onTerminationOrNow { monitoringJob.cancel() }
     }
 
     init {

components/ide/jetbrains/image/status/main.go

@@ -88,7 +88,7 @@ func main() {
 	if err != nil {
 		log.WithError(err).Error("failed to configure backend Xmx")
 	}
-	go run(wsInfo)
+	go run(wsInfo, alias)
 
 	http.HandleFunc("/joinLink", func(w http.ResponseWriter, r *http.Request) {
 		backendPort := r.URL.Query().Get("backendPort")
@@ -224,11 +224,12 @@ func resolveWorkspaceInfo(ctx context.Context) (*supervisor.ContentStatusRespons
 	return nil, nil, errors.New("failed with attempt 10 times")
 }
 
-func run(wsInfo *supervisor.WorkspaceInfoResponse) {
+func run(wsInfo *supervisor.WorkspaceInfoResponse, alias string) {
 	var args []string
 	args = append(args, "run")
 	args = append(args, wsInfo.GetCheckoutLocation())
 	cmd := remoteDevServerCmd(args)
+	cmd.Env = append(cmd.Env, "JETBRAINS_GITPOD_BACKEND_KIND="+alias)
 	workspaceUrl, err := url.Parse(wsInfo.WorkspaceUrl)
 	if err == nil {
 		cmd.Env = append(cmd.Env, "JETBRAINS_GITPOD_WORKSPACE_HOST="+workspaceUrl.Hostname())