[jb] experimental support of warm up in prebuilds

akosyakov · akosyakov · commit 75330d3b784c · 2022-04-21T08:28:38.000Z
diff --git a/components/gitpod-protocol/data/gitpod-schema.json b/components/gitpod-protocol/data/gitpod-schema.json
@@ -271,6 +271,22 @@
                             "items": {
                                 "type": "string"
                             }
+                        },
+                        "prebuilds": {
+                            "type": "object",
+                            "description": "Enable warming up of IntelliJ in prebuilds.",
+                            "additionalProperties": false,
+                            "properties": {
+                                "version": {
+                                    "type": "string",
+                                    "enum": [
+                                        "stable",
+                                        "latest",
+                                        "both"
+                                    ],
+                                    "description": "Whether only stable, latest or both versions should be warmed up. Default is stable only."
+                                }
+                            }
                         }
                     }
                 },
@@ -285,6 +301,22 @@
                             "items": {
                                 "type": "string"
                             }
+                        },
+                        "prebuilds": {
+                            "type": "object",
+                            "description": "Enable warming up of GoLand in prebuilds.",
+                            "additionalProperties": false,
+                            "properties": {
+                                "version": {
+                                    "type": "string",
+                                    "enum": [
+                                        "stable",
+                                        "latest",
+                                        "both"
+                                    ],
+                                    "description": "Whether only stable, latest or both versions should be warmed up. Default is stable only."
+                                }
+                            }
                         }
                     }
                 },
@@ -299,6 +331,22 @@
                             "items": {
                                 "type": "string"
                             }
+                        },
+                        "prebuilds": {
+                            "type": "object",
+                            "description": "Enable warming up of PyCharm in prebuilds.",
+                            "additionalProperties": false,
+                            "properties": {
+                                "version": {
+                                    "type": "string",
+                                    "enum": [
+                                        "stable",
+                                        "latest",
+                                        "both"
+                                    ],
+                                    "description": "Whether only stable, latest or both versions should be warmed up. Default is stable only."
+                                }
+                            }
                         }
                     }
                 },
@@ -313,6 +361,22 @@
                             "items": {
                                 "type": "string"
                             }
+                        },
+                        "prebuilds": {
+                            "type": "object",
+                            "description": "Enable warming up of PhpStorm in prebuilds.",
+                            "additionalProperties": false,
+                            "properties": {
+                                "version": {
+                                    "type": "string",
+                                    "enum": [
+                                        "stable",
+                                        "latest",
+                                        "both"
+                                    ],
+                                    "description": "Whether only stable, latest or both versions should be warmed up. Default is stable only."
+                                }
+                            }
                         }
                     }
                 }
diff --git a/components/gitpod-protocol/go/gitpod-config-types.go b/components/gitpod-protocol/go/gitpod-config-types.go
diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts
@@ -539,6 +539,19 @@ export interface VSCodeConfig {
     extensions?: string[];
 }
 
+export interface JetBrainsConfig {
+    intellij?: JetBrainsProductConfig;
+    goland?: JetBrainsProductConfig;
+    pycharm?: JetBrainsProductConfig;
+    phpstorm?: JetBrainsProductConfig;
+}
+export interface JetBrainsProductConfig {
+    prebuilds?: JetBrainsPrebuilds;
+}
+export interface JetBrainsPrebuilds {
+    version?: "stable" | "latest" | "both";
+}
+
 export interface RepositoryCloneInformation {
     url: string;
     checkoutLocation?: string;
@@ -555,6 +568,7 @@ export interface WorkspaceConfig {
     gitConfig?: { [config: string]: string };
     github?: GithubAppConfig;
     vscode?: VSCodeConfig;
+    jetbrains?: JetBrainsConfig;
 
     /** deprecated. Enabled by default **/
     experimentalNetwork?: boolean;
diff --git a/components/gitpod-protocol/src/workspace-instance.ts b/components/gitpod-protocol/src/workspace-instance.ts
@@ -207,6 +207,7 @@ export interface WorkspaceInstanceRepoStatus {
 // ConfigurationIdeConfig ide config of WorkspaceInstanceConfiguration
 export interface ConfigurationIdeConfig {
     useLatest?: boolean;
+    desktopIdeAlias?: string;
 }
 
 // WorkspaceInstanceConfiguration contains all per-instance configuration
diff --git a/components/server/src/container-module.ts b/components/server/src/container-module.ts
@@ -97,10 +97,12 @@ import { LocalMessageBroker, LocalRabbitMQBackedMessageBroker } from "./messagin
 import { contentServiceBinder } from "@gitpod/content-service/lib/sugar";
 import { ReferrerPrefixParser } from "./workspace/referrer-prefix-context-parser";
 import { InstallationAdminTelemetryDataProvider } from "./installation-admin/telemetry-data-provider";
+import { IDEService } from "./ide-service";
 
 export const productionContainerModule = new ContainerModule((bind, unbind, isBound, rebind) => {
     bind(Config).toConstantValue(ConfigFile.fromFile());
     bind(IDEConfigService).toSelf().inSingletonScope();
+    bind(IDEService).toSelf().inSingletonScope();
 
     bind(UserService).toSelf().inSingletonScope();
     bind(UserDeletionService).toSelf().inSingletonScope();
diff --git a/components/server/src/ide-service.ts b/components/server/src/ide-service.ts
@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) 2022 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { JetBrainsConfig, TaskConfig, Workspace, WorkspaceInstance } from "@gitpod/gitpod-protocol";
+import { injectable } from "inversify";
+
+@injectable()
+export class IDEService {
+    resolveGitpodTasks(ws: Workspace, instance: WorkspaceInstance): TaskConfig[] {
+        const tasks: TaskConfig[] = [];
+        if (ws.config.tasks) {
+            tasks.push(...ws.config.tasks);
+        }
+        const desktopIdeAlias = instance.configuration?.ideConfig?.desktopIdeAlias;
+        // TODO(ak) layer JB products on prebuild workspaces and move logic to corresponding images
+        if (
+            ws.type === "prebuild" &&
+            ws.config.jetbrains &&
+            desktopIdeAlias &&
+            desktopIdeAlias in ws.config.jetbrains
+        ) {
+            const alias = desktopIdeAlias as keyof JetBrainsConfig;
+            const prebuilds = ws.config.jetbrains[alias]?.prebuilds;
+            let productCode;
+            if (alias === "intellij") {
+                productCode = "IIU";
+            } else if (alias === "goland") {
+                productCode = "GO";
+            } else if (alias === "pycharm") {
+                productCode = "PCP";
+            } else if (alias === "phpstorm") {
+                productCode = "PS";
+            }
+            if (prebuilds) {
+                const warmUpStable =
+                    prebuilds.version === "latest"
+                        ? ""
+                        : `
+# index stable release
+JB_BACKEND_DOWNLOAD_URL="https://download.jetbrains.com/product?type=release&distribution=linux&code=${productCode}"
+echo "$JB_BACKEND_DOWNLOAD_URL"
+
+# download JB backend
+mkdir /tmp/backend && cd /tmp/backend || exit 1
+curl -sSLo backend.tar.gz "$JB_BACKEND_DOWNLOAD_URL" && tar -xf backend.tar.gz --strip-components=1 && rm backend.tar.gz
+
+# config JB system config and caches aligned with runtime
+printf '\nshared.indexes.download.auto.consent=true' >> "/tmp/backend/bin/idea.properties"
+unset JAVA_TOOL_OPTIONS
+export IJ_HOST_CONFIG_BASE_DIR=/workspace/.config/JetBrains
+export IJ_HOST_SYSTEM_BASE_DIR=/workspace/.cache/JetBrains
+
+# start JB backend in indexing mode
+/tmp/backend/bin/remote-dev-server.sh warmup "$GITPOD_REPO_ROOT"
+`;
+                const warmUpLatest =
+                    prebuilds.version === "stable"
+                        ? ""
+                        : `
+# index latest release
+JB_BACKEND_DOWNLOAD_URL="https://download.jetbrains.com/product?type=release,eap,rc&distribution=linux&code=${productCode}"
+echo "$JB_BACKEND_DOWNLOAD_URL"
+
+# download JB backend
+mkdir /tmp/backend-latest && cd /tmp/backend-latest || exit 1
+curl -sSLo backend-latest.tar.gz "$JB_BACKEND_DOWNLOAD_URL" && tar -xf backend-latest.tar.gz --strip-components=1 && rm backend-latest.tar.gz
+
+# config JB system config and caches aligned with runtime
+printf '\nshared.indexes.download.auto.consent=true' >> "/tmp/backend-latest/bin/idea.properties"
+unset JAVA_TOOL_OPTIONS
+export IJ_HOST_CONFIG_BASE_DIR=/workspace/.config/JetBrains-latest
+export IJ_HOST_SYSTEM_BASE_DIR=/workspace/.cache/JetBrains-latest
+
+# start JB backend in indexing mode
+/tmp/backend-latest/bin/remote-dev-server.sh warmup "$GITPOD_REPO_ROOT"
+`;
+                tasks.push({
+                    init: `| ${warmUpStable} ${warmUpLatest}`,
+                });
+            }
+        }
+        return tasks;
+    }
+}
diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts
@@ -109,6 +109,7 @@ import { Deferred } from "@gitpod/gitpod-protocol/lib/util/deferred";
 import { ExtendedUser } from "@gitpod/ws-manager/lib/constraints";
 import { increaseFailedInstanceStartCounter, increaseSuccessfulInstanceStartCounter } from "../prometheus-metrics";
 import { ContextParser } from "./context-parser-service";
+import { IDEService } from "../ide-service";
 
 export interface StartWorkspaceOptions {
     rethrow?: boolean;
@@ -119,6 +120,7 @@ export interface StartWorkspaceOptions {
 const MAX_INSTANCE_START_RETRIES = 2;
 const INSTANCE_START_RETRY_INTERVAL_SECONDS = 2;
 
+// TODO(ak) move to IDE service
 export const migrationIDESettings = (user: User) => {
     if (!user?.additionalData?.ideSettings || user.additionalData.ideSettings.settingVersion === "2.0") {
         return;
@@ -145,6 +147,7 @@ export const migrationIDESettings = (user: User) => {
     return newIDESettings;
 };
 
+// TODO(ak) move to IDE service
 export const chooseIDE = (
     ideChoice: string,
     ideOptions: IDEOptions,
@@ -153,15 +156,17 @@ export const chooseIDE = (
 ) => {
     const defaultIDEOption = ideOptions.options[ideOptions.defaultIde];
     const defaultIdeImage = useLatest ? defaultIDEOption.latestImage ?? defaultIDEOption.image : defaultIDEOption.image;
-    const data: { desktopIdeImage?: string; ideImage: string } = {
+    const data: { desktopIdeImage?: string; desktopIdeAlias?: string; ideImage: string } = {
         ideImage: defaultIdeImage,
     };
     const chooseOption = ideOptions.options[ideChoice];
     const isDesktopIde = chooseOption.type === "desktop";
     if (isDesktopIde) {
         data.desktopIdeImage = useLatest ? chooseOption?.latestImage ?? chooseOption?.image : chooseOption?.image;
-        if (hasIdeSettingPerm) {
-            data.desktopIdeImage = data.desktopIdeImage || ideChoice;
+        if (data.desktopIdeImage) {
+            data.desktopIdeAlias = ideChoice;
+        } else if (hasIdeSettingPerm) {
+            data.desktopIdeImage = ideChoice;
         }
     } else {
         data.ideImage = useLatest ? chooseOption?.latestImage ?? chooseOption?.image : chooseOption?.image;
@@ -181,6 +186,7 @@ export class WorkspaceStarter {
     @inject(WorkspaceManagerClientProvider) protected readonly clientProvider: WorkspaceManagerClientProvider;
     @inject(Config) protected readonly config: Config;
     @inject(IDEConfigService) private readonly ideConfigService: IDEConfigService;
+    @inject(IDEService) private readonly ideService: IDEService;
     @inject(TracedWorkspaceDB) protected readonly workspaceDb: DBWithTracing<WorkspaceDB>;
     @inject(TracedUserDB) protected readonly userDB: DBWithTracing<UserDB>;
     @inject(TokenProvider) protected readonly tokenProvider: TokenProvider;
@@ -598,6 +604,7 @@ export class WorkspaceStarter {
         excludeFeatureFlags: NamedWorkspaceFeatureFlag[],
         ideConfig: IDEConfig,
     ): Promise<WorkspaceInstance> {
+        //#endregion IDE resolution TODO(ak) move to IDE service
         // TODO: Compatible with ide-config not deployed, need revert after ide-config deployed
         delete ideConfig.ideOptions.options["code-latest"];
         delete ideConfig.ideOptions.options["code-desktop-insiders"];
@@ -631,13 +638,15 @@ export class WorkspaceStarter {
             );
             configuration.ideImage = choose.ideImage;
             configuration.desktopIdeImage = choose.desktopIdeImage;
+            configuration.ideConfig!.desktopIdeAlias = choose.desktopIdeAlias;
         }
 
         const referrerIde = this.resolveReferrerIDE(workspace, user, ideConfig);
         if (referrerIde) {
             configuration.desktopIdeImage = useLatest
                 ? referrerIde.option.latestImage ?? referrerIde.option.image
                 : referrerIde.option.image;
+            configuration.ideConfig!.desktopIdeAlias = referrerIde.id;
             if (!user.additionalData?.ideSettings) {
                 // A user does not have IDE settings configured yet configure it with a referrer ide as default.
                 const additionalData = user?.additionalData || {};
@@ -654,6 +663,7 @@ export class WorkspaceStarter {
                     });
             }
         }
+        //#endregion
 
         let featureFlags: NamedWorkspaceFeatureFlag[] = workspace.config._featureFlags || [];
         featureFlags = featureFlags.concat(this.config.workspaceDefaults.defaultFeatureFlags);
@@ -706,6 +716,7 @@ export class WorkspaceStarter {
         return instance;
     }
 
+    // TODO(ak) move to IDE service
     protected resolveReferrerIDE(
         workspace: Workspace,
         user: User,
@@ -1129,12 +1140,13 @@ export class WorkspaceStarter {
         envvars.push(contextEnv);
 
         log.debug("Workspace config", workspace.config);
-        if (!!workspace.config.tasks) {
-            // The task config is interpreted by Theia only, there's little point in transforming it into something
+        const tasks = this.ideService.resolveGitpodTasks(workspace, instance);
+        if (tasks.length) {
+            // The task config is interpreted by supervisor only, there's little point in transforming it into something
             // wsman understands and back into the very same structure.
             const ev = new EnvironmentVariable();
             ev.setName("GITPOD_TASKS");
-            ev.setValue(JSON.stringify(workspace.config.tasks));
+            ev.setValue(JSON.stringify(tasks));
             envvars.push(ev);
         }
 

Original file line number	Diff line number	Diff line change
`@@ -207,6 +207,7 @@ export interface WorkspaceInstanceRepoStatus {`
`207`	`207`	`// ConfigurationIdeConfig ide config of WorkspaceInstanceConfiguration`
`208`	`208`	`export interface ConfigurationIdeConfig {`
`209`	`209`	`useLatest?: boolean;`
	`210`	`+ desktopIdeAlias?: string;`
`210`	`211`	`}`
`211`	`212`
`212`	`213`	`// WorkspaceInstanceConfiguration contains all per-instance configuration`