Implement incremental prebuilds

Author: jankeromnes

chart/templates/server-deployment.yaml

@@ -202,6 +202,14 @@ spec:
           value: {{ $comp.defaultBaseImageRegistryWhitelist | toJson | quote }}
         - name: GITPOD_DEFAULT_FEATURE_FLAGS
           value: {{ $comp.defaultFeatureFlags | toJson | quote }}
+    {{- if $comp.incrementalPrebuilds.repositoryPasslist }}
+        - name: INCREMENTAL_PREBUILDS_REPO_PASSLIST
+          value: {{ $comp.incrementalPrebuilds.repositoryPasslist | toJson | quote }}
+    {{- end }}
+    {{- if $comp.incrementalPrebuilds.commitHistory }}
+        - name: INCREMENTAL_PREBUILDS_COMMIT_HISTORY
+          value: {{ $comp.incrementalPrebuilds.commitHistory | quote }}
+    {{- end }}
         - name: AUTH_PROVIDERS_CONFIG
           valueFrom:
             configMapKeyRef:

chart/values.yaml

@@ -280,6 +280,9 @@ components:
     wsman: []
     defaultBaseImageRegistryWhitelist: []
     defaultFeatureFlags: []
+    incrementalPrebuilds:
+      repositoryPasslist: []
+      commitHistory: 100
     ports:
       http:
         expose: true

components/gitpod-db/src/typeorm/migration/README.md

@@ -4,6 +4,7 @@ To create a new migration file, run this command in the `gitpod-db` component di
 
 ```
 yarn typeorm migrations:create -n NameOfYourMigration
+leeway run components:update-license-header
 ```
 
 Then, simply populate the `up` and `down` methods in the generated migration file.

components/gitpod-protocol/src/protocol.ts

@@ -813,6 +813,7 @@ export namespace SnapshotContext {
 
 export interface StartPrebuildContext extends WorkspaceContext {
     actual: WorkspaceContext;
+    commitHistory?: string[];
 }
 
 export namespace StartPrebuildContext {

components/server/ee/src/prebuilds/prebuild-manager.ts

@@ -16,6 +16,7 @@ import { StartPrebuildResult } from './github-app';
 import { WorkspaceFactory } from '../../../src/workspace/workspace-factory';
 import { ConfigProvider } from '../../../src/workspace/config-provider';
 import { WorkspaceStarter } from '../../../src/workspace/workspace-starter';
+import { Env } from '../../../src/env';
 
 export class WorkspaceRunningError extends Error {
     constructor(msg: string, public instance: WorkspaceInstance) {
@@ -30,6 +31,7 @@ export class PrebuildManager {
     @inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter;
     @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
     @inject(ConfigProvider) protected readonly configProvider: ConfigProvider;
+    @inject(Env) protected env: Env;
 
     async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise<boolean> {
         const span = TraceContext.startSpan("hasPrebuilds", ctx);
@@ -75,6 +77,11 @@ export class PrebuildManager {
                 actual
             };
 
+            if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl)) {
+                const maxDepth = this.env.incrementalPrebuildsCommitHistory;
+                prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth);
+            }
+
             log.debug("Created prebuild context", prebuildContext);
 
             const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL);
@@ -137,6 +144,12 @@ export class PrebuildManager {
         return true;
     }
 
+    protected shouldPrebuildIncrementally(cloneUrl: string): boolean {
+        const trimRepoUrl = (url: string) => url.replace(/\/$/, '').replace(/\.git$/, '');
+        const repoUrl = trimRepoUrl(cloneUrl);
+        return this.env.incrementalPrebuildsRepositoryPassList.some(url => trimRepoUrl(url) === repoUrl);
+    }
+
     async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise<WorkspaceConfig | undefined> {
         const span = TraceContext.startSpan("fetchConfig", ctx);
         span.setTag("contextURL", contextURL);

components/server/ee/src/workspace/gitpod-server-impl.ts

@@ -549,7 +549,7 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
 
             const logCtx: LogContext = { userId: user.id };
             const cloneUrl = context.repository.cloneUrl;
-            const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(context.repository.cloneUrl, context.revision);
+            const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision);
             const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace };
             log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload);
             if (!prebuiltWorkspace) {
@@ -568,6 +568,10 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
                 if (mode === CreateWorkspaceMode.ForceNew) {
                     // in force mode we ignore running prebuilds as we want to start a workspace as quickly as we can.
                     return;
+                    // TODO(janx): Fall back to parent prebuild instead, if it's available:
+                    //   const buildWorkspace = await this.workspaceDb.trace({span}).findById(prebuiltWorkspace.buildWorkspaceId);
+                    //   const parentPrebuild = await this.workspaceDb.trace({span}).findPrebuildByID(buildWorkspace.basedOnPrebuildId);
+                    // Also, make sure to initialize it by both printing the parent prebuild logs AND re-runnnig the before/init/prebuild tasks.
                 }
 
                 let result: WorkspaceCreationResult = {

components/server/ee/src/workspace/workspace-factory.ts

@@ -8,7 +8,7 @@ import * as uuidv4 from 'uuid/v4';
 import { WorkspaceFactory } from "../../../src/workspace/workspace-factory";
 import { injectable, inject } from "inversify";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
-import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild } from "@gitpod/gitpod-protocol";
+import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig } from "@gitpod/gitpod-protocol";
 import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { LicenseEvaluator } from '@gitpod/licensor/lib';
 import { Feature } from '@gitpod/licensor/lib/api';
@@ -56,7 +56,62 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
                 }
             }
 
-            let ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
+            const config = await this.configProvider.fetchConfig({span}, user, context.actual);
+            const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config);
+
+            // Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on.
+            let ws;
+            for (const parent of (context.commitHistory || [])) {
+                const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent);
+                if (!parentPrebuild) {
+                    continue;
+                }
+                if (parentPrebuild.state !== 'available') {
+                    continue;
+                }
+                log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild);
+                const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId);
+                if (!buildWorkspace) {
+                    continue;
+                }
+                if (!!buildWorkspace.basedOnPrebuildId) {
+                    continue;
+                }
+                if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) {
+                    log.debug(`Skipping parent prebuild: Outdated image`, {
+                        imageSource,
+                        parentImageSource: buildWorkspace.imageSource,
+                    });
+                    continue;
+                }
+                const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
+                    .map(task => Object.keys(task)
+                        .filter(key => ['before', 'init', 'prebuild'].includes(key))
+                        // @ts-ignore
+                        .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
+                    .filter(task => Object.keys(task).length > 0));
+                const prebuildTasks = filterPrebuildTasks(config.tasks);
+                const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks);
+                if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
+                    log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
+                        prebuildTasks,
+                        parentPrebuildTasks,
+                    });
+                    continue;
+                }
+                const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
+                    title: `Incremental prebuild of "${commitContext.title}"`,
+                    originalContext: commitContext,
+                    prebuiltWorkspace: parentPrebuild,
+                }
+                ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
+                break;
+            }
+
+            if (!ws) {
+                // No suitable parent prebuild was found -- create a (fresh) full prebuild.
+                ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
+            }
             ws.type = "prebuild";
             ws = await this.db.trace({span}).store(ws);
 
@@ -82,7 +137,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
 
     protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise<Workspace> {
         this.requireEELicense(Feature.FeaturePrebuild);
-        const span = TraceContext.startSpan("createForStartPrebuild", ctx);
+        const span = TraceContext.startSpan("createForPrebuiltWorkspace", ctx);
 
         const fallback = await this.fallbackIfOutPrebuildTime(ctx, user, context, normalizedContextURL);
         if (!!fallback) {

components/server/src/bitbucket/bitbucket-context-parser.spec.ts

@@ -451,6 +451,13 @@ class TestBitbucketContextParser {
             "title": "gitpod/clu-sample-repo - master"
         })
     }
+
+    @test public async testFetchCommitHistory() {
+        const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/sample-repository', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
+        expect(result).to.deep.equal([
+            'da2119f51b0e744cb6b36399f8433b477a4174ef',
+        ])
+    }
 }
 
 module.exports = new TestBitbucketContextParser();

components/server/src/bitbucket/bitbucket-context-parser.ts

@@ -259,4 +259,27 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo
 
         return result;
     }
+
+    public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
+        const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx);
+        try {
+            // TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
+            // The additional property 'page' may be helfpul.
+            const api = await this.api(user);
+            const { owner, repoName } = await this.parseURL(user, contextUrl);
+            const result = await api.repositories.listCommitsAt({
+                workspace: owner,
+                repo_slug: repoName,
+                revision: sha,
+                pagelen: maxDepth,
+            });
+            return result.data.values.slice(1).map((v: Schema.Commit) => v.hash);
+        } catch (e) {
+            span.log({ error: e });
+            log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e);
+            throw e;
+        } finally {
+            span.finish();
+        }
+    }
 }

components/server/src/env.ts

@@ -46,16 +46,25 @@ export class Env extends AbstractComponentEnv {
     })()
 
     readonly previewFeatureFlags: NamedWorkspaceFeatureFlag[] = (() => {
-        const value = process.env.EXPERIMENTAL_FEATURE_FLAGS;
-        if (!value) {
-          return [];
+        return this.parseStringArray('EXPERIMENTAL_FEATURE_FLAGS') as NamedWorkspaceFeatureFlag[];
+    })();
+
+    protected parseStringArray(name: string): string[] {
+        const json = process.env[name];
+        if (!json) {
+            return [];
         }
-        const flags = JSON.parse(value);
-        if (!Array.isArray(flags)) {
-          throw new Error(`EXPERIMENTAL_FEATURE_FLAGS should be an Array: ${value}`);
+        let value;
+        try {
+            value = JSON.parse(json);
+        } catch (error) {
+            throw new Error(`Could not parse ${name}: ${error}`);
         }
-        return flags;
-    })();
+        if (!Array.isArray(value) || value.some(e => typeof e !== 'string')) {
+            throw `${name} should be an array of string: ${json}`;
+        }
+        return value;
+    }
 
     readonly gitpodRegion: string = process.env.GITPOD_REGION || 'unknown';
 
@@ -111,6 +120,15 @@ export class Env extends AbstractComponentEnv {
     // maxConcurrentPrebuildsPerRef is the maximum number of prebuilds we allow per ref type at any given time
     readonly maxConcurrentPrebuildsPerRef = Number.parseInt(process.env.MAX_CONCUR_PREBUILDS_PER_REF || '10', 10) || 10;
 
+    readonly incrementalPrebuildsRepositoryPassList: string[] = (() => {
+        try {
+            return this.parseStringArray('INCREMENTAL_PREBUILDS_REPO_PASSLIST');
+        } catch (error) {
+            console.error(error);
+            return [];
+        }
+    })()
+    readonly incrementalPrebuildsCommitHistory: number = Number.parseInt(process.env.INCREMENTAL_PREBUILDS_COMMIT_HISTORY || '100', 10) || 100;
 
     protected gitpodLayernameFromFilesystem: string | null | undefined;
     protected readGitpodLayernameFromFilesystem(): string | undefined {
@@ -140,19 +158,10 @@ export class Env extends AbstractComponentEnv {
 
     readonly blockNewUsers: boolean = this.parseBool("BLOCK_NEW_USERS");
     readonly blockNewUsersPassList: string[] = (() => {
-        const l = process.env.BLOCK_NEW_USERS_PASSLIST;
-        if (!l) {
-            return [];
-        }
         try {
-            const res = JSON.parse(l);
-            if (!Array.isArray(res) || res.some(e => typeof e !== 'string')) {
-                console.error("BLOCK_NEW_USERS_PASSLIST is not an array of string");
-                return [];
-            }
-            return res;
-        } catch (err) {
-            console.error("cannot parse BLOCK_NEW_USERS_PASSLIST", err);
+            return this.parseStringArray('BLOCK_NEW_USERS_PASSLIST');
+        } catch (error) {
+            console.error(error);
             return [];
         }
     })();
@@ -164,26 +173,17 @@ export class Env extends AbstractComponentEnv {
 
     /** defaultBaseImageRegistryWhitelist is the list of registryies users get acces to by default */
     readonly defaultBaseImageRegistryWhitelist: string[] = (() => {
-        const wljson = process.env.GITPOD_BASEIMG_REGISTRY_WHITELIST;
-        if (!wljson) {
-            return [];
-        }
-
-        return JSON.parse(wljson);
+        return this.parseStringArray('GITPOD_BASEIMG_REGISTRY_WHITELIST');
     })()
 
     readonly defaultFeatureFlags: NamedWorkspaceFeatureFlag[] = (() => {
-        const json = process.env.GITPOD_DEFAULT_FEATURE_FLAGS;
-        if (!json) {
-            return [];
-        }
-
-        let r = JSON.parse(json);
-        if (!Array.isArray(r)) {
+        try {
+            const r = (this.parseStringArray('GITPOD_DEFAULT_FEATURE_FLAGS') as NamedWorkspaceFeatureFlag[]);
+            return r.filter(e => e in WorkspaceFeatureFlags);
+        } catch (error) {
+            console.error(error);
             return [];
         }
-        r = r.filter(e => e in WorkspaceFeatureFlags);
-        return r;
     })();
 
     /** defaults to: false */