Implement incremental prebuilds

chart/templates/server-deployment.yaml

@@ -202,6 +202,14 @@ spec:
           value: {{ $comp.defaultBaseImageRegistryWhitelist | toJson | quote }}
         - name: GITPOD_DEFAULT_FEATURE_FLAGS
           value: {{ $comp.defaultFeatureFlags | toJson | quote }}
+    {{- if $comp.incrementalPrebuilds.repositoryPasslist }}
+        - name: INCREMENTAL_PREBUILDS_REPO_PASSLIST
+          value: {{ $comp.incrementalPrebuilds.repositoryPasslist | toJson | quote }}
+    {{- end }}
+    {{- if $comp.incrementalPrebuilds.commitHistory }}
+        - name: INCREMENTAL_PREBUILDS_COMMIT_HISTORY
+          value: {{ $comp.incrementalPrebuilds.commitHistory | quote }}
+    {{- end }}
         - name: AUTH_PROVIDERS_CONFIG
           valueFrom:
             configMapKeyRef:

chart/values.yaml

@@ -280,6 +280,9 @@ components:
     wsman: []
     defaultBaseImageRegistryWhitelist: []
     defaultFeatureFlags: []
+    incrementalPrebuilds:
+      repositoryPasslist: []
+      commitHistory: 100
     ports:
       http:
         expose: true

components/content-service/pkg/initializer/git.go

@@ -93,21 +93,25 @@ func (ws *GitInitializer) realizeCloneTarget(ctx context.Context) (err error) {
 
 	// checkout branch
 	if ws.TargetMode == RemoteBranch {
-		// create local branch based on remote
+		// create local branch based on specific remote branch
 		if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/"+ws.CloneTarget); err != nil {
 			return err
 		}
 	} else if ws.TargetMode == LocalBranch {
-		if err := ws.Git(ctx, "checkout", "-b", ws.CloneTarget); err != nil {
+		// checkout local branch based on remote HEAD
+		if err := ws.Git(ctx, "checkout", "-B", ws.CloneTarget, "origin/HEAD"); err != nil {
 			return err
 		}
 	} else if ws.TargetMode == RemoteCommit {
 		// checkout specific commit
 		if err := ws.Git(ctx, "checkout", ws.CloneTarget); err != nil {
 			return err
 		}
-	} else { //nolint:staticcheck
-		// nothing to do - we're already on the remote branch
+	} else {
+		// update to remote HEAD
+		if err := ws.Git(ctx, "reset", "--hard", "origin/HEAD"); err != nil {
+			return err
+		}
 	}
 	return nil
 }

components/content-service/pkg/initializer/prebuild.go

@@ -110,7 +110,12 @@ func (p *PrebuildInitializer) Run(ctx context.Context, mappings []archive.IDMapp
 		// If any of these cleanup operations fail that's no reason to fail ws initialization.
 		// It just results in a slightly degraded state.
 		if didStash {
-			_ = p.Git.Git(ctx, "stash", "pop")
+			err = p.Git.Git(ctx, "stash", "pop")
+			if err != nil {
+				// If restoring the stashed changes produces merge conflicts on the new Git ref, simply
+				// throw them away (they'll remain in the stash, but are likely outdated anyway).
+				_ = p.Git.Git(ctx, "reset", "--hard")
+			}
 		}
 
 		log.Debug("prebuild initializer Git operations complete")

components/dashboard/src/start/StartWorkspace.tsx

@@ -149,7 +149,7 @@ export default class StartWorkspace extends React.Component<StartWorkspaceProps,
 
     // Successfully stopped and headless: the prebuild is done, let's try to use it!
     if (!error && workspaceInstance.status.phase === 'stopped' && this.state.workspace?.type !== 'regular') {
-      const contextUrl = this.state.workspace?.contextURL.replace('prebuild/', '')!;
+      const contextUrl = this.state.workspace?.contextURL.replace('incremental-prebuild/', '').replace('prebuild/', '')!;
       this.redirectTo(gitpodHostUrl.withContext(contextUrl).toString());
     }
 

components/gitpod-db/src/typeorm/migration/README.md

@@ -4,6 +4,7 @@ To create a new migration file, run this command in the `gitpod-db` component di
 
 ```
 yarn typeorm migrations:create -n NameOfYourMigration
+leeway run components:update-license-header
 ```
 
 Then, simply populate the `up` and `down` methods in the generated migration file.

components/gitpod-protocol/src/protocol.ts

@@ -813,6 +813,7 @@ export namespace SnapshotContext {
 
 export interface StartPrebuildContext extends WorkspaceContext {
     actual: WorkspaceContext;
+    commitHistory?: string[];
 }
 
 export namespace StartPrebuildContext {

components/server/ee/src/container-module.ts

@@ -31,6 +31,7 @@ import { GitLabApp } from "./prebuilds/gitlab-app";
 import { BitbucketApp } from "./prebuilds/bitbucket-app";
 import { IPrefixContextParser } from "../../src/workspace/context-parser";
 import { StartPrebuildContextParser } from "./prebuilds/start-prebuild-context-parser";
+import { StartIncrementalPrebuildContextParser } from "./prebuilds/start-incremental-prebuild-context-parser";
 import { WorkspaceFactory } from "../../src/workspace/workspace-factory";
 import { WorkspaceFactoryEE } from "./workspace/workspace-factory";
 import { MonitoringEndpointsAppEE } from "./monitoring-endpoint-ee";
@@ -52,6 +53,7 @@ export const productionEEContainerModule = new ContainerModule((bind, unbind, is
     bind(PrebuildRateLimiter).toSelf().inSingletonScope();
     bind(PrebuildQueueMaintainer).toSelf().inSingletonScope();
     bind(IPrefixContextParser).to(StartPrebuildContextParser).inSingletonScope();
+    bind(IPrefixContextParser).to(StartIncrementalPrebuildContextParser).inSingletonScope();
     bind(GithubApp).toSelf().inSingletonScope();
     bind(GithubAppRules).toSelf().inSingletonScope();
     bind(PrebuildStatusMaintainer).toSelf().inSingletonScope();

components/server/ee/src/prebuilds/prebuild-manager.ts

@@ -16,6 +16,7 @@ import { StartPrebuildResult } from './github-app';
 import { WorkspaceFactory } from '../../../src/workspace/workspace-factory';
 import { ConfigProvider } from '../../../src/workspace/config-provider';
 import { WorkspaceStarter } from '../../../src/workspace/workspace-starter';
+import { Env } from '../../../src/env';
 
 export class WorkspaceRunningError extends Error {
     constructor(msg: string, public instance: WorkspaceInstance) {
@@ -30,6 +31,7 @@ export class PrebuildManager {
     @inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter;
     @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
     @inject(ConfigProvider) protected readonly configProvider: ConfigProvider;
+    @inject(Env) protected env: Env;
 
     async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise<boolean> {
         const span = TraceContext.startSpan("hasPrebuilds", ctx);
@@ -75,6 +77,11 @@ export class PrebuildManager {
                 actual
             };
 
+            if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl)) {
+                const maxDepth = this.env.incrementalPrebuildsCommitHistory;
+                prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit, maxDepth);
+            }
+
             log.debug("Created prebuild context", prebuildContext);
 
             const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL);
@@ -137,6 +144,12 @@ export class PrebuildManager {
         return true;
     }
 
+    protected shouldPrebuildIncrementally(cloneUrl: string): boolean {
+        const trimRepoUrl = (url: string) => url.replace(/\/$/, '').replace(/\.git$/, '');
+        const repoUrl = trimRepoUrl(cloneUrl);
+        return this.env.incrementalPrebuildsRepositoryPassList.some(url => trimRepoUrl(url) === repoUrl);
+    }
+
     async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise<WorkspaceConfig | undefined> {
         const span = TraceContext.startSpan("fetchConfig", ctx);
         span.setTag("contextURL", contextURL);

components/server/ee/src/prebuilds/start-incremental-prebuild-context-parser.ts

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the Gitpod Enterprise Source Code License,
+ * See License.enterprise.txt in the project root folder.
+ */
+
+import { User, WorkspaceContext, StartPrebuildContext, CommitContext } from "@gitpod/gitpod-protocol";
+import { inject, injectable } from "inversify";
+import { URL } from "url";
+import { Env } from '../../../src/env';
+import { HostContextProvider } from "../../../src/auth/host-context-provider";
+import { IPrefixContextParser } from "../../../src/workspace/context-parser";
+
+@injectable()
+export class StartIncrementalPrebuildContextParser implements IPrefixContextParser {
+    @inject(Env) protected env: Env;
+    @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
+    static PREFIX = 'incremental-prebuild/';
+
+    findPrefix(user: User, context: string): string | undefined {
+        if (context.startsWith(StartIncrementalPrebuildContextParser.PREFIX)) {
+            return StartIncrementalPrebuildContextParser.PREFIX;
+        }
+    }
+
+    public async handle(user: User, prefix: string, context: WorkspaceContext): Promise<WorkspaceContext> {
+        if (!CommitContext.is(context)) {
+            throw new Error("can only start incremental prebuilds on a commit context")
+        }
+
+        const host = new URL(context.repository.cloneUrl).hostname;
+        const hostContext = this.hostContextProvider.get(host);
+        const maxDepth = this.env.incrementalPrebuildsCommitHistory;
+        const result: StartPrebuildContext = {
+            title: `Prebuild of "${context.title}"`,
+            actual: context,
+            commitHistory: await (hostContext?.contextParser?.fetchCommitHistory({}, user, context.repository.cloneUrl, context.revision, maxDepth) || [])
+        };
+        return result;
+    }
+
+}

components/server/ee/src/workspace/gitpod-server-impl.ts

@@ -549,7 +549,7 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
 
             const logCtx: LogContext = { userId: user.id };
             const cloneUrl = context.repository.cloneUrl;
-            const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(context.repository.cloneUrl, context.revision);
+            const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision);
             const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace };
             log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload);
             if (!prebuiltWorkspace) {
@@ -568,6 +568,10 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
                 if (mode === CreateWorkspaceMode.ForceNew) {
                     // in force mode we ignore running prebuilds as we want to start a workspace as quickly as we can.
                     return;
+                    // TODO(janx): Fall back to parent prebuild instead, if it's available:
+                    //   const buildWorkspace = await this.workspaceDb.trace({span}).findById(prebuiltWorkspace.buildWorkspaceId);
+                    //   const parentPrebuild = await this.workspaceDb.trace({span}).findPrebuildByID(buildWorkspace.basedOnPrebuildId);
+                    // Also, make sure to initialize it by both printing the parent prebuild logs AND re-runnnig the before/init/prebuild tasks.
                 }
 
                 let result: WorkspaceCreationResult = {

components/server/ee/src/workspace/workspace-factory.ts

@@ -8,7 +8,7 @@ import * as uuidv4 from 'uuid/v4';
 import { WorkspaceFactory } from "../../../src/workspace/workspace-factory";
 import { injectable, inject } from "inversify";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
-import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild } from "@gitpod/gitpod-protocol";
+import { User, StartPrebuildContext, Workspace, CommitContext, PrebuiltWorkspaceContext, WorkspaceContext, WithSnapshot, WithPrebuild, TaskConfig } from "@gitpod/gitpod-protocol";
 import { log } from '@gitpod/gitpod-protocol/lib/util/logging';
 import { LicenseEvaluator } from '@gitpod/licensor/lib';
 import { Feature } from '@gitpod/licensor/lib/api';
@@ -56,7 +56,62 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
                 }
             }
 
-            let ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
+            const config = await this.configProvider.fetchConfig({span}, user, context.actual);
+            const imageSource = await this.imageSourceProvider.getImageSource(ctx, user, context.actual, config);
+
+            // Walk back the commit history to find suitable parent prebuild to start an incremental prebuild on.
+            let ws;
+            for (const parent of (context.commitHistory || [])) {
+                const parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent);
+                if (!parentPrebuild) {
+                    continue;
+                }
+                if (parentPrebuild.state !== 'available') {
+                    continue;
+                }
+                log.debug(`Considering parent prebuild for ${commitContext.revision}`, parentPrebuild);
+                const buildWorkspace = await this.db.trace({span}).findById(parentPrebuild.buildWorkspaceId);
+                if (!buildWorkspace) {
+                    continue;
+                }
+                if (!!buildWorkspace.basedOnPrebuildId) {
+                    continue;
+                }
+                if (JSON.stringify(imageSource) !== JSON.stringify(buildWorkspace.imageSource)) {
+                    log.debug(`Skipping parent prebuild: Outdated image`, {
+                        imageSource,
+                        parentImageSource: buildWorkspace.imageSource,
+                    });
+                    continue;
+                }
+                const filterPrebuildTasks = (tasks: TaskConfig[] = []) => (tasks
+                    .map(task => Object.keys(task)
+                        .filter(key => ['before', 'init', 'prebuild'].includes(key))
+                        // @ts-ignore
+                        .reduce((obj, key) => ({ ...obj, [key]: task[key] }), {}))
+                    .filter(task => Object.keys(task).length > 0));
+                const prebuildTasks = filterPrebuildTasks(config.tasks);
+                const parentPrebuildTasks = filterPrebuildTasks(buildWorkspace.config.tasks);
+                if (JSON.stringify(prebuildTasks) !== JSON.stringify(parentPrebuildTasks)) {
+                    log.debug(`Skipping parent prebuild: Outdated prebuild tasks`, {
+                        prebuildTasks,
+                        parentPrebuildTasks,
+                    });
+                    continue;
+                }
+                const incrementalPrebuildContext: PrebuiltWorkspaceContext = {
+                    title: `Incremental prebuild of "${commitContext.title}"`,
+                    originalContext: commitContext,
+                    prebuiltWorkspace: parentPrebuild,
+                }
+                ws = await this.createForPrebuiltWorkspace({span}, user, incrementalPrebuildContext, normalizedContextURL);
+                break;
+            }
+
+            if (!ws) {
+                // No suitable parent prebuild was found -- create a (fresh) full prebuild.
+                ws = await this.createForCommit({span}, user, commitContext, normalizedContextURL);
+            }
             ws.type = "prebuild";
             ws = await this.db.trace({span}).store(ws);
 
@@ -82,7 +137,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
 
     protected async createForPrebuiltWorkspace(ctx: TraceContext, user: User, context: PrebuiltWorkspaceContext, normalizedContextURL: string): Promise<Workspace> {
         this.requireEELicense(Feature.FeaturePrebuild);
-        const span = TraceContext.startSpan("createForStartPrebuild", ctx);
+        const span = TraceContext.startSpan("createForPrebuiltWorkspace", ctx);
 
         const fallback = await this.fallbackIfOutPrebuildTime(ctx, user, context, normalizedContextURL);
         if (!!fallback) {

components/server/src/bitbucket/bitbucket-context-parser.spec.ts

@@ -451,6 +451,13 @@ class TestBitbucketContextParser {
             "title": "gitpod/clu-sample-repo - master"
         })
     }
+
+    @test public async testFetchCommitHistory() {
+        const result = await this.parser.fetchCommitHistory({}, this.user, 'https://bitbucket.org/gitpod/sample-repository', 'dd0aef8097a7c521b8adfced795fcf96c9e598ef', 100);
+        expect(result).to.deep.equal([
+            'da2119f51b0e744cb6b36399f8433b477a4174ef',
+        ])
+    }
 }
 
 module.exports = new TestBitbucketContextParser();

components/server/src/bitbucket/bitbucket-context-parser.ts

@@ -259,4 +259,27 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo
 
         return result;
     }
+
+    public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string, maxDepth: number): Promise<string[]> {
+        const span = TraceContext.startSpan("BitbucketContextParser.fetchCommitHistory", ctx);
+        try {
+            // TODO(janx): To get more results than Bitbucket API's max pagelen (seems to be 100), pagination should be handled.
+            // The additional property 'page' may be helfpul.
+            const api = await this.api(user);
+            const { owner, repoName } = await this.parseURL(user, contextUrl);
+            const result = await api.repositories.listCommitsAt({
+                workspace: owner,
+                repo_slug: repoName,
+                revision: sha,
+                pagelen: maxDepth,
+            });
+            return result.data.values.slice(1).map((v: Schema.Commit) => v.hash);
+        } catch (e) {
+            span.log({ error: e });
+            log.error({ userId: user.id }, "Error fetching Bitbucket commit history", e);
+            throw e;
+        } finally {
+            span.finish();
+        }
+    }
 }