Skip to content

Commit fa33820

Browse files
jankeromnesjankeromnes
committed
v2
1 parent dc64a9d commit fa33820

File tree

12 files changed

+97
-93
lines changed

12 files changed

+97
-93
lines changed

‎components/gitpod-db/src/typeorm/entity/db-prebuilt-workspace.ts

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,4 @@ export class DBPrebuiltWorkspace implements PrebuiltWorkspace {
5050
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED
5151
})
5252
error?: string;
53-
54-
@Column({
55-
...TypeORM.UUID_COLUMN_TYPE,
56-
default: '',
57-
transformer: Transformer.MAP_EMPTY_STR_TO_UNDEFINED
58-
})
59-
@Index('ind_parentPrebuildId')
60-
parentPrebuildId?: string;
6153
}

‎components/gitpod-db/src/typeorm/migration/1619702180986-IncrementalPrebuilds.ts

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎components/gitpod-protocol/src/protocol.ts

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -634,7 +634,6 @@ export interface PrebuiltWorkspace {
634634
state: PrebuiltWorkspaceState;
635635
error?: string;
636636
snapshot?: string;
637-
parentPrebuildId?: string;
638637
}
639638

640639
export namespace PrebuiltWorkspace {
@@ -811,6 +810,7 @@ export namespace SnapshotContext {
811810

812811
export interface StartPrebuildContext extends WorkspaceContext {
813812
actual: WorkspaceContext;
813+
commitHistory?: string[];
814814
}
815815

816816
export namespace StartPrebuildContext {
@@ -878,9 +878,6 @@ export interface Commit {
878878

879879
// refType is only set if ref is present (and not for old workspaces, before this feature was added)
880880
refType?: RefType
881-
882-
// Might contain the recent commit history for this commit (used for incremental prebuilds)
883-
history?: string[]
884881
}
885882

886883
export interface CommitContext extends WorkspaceContext, Commit {

‎components/server/ee/src/prebuilds/prebuild-manager.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { StartPrebuildResult } from './github-app';
1616
import { WorkspaceFactory } from '../../../src/workspace/workspace-factory';
1717
import { ConfigProvider } from '../../../src/workspace/config-provider';
1818
import { WorkspaceStarter } from '../../../src/workspace/workspace-starter';
19+
import { Env } from '../../../src/env';
1920

2021
export class WorkspaceRunningError extends Error {
2122
constructor(msg: string, public instance: WorkspaceInstance) {
@@ -30,6 +31,7 @@ export class PrebuildManager {
3031
@inject(WorkspaceStarter) protected readonly workspaceStarter: WorkspaceStarter;
3132
@inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider;
3233
@inject(ConfigProvider) protected readonly configProvider: ConfigProvider;
34+
@inject(Env) protected env: Env;
3335

3436
async hasAutomatedPrebuilds(ctx: TraceContext, cloneURL: string): Promise<boolean> {
3537
const span = TraceContext.startSpan("hasPrebuilds", ctx);
@@ -75,6 +77,10 @@ export class PrebuildManager {
7577
actual
7678
};
7779

80+
if (this.shouldPrebuildIncrementally(actual.repository.cloneUrl)) {
81+
prebuildContext.commitHistory = await contextParser.fetchCommitHistory({ span }, user, contextURL, commit);
82+
}
83+
7884
log.debug("Created prebuild context", prebuildContext);
7985

8086
const workspace = await this.workspaceFactory.createForContext({span}, user, prebuildContext, contextURL);
@@ -137,6 +143,12 @@ export class PrebuildManager {
137143
return true;
138144
}
139145

146+
protected shouldPrebuildIncrementally(cloneUrl: string): boolean {
147+
const trimRepoUrl = (url: string) => url.replace(/\/$/, '').replace(/\.git$/, '');
148+
const repoUrl = trimRepoUrl(cloneUrl);
149+
return this.env.incrementalPrebuildsRepositoryPassList.some(url => trimRepoUrl(url) === repoUrl);
150+
}
151+
140152
async fetchConfig(ctx: TraceContext, user: User, contextURL: string): Promise<WorkspaceConfig | undefined> {
141153
const span = TraceContext.startSpan("fetchConfig", ctx);
142154
span.setTag("contextURL", contextURL);

‎components/server/ee/src/workspace/gitpod-server-impl.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,6 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
549549

550550
const logCtx: LogContext = { userId: user.id };
551551
const cloneUrl = context.repository.cloneUrl;
552-
// TODO: Use `context.history` to find the last successful (full/incremental) prebuild
553552
const prebuiltWorkspace = await this.workspaceDb.trace({ span }).findPrebuiltWorkspaceByCommit(cloneUrl, context.revision);
554553
const logPayload = { mode, cloneUrl, commit: context.revision, prebuiltWorkspace };
555554
log.debug(logCtx, "Looking for prebuilt workspace: ", logPayload);
@@ -568,7 +567,7 @@ export class GitpodServerEEImpl<C extends GitpodClient, S extends GitpodServer>
568567
} else if (prebuiltWorkspace.state === 'queued' || prebuiltWorkspace.state === 'building') {
569568
if (mode === CreateWorkspaceMode.ForceNew) {
570569
// in force mode we ignore running prebuilds as we want to start a workspace as quickly as we can.
571-
// TODO: Fall back to `prebuiltWorkspace.parentPrebuildId` if it's available (i.e. finished successfully)
570+
// TODO(optimization): Fall back to `getWsById(prebuiltWorkspace.buildWorkspaceId).basedOnPrebuildId` if it's available (i.e. finished successfully)
572571
return;
573572
}
574573

‎components/server/ee/src/workspace/workspace-factory.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
6060
let parentPrebuild;
6161

6262
// Find an older full prebuild to start an incremental prebuild
63-
for (const parent of (commitContext.history || [])) {
63+
for (const parent of (context.commitHistory || [])) {
6464
parentPrebuild = await this.db.trace({span}).findPrebuiltWorkspaceByCommit(commitContext.repository.cloneUrl, parent);
6565
if (!!parentPrebuild) {
6666
log.debug(`Found parent prebuild for ${commitContext.revision}`, parentPrebuild);
@@ -88,8 +88,7 @@ export class WorkspaceFactoryEE extends WorkspaceFactory {
8888
cloneURL: commitContext.repository.cloneUrl,
8989
commit: commitContext.revision,
9090
state: "queued",
91-
creationTime: new Date().toISOString(),
92-
... (!!parentPrebuild ? { parentPrebuildId: parentPrebuild.id } : {})
91+
creationTime: new Date().toISOString()
9392
});
9493

9594
log.debug({ userId: user.id, workspaceId: ws.id }, `Registered workspace prebuild: ${pws.id} for ${commitContext.repository.cloneUrl}:${commitContext.revision}`);

‎components/server/src/bitbucket/bitbucket-context-parser.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,4 +259,9 @@ export class BitbucketContextParser extends AbstractContextParser implements ICo
259259

260260
return result;
261261
}
262+
263+
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string): Promise<string[]> {
264+
// FIXME
265+
return [];
266+
}
262267
}

‎components/server/src/github/github-context-parser.ts

Lines changed: 54 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte
7878
defaultBranchRef {
7979
name,
8080
target {
81-
oid,
82-
... on Commit {
83-
${this.commitHistoryProperty()}
84-
}
81+
oid
8582
}
8683
},
8784
}
@@ -102,7 +99,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
10299
ref,
103100
refType,
104101
revision: defaultBranch && defaultBranch.target.oid || '',
105-
history: defaultBranch && this.toCommitHistory(defaultBranch.target) || undefined,
106102
repository: this.toRepository(host, result.data.repository)
107103
}
108104
} catch (e) {
@@ -137,19 +133,13 @@ export class GithubContextParser extends AbstractContextParser implements IConte
137133
}
138134
}
139135
commit: object(expression: "${branchNameOrCommitHash}") {
140-
oid,
141-
... on Commit {
142-
${this.commitHistoryProperty()}
143-
}
136+
oid
144137
}
145138
ref(qualifiedName: "${branchNameOrCommitHash}") {
146139
name
147140
prefix
148141
target {
149-
oid,
150-
... on Commit {
151-
${this.commitHistoryProperty()}
152-
}
142+
oid
153143
}
154144
}
155145
}
@@ -172,7 +162,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
172162
path,
173163
title: `${owner}/${repoName} - ${repo.ref.name}`,
174164
revision: repo.ref.target.oid,
175-
history: this.toCommitHistory(repo.ref.target),
176165
repository
177166
};
178167
}
@@ -184,7 +173,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
184173
path,
185174
title: `${owner}/${repoName} - ${shortRevision}:${path}`,
186175
revision,
187-
history: this.toCommitHistory(repo.commit),
188176
repository
189177
};
190178
}
@@ -227,8 +215,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte
227215
object(oid: "${sha}") {
228216
oid,
229217
... on Commit {
230-
messageHeadline,
231-
${this.commitHistoryProperty()}
218+
messageHeadline
232219
}
233220
}
234221
${this.repoProperties()}
@@ -260,7 +247,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
260247
title: `${owner}/${repoName} - ${commit.messageHeadline}`,
261248
owner,
262249
revision: sha,
263-
history: this.toCommitHistory(commit),
264250
repository: this.toRepository(host, result.data.repository),
265251
};
266252
} catch (e) {
@@ -286,10 +272,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte
286272
${this.repoProperties()}
287273
}
288274
target {
289-
oid,
290-
... on Commit {
291-
${this.commitHistoryProperty()}
292-
}
275+
oid
293276
}
294277
}
295278
baseRef {
@@ -328,7 +311,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
328311
ref: pr.headRef.name,
329312
refType: "branch",
330313
revision: pr.headRef.target.oid,
331-
history: this.toCommitHistory(pr.headRef.target),
332314
nr: pullRequestNr,
333315
base: {
334316
repository: this.toRepository(host, pr.baseRef.repository),
@@ -358,10 +340,7 @@ export class GithubContextParser extends AbstractContextParser implements IConte
358340
defaultBranchRef {
359341
name,
360342
target {
361-
oid,
362-
... on Commit {
363-
${this.commitHistoryProperty()}
364-
}
343+
oid
365344
}
366345
},
367346
}
@@ -394,7 +373,6 @@ export class GithubContextParser extends AbstractContextParser implements IConte
394373
ref,
395374
refType,
396375
revision: branchRef && branchRef.target.oid || '',
397-
history: branchRef && this.toCommitHistory(branchRef.target) || undefined,
398376
repository: this.toRepository(host, result.data.repository)
399377
};
400378
} catch (e) {
@@ -439,32 +417,57 @@ export class GithubContextParser extends AbstractContextParser implements IConte
439417
`;
440418
}
441419

442-
/**
443-
* Returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log`.
444-
*/
445-
protected toCommitHistory(commit: any): string[] | undefined {
446-
return commit.history?.edges.slice(1).map((e: any) => e.node.oid) || undefined
447-
}
420+
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, sha: string): Promise<string[]> {
421+
const span = TraceContext.startSpan("GithubContextParser.fetchCommitHistory", ctx);
422+
const maxDepth = 100;
423+
const maxAgeMs = 1000 * 3600 * 24 * 10;
448424

449-
/**
450-
* Fetches the commit history of a commit (used to find a relevant base for incremental prebuilds)
451-
*
452-
* FIXME: To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
453-
* These additional history properties may be helfpul:
454-
* totalCount,
455-
* pageInfo {
456-
* haxNextPage,
457-
* }
458-
* */
459-
protected commitHistoryProperty(maxDepth: number = 100, maxAgeMs: number = 1000 * 3600 * 24 * 10): string {
460-
return `
461-
history(first: ${maxDepth}, since: "${new Date(Date.now() - maxAgeMs).toISOString()}") {
462-
edges {
463-
node {
464-
oid
425+
try {
426+
if (sha.length != 40) {
427+
throw new Error(`Invalid commit ID ${sha}.`);
428+
}
429+
430+
// FIXME: To get more results than GitHub API's max page size (seems to be 100), pagination should be handled.
431+
// These additional history properties may be helfpul:
432+
// totalCount,
433+
// pageInfo {
434+
// haxNextPage,
435+
// },
436+
const { owner, repoName } = await this.parseURL(user, contextUrl);
437+
const result: any = await this.githubQueryApi.runQuery(user, `
438+
query {
439+
repository(name: "${repoName}", owner: "${owner}") {
440+
object(oid: "${sha}") {
441+
... on Commit {
442+
history(first: ${maxDepth}, since: "${new Date(Date.now() - maxAgeMs).toISOString()}") {
443+
edges {
444+
node {
445+
oid
446+
}
447+
}
448+
}
449+
}
450+
}
465451
}
466452
}
453+
`);
454+
span.log({"request.finished": ""});
455+
456+
if (result.data.repository === null) {
457+
throw await NotFoundError.create(await this.tokenHelper.getCurrentToken(user), user, this.config.host, owner, repoName);
467458
}
468-
`;
459+
460+
const commit = result.data.repository.object;
461+
if (commit === null) {
462+
throw new Error(`Couldn't find commit ${sha} in repository ${owner}/${repoName}.`);
463+
}
464+
465+
return commit.history.edges.slice(1).map((e: any) => e.node.oid) || [];
466+
} catch (e) {
467+
span.log({"error": e});
468+
throw e;
469+
} finally {
470+
span.finish();
471+
}
469472
}
470473
}

‎components/server/src/gitlab/gitlab-context-parser.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,4 +339,9 @@ export class GitlabContextParser extends AbstractContextParser implements IConte
339339
repository,
340340
};
341341
}
342+
343+
public async fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string): Promise<string[]> {
344+
// FIXME
345+
return [];
346+
}
342347
}

‎components/server/src/workspace/context-parser.ts

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ import { AuthProviderParams } from "../auth/auth-provider";
1111
import { URLSearchParams, URL } from "url";
1212

1313
export interface IContextParser {
14-
normalize?(contextURL: string): string | undefined
15-
canHandle(user: User, context: string): boolean
16-
handle(ctx: TraceContext, user: User, context: string): Promise<WorkspaceContext>
14+
normalize?(contextUrl: string): string | undefined
15+
canHandle(user: User, contextUrl: string): boolean
16+
handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>
17+
fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string): Promise<string[]>
1718
}
1819
export const IContextParser = Symbol("IContextParser")
1920

@@ -73,7 +74,14 @@ export abstract class AbstractContextParser implements IContextParser {
7374
return lastSegment && urlSegment.endsWith('.git') ? urlSegment.substring(0, urlSegment.length - '.git'.length) : urlSegment;
7475
}
7576

76-
public abstract handle(ctx: TraceContext, user: User, context: string): Promise<WorkspaceContext>;
77+
public abstract handle(ctx: TraceContext, user: User, contextUrl: string): Promise<WorkspaceContext>;
78+
79+
/**
80+
* Fetches the commit history of a commit (used to find a relevant parent prebuild for incremental prebuilds).
81+
*
82+
* @returns the linear commit history starting from (but excluding) the given commit, in the same order as `git log`
83+
*/
84+
public abstract fetchCommitHistory(ctx: TraceContext, user: User, contextUrl: string, commit: string): Promise<string[]>;
7785
}
7886

7987
export interface URLParts {

0 commit comments

Comments
 (0)