From b18a8e06a712accdad70d101fb704a0b21056ebf Mon Sep 17 00:00:00 2001 From: Alex Tugarev Date: Fri, 21 Jul 2023 13:16:28 +0200 Subject: [PATCH] Revert "[server] remove definitely-gp (#18278) (#18316)" This reverts commit aeb688404b41ac9a4d2e8b0dbb96dbbd21c7a546. --- .../gitpod-protocol/go/gitpod-service.go | 1 + components/gitpod-protocol/src/protocol.ts | 3 +- .../server/src/workspace/config-provider.ts | 175 +++++++++++++++--- .../src/workspace/image-source-provider.ts | 25 ++- 4 files changed, 176 insertions(+), 28 deletions(-) diff --git a/components/gitpod-protocol/go/gitpod-service.go b/components/gitpod-protocol/go/gitpod-service.go index 082c9348bd9b5c..457c1da0c80971 100644 --- a/components/gitpod-protocol/go/gitpod-service.go +++ b/components/gitpod-protocol/go/gitpod-service.go @@ -1702,6 +1702,7 @@ type WorkspaceConfig struct { // Where the config object originates from. // // repo - from the repository + // definitely-gp - from github.com/gitpod-io/definitely-gp // derived - computed based on analyzing the repository // default - our static catch-all default config Origin string `json:"_origin,omitempty"` diff --git a/components/gitpod-protocol/src/protocol.ts b/components/gitpod-protocol/src/protocol.ts index 737997aa63c63e..f64d41f7046537 100644 --- a/components/gitpod-protocol/src/protocol.ts +++ b/components/gitpod-protocol/src/protocol.ts @@ -970,11 +970,12 @@ export interface WorkspaceConfig { * Where the config object originates from. * * repo - from the repository + * definitly-gp - from github.com/gitpod-io/definitely-gp * derived - computed based on analyzing the repository * additional-content - config comes from additional content, usually provided through the project's configuration * default - our static catch-all default config */ - _origin?: "repo" | "derived" | "additional-content" | "default"; + _origin?: "repo" | "definitely-gp" | "derived" | "additional-content" | "default"; /** * Set of automatically infered feature flags. That's not something the user can set, but diff --git a/components/server/src/workspace/config-provider.ts b/components/server/src/workspace/config-provider.ts index d7f043dda82a8b..d1c99e393641fa 100644 --- a/components/server/src/workspace/config-provider.ts +++ b/components/server/src/workspace/config-provider.ts @@ -4,42 +4,55 @@ * See License.AGPL.txt in the project root for license information. */ -import * as crypto from "crypto"; import { inject, injectable } from "inversify"; +import fetch from "node-fetch"; import * as path from "path"; +import * as crypto from "crypto"; +import { log, LogContext } from "@gitpod/gitpod-protocol/lib/util/logging"; import { - AdditionalContentContext, - Commit, + User, + WorkspaceConfig, CommitContext, + Repository, + ImageConfigString, ExternalImageConfigFile, ImageConfigFile, - ImageConfigString, + Commit, NamedWorkspaceFeatureFlag, - ProjectConfig, - Repository, - User, + AdditionalContentContext, WithDefaultConfig, - WorkspaceConfig, + ProjectConfig, } from "@gitpod/gitpod-protocol"; import { GitpodFileParser } from "@gitpod/gitpod-protocol/lib/gitpod-file-parser"; -import { log, LogContext } from "@gitpod/gitpod-protocol/lib/util/logging"; -import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; +import { MaybeContent } from "../repohost/file-provider"; +import { ConfigurationService } from "../config/configuration-service"; import { HostContextProvider } from "../auth/host-context-provider"; +import { AuthorizationService } from "../user/authorization-service"; +import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing"; import { Config } from "../config"; -import { ConfigurationService } from "../config/configuration-service"; +import { EntitlementService } from "../billing/entitlement-service"; +import { TeamDB } from "@gitpod/gitpod-db/lib"; const POD_PATH_WORKSPACE_BASE = "/workspace"; @injectable() export class ConfigProvider { - constructor( - @inject(GitpodFileParser) private readonly gitpodParser: GitpodFileParser, - @inject(HostContextProvider) private readonly hostContextProvider: HostContextProvider, - @inject(Config) private readonly config: Config, - @inject(ConfigurationService) private readonly configurationService: ConfigurationService, - ) {} + static readonly DEFINITELY_GP_REPO: Repository = { + host: "github.com", + owner: "gitpod-io", + name: "definitely-gp", + cloneUrl: "https://github.com/gitpod-io/definitely-gp", + }; + + @inject(GitpodFileParser) protected readonly gitpodParser: GitpodFileParser; + @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; + @inject(AuthorizationService) protected readonly authService: AuthorizationService; + @inject(Config) protected readonly config: Config; + @inject(ConfigurationService) protected readonly configurationService: ConfigurationService; + @inject(EntitlementService) protected readonly entitlementService: EntitlementService; + @inject(TeamDB) protected readonly teamDB: TeamDB; public async fetchConfig( ctx: TraceContext, @@ -83,10 +96,15 @@ export class ConfigProvider { config.image = this.config.workspaceDefaults.workspaceImage; } else if (ImageConfigFile.is(config.image)) { const dockerfilePath = [configBasePath, config.image.file].filter((s) => !!s).join("/"); - const repo = commit.repository; - const rev = commit.revision; + let repo = commit.repository; + let rev = commit.revision; const image = config.image!; + if (config._origin === "definitely-gp") { + repo = ConfigProvider.DEFINITELY_GP_REPO; + rev = "master"; + image.file = dockerfilePath; + } if (!(AdditionalContentContext.is(commit) && commit.additionalFiles[dockerfilePath])) { config.image = { ...image, @@ -125,7 +143,7 @@ export class ConfigProvider { } } - private async fetchCustomConfig( + protected async fetchCustomConfig( ctx: TraceContext, user: User, commit: CommitContext, @@ -136,7 +154,7 @@ export class ConfigProvider { try { let customConfig: WorkspaceConfig | undefined; - const configBasePath = ""; + let configBasePath = ""; if (AdditionalContentContext.is(commit) && commit.additionalFiles[".gitpod.yml"]) { customConfigString = commit.additionalFiles[".gitpod.yml"]; const parseResult = this.gitpodParser.parse(customConfigString); @@ -165,6 +183,21 @@ export class ConfigProvider { customConfigString = await contextRepoConfig; let origin: WorkspaceConfig["_origin"] = "repo"; + if (!customConfigString) { + /* We haven't found a Gitpod configuration file in the context repo - check definitely-gp. + * + * In case we had found a config file here, we'd still be checking the definitely GP repo, just to save some time. + * While all those checks will be in vain, they should not leak memory either as they'll simply + * be resolved and garbage collected. + */ + const definitelyGpConfig = this.fetchExternalGitpodFileContent({ span }, commit.repository); + const { content, basePath } = await definitelyGpConfig; + customConfigString = content; + // We do not only care about the config itself but also where we got it from + configBasePath = basePath; + origin = "definitely-gp"; + } + if (!customConfigString) { const inferredConfig = this.configurationService.guessRepositoryConfiguration( { span }, @@ -215,7 +248,7 @@ export class ConfigProvider { }; } - private async fetchWorkspaceImageSourceDocker( + protected async fetchWorkspaceImageSourceDocker( ctx: TraceContext, repository: Repository, revisionOrTagOrBranch: string, @@ -254,7 +287,101 @@ export class ConfigProvider { } } - private async validateConfig(config: WorkspaceConfig, user: User): Promise { + protected async fillInDefaultLocations( + cfg: WorkspaceConfig | undefined, + inferredConfig: Promise, + ): Promise { + if (!cfg) { + // there is no config - return + return; + } + + if (!cfg.checkoutLocation) { + const inferredCfg = await inferredConfig; + if (inferredCfg) { + cfg.checkoutLocation = inferredCfg.checkoutLocation; + } + } + if (!cfg.workspaceLocation) { + const inferredCfg = await inferredConfig; + if (inferredCfg) { + cfg.workspaceLocation = inferredCfg.workspaceLocation; + } + } + } + + protected async fetchExternalGitpodFileContent( + ctx: TraceContext, + repository: Repository, + ): Promise<{ content: MaybeContent; basePath: string }> { + const span = TraceContext.startSpan("fetchExternalGitpodFileContent", ctx); + span.setTag("repo", `${repository.owner}/${repository.name}`); + + if (this.config.definitelyGpDisabled) { + span.finish(); + return { + content: undefined, + basePath: `${repository.name}`, + }; + } + + try { + const ownerConfigBasePath = `${repository.name}/${repository.owner}`; + const baseConfigBasePath = `${repository.name}`; + + const possibleConfigs = [ + [this.fetchDefinitelyGpContent({ span }, `${ownerConfigBasePath}/.gitpod.yml`), ownerConfigBasePath], + [this.fetchDefinitelyGpContent({ span }, `${ownerConfigBasePath}/.gitpod`), ownerConfigBasePath], + [this.fetchDefinitelyGpContent({ span }, `${baseConfigBasePath}/.gitpod.yml`), baseConfigBasePath], + [this.fetchDefinitelyGpContent({ span }, `${baseConfigBasePath}/.gitpod`), baseConfigBasePath], + ]; + for (const [configPromise, basePath] of possibleConfigs) { + const ownerConfig = await configPromise; + if (ownerConfig !== undefined) { + return { + content: ownerConfig, + basePath: basePath as string, + }; + } + } + return { + content: undefined, + basePath: baseConfigBasePath, + }; + } catch (e) { + TraceContext.setError({ span }, e); + throw e; + } finally { + span.finish(); + } + } + + protected async fetchDefinitelyGpContent(ctx: TraceContext, filePath: string) { + const span = TraceContext.startSpan("fetchDefinitelyGpContent", ctx); + span.setTag("filePath", filePath); + + try { + const url = `https://raw.githubusercontent.com/gitpod-io/definitely-gp/master/${filePath}`; + const response = await fetch(url, { + timeout: 10000, + method: "GET", + }); + let content; + if (response.ok) { + try { + content = await response.text(); + } catch {} + } + return content; + } catch (e) { + TraceContext.setError({ span }, e); + throw e; + } finally { + span.finish(); + } + } + + protected async validateConfig(config: WorkspaceConfig, user: User): Promise { // Make sure the projectRoot does not leave POD_PATH_WORKSPACE_BASE as that's a common // assumption throughout the code (e.g. ws-daemon) const checkoutLocation = config.checkoutLocation; @@ -280,7 +407,7 @@ export class ConfigProvider { } } - private leavesWorkspaceBase(normalizedPath: string) { + protected leavesWorkspaceBase(normalizedPath: string) { const pathSegments = normalizedPath.split(path.sep); return normalizedPath.includes("..") || pathSegments.slice(0, 2).join("/") != POD_PATH_WORKSPACE_BASE; } diff --git a/components/server/src/workspace/image-source-provider.ts b/components/server/src/workspace/image-source-provider.ts index 447be98d31a56b..f21505ea78fc36 100644 --- a/components/server/src/workspace/image-source-provider.ts +++ b/components/server/src/workspace/image-source-provider.ts @@ -14,6 +14,7 @@ import { WorkspaceImageSourceReference, WorkspaceImageSourceDocker, ImageConfigFile, + ExternalImageConfigFile, User, AdditionalContentContext, } from "@gitpod/gitpod-protocol"; @@ -21,7 +22,7 @@ import { createHash } from "crypto"; @injectable() export class ImageSourceProvider { - constructor(@inject(HostContextProvider) private readonly hostContextProvider: HostContextProvider) {} + @inject(HostContextProvider) protected readonly hostContextProvider: HostContextProvider; public async getImageSource( ctx: TraceContext, @@ -35,7 +36,25 @@ export class ImageSourceProvider { let result: WorkspaceImageSource; const imgcfg = config.image; - if (ImageConfigFile.is(imgcfg)) { + if (ExternalImageConfigFile.is(imgcfg)) { + // we're asked to pull the Dockerfile from a repo possibly different than the one we're opening a workspace for (e.g. definitely-gp). + const repository = imgcfg.externalSource.repository; + const hostContext = this.hostContextProvider.get(repository.host); + if (!hostContext || !hostContext.services) { + throw new Error(`Cannot fetch workspace image source for host: ${repository.host}`); + } + const lastDockerFileSha = await hostContext.services.fileProvider.getLastChangeRevision( + repository, + imgcfg.externalSource.revision, + user, + imgcfg.file, + ); + result = { + dockerFilePath: imgcfg.file, + dockerFileSource: imgcfg.externalSource, + dockerFileHash: lastDockerFileSha, + }; + } else if (ImageConfigFile.is(imgcfg)) { // if a dockerfile sits in the additional content we use its contents sha if ( AdditionalContentContext.is(context) && @@ -81,7 +100,7 @@ export class ImageSourceProvider { } } - private getContentSHA(contents: string): string { + protected getContentSHA(contents: string): string { return createHash("sha256").update(contents).digest("hex"); } }