make vm support ssh gateway

Author: iQQBot

.werft/jobs/build/deploy-to-preview-environment.ts

@@ -317,7 +317,11 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     await waitUntilAllPodsAreReady(deploymentConfig.namespace, installer.options.kubeconfigPath, { slice: installerSlices.DEPLOYMENT_WAITING })
     werft.done(installerSlices.DEPLOYMENT_WAITING);
 
-    await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, !withVM, installer.options.kubeconfigPath)
+    if (!withVM) {
+        await addDNSRecord(werft, deploymentConfig.namespace, deploymentConfig.domain, !withVM, installer.options.kubeconfigPath)
+    } else {
+        await addVMDNSRecord(werft, destname, domain)
+    }
     addAgentSmithToken(werft, deploymentConfig.namespace, installer.options.kubeconfigPath, tokenHash)
 
     // TODO: Fix sweeper, it does not appear to be doing clean-up
@@ -676,6 +680,35 @@ async function addDNSRecord(werft: Werft, namespace: string, domain: string, isL
     werft.done(installerSlices.DNS_ADD_RECORD);
 }
 
+async function addVMDNSRecord(werft: Werft, name: string, domain: string) {
+    const ingressIP = getHarvesterIngressIP()
+    let proxyLBIP = null
+    werft.log(installerSlices.DNS_ADD_RECORD, "Getting loadbalancer IP");
+    for (let i = 0; i < 60; i++) {
+        try {
+            let lb = exec(`kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} -n loadbalancers get service lb-${name} -o=jsonpath='{.status.loadBalancer.ingress[0].ip}'`, { silent: true })
+            if (lb.length > 4) {
+                proxyLBIP = lb.toString()
+                break
+            }
+            await sleep(1000)
+        } catch (err) {
+            await sleep(1000)
+        }
+    }
+    if (proxyLBIP == null) {
+        werft.fail(installerSlices.DNS_ADD_RECORD, new Error("Can't get loadbalancer IP"));
+    }
+    werft.log(installerSlices.DNS_ADD_RECORD, "Get loadbalancer IP: " + proxyLBIP);
+
+    await Promise.all([
+        createDNSRecord(domain, 'preview-gitpod-dev-com', ingressIP, installerSlices.DNS_ADD_RECORD),
+        createDNSRecord(`*.${domain}`, 'preview-gitpod-dev-com', ingressIP, installerSlices.DNS_ADD_RECORD),
+        createDNSRecord(`*.ws.${domain}`, 'preview-gitpod-dev-com', proxyLBIP, installerSlices.DNS_ADD_RECORD),
+    ])
+    werft.done(installerSlices.DNS_ADD_RECORD);
+}
+
 export async function issueMetaCerts(werft: Werft, certName: string, certsNamespace: string, domain: string, withVM: boolean, slice: string) {
     const additionalSubdomains: string[] = ["", "*.", `*.ws${withVM ? '' : '-dev'}.`]
     var metaClusterCertParams = new IssueCertificateParams();
@@ -720,6 +753,11 @@ function getCoreDevIngressIP(): string {
     return "104.199.27.246";
 }
 
+// returns the static IP address
+function getHarvesterIngressIP(): string {
+    return "159.69.172.117";
+}
+
 function metaEnv(_parent?: ExecOptions): ExecOptions {
     return env("", _parent);
 }

.werft/jobs/build/prepare.ts

@@ -5,6 +5,8 @@ import * as VM from '../../vm/vm'
 import { CORE_DEV_KUBECONFIG_PATH, GCLOUD_SERVICE_ACCOUNT_PATH, HARVESTER_KUBECONFIG_PATH } from "./const";
 import { issueMetaCerts } from './deploy-to-preview-environment';
 import { JobConfig } from './job-config';
+import * as Manifests from '../../vm/manifests';
+import { sleep } from '../../util/util';
 
 const phaseName = "prepare";
 const prepareSlices = {
@@ -86,6 +88,7 @@ function decideHarvesterVMCreation(werft: Werft, config: JobConfig) {
     } else {
         werft.currentPhaseSpan.setAttribute("werft.harvester.created_vm", false)
     }
+    applyLoadBalancer({ name: config.previewEnvironment.destname })
     werft.done(prepareSlices.BOOT_VM)
 }
 
@@ -108,3 +111,30 @@ function createVM(werft: Werft, config: JobConfig) {
     VM.startVM({ name: config.previewEnvironment.destname })
     werft.currentPhaseSpan.setAttribute("werft.harvester.created_vm", true)
 }
+
+function applyLoadBalancer(option: { name: string }) {
+    const namespace = `preview-${option.name}`;
+    function kubectlApplyManifest(manifest: string, options?: { validate?: boolean }) {
+        exec(`
+            cat <<EOF | kubectl --kubeconfig ${CORE_DEV_KUBECONFIG_PATH} apply --validate=${!!options?.validate} -f -
+${manifest}
+EOF
+        `);
+    }
+    function getVMServiceIP() {
+        let ip = exec(
+            `kubectl --kubeconfig ${HARVESTER_KUBECONFIG_PATH} -n ${namespace} get service proxy -o=jsonpath='{.spec.clusterIP}'`,
+            { silent: true },
+        );
+        if (ip.length > 4) {
+            return ip;
+        }
+        return null;
+    }
+    let forwardIP = getVMServiceIP();
+    if (forwardIP == null) {
+        throw new Error("Failed to get VM IP");
+    }
+    kubectlApplyManifest(Manifests.LBDeployManifest({ name: option.name, destIP: forwardIP }));
+    kubectlApplyManifest(Manifests.LBServiceManifest({ name: option.name }));
+}

.werft/util/gcloud.ts

@@ -39,7 +39,10 @@ function getExternalIp(name: string, region = "europe-west1") {
 export async function createDNSRecord(domain: string, dnsZone: string, IP: string, slice: string): Promise<void> {
     const werft = getGlobalWerftInstance()
 
-    const dnsClient = new DNS({ projectId: 'gitpod-dev', keyFilename: GCLOUD_SERVICE_ACCOUNT_PATH })
+    const dnsClient = new DNS({
+        projectId: dnsZone === "gitpod-dev.com" ? "gitpod-dev" : "gitpod-core-dev",
+        keyFilename: GCLOUD_SERVICE_ACCOUNT_PATH,
+    });
     const zone = dnsClient.zone(dnsZone)
 
     if (!(await matchesExistingRecord(zone, domain, IP))) {

.werft/vm/manifests.ts

@@ -140,6 +140,102 @@ spec:
 `
 }
 
+type LBServiceManifestOptions = {
+  name: string
+}
+
+export function LBServiceManifest({ name }: LBServiceManifestOptions) {
+    return `
+apiVersion: v1
+kind: Service
+metadata:
+  name: lb-${name}
+  namespace: loadbalancers
+spec:
+  ports:
+    - name: ssh-gateway
+      protocol: TCP
+      port: 22
+      targetPort: 22
+    - name: https
+      protocol: TCP
+      port: 443
+      targetPort: 443
+  selector:
+    gitpod.io/lbName: ${name}
+  type: LoadBalancer
+  `
+}
+
+type LBDeployManifestOptions = {
+  name: string
+  destIP: string
+}
+
+export function LBDeployManifest({ name, destIP }: LBDeployManifestOptions) {
+  return `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lb-${name}
+  namespace: loadbalancers
+  labels:
+    gitpod.io/lbName: ${name}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      gitpod.io/lbName: ${name}
+  template:
+    metadata:
+      name: lb
+      labels:
+        gitpod.io/lbName: ${name}
+    spec:
+      containers:
+        - name: lb-port-22
+          image: rancher/klipper-lb:v0.3.4
+          ports:
+            - name: lb-port-22
+              containerPort: 22
+              protocol: TCP
+          env:
+            - name: SRC_PORT
+              value: '22'
+            - name: DEST_PROTO
+              value: TCP
+            - name: DEST_PORT
+              value: '22'
+            - name: DEST_IPS
+              value: ${destIP}
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+        - name: lb-port-443
+          image: rancher/klipper-lb:v0.3.4
+          ports:
+            - name: lb-port-443
+              containerPort: 443
+              protocol: TCP
+          env:
+            - name: SRC_PORT
+              value: '443'
+            - name: DEST_PROTO
+              value: TCP
+            - name: DEST_PORT
+              value: '443'
+            - name: DEST_IPS
+              value: ${destIP}
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+      serviceAccount: proxy
+      enableServiceLinks: false
+`
+}
+
 type UserDataSecretManifestOptions = {
   vmName: string
   namespace: string,