[server] clean up DB tests (#18333)

Author: svenefftinge

.github/workflows/build.yml

@@ -240,6 +240,7 @@ jobs:
             $TEST \
             -Dversion=$VERSION \
             --docker-build-options network=host \
+            --max-concurrent-tasks 1 \
             -DlocalAppVersion=$VERSION \
             -DSEGMENT_IO_TOKEN=$SEGMENT_IO_TOKEN \
             -DpublishToNPM="${PUBLISH_TO_NPM}" \

components/gitpod-db/.vscode/launch.json

@@ -8,17 +8,15 @@
                 "DB_PORT": "23306",
                 "DB_ENCRYPTION_KEYS": "[{\"name\":\"general\",\"version\":1,\"primary\":true,\"material\":\"5vRrp0H4oRgdkPnX1qQcS54Q0xggr6iyho42IQ1rO+c=\"}]"
             },
-            "name": "Run Mocha Test in Editor",
+            "name": "Run all Tests - in Editor",
             "program": "${workspaceFolder}/node_modules/.bin/_mocha",
             "args": [
                 "--opts",
                 "${workspaceFolder}/mocha.opts",
                 "--colors",
                 "--timeout",
                 "999999",
-                "${file}",
-                "-g",
-                "${selectedText}"
+                "${file}"
             ],
             "internalConsoleOptions": "openOnSessionStart"
         }

components/gitpod-db/package.json

@@ -35,6 +35,7 @@
   "devDependencies": {
     "@testdeck/mocha": "^0.3.3",
     "@types/chai": "^4.2.2",
+    "@types/mocha": "^2.2.45",
     "@types/mysql": "^2.15.0",
     "@types/uuid": "^8.3.1",
     "@typescript-eslint/eslint-plugin": "^4.5.0",

components/gitpod-db/src/auth-provider-entry.spec.db.ts

@@ -4,38 +4,30 @@
  * See License.AGPL.txt in the project root for license information.
  */
 
-import { suite, test, timeout } from "@testdeck/mocha";
 import { testContainer } from "./test-container";
 import { TypeORM } from "./typeorm/typeorm";
-import { AuthProviderEntryDB } from ".";
 import { DBAuthProviderEntry } from "./typeorm/entity/db-auth-provider-entry";
 import { DeepPartial } from "@gitpod/gitpod-protocol/lib/util/deep-partial";
-import * as chai from "chai";
-const expect = chai.expect;
+import { resetDB } from "./test/reset-db";
+import { AuthProviderEntryDB } from "./auth-provider-entry-db";
+import { expect } from "chai";
+import "mocha";
 
-@suite(timeout(10000))
-export class AuthProviderEntryDBSpec {
-    private readonly db = testContainer.get<AuthProviderEntryDB>(AuthProviderEntryDB);
+const container = testContainer.createChild();
 
-    // TODO(gpl) Since the upgrade to ts-node 10.4.0 establishing the inital connection takes up to 25s
-    @timeout(30000)
-    async before() {
-        await this.clear();
-    }
+describe("AuthProviderEntryDBSpec", async () => {
+    let db: AuthProviderEntryDB;
 
-    async after() {
-        await this.clear();
-    }
+    beforeEach(async () => {
+        db = container.get<AuthProviderEntryDB>(AuthProviderEntryDB);
+    });
 
-    async clear() {
-        const typeorm = testContainer.get<TypeORM>(TypeORM);
-        const manager = await typeorm.getConnection();
-        console.log("before delete");
-        await manager.getRepository(DBAuthProviderEntry).delete({});
-        console.log("after delete");
-    }
+    afterEach(async () => {
+        const typeorm = container.get<TypeORM>(TypeORM);
+        await resetDB(typeorm);
+    });
 
-    protected authProvider(ap: DeepPartial<DBAuthProviderEntry> = {}): DBAuthProviderEntry {
+    function authProvider(ap: DeepPartial<DBAuthProviderEntry> = {}): DBAuthProviderEntry {
         const ownerId = "1234";
         const host = "github.com";
         return {
@@ -63,66 +55,63 @@ export class AuthProviderEntryDBSpec {
         };
     }
 
-    @test() public async testFindAll() {
-        console.log("start findAll");
-        const ap1 = this.authProvider({ id: "1", oauthRevision: "rev1" });
-        const ap2 = this.authProvider({ id: "2", oauthRevision: "rev2" });
-        await this.db.storeAuthProvider(ap1, false);
-        await this.db.storeAuthProvider(ap2, false);
+    it("should findAll", async () => {
+        const ap1 = authProvider({ id: "1", oauthRevision: "rev1" });
+        const ap2 = authProvider({ id: "2", oauthRevision: "rev2" });
+        await db.storeAuthProvider(ap1, false);
+        await db.storeAuthProvider(ap2, false);
 
-        const all = await this.db.findAll();
+        const all = await db.findAll();
         expect(all, "findAll([])").to.deep.equal([ap1, ap2]);
-        expect(await this.db.findAll([ap1.oauthRevision!, ap2.oauthRevision!]), "findAll([ap1, ap2])").to.be.empty;
-        expect(await this.db.findAll([ap1.oauthRevision!]), "findAll([ap1])").to.deep.equal([ap2]);
-    }
+        expect(await db.findAll([ap1.oauthRevision!, ap2.oauthRevision!]), "findAll([ap1, ap2])").to.be.empty;
+        expect(await db.findAll([ap1.oauthRevision!]), "findAll([ap1])").to.deep.equal([ap2]);
+    }).timeout(30000); // this test is sometimes slow because it is the first one and ts-node needs to compile
 
-    @test() public async findAllHosts() {
-        const ap1 = this.authProvider({ id: "1", oauthRevision: "rev1", host: "foo" });
-        const ap2 = this.authProvider({ id: "2", oauthRevision: "rev2", host: "BAR" });
-        await this.db.storeAuthProvider(ap1, false);
-        await this.db.storeAuthProvider(ap2, false);
+    it("should findAllHosts", async () => {
+        const ap1 = authProvider({ id: "1", oauthRevision: "rev1", host: "foo" });
+        const ap2 = authProvider({ id: "2", oauthRevision: "rev2", host: "BAR" });
+        await db.storeAuthProvider(ap1, false);
+        await db.storeAuthProvider(ap2, false);
 
-        const all = await this.db.findAllHosts();
+        const all = await db.findAllHosts();
         expect(all, "findAllHosts([])").to.deep.equal(["foo", "bar"]);
-    }
+    });
 
-    @test() public async oauthRevision() {
-        const ap = this.authProvider({ id: "1" });
-        await this.db.storeAuthProvider(ap, true);
+    it("should oauthRevision", async () => {
+        const ap = authProvider({ id: "1" });
+        await db.storeAuthProvider(ap, true);
 
-        const loadedAp = await this.db.findByHost(ap.host);
+        const loadedAp = await db.findByHost(ap.host);
         expect(loadedAp, "findByHost()").to.deep.equal(ap);
         expect(loadedAp?.oauthRevision, "findByHost()").to.equal(
             "3d1390670fd19c27157d046960c3d7c46df81db642302dea1a9fe86cf0594361",
         );
-    }
+    });
 
-    @test() public async findByOrgId() {
-        const ap1 = this.authProvider({ id: "1", organizationId: "O1", host: "H1" });
-        const ap2 = this.authProvider({ id: "2", organizationId: "O1", host: "H2" });
-        const ap3 = this.authProvider({ id: "3", organizationId: "O2", host: "H1" });
+    it("should findByOrgId()", async () => {
+        const ap1 = authProvider({ id: "1", organizationId: "O1", host: "H1" });
+        const ap2 = authProvider({ id: "2", organizationId: "O1", host: "H2" });
+        const ap3 = authProvider({ id: "3", organizationId: "O2", host: "H1" });
 
-        await this.db.storeAuthProvider(ap1, false);
-        await this.db.storeAuthProvider(ap2, false);
-        await this.db.storeAuthProvider(ap3, false);
+        await db.storeAuthProvider(ap1, false);
+        await db.storeAuthProvider(ap2, false);
+        await db.storeAuthProvider(ap3, false);
 
-        const results = await this.db.findByOrgId("O1");
+        const results = await db.findByOrgId("O1");
         expect(results.length).to.equal(2);
         expect(results).to.deep.contain(ap1);
         expect(results).to.deep.contain(ap2);
-    }
+    });
 
-    @test() public async findByUserId() {
-        const ap1 = this.authProvider({ id: "1", ownerId: "owner1" });
-        const ap2 = this.authProvider({ id: "2", ownerId: "owner1", organizationId: "org1" });
+    it("should findByUserId", async () => {
+        const ap1 = authProvider({ id: "1", ownerId: "owner1" });
+        const ap2 = authProvider({ id: "2", ownerId: "owner1", organizationId: "org1" });
 
-        await this.db.storeAuthProvider(ap1, false);
-        await this.db.storeAuthProvider(ap2, false);
+        await db.storeAuthProvider(ap1, false);
+        await db.storeAuthProvider(ap2, false);
 
-        const results = await this.db.findByUserId("owner1");
+        const results = await db.findByUserId("owner1");
         expect(results.length).to.equal(1);
         expect(results).to.deep.contain(ap1);
-    }
-}
-
-module.exports = AuthProviderEntryDBSpec;
+    });
+});

components/gitpod-db/src/blocked-repository-db.spec.db.ts

@@ -11,7 +11,7 @@ import { suite, test, timeout } from "@testdeck/mocha";
 import { testContainer } from "./test-container";
 import { TypeORMBlockedRepositoryDBImpl } from "./typeorm/blocked-repository-db-impl";
 import { TypeORM } from "./typeorm/typeorm";
-import { DBBlockedRepository } from "./typeorm/entity/db-blocked-repository";
+import { resetDB } from "./test/reset-db";
 
 @suite
 class BlockedRepositoryDBSpec {
@@ -79,8 +79,7 @@ class BlockedRepositoryDBSpec {
 
     async wipeRepo() {
         const typeorm = testContainer.get<TypeORM>(TypeORM);
-        const manager = await typeorm.getConnection();
-        await manager.getRepository(DBBlockedRepository).delete({});
+        await resetDB(typeorm);
     }
 }
 

components/gitpod-db/src/container-module.ts

@@ -46,12 +46,20 @@ import { LinkedInProfileDBImpl } from "./typeorm/linked-in-profile-db-impl";
 import { LinkedInProfileDB } from "./linked-in-profile-db";
 import { DataCache, DataCacheNoop } from "./data-cache";
 import { TracingManager } from "@gitpod/gitpod-protocol/lib/util/tracing";
+import { EncryptionService, GlobalEncryptionService } from "@gitpod/gitpod-protocol/lib/encryption/encryption-service";
 
 // THE DB container module that contains all DB implementations
 export const dbContainerModule = (cacheClass = DataCacheNoop) =>
     new ContainerModule((bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
         bind(Config).toSelf().inSingletonScope();
-        bind(TypeORM).toSelf().inSingletonScope();
+        bind(TypeORM)
+            .toSelf()
+            .inSingletonScope()
+            .onActivation((ctx, orm) => {
+                // HACK we need to initialize the global encryption service.
+                GlobalEncryptionService.encryptionService = ctx.container.get(EncryptionService);
+                return orm;
+            });
         bind(DBWithTracing).toSelf().inSingletonScope();
         bind(TracingManager).toSelf().inSingletonScope();
         bind(DataCache).to(cacheClass).inSingletonScope();

components/gitpod-db/src/email-domain-filter-db.spec.db.ts

@@ -9,7 +9,7 @@ import { suite, test, timeout } from "@testdeck/mocha";
 import { testContainer } from "./test-container";
 import { TypeORM } from "./typeorm/typeorm";
 import { EmailDomainFilterDB } from "./email-domain-filter-db";
-import { DBEmailDomainFilterEntry } from "./typeorm/entity/db-email-domain-filter-entry";
+import { resetDB } from "./test/reset-db";
 const expect = chai.expect;
 
 @suite
@@ -27,9 +27,7 @@ export class EmailDomainFilterDBSpec {
     }
 
     protected async clear() {
-        const connection = await this.typeORM.getConnection();
-        const manager = connection.manager;
-        await manager.clear(DBEmailDomainFilterEntry);
+        await resetDB(this.typeORM);
     }
 
     @test public async filterSimple() {

components/gitpod-db/src/project-db.spec.db.ts

@@ -4,16 +4,15 @@
  * See License.AGPL.txt in the project root for license information.
  */
 
-import * as chai from "chai";
-const expect = chai.expect;
+import { Project } from "@gitpod/gitpod-protocol";
 import { suite, test } from "@testdeck/mocha";
-import { TypeORM } from "./typeorm/typeorm";
-import { TypeORMUserDBImpl } from "./typeorm/user-db-impl";
+import * as chai from "chai";
 import { testContainer } from "./test-container";
+import { resetDB } from "./test/reset-db";
 import { ProjectDBImpl } from "./typeorm/project-db-impl";
-import { DBProject } from "./typeorm/entity/db-project";
-import { DBUser } from "./typeorm/entity/db-user";
-import { Project } from "@gitpod/gitpod-protocol";
+import { TypeORM } from "./typeorm/typeorm";
+import { TypeORMUserDBImpl } from "./typeorm/user-db-impl";
+const expect = chai.expect;
 
 @suite
 class ProjectDBSpec {
@@ -30,9 +29,7 @@ class ProjectDBSpec {
 
     async wipeRepo() {
         const typeorm = testContainer.get<TypeORM>(TypeORM);
-        const manager = await typeorm.getConnection();
-        await manager.getRepository(DBProject).delete({});
-        await manager.getRepository(DBUser).delete({});
+        await resetDB(typeorm);
     }
 
     @test()

components/gitpod-db/src/team-db.spec.db.ts

@@ -13,11 +13,8 @@ import { testContainer } from "./test-container";
 import { TeamDBImpl } from "./typeorm/team-db-impl";
 import { TypeORMUserDBImpl } from "./typeorm/user-db-impl";
 import { TypeORM } from "./typeorm/typeorm";
-import { DBTeam } from "./typeorm/entity/db-team";
-import { DBTeamMembership } from "./typeorm/entity/db-team-membership";
-import { DBUser } from "./typeorm/entity/db-user";
-import { DBIdentity } from "./typeorm/entity/db-identity";
 import { Connection } from "typeorm";
+import { resetDB } from "./test/reset-db";
 
 @suite
 class TeamDBSpec {
@@ -34,12 +31,7 @@ class TeamDBSpec {
 
     async wipeRepo() {
         const typeorm = testContainer.get<TypeORM>(TypeORM);
-        const manager = await typeorm.getConnection();
-        await manager.getRepository(DBTeam).delete({});
-        await manager.getRepository(DBTeamMembership).delete({});
-        await manager.getRepository(DBUser).delete({});
-        await manager.getRepository(DBIdentity).delete({});
-        await manager.query("delete from d_b_oidc_client_config;");
+        await resetDB(typeorm);
     }
 
     @test(timeout(10000))

components/gitpod-db/src/test/reset-db.ts

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { DBUser } from "../typeorm/entity/db-user";
+import { TypeORM } from "../typeorm/typeorm";
+import { isBuiltinUser } from "../user-db";
+
+export async function resetDB(typeorm: TypeORM) {
+    const conn = await typeorm.getConnection();
+    const users = await conn.getRepository(DBUser).find();
+    // delete all users except the builtin users
+    conn.getRepository(DBUser).remove(users.filter((u) => !isBuiltinUser(u.id)));
+
+    const deletions = conn.entityMetadatas
+        .filter((meta) => meta.tableName !== "d_b_user")
+        .map((meta) => {
+            return conn.getRepository(meta.name).clear();
+        });
+
+    await Promise.all([
+        // delete all other entities
+        ...deletions,
+
+        // we don't have a typeorm entity for this table
+        conn.query("DELETE FROM d_b_oidc_client_config;"),
+    ]);
+}

components/gitpod-db/src/typeorm/auth-code-repository-db.spec.db.ts

@@ -10,9 +10,9 @@ import { AuthCodeRepositoryDB } from "./auth-code-repository-db";
 import { UserDB } from "../user-db";
 import { DBOAuthAuthCodeEntry } from "./entity/db-oauth-auth-code";
 import { TypeORM } from "./typeorm";
-import { DBUser } from "./entity/db-user";
 import * as chai from "chai";
 import { User } from "@gitpod/gitpod-protocol";
+import { resetDB } from "../test/reset-db";
 const expect = chai.expect;
 
 @suite(timeout(10000))
@@ -30,9 +30,7 @@ export class AuthCodeRepositoryDBSpec {
 
     async wipeRepo() {
         const typeorm = testContainer.get<TypeORM>(TypeORM);
-        const manager = await typeorm.getConnection();
-        await manager.getRepository(DBOAuthAuthCodeEntry).delete({});
-        await manager.getRepository(DBUser).delete({});
+        await resetDB(typeorm);
     }
 
     @test()

components/gitpod-db/src/typeorm/code-sync-resource-db.spec.db.ts

@@ -10,6 +10,8 @@ import { suite, test, timeout } from "@testdeck/mocha";
 import { testContainer } from "../test-container";
 import { CodeSyncResourceDB } from "./code-sync-resource-db";
 import { IUserDataManifest, SyncResource } from "./entity/db-code-sync-resource";
+import { resetDB } from "../test/reset-db";
+import { TypeORM } from "./typeorm";
 const expect = chai.expect;
 
 @suite(timeout(10000))
@@ -23,8 +25,7 @@ export class CodeSyncResourceDBSpec {
     }
 
     async after(): Promise<void> {
-        await this.db.deleteSettingsSyncResources(this.userId, () => Promise.resolve());
-        await this.db.deleteCollection(this.userId, undefined, () => Promise.resolve());
+        await resetDB(testContainer.get<TypeORM>(TypeORM));
     }
 
     @test()