(EAI-585): Move MongoDB data sources to implementation package (#542)

* move mongodb data sources to implementation package

* move test data

* Remove files that shouldn't be present

* remove for naming

* rename

* master --> main

* master --> main more

* clean up imports

Author: mongodben

packages/ingest-mongodb-public/jest.config.cjs

@@ -1,5 +1,6 @@
 module.exports = {
   preset: "ts-jest",
   testEnvironment: "node",
+  setupFiles: ["<rootDir>/src/test/jestSetUp.ts"],
   testPathIgnorePatterns: ["<rootDir>/build"],
 };

packages/ingest-mongodb-public/src/PublicIngestEnvVars.ts

@@ -17,8 +17,18 @@ export const PUBLIC_INGEST_MONGODB_DOCS_META_ENV_VARS = {
   MONGODB_META_DATABASE_NAME: "",
 };
 
+export const INGEST_DEVCENTER_ENV_VARS = {
+  DEVCENTER_CONNECTION_URI: "",
+};
+
+export const INGEST_MDBU_ENV_VARS = {
+  UNIVERSITY_DATA_API_KEY: "",
+};
+
 export const PUBLIC_INGEST_ENV_VARS = {
   ...CORE_CHATBOT_APP_ENV_VARS,
   ...CORE_OPENAI_EMBEDDING_ENV_VARS,
   ...INGEST_ENV_VARS,
+  ...INGEST_DEVCENTER_ENV_VARS,
+  ...INGEST_MDBU_ENV_VARS,
 };

packages/ingest-mongodb-public/src/meta.config.ts

@@ -1,8 +1,4 @@
-import {
-  Config,
-  INGEST_ENV_VARS,
-  makeIngestMetaStore,
-} from "mongodb-rag-ingest";
+import { Config, makeIngestMetaStore } from "mongodb-rag-ingest";
 import { standardChunkFrontMatterUpdater } from "mongodb-rag-ingest/embed";
 import {
   assertEnvVars,
@@ -12,12 +8,12 @@ import {
   OpenAIClient,
   AzureKeyCredential,
 } from "mongodb-rag-core";
-import { snootyDataApiBaseUrl } from "./sources/snooty";
-import { makeSnootyDataSource } from "mongodb-rag-ingest/sources/snooty";
+import { makeSnootyDataSource } from "./sources/snooty";
 import {
   PUBLIC_INGEST_ENV_VARS,
   PUBLIC_INGEST_MONGODB_DOCS_META_ENV_VARS,
 } from "./PublicIngestEnvVars";
+import { snootyDataApiBaseUrl } from "./sources/snootySources";
 
 const {
   OPENAI_ENDPOINT,

packages/mongodb-rag-ingest/src/sources/DevCenterDataSource.test.ts renamed to packages/ingest-mongodb-public/src/sources/DevCenterDataSource.test.ts

@@ -1,14 +1,16 @@
-import { strict as assert } from "assert";
 import Path from "path";
 import fs from "fs";
 import {
   makeDevCenterPageBody,
   makeDevCenterDataSource,
   makeDevCenterPage,
 } from "./DevCenterDataSource";
-
+import { assertEnvVars } from "mongodb-rag-core";
+import { INGEST_DEVCENTER_ENV_VARS } from "../PublicIngestEnvVars";
 import "dotenv/config";
 
+const { DEVCENTER_CONNECTION_URI } = assertEnvVars(INGEST_DEVCENTER_ENV_VARS);
+
 const SRC_ROOT = Path.resolve(__dirname, "../");
 
 const devCenterDoc = JSON.parse(
@@ -21,18 +23,14 @@ const devCenterDoc = JSON.parse(
 );
 describe("DevCenterDataSource", () => {
   jest.setTimeout(90000);
-  const { DEVCENTER_CONNECTION_URI } = process.env;
   it("loads pages from dev center", async () => {
-    assert(
-      DEVCENTER_CONNECTION_URI !== undefined,
-      "env var DEVCENTER_CONNECTION_URI not defined. Did you copy .env.example to .env and fill it in?"
-    );
     const source = await makeDevCenterDataSource({
       type: "devcenter",
       name: "devcenter",
       collectionName: "search_content_prod",
       databaseName: "devcenter",
       baseUrl: "https://example.com/developer",
+      connectionUri: DEVCENTER_CONNECTION_URI,
     });
 
     const pages = await source.fetchPages();

packages/mongodb-rag-ingest/src/sources/DevCenterDataSource.ts renamed to packages/ingest-mongodb-public/src/sources/DevCenterDataSource.ts

@@ -1,17 +1,18 @@
-import { MongoClient } from "mongodb-rag-core";
+import { logger, MongoClient, Page } from "mongodb-rag-core";
 import { strict as assert } from "assert";
 import { convert } from "html-to-text";
-import { Page, assertEnvVars, logger } from "mongodb-rag-core";
-import { INGEST_ENV_VARS } from "../IngestEnvVars";
-import { removeMarkdownImagesAndLinks } from "./removeMarkdownImagesAndLinks";
-import { DataSource } from "./DataSource";
-import { ProjectBase } from "./ProjectBase";
+import {
+  DataSource,
+  ProjectBase,
+  removeMarkdownImagesAndLinks,
+} from "mongodb-rag-ingest/sources";
 
 export type DevCenterProjectConfig = ProjectBase & {
   type: "devcenter";
   databaseName: string;
   collectionName: string;
   baseUrl: string;
+  connectionUri: string;
 };
 
 // This type is based on what's in the DevCenter search_content_prod collection
@@ -34,13 +35,12 @@ export const makeDevCenterDataSource = async ({
   databaseName,
   collectionName,
   baseUrl,
+  connectionUri,
 }: DevCenterProjectConfig): Promise<DataSource> => {
-  const { DEVCENTER_CONNECTION_URI } = assertEnvVars(INGEST_ENV_VARS);
-
   return {
     name,
     async fetchPages() {
-      const client = await new MongoClient(DEVCENTER_CONNECTION_URI).connect();
+      const client = await new MongoClient(connectionUri).connect();
       try {
         const db = client.db(databaseName);
         const collection = db.collection<DevCenterEntry>(collectionName);

packages/ingest-mongodb-public/src/sources/index.ts

@@ -2,28 +2,35 @@ import { strict as assert } from "assert";
 import { Page, extractFrontMatter } from "mongodb-rag-core";
 import {
   DataSource,
-  makeDevCenterDataSource,
-  DevCenterProjectConfig,
   makeGitDataSource,
-  HandleHtmlPageFuncOptions,
-  handleHtmlDocument,
   MakeMdOnGithubDataSourceParams,
   makeMdOnGithubDataSource,
   removeMarkdownImagesAndLinks,
-  MakeMongoDbUniversityDataSourceParams,
-  makeMongoDbUniversityDataSource,
-  filterOnlyPublicActiveTiCatalogItems,
 } from "mongodb-rag-ingest/sources";
-import { prepareSnootySources } from "mongodb-rag-ingest/sources/snooty";
 import { prismaSourceConstructor } from "./prisma";
 import { wiredTigerSourceConstructor } from "./wiredTiger";
 import { mongooseSourceConstructor } from "./mongoose";
 import { practicalAggregationsDataSource } from "./practicalAggregations";
 import {
+  makeSnootyDataSources,
   snootyDataApiBaseUrl,
   snootyProjectConfig,
-  makeSnootyDataSources,
-} from "./snooty";
+} from "./snootySources";
+
+import { assertEnvVars } from "mongodb-rag-core";
+import { PUBLIC_INGEST_ENV_VARS } from "../PublicIngestEnvVars";
+import {
+  DevCenterProjectConfig,
+  makeDevCenterDataSource,
+} from "./DevCenterDataSource";
+import {
+  MakeMongoDbUniversityDataSourceParams,
+  filterOnlyPublicActiveTiCatalogItems,
+  makeMongoDbUniversityDataSource,
+} from "./mongodb-university";
+const { DEVCENTER_CONNECTION_URI, UNIVERSITY_DATA_API_KEY } = assertEnvVars(
+  PUBLIC_INGEST_ENV_VARS
+);
 
 /**
   Async constructor for specific data sources -- parameters baked in.
@@ -36,10 +43,11 @@ export const devCenterProjectConfig: DevCenterProjectConfig = {
   collectionName: "search_content_prod",
   databaseName: "devcenter",
   baseUrl: "https://www.mongodb.com/developer",
+  connectionUri: DEVCENTER_CONNECTION_URI,
 };
 
 const mongoDbUniversitySourceConstructor = async () => {
-  const universityDataApiKey = process.env.UNIVERSITY_DATA_API_KEY;
+  const universityDataApiKey = UNIVERSITY_DATA_API_KEY;
   assert(!!universityDataApiKey, "UNIVERSITY_DATA_API_KEY required");
   const universityConfig: MakeMongoDbUniversityDataSourceParams = {
     sourceName: "mongodb-university",

packages/mongodb-rag-ingest/src/sources/mongodb-university/MongoDbUniversityDataApiClient.test.ts renamed to packages/ingest-mongodb-public/src/sources/mongodb-university/MongoDbUniversityDataApiClient.test.ts

@@ -1,5 +1,5 @@
 import { PageMetadata } from "mongodb-rag-core";
-import { DataSource } from "../DataSource";
+import { DataSource } from "../../../../mongodb-rag-ingest/src/sources/DataSource";
 import { makeUniversityPages } from "./makeUniversityPages";
 import {
   TiCatalogItem,