(EAI-1003) get available versions for data source

packages/ingest-mongodb-public/src/sources/mongoose.ts

@@ -26,7 +26,7 @@ export const mongooseSourceConstructor = async () => {
     metadata: {
       productName: "Mongoose ODM",
       tags: ["node.js", "community library", "mongoose", "odm"],
-      version: "v7.x (current)",
+      versionLabel: "v7.x (current)",
     },
   });
 };

packages/ingest-mongodb-public/src/sources/snooty/SnootyProjectsInfo.test.ts

@@ -5,7 +5,7 @@ import {
   overrideCurrentVersion,
   prepareSnootySources,
 } from "./SnootyProjectsInfo";
-import { Page } from "mongodb-rag-core";
+import { PageMetadata } from "mongodb-rag-core";
 
 const snootyDataApiBaseUrl = "https://snooty-data-api.mongodb.com/prod/";
 
@@ -128,7 +128,7 @@ describe("SnootyProjectsInfo", () => {
       });
       assert(snootySources !== undefined);
       const pages = await snootySources[0].fetchPages();
-      const versions = pages.map((page) => (page.metadata?.version as Page['version']));
+      const versions = pages.map((page) => (page.metadata?.version as PageMetadata['version']));
       const originalCurrentVersion = versions.find((version) => version?.isCurrent);
       const currentVersionOverride = versions.find((version) => !version?.isCurrent)?.label;
 
@@ -141,7 +141,7 @@ describe("SnootyProjectsInfo", () => {
         snootyDataApiBaseUrl,
       });
       const pagesAfterOverride = await sourcesAfterOverride[0].fetchPages();
-      const versionsAfterOverride = pagesAfterOverride.map((page) => (page.metadata?.version as Page['version']));
+      const versionsAfterOverride = pagesAfterOverride.map((page) => (page.metadata?.version as PageMetadata['version']));
       const currentVersionAfterOverride = versionsAfterOverride.find((version) => version?.isCurrent);
 
       // Check that the current version is the one we set

packages/mongodb-artifact-generator/src/test/MockPageStore.ts

@@ -11,6 +11,9 @@ export const makeMockPageStore = (): PageStore => {
     },
     async deletePages() {
       return;
-    }
+    },
+    async getDataSourceVersions() {
+      return [];
+    },
   };
 };

packages/mongodb-rag-core/src/contentStore/MongoDbPageStore.test.ts

@@ -333,4 +333,64 @@ describe("MongoDbPageStore", () => {
       expect(missingUrls).toEqual([]);
     });
   });
+
+  describe("getDataSourceVersions", () => {
+    const moviePagesWithVersion = moviePages.map((page, index) => ({
+      ...page,
+      sourceName: `movie-source`,
+      metadata: {
+        version: {
+          label: `${index % 2 ? 1 : 2}`,
+          isCurrent: index % 2 ? false : true,
+        },
+      },
+    }));
+    beforeEach(async () => {
+      assert(store);
+      await store.updatePages(moviePagesWithVersion);
+    });
+    afterAll(async () => {
+      assert(store);
+      await store.deletePages({
+        dataSources: ["movie-source", "another-movie-source"],
+        permanent: true,
+      });
+    });
+
+    it("returns list of versions for a specific data source", async () => {
+      assert(store);
+      const dataSourceVersions = await store.getDataSourceVersions({
+        dataSources: ["movie-source"],
+      });
+
+      expect(dataSourceVersions.length).toBe(1);
+      const movieSourceVersions = dataSourceVersions[0];
+      expect(movieSourceVersions.sourceName).toBe("movie-source");
+      expect(movieSourceVersions.versions.length).toBe(2);
+      expect(movieSourceVersions.versions[0]).toMatchObject({
+        label: "1",
+        isCurrent: false,
+      });
+      expect(movieSourceVersions.versions[1]).toMatchObject({
+        label: "2",
+        isCurrent: true,
+      });
+    });
+
+    it("returns list of versions for all data sources", async () => {
+      assert(store);
+      // add another versioned data source
+      const anotherMoviePagesWithVersion = moviePagesWithVersion.map(
+        (page) => ({
+          ...page,
+          sourceName: `another-movie-source`,
+        })
+      );
+      await store.updatePages(anotherMoviePagesWithVersion);
+
+      const dataSourceVersions = await store.getDataSourceVersions();
+
+      expect(dataSourceVersions.length).toBe(2);
+    });
+  });
 });

packages/mongodb-rag-core/src/contentStore/MongoDbPageStore.ts

@@ -9,6 +9,7 @@ import {
   LoadPagesArgs,
   PageStore,
   PersistedPage,
+  SourceVersions,
 } from "./Page";
 import { Filter, Document } from "mongodb";
 
@@ -29,6 +30,9 @@ export type MongoDbPageStore = DatabaseConnection &
       expectedUrls: string[];
       urlTransformer?: (url: string) => string;
     }): Promise<string[]>;
+    getDataSourceVersions(args: {
+      dataSources: string[];
+    }): Promise<SourceVersions[]>;
     metadata: {
       databaseName: string;
       collectionName: string;
@@ -145,6 +149,42 @@ export function makeMongoDbPageStore({
       );
       return results.filter((url) => url !== null) as string[];
     },
+    async getDataSourceVersions(args?: {
+      dataSources: string[];
+    }): Promise<SourceVersions[]> {
+      const pipeline = [
+        {
+          $match: {
+            ...(args && { sourceName: { $in: args.dataSources } }), // Filter by data sources if provided
+            action: { $ne: "deleted" }, // Exclude deleted pages
+            "metadata.version.label": { $exists: true }, // Exclude unversioned pages
+          },
+        },
+        {
+          $group: {
+            _id: "$sourceName", // Group by sourceName
+            versions: {
+              $addToSet: "$metadata.version", // Collect unique versions
+            },
+          },
+        },
+        {
+          $project: {
+            _id: 0,
+            sourceName: "$_id",
+            versions: {
+              $sortArray: {
+                input: "$versions",
+                sortBy: {
+                  label: 1, // sort by label in ascending order
+                },
+              },
+            },
+          },
+        },
+      ];
+      return pagesCollection.aggregate<SourceVersions>(pipeline).toArray();
+    },
     async init() {
       await pagesCollection.createIndex({ url: 1 });
       await pagesCollection.createIndex({ sourceName: 1 });

packages/mongodb-rag-core/src/contentStore/Page.ts

@@ -27,26 +27,27 @@ export type Page = {
    */
   sourceName: string;
 
-  /**
-    The version of the page. This is relevant for versioned docs.
-    If the page is not versioned, this field should be undefined.
-   */
-  version?: {
-    label: string;
-    isCurrent: boolean;
-  }
-
   /**
     Arbitrary metadata for page.
    */
   metadata?: PageMetadata;
 };
 
+interface VersionInfo {
+  isCurrent: boolean;
+  label: string;
+}
+
 export type PageMetadata = {
   /**
     Arbitrary tags.
    */
   tags?: string[];
+  /**
+    The version of the page. This is relevant for versioned docs.
+    If the page is not versioned, this field should be undefined.
+   */
+  version?: VersionInfo;
   /**
     Page-level metadata. Should not be chunked.
    */
@@ -112,6 +113,10 @@ export type DeletePagesArgs = {
   inverse?: boolean;
 };
 
+export interface SourceVersions {
+  [sourceName: string]: VersionInfo[];
+}
+
 /**
   Data store for {@link Page} objects.
  */
@@ -137,6 +142,13 @@ export type PageStore = {
    */
   deletePages(args?: DeletePagesArgs): Promise<void>;
 
+  /**
+    Gets a list of versions for dataSources.
+   */
+  getDataSourceVersions(args?: {
+    dataSources: string[];
+  }): Promise<SourceVersions[]>;
+
   /**
     Close connection to data store.
    */

packages/mongodb-rag-core/src/dataSources/handleHtmlDocument.test.ts

@@ -20,7 +20,7 @@ const options: HandleHtmlPageFuncOptions = {
     `https://example.com/${pathInRepo}`.replace(/index\.html$/, "testing.html"),
   metadata: {
     productName: "Java Reactive Streams Driver",
-    version: javaVersion,
+    versionLabel: javaVersion,
   },
   extractMetadata: () => ({
     foo: "bar",
@@ -66,7 +66,7 @@ describe("handleHtmlDocument()", () => {
   it("should extract metadata from DOM", () => {
     expect(page.metadata).toMatchObject({
       foo: "bar",
-      version: "4.10",
+      versionLabel: "4.10",
       productName: "Java Reactive Streams Driver",
     });
   });

packages/mongodb-rag-core/src/test/MockPageStore.ts

@@ -12,5 +12,8 @@ export const makeMockPageStore = (): PageStore => {
     async deletePages(_args) {
       return;
     },
+    async getDataSourceVersions(_args) {
+      return [];
+    },
   };
 };