(EAI-473) Ingest OM/CM Docs (#492)

Author: nlarew

packages/chatbot-server-mongodb-public/evalCases/conversations.yml

@@ -1315,3 +1315,78 @@
     - atlas_cli
     - hallucination
     - user_feedback
+# Ops Manager / Cloud Manager FAQ
+- name: should understand OM/CM alerts
+  messages:
+    - content: Is there a way to generate an Ops Manager alert for replica sets with an even number of nodes?
+      role: user
+  expectation: >
+    The ASSISTANT should say that Ops Manager does not provide a built-in alert condition specifically for detecting replica sets with an even number of voting members. However, the user can create a custom alert condition to monitor the number of voting members in a replica set.
+  tags:
+    - ops-manager
+    - cloud-manager
+    - alerts
+- name: should understand OM/CM user account settings
+  messages:
+    - content: Can I reset my password in ops manager?
+      role: user
+  expectation: >
+    The ASSISTANT should suggest editing the password in the profile section of the user's account settings.
+  tags:
+    - ops-manager
+    - cloud-manager
+- name: should understand OM/CM organization & project user management
+  messages:
+    - content: How do I add a user to my organization or project in ops manager?
+      role: user
+  expectation: >
+    The ASSISTANT should mention user invitations, emails, and required
+    permissions. It should note that the inviter must have the Project Owner
+    role for project invitations or the Organization Owner role for organization
+    invitations.
+  tags:
+    - ops-manager
+    - cloud-manager
+- name: should understand OM/CM one-time passcodes
+  messages:
+    - content: Can I configure multiple Google Authenticator apps to use the same account in ops manager?
+      role: user
+  expectation: >
+    The ASSISTANT should say that the user can configure multiple Google Authenticator apps to use the same account by selecting the /Can't scan the barcode\?/i option during the procedure to Configure Two-Factor Authentication.
+  tags:
+    - ops-manager
+    - cloud-manager
+- name: should understand OM/CM project administration
+  messages:
+    - content: How do I delete my Ops Manager project?
+      role: user
+    - content: |
+        A user with the Project Owner access for the projects can close a project.
+
+        You can delete a project from the project's Settings page. For details, see Delete One Project.
+
+        The operation cannot be undone.
+      role: assistant
+  expectation: >
+    The ASSISTANT should say that a user with the Project Owner role can delete a project from the Projects list in the Ops Manager UI.
+  tags:
+    - ops-manager
+    - cloud-manager
+- name: should understand OM/CM agents and plans
+  messages:
+    - content: How does Ops Manager manage MongoDB deployments?
+      role: user
+  expectation: >
+    The ASSISTANT should describe how Ops Manager uses automation agents that follow plans to manage MongoDB deployments.
+  tags:
+    - ops-manager
+    - cloud-manager
+- name: should understand OM/CM maintenance processes
+  messages:
+    - content: How does Ops Manager perform maintenance on cluster nodes?
+      role: user
+  expectation: >
+    The ASSISTANT should say that Ops Manager uses rolling restarts to perform maintenance on cluster nodes.
+  tags:
+    - ops-manager
+    - cloud-manager

packages/chatbot-server-mongodb-public/src/conversations.eval.ts

@@ -119,42 +119,45 @@ Eval("mongodb-chatbot-conversations", {
     const faqCases = getConversationsEvalCasesFromYaml(
       fs.readFileSync(path.resolve(basePath, "faq_conversations.yml"), "utf8")
     );
-    return [...miscCases, ...faqCases].map((evalCase) => {
-      const prevConversationMessages = evalCase.messages.slice(0, -1).map(
-        (m) =>
-          ({
-            content: m.content,
-            role: m.role,
-            id: new ObjectId(),
-            createdAt: new Date(),
-          } satisfies Message)
-      );
-      prevConversationMessages.unshift({
-        ...systemPrompt,
-        id: new ObjectId(),
-        createdAt: new Date(),
-      } satisfies Message);
-      const latestMessageText = evalCase.messages.at(-1)?.content;
-      assert(latestMessageText, "No latest message text found");
-      return {
-        name: evalCase.name,
-        tags: evalCase.tags as MongoDbTag[],
-        input: {
-          latestMessageText,
-          previousConversation: {
-            messages: prevConversationMessages,
-            _id: new ObjectId(),
-            createdAt: new Date(),
+    return [...miscCases, ...faqCases]
+      .filter((evalCase) => evalCase.tags?.includes("ops-manager"))
+      .map((evalCase) => {
+        const prevConversationMessages = evalCase.messages.slice(0, -1).map(
+          (m) =>
+            ({
+              content: m.content,
+              role: m.role,
+              id: new ObjectId(),
+              createdAt: new Date(),
+            } satisfies Message)
+        );
+        prevConversationMessages.unshift({
+          ...systemPrompt,
+          id: new ObjectId(),
+          createdAt: new Date(),
+        } satisfies Message);
+        const latestMessageText = evalCase.messages.at(-1)?.content;
+        assert(latestMessageText, "No latest message text found");
+        return {
+          name: evalCase.name,
+          tags: evalCase.tags as MongoDbTag[],
+          input: {
+            latestMessageText,
+            previousConversation: {
+              messages: prevConversationMessages,
+              _id: new ObjectId(),
+              createdAt: new Date(),
+            },
           },
-        },
-        expected: null,
-        metadata: null,
-      } satisfies ConversationEvalCase;
-    });
+          expected: null,
+          metadata: null,
+        } satisfies ConversationEvalCase;
+      });
   },
-  experimentName: "mongodb-chatbot-latest",
+  experimentName: "mongodb-chatbot-omcm-latest",
   metadata: {
-    description: "Evaluates how well the MongoDB AI Chatbot RAG pipeline works",
+    description:
+      "Evaluates how well the MongoDB AI Chatbot RAG pipeline works for Ops/Cloud Manager questions",
   },
   maxConcurrency: 2,
   async task(input): Promise<ConversationTaskOutput> {