(EAI-1023, EAI-1024, EAI-1025): Scrubbed messages with redaction (#701)

* start on redaction

* add routes to trace handlers

* compilin

* update env and tests

* remove scrubbing and analysis from scripts dir

* update env vars

* fix ci errs

---------

Co-authored-by: Ben Perlmutter <[email protected]>

Author: mongodben

.drone.yml

@@ -35,7 +35,9 @@ steps:
       VECTOR_SEARCH_INDEX_NAME: vector_index
       OPENAI_CHAT_COMPLETION_MODEL_VERSION: 2023-06-01-preview
       OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: gpt-4o-mini
+      OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT: gpt-4.1-mini
       OPENAI_API_VERSION: "2024-06-01"
+      OPENAI_RESOURCE_NAME: "docs-ai-chatbot-useast2resource-sandbox"
       BRAINTRUST_TEXT_TO_DRIVER_PROJECT_NAME: "text-to-node-js-driver-benchmark"
       BRAINTRUST_API_KEY:
         from_secret: braintrust_api_key

package-lock.json

@@ -1,5 +1,6 @@
 PORT=3000
 OPENAI_ENDPOINT="https://<resource_name>.openai.azure.com/"
+OPENAI_RESOURCE_NAME="<resource_name>"
 OPENAI_API_KEY="<api_key>"
 OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT="docs-chatbot-embedding-ada-002"
 OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT="text-embedding-3-small"
@@ -13,6 +14,7 @@ VECTOR_SEARCH_INDEX_NAME="<index_name>"
 NODE_ENV="development"
 ALLOWED_ORIGINS="http://example.com,http://localhost:5173"
 OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT="<deployment name>"
+OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT="<deployment name>"
 OPENAI_API_VERSION="2024-06-01"
 JUDGE_EMBEDDING_MODEL="text-embedding-3-small"
 JUDGE_LLM="gpt-4o-mini"

packages/chatbot-server-mongodb-public/.env.example

@@ -13,6 +13,8 @@ env:
   OPENAI_CHAT_COMPLETION_DEPLOYMENT: gpt-4o
   OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT: "docs-chatbot-embedding-ada-002"
   OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT: "text-embedding-3-small"
+  OPENAI_RESOURCE_NAME: "docs-ai-chatbot-useast2resource"
+  OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT: gpt-4.1-mini
   JUDGE_LLM: "gpt-4o-mini"
   JUDGE_EMBEDDING_MODEL: "text-embedding-3-small"
   SLACK_COMMENT_CONVERSATION_ID: "C07A68V63EH"

packages/chatbot-server-mongodb-public/environments/production.yml

@@ -13,6 +13,8 @@ env:
   OPENAI_CHAT_COMPLETION_DEPLOYMENT: gpt-4o
   OPENAI_VERIFIED_ANSWER_EMBEDDING_DEPLOYMENT: "docs-chatbot-embedding-ada-002"
   OPENAI_RETRIEVAL_EMBEDDING_DEPLOYMENT: "text-embedding-3-small"
+  OPENAI_RESOURCE_NAME: "docs-ai-chatbot-useast2resource-sandbox"
+  OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT: gpt-4.1-mini
   BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME: "chatbot-responses-staging"
   JUDGE_LLM: "gpt-4o-mini"
   JUDGE_EMBEDDING_MODEL: "text-embedding-3-small"

packages/chatbot-server-mongodb-public/environments/staging.yml

@@ -1,9 +1,12 @@
 {
   "preset": "ts-jest",
-  "setupFiles": ["<rootDir>/src/jestEnvSetUp.ts"],
-  "setupFilesAfterEnv": ["<rootDir>/src/jestSetUp.ts"],
+  "testEnvironment": "node",
+  "setupFiles": [
+    "<rootDir>/src/test/jestSetUp.ts"
+  ],
   "testPathIgnorePatterns": [
-    "<rootDir>/build",
-    "<rootDir>/src/llmQualitativeTests"
-  ]
-}
+    "<rootDir>/build"
+  ],
+  "globalSetup": "<rootDir>/src/test/globalSetup.ts",
+  "globalTeardown": "<rootDir>/src/test/globalTeardown.ts"
+}

packages/chatbot-server-mongodb-public/jest.config.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@segment/analytics-node": "^2.2.1",
     "@slack/web-api": "^7.8.0",
+    "ahocorasick": "^1.0.2",
     "common-tags": "^1.8.2",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.0.3",
@@ -47,6 +48,7 @@
     "@babel/preset-env": "^7.22.5",
     "@babel/preset-typescript": "^7.22.5",
     "@lg-chat/rich-links": "^1.2.0",
+    "@types/ahocorasick": "^1.0.0",
     "@types/common-tags": "^1.8.1",
     "@types/cookie-parser": "^1.4.6",
     "@types/express": "^4.17.21",
@@ -60,6 +62,7 @@
     "eslint-config-prettier": "^8.8.0",
     "eslint-plugin-jest": "^27.2.1",
     "jest": "^29.6.1",
+    "mongodb-memory-server": "^10.1.4",
     "node-mocks-http": "^1.16.2",
     "nodemon": "^3.0.1",
     "prettier": "^2.8.7",

packages/chatbot-server-mongodb-public/package.json

@@ -2,12 +2,28 @@ const MODEL_JUDGE_ENV_VARS = {
   JUDGE_LLM: "",
   JUDGE_EMBEDDING_MODEL: "",
 };
+
 export const EVAL_ENV_VARS = {
   ...MODEL_JUDGE_ENV_VARS,
 };
 
+export const ANALYZER_ENV_VARS = {
+  OPENAI_ANALYZER_CHAT_COMPLETION_DEPLOYMENT: "",
+};
+
 export const TRACING_ENV_VARS = {
   ...MODEL_JUDGE_ENV_VARS,
+  ...ANALYZER_ENV_VARS,
+};
+
+export const AZURE_OPENAI_ENV_VARS = {
+  OPENAI_API_KEY: "",
+  OPENAI_API_VERSION: "",
+  OPENAI_RESOURCE_NAME: "",
+};
+
+export const PREPROCESSOR_ENV_VARS = {
+  OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT: "",
 };
 
 export const SLACK_ENV_VARS = {