Lotte/create qa chatbot experiments script

components/qaChatbot/apiHandler.ts

@@ -1,25 +1,22 @@
-import { openai, OpenAIResponsesProviderOptions } from "@ai-sdk/openai";
 import {
-  streamText,
   UIMessage,
   experimental_createMCPClient as createMCPClient,
   MCPTransport,
-  stepCountIs,
 } from "ai";
 import {
   observe,
   startActiveObservation,
   updateActiveObservation,
   updateActiveTrace,
+  getActiveTraceId,
 } from "@langfuse/tracing";
 import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp";
 import { LangfuseClient } from "@langfuse/client";
-import { getActiveTraceId } from "@langfuse/tracing";
+import { chatbotStep } from "./chatbotCore";
 import { after } from "next/server";
 import { flush } from "@/src/instrumentation";
-import { trace } from "@opentelemetry/api";
 
-const langfuseClient = new LangfuseClient({
+export const langfuseClient = new LangfuseClient({
   baseUrl: process.env.NEXT_PUBLIC_EU_LANGFUSE_BASE_URL,
   publicKey: process.env.NEXT_PUBLIC_EU_LANGFUSE_PUBLIC_KEY,
   secretKey: process.env.EU_LANGFUSE_SECRET_KEY,
@@ -52,30 +49,10 @@ export const handler = async (req: Request) => {
     input: inputText,
   });
 
-  // Get prompt from Langfuse
   const prompt = await tracedGetPrompt("langfuse-docs-assistant-chat", {
     type: "chat",
   });
 
-  const reasoningSummary = prompt.config
-    .reasoningSummary as 'low' | 'medium' | 'high' | undefined;
-  const textVerbosity = prompt.config
-    .textVerbosity as 'low' | 'medium' | 'high' | undefined;
-  const reasoningEffort = prompt.config
-    .reasoningEffort as 'low' | 'medium' | 'high' | undefined;
-
-  // Convert UI messages format of Vercel AI SDK to: [{ role: "user", content: "text..." }]
-  const chatHistory = messages.map((msg) => ({
-    role: msg.role,
-    content: msg.parts
-      .filter((part) => part.type === "text")
-      .map((part) => part.text)
-      .join(""),
-  }));
-
-  // Compile the prompt with chat history as a message placeholder
-  const compiledPrompt = prompt.compile({}, { chat_history: chatHistory });
-
   // Initialize MCP client using Streamable HTTP transport (works with our MCP server)
   const mcpClient = await startActiveObservation(
     "create-mcp-client",
@@ -90,42 +67,7 @@ export const handler = async (req: Request) => {
     },
   );
 
-  // Discover all tools exposed by the MCP server
-  const tools = await mcpClient.tools();
-
-
-  const result = streamText({
-    model: openai(String(prompt.config.model)),
-    providerOptions: {
-      openai: {
-        reasoningSummary,
-        textVerbosity,
-        reasoningEffort,
-      } satisfies OpenAIResponsesProviderOptions,
-    },
-    messages: compiledPrompt,
-    tools,
-    stopWhen: stepCountIs(10),
-    experimental_telemetry: {
-      isEnabled: true,
-      metadata: { langfusePrompt: prompt.toJSON() },
-    },
-    onFinish: async (result) => {
-      await mcpClient.close();
-
-      const latestText = Array.isArray((result as any).content)
-        ? [...((result as any).content as Array<any>)]
-            .reverse()
-            .find((part: any) => part?.type === "text")?.text
-        : (result as any).content;
-
-      updateActiveObservation({ output: latestText }, { asType: "generation" });
-      updateActiveTrace({
-        output: latestText,
-      });
-      trace.getActiveSpan().end();
-    },
-  });
+  const result = await chatbotStep(prompt, messages, mcpClient);
 
   // Schedule flush after request is finished
   after(async () => await flush());

components/qaChatbot/chatbotCore.ts

@@ -0,0 +1,83 @@
+import { openai, OpenAIResponsesProviderOptions } from "@ai-sdk/openai";
+import {
+  streamText,
+  UIMessage,
+  experimental_createMCPClient as createMCPClient,
+  stepCountIs,
+} from "ai";
+import {
+  updateActiveObservation,
+  updateActiveTrace,
+} from "@langfuse/tracing";
+import { trace } from "@opentelemetry/api";
+import { langfuseClient } from "./apiHandler";
+
+// Type declarations
+export type Prompt = Awaited<ReturnType<typeof langfuseClient.prompt.get>>;
+export type MCPClient = Awaited<ReturnType<typeof createMCPClient>>;
+
+export interface PromptConfig {
+  model?: string;
+  reasoningSummary?: 'low' | 'medium' | 'high';
+  textVerbosity?: 'low' | 'medium' | 'high';
+  reasoningEffort?: 'low' | 'medium' | 'high';
+  [key: string]: unknown; // Allow other properties
+}
+
+export const getProcessedPrompt = (prompt: Prompt, messages: UIMessage[]) => {
+  const promptConfig = prompt.config as PromptConfig;
+
+  // Convert UI messages format of Vercel AI SDK to: [{ role: "user", content: "text..." }]
+  const chatHistory = messages.map((msg) => ({
+    role: msg.role,
+    content: msg.parts
+      .filter((part) => part.type === "text")
+      .map((part) => part.text)
+      .join(""),
+  }));
+
+  // Compile the prompt with chat history as a message placeholder
+  const compiledPrompt = prompt.compile({}, { chat_history: chatHistory });
+  return { compiledPrompt, promptConfig };
+}
+
+export const chatbotStep = async (prompt: Prompt, messages: UIMessage[], mcpClient: MCPClient) => {
+  const { compiledPrompt, promptConfig } = getProcessedPrompt(prompt, messages);
+  // Discover all tools exposed by the MCP server
+  const tools = await mcpClient.tools();
+  const result = streamText({
+    model: openai(String(promptConfig.model)),
+    providerOptions: {
+      openai: {
+        reasoningSummary: promptConfig.reasoningSummary,
+        textVerbosity: promptConfig.textVerbosity,
+        reasoningEffort: promptConfig.reasoningEffort,
+      } satisfies OpenAIResponsesProviderOptions,
+    },
+    messages: compiledPrompt,
+    tools,
+    stopWhen: stepCountIs(10),
+    experimental_telemetry: {
+      isEnabled: true,
+      metadata: { langfusePrompt: prompt.toJSON() },
+    }, 
+    onFinish: async (result) => {
+      await mcpClient.close();
+
+      const latestText = Array.isArray((result as any).content)
+        ? [...((result as any).content as Array<any>)]
+            .reverse()
+            .find((part: any) => part?.type === "text")?.text
+        : (result as any).content;
+
+      updateActiveObservation({ output: latestText }, { asType: "generation" });
+      updateActiveTrace({
+        output: latestText,
+      });
+      trace.getActiveSpan().end();
+    }
+  });
+
+  return result;
+}
+

package.json

@@ -16,7 +16,8 @@
     "postbuild:llms-txt": "node scripts/generate_llms_txt.js",
     "link-check": "node scripts/check-links.js",
     "sitemap-check": "node scripts/check-sitemap-links.js",
-    "nuke": "rm -rf .next && rm -rf node_modules && pnpm install"
+    "nuke": "rm -rf .next && rm -rf node_modules && pnpm install",
+    "experiment:chatbot": "tsx scripts/run-chatbot-experiment.ts"
   },
   "engines": {
     "node": "22"
@@ -30,12 +31,13 @@
     "@hookform/resolvers": "^3.9.1",
     "@icons-pack/react-simple-icons": "^13.7.0",
     "@inkeep/cxkit-react": "^0.5.98",
-    "@langfuse/client": "4.0.0",
-    "@langfuse/core": "4.0.0",
-    "@langfuse/otel": "4.0.0",
-    "@langfuse/tracing": "4.0.0",
+    "@langfuse/client": "4.4.2",
+    "@langfuse/core": "4.4.2",
+    "@langfuse/otel": "4.4.2",
+    "@langfuse/tracing": "4.4.2",
     "@modelcontextprotocol/sdk": "^1.13.2",
     "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/sdk-node": "^0.208.0",
     "@opentelemetry/sdk-trace-node": "^2.0.1",
     "@radix-ui/react-accordion": "^1.2.3",
     "@radix-ui/react-avatar": "^1.1.3",
@@ -66,6 +68,7 @@
     "autoprefixer": "^10.4.21",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
+    "dotenv": "^16.4.7",
     "embla-carousel-react": "^8.6.0",
     "framer-motion": "^11.15.0",
     "geist": "^1.3.1",
@@ -81,6 +84,7 @@
     "next-sitemap": "^4.2.3",
     "nextra": "3.0.15",
     "nextra-theme-docs": "3.0.15",
+    "openai": "^4.75.0",
     "openai-edge": "^1.2.2",
     "postcss": "^8.4.49",
     "posthog-js": "^1.203.1",
@@ -110,6 +114,7 @@
     "cross-env": "^7.0.3",
     "markdown-link-check": "^3.13.6",
     "puppeteer": "^24.23.0",
+    "tsx": "^4.19.2",
     "typescript": "^5.6.3",
     "xml2js": "^0.6.2"
   },