Merge branch 'main' into tru-upload-more-file-types

Author: chrarnoldus

.changeset/bright-radios-kiss.md

@@ -0,0 +1,74 @@
+name: Evals
+
+on:
+    pull_request:
+        types: [labeled]
+    workflow_dispatch:
+
+env:
+    DOCKER_BUILDKIT: 1
+    COMPOSE_DOCKER_CLI_BUILD: 1
+
+jobs:
+    evals:
+        # Run if triggered manually or if PR has 'evals' label.
+        if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'evals')
+        runs-on: blacksmith-16vcpu-ubuntu-2404
+        timeout-minutes: 45
+
+        defaults:
+            run:
+                working-directory: packages/evals
+
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Create environment
+              run: |
+                  cat > .env.local << EOF
+                  OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY || 'test-key-for-build' }}
+                  EOF
+
+                  cat > .env.development << EOF
+                  NODE_ENV=development
+                  DATABASE_URL=postgresql://postgres:password@db:5432/evals_development
+                  REDIS_URL=redis://redis:6379
+                  HOST_EXECUTION_METHOD=docker
+                  EOF
+
+            - name: Build image
+              uses: docker/build-push-action@v5
+              with:
+                  context: .
+                  file: packages/evals/Dockerfile.runner
+                  tags: evals-runner:latest
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max
+                  push: false
+                  load: true
+
+            - name: Tag image
+              run: docker tag evals-runner:latest evals-runner
+
+            - name: Start containers
+              run: |
+                  docker compose up -d db redis
+                  timeout 60 bash -c 'until docker compose exec -T db pg_isready -U postgres; do sleep 2; done'
+                  timeout 60 bash -c 'until docker compose exec -T redis redis-cli ping | grep -q PONG; do sleep 2; done'
+                  docker compose run --rm runner sh -c 'nc -z db 5432 && echo "✓ Runner -> Database connection successful"'
+                  docker compose run --rm runner sh -c 'nc -z redis 6379 && echo "✓ Runner -> Redis connection successful"'
+                  docker compose run --rm runner docker ps
+
+            - name: Run database migrations
+              run: docker compose run --rm runner pnpm --filter @roo-code/evals db:migrate
+
+            - name: Run evals
+              run: docker compose run --rm runner pnpm --filter @roo-code/evals cli --ci
+
+            - name: Cleanup
+              if: always()
+              run: docker compose down -v --remove-orphans

.github/workflows/evals.yml

@@ -43,3 +43,4 @@ logs
 .idea/
 .qodo/
 .vercel
+.roo/mcp.json

.gitignore

@@ -1,5 +1,78 @@
 # kilo-code
 
+## [v4.36.0]
+
+- [#690](https://github.com/Kilo-Org/kilocode/pull/690) [`9b1451a`](https://github.com/Kilo-Org/kilocode/commit/9b1451a47bd2bc567646a4a0c2a12b42826ab9d1) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Include changes from Roo Code v3.19.7:
+
+    - Fix McpHub sidebar focus behavior to prevent unwanted focus grabbing
+    - Disable checkpoint functionality when nested git repositories are detected to prevent conflicts
+    - Remove unused Storybook components and dependencies to reduce bundle size
+    - Add data-testid ESLint rule for improved testing standards (thanks @elianiva!)
+    - Update development dependencies including eslint, knip, @types/node, i18next, fast-xml-parser, and @google/genai
+    - Improve CI infrastructure with GitHub Actions and Blacksmith runner migrations
+    - Replace explicit caching with implicit caching to reduce latency for Gemini models
+    - Clarify that the default concurrent file read limit is 15 files (thanks @olearycrew!)
+    - Fix copy button logic (thanks @samhvw8!)
+    - Fade buttons on history preview if no interaction in progress (thanks @sachasayan!)
+    - Allow MCP server refreshing, fix state changes in MCP server management UI view (thanks @taylorwilsdon!)
+    - Remove unnecessary npx usage in some npm scripts (thanks @user202729!)
+    - Bug fix for trailing slash error when using LiteLLM provider (thanks @kcwhite!)
+    - Fix Gemini 2.5 Pro Preview thinking budget bug
+    - Add Gemini Pro 06-05 model support (thanks @daniel-lxs and @shariqriazz!)
+    - Fix reading PDF, DOCX, and IPYNB files in read_file tool (thanks @samhvw8!)
+    - Fix Mermaid CSP errors with enhanced bundling strategy (thanks @KJ7LNW!)
+    - Improve model info detection for custom Bedrock ARNs (thanks @adamhill!)
+    - Add OpenAI Compatible embedder for codebase indexing (thanks @SannidhyaSah!)
+    - Fix multiple memory leaks in ChatView component (thanks @kiwina!)
+    - Fix WorkspaceTracker resource leaks by disposing FileSystemWatcher (thanks @kiwina!)
+    - Fix RooTips setTimeout cleanup to prevent state updates on unmounted components (thanks @kiwina!)
+    - Fix FileSystemWatcher leak in RooIgnoreController (thanks @kiwina!)
+    - Fix clipboard memory leak by clearing setTimeout in useCopyToClipboard (thanks @kiwina!)
+    - Fix ClineProvider instance cleanup (thanks @xyOz-dev!)
+    - Enforce codebase_search as primary tool for code understanding tasks (thanks @hannesrudolph!)
+    - Improve Docker setup for evals
+    - Move evals into pnpm workspace, switch from SQLite to Postgres
+    - Refactor MCP to use getDefaultEnvironment for stdio client transport (thanks @samhvw8!)
+    - Get rid of "partial" component in names referencing not necessarily partial messages (thanks @wkordalski!)
+    - Improve feature request template (thanks @elianiva!)
+
+- [#592](https://github.com/Kilo-Org/kilocode/pull/592) [`68c3d6e`](https://github.com/Kilo-Org/kilocode/commit/68c3d6e7a1250e08e2bd2b9cbbbd6b4312bad045) Thanks [@chrarnoldus](https://github.com/chrarnoldus)! - Workflow and rules configuration screen added
+
+### Patch Changes
+
+- [#697](https://github.com/Kilo-Org/kilocode/pull/697) [`9514f22`](https://github.com/Kilo-Org/kilocode/commit/9514f22a9d77b2d838ddcb97b5f2c5909aaea68a) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Add correct path to walkthrough files to show walkthrough on first load (thanks for the report @adamhill!)
+
+## [v4.35.1]
+
+- [#695](https://github.com/Kilo-Org/kilocode/pull/695) [`a7910eb`](https://github.com/Kilo-Org/kilocode/commit/a7910eba54a4ede296bfa82beddae71a1d9f77c5) Thanks [@kevinvandijk](https://github.com/kevinvandijk)! - Fix: Feedback button overlaps new mode creation dialog
+
+- [#693](https://github.com/Kilo-Org/kilocode/pull/693) [`2a9edf8`](https://github.com/Kilo-Org/kilocode/commit/2a9edf85ca2062d0b296430348ebac967f28febb) Thanks [@hassoncs](https://github.com/hassoncs)! - Temporarily remove .kilocode/rule loading for commit message generation until it works better
+
+## [v4.35.0]
+
+- [#633](https://github.com/Kilo-Org/kilocode/pull/633) [`347cf9e`](https://github.com/Kilo-Org/kilocode/commit/347cf9e6dc10d5b8706af5e111ccc854f7742566) Thanks [@hassoncs](https://github.com/hassoncs)! - # AI-Powered Git Commit Message Generation
+
+    Automatically generate meaningful Git commit messages using AI
+
+    ## How It Works
+
+    1. Stage your changes in Git as usual
+    2. Click the [KILO] square icon in the Source Control panel
+    3. The AI analyzes your staged changes and generates an appropriate commit message
+    4. The generated message is automatically populated in the commit input box
+
+- [#638](https://github.com/Kilo-Org/kilocode/pull/638) [`3d2e749`](https://github.com/Kilo-Org/kilocode/commit/3d2e749d51797681c018bc390757fdabefd60620) Thanks [@tru-kilo](https://github.com/tru-kilo)! - Added ability to favorite tasks
+
+## [v4.34.1]
+
+### Patch Changes
+
+- [#612](https://github.com/Kilo-Org/kilocode/pull/612) [`793cfdd`](https://github.com/Kilo-Org/kilocode/commit/793cfdd4fc1411c63c818e14b0b6ca8c5225a859) Thanks [@HadesArchitect](https://github.com/HadesArchitect)! - - #611 Customer Support Visibility (Added links to contact customer support)
+
+- [#672](https://github.com/Kilo-Org/kilocode/pull/672) [`c3d955c`](https://github.com/Kilo-Org/kilocode/commit/c3d955c2280258601d5f4b05101710e34d540075) Thanks [@chrarnoldus](https://github.com/chrarnoldus)! - Fixed response times for gemini-2.5-pro-preview being very slow (minutes instead of seconds)
+
+- [#671](https://github.com/Kilo-Org/kilocode/pull/671) [`e0a3740`](https://github.com/Kilo-Org/kilocode/commit/e0a37406fe8102b1acd4f8e9005652e828a14e36) Thanks [@chrarnoldus](https://github.com/chrarnoldus)! - OpenRouter bring-your-own-key models now have much more accurate cost estimates.
+
 ## [v4.34.0]
 
 ### Minor Changes

CHANGELOG.md

@@ -5,7 +5,7 @@
 	"scripts": {
 		"lint": "next lint",
 		"check-types": "tsc -b",
-		"dev": "scripts/check-services.sh && next dev --turbopack",
+		"dev": "scripts/check-services.sh && next dev",
 		"format": "prettier --write src",
 		"build": "next build",
 		"start": "next start"

apps/web-evals/package.json

@@ -0,0 +1,22 @@
+"use server"
+
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+import { exerciseLanguages, listDirectories } from "@roo-code/evals"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url)) // <repo>/apps/web-evals/src/actions
+
+const EVALS_REPO_PATH = path.resolve(__dirname, "../../../../../evals")
+
+export const getExercises = async () => {
+	const result = await Promise.all(
+		exerciseLanguages.map(async (language) => {
+			const languagePath = path.join(EVALS_REPO_PATH, language)
+			const exercises = await listDirectories(__dirname, languagePath)
+			return exercises.map((exercise) => `${language}/${exercise}`)
+		}),
+	)
+
+	return result.flat()
+}

apps/web-evals/src/actions/exercises.ts

@@ -0,0 +1,8 @@
+"use server"
+
+import { redisClient } from "@/lib/server/redis"
+
+export const getHeartbeat = async (runId: number) => {
+	const redis = await redisClient()
+	return redis.get(`heartbeat:${runId}`)
+}

apps/web-evals/src/actions/heartbeat.ts

@@ -1,6 +1,6 @@
 "use server"
 
-import { redisClient } from "./redis"
+import { redisClient } from "@/lib/server/redis"
 
 export const getRunners = async (runId: number) => {
 	const redis = await redisClient()

apps/web-evals/src/lib/server/runners.ts renamed to apps/web-evals/src/actions/runners.ts

@@ -1,7 +1,9 @@
 "use server"
 
-import { spawn } from "child_process"
+import * as path from "path"
 import fs from "fs"
+import { fileURLToPath } from "url"
+import { spawn } from "child_process"
 
 import { revalidatePath } from "next/cache"
 import pMap from "p-map"
@@ -12,11 +14,12 @@ import {
 	createRun as _createRun,
 	deleteRun as _deleteRun,
 	createTask,
+	getExercisesForLanguage,
 } from "@roo-code/evals"
 
 import { CreateRun } from "@/lib/schemas"
 
-import { getExercisesForLanguage } from "./exercises"
+const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals")
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 export async function createRun({ suite, exercises = [], systemPrompt, ...values }: CreateRun) {
@@ -37,9 +40,9 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 		}
 	} else {
 		for (const language of exerciseLanguages) {
-			const exercises = await getExercisesForLanguage(language)
+			const exercises = await getExercisesForLanguage(EVALS_REPO_PATH, language)
 
-			await pMap(exercises, (exercise) => createTask({ ...values, runId: run.id, language, exercise }), {
+			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
 				concurrency: 10,
 			})
 		}
@@ -52,9 +55,10 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 
 		const dockerArgs = [
 			`--name evals-controller-${run.id}`,
-			"--rm",
+			// "--rm",
 			"--network evals_default",
 			"-v /var/run/docker.sock:/var/run/docker.sock",
+			"-v /tmp/evals:/var/log/evals",
 			"-e HOST_EXECUTION_METHOD=docker",
 		]
 

apps/web-evals/src/lib/server/runs.ts renamed to apps/web-evals/src/actions/runs.ts

@@ -0,0 +1,24 @@
+import { NextResponse } from "next/server"
+
+export async function GET() {
+	try {
+		return NextResponse.json(
+			{
+				status: "healthy",
+				timestamp: new Date().toISOString(),
+				uptime: process.uptime(),
+				environment: process.env.NODE_ENV || "production",
+			},
+			{ status: 200 },
+		)
+	} catch (error) {
+		return NextResponse.json(
+			{
+				status: "unhealthy",
+				timestamp: new Date().toISOString(),
+				error: error instanceof Error ? error.message : "Unknown error",
+			},
+			{ status: 503 },
+		)
+	}
+}

apps/web-evals/src/lib/server/tasks.ts renamed to apps/web-evals/src/actions/tasks.ts

@@ -7,7 +7,7 @@ import { Ellipsis, Rocket } from "lucide-react"
 
 import type { Run, TaskMetrics } from "@roo-code/evals"
 
-import { deleteRun } from "@/lib/server/runs"
+import { deleteRun } from "@/actions/runs"
 import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
 import {
 	Button,