chore: add expectAria to perform

Author: pavelfeldman

examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts

@@ -1,4 +1,4 @@
-import { test } from '../fixtures';
+import { test, expect } from '../fixtures';
 
 test.use({
   agent: {
@@ -16,6 +16,8 @@ test.describe('Adding New Todos', () => {
     await page.perform(`Add "Walk the dog" todo item`);
     await page.perform(`Add "Read a book" todo item`);
 
+    await page.perform('Ensure all three todos appear in the list in order of creation. Use browser_expect_list_visible to verify the list. Do not report_result right away, call browser_expect_list_visible tool. You must.');
+
     await page.perform(`Ensure each todo has an unchecked checkbox`);
     await page.perform(`Ensure counter shows "3 items left" (plural)`);
     await page.perform(`Ensure input field is cleared`);

examples/todomvc/tests/adding-new-todos/add-multiple-todos-2.spec.ts-cache.json

@@ -108,5 +108,16 @@
         "intent": "Check that the input field is cleared (empty)."
       }
     ]
+  },
+  "Ensure all three todos appear in the list in order of creation. Use browser_expect_list_visible to verify the list. Do not report_result right away, call browser_expect_list_visible tool. You must.": {
+    "timestamp": 1766528380006,
+    "actions": [
+      {
+        "method": "expectAria",
+        "template": "- list:\n  - listitem: Buy groceries\n  - listitem: Walk the dog\n  - listitem: Read a book",
+        "code": "await expect(page.locator('body')).toMatchAria(- list:\n  - listitem: Buy groceries\n  - listitem: Walk the dog\n  - listitem: Read a book);",
+        "intent": "Ensure all three todos (Buy groceries, Walk the dog, Read a book) appear in the list in order of creation."
+      }
+    ]
   }
 }

packages/injected/src/ariaSnapshot.ts

@@ -16,10 +16,10 @@
 
 import { ariaPropsEqual } from '@isomorphic/ariaSnapshot';
 import { escapeRegExp, longestCommonSubstring, normalizeWhiteSpace } from '@isomorphic/stringUtils';
+import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from '@isomorphic/yaml';
 
 import { computeBox, getElementComputedStyle, isElementVisible } from './domUtils';
 import * as roleUtils from './roleUtils';
-import { yamlEscapeKeyIfNeeded, yamlEscapeValueIfNeeded } from './yaml';
 
 import type { AriaProps, AriaRegex, AriaTextValue, AriaRole, AriaTemplateNode } from '@isomorphic/ariaSnapshot';
 import type { Box } from './domUtils';

packages/playwright-core/src/server/agent/actionRunner.ts

@@ -16,7 +16,9 @@
 
 import { serializeExpectedTextValues } from '../utils/expectUtils';
 import { monotonicTime } from '../../utils/isomorphic/time';
+import { parseAriaSnapshotUnsafe } from '../../utils/isomorphic/ariaSnapshot';
 import { ProgressController } from '../progress';
+import { yaml } from '../../utilsBundle';
 
 import type * as actions from './actions';
 import type { Page } from '../page';
@@ -75,7 +77,7 @@ async function innerRunAction(progress: Progress, page: Page, action: actions.Ac
       break;
     case 'expectVisible': {
       const result = await frame.expect(progress, action.selector, { expression: 'to.be.visible', isNot: false });
-      if (result.errorMessage)
+      if (!result.matches)
         throw new Error(result.errorMessage);
       break;
     }
@@ -94,6 +96,13 @@ async function innerRunAction(progress: Progress, page: Page, action: actions.Ac
         throw new Error(result.errorMessage);
       break;
     }
+    case 'expectAria': {
+      const expectedValue = parseAriaSnapshotUnsafe(yaml, action.template);
+      const result = await frame.expect(progress, 'body', { expression: 'to.match.aria', expectedValue, isNot: false });
+      if (!result.matches)
+        throw new Error(result.errorMessage);
+      break;
+    }
   }
 }
 
@@ -110,6 +119,7 @@ export function generateActionTimeout(action: actions.Action): number {
       return 5000;
     case 'expectVisible':
     case 'expectValue':
+    case 'expectAria':
       return 1;  // one shot
   }
 }
@@ -127,6 +137,7 @@ export function performActionTimeout(action: actions.Action): number {
       return 0;  // no timeout
     case 'expectVisible':
     case 'expectValue':
+    case 'expectAria':
       return 5000;  // default expect timeout.
   }
 }

packages/playwright-core/src/server/agent/actions.ts

@@ -77,7 +77,24 @@ export type ExpectValue = {
   value: string;
 };
 
-export type Action = ClickAction | DragAction | HoverAction | SelectOptionAction | PressAction | PressSequentiallyAction | FillAction | SetChecked | ExpectVisible | ExpectValue;
+export type ExpectAria = {
+  method: 'expectAria';
+  template: string;
+};
+
+export type Action =
+  | ClickAction
+  | DragAction
+  | HoverAction
+  | SelectOptionAction
+  | PressAction
+  | PressSequentiallyAction
+  | FillAction
+  | SetChecked
+  | ExpectVisible
+  | ExpectValue
+  | ExpectAria;
+
 export type ActionWithCode = Action & {
   code: string;
   intent?: string;

packages/playwright-core/src/server/agent/agent.ts

@@ -40,7 +40,17 @@ export async function pagePerform(progress: Progress, page: Page, options: chann
   if (await cachedPerform(progress, context, options))
     return { turns: 0, inputTokens: 0, outputTokens: 0 };
 
-  const { usage } = await perform(progress, context, options.task, undefined, options);
+  const task = `
+### Instructions
+- Perform the following task on the page.
+- Your reply should be a tool call that performs action the page.
+- If you are asked to verify / assert something, don't just examine the snapshot, generate action that verifies / asserts the condition using tool that starts with "browser_expect_".
+
+### Task
+${options.task}
+`;
+
+  const { usage } = await perform(progress, context, task, undefined, options);
   await updateCache(context, options);
   return usage;
 }

packages/playwright-core/src/server/agent/codegen.ts

@@ -15,7 +15,7 @@
  */
 
 import { asLocator } from '../../utils/isomorphic/locatorGenerators';
-import { escapeWithQuotes, formatObjectOrVoid } from '../../utils/isomorphic/stringUtils';
+import { escapeTemplateString, escapeWithQuotes, formatObjectOrVoid } from '../../utils/isomorphic/stringUtils';
 
 import type * as actions from './actions';
 import type { Language } from '../../utils/isomorphic/locatorGenerators';
@@ -73,6 +73,9 @@ export async function generateCode(sdkLanguage: Language, action: actions.Action
         return `await expect(page.${locator}).toBeChecked({ checked: ${action.value === 'true'} });`;
       return `await expect(page.${locator}).toHaveValue(${escapeWithQuotes(action.value)});`;
     }
+    case 'expectAria': {
+      return `await expect(page.locator('body')).toMatchAria(\`\n${escapeTemplateString(action.template)}\n\`);`;
+    }
   }
   // @ts-expect-error
   throw new Error('Unknown action ' + action.method);

packages/playwright-core/src/server/agent/tools.ts

@@ -16,6 +16,7 @@
 
 import { z } from '../../mcpBundle';
 import { getByRoleSelector, getByTextSelector } from '../../utils/isomorphic/locatorUtils';
+import { yamlEscapeValueIfNeeded } from '../../utils/isomorphic/yaml';
 
 import type zod from 'zod';
 import type * as loopTypes from '@lowire/loop';
@@ -315,6 +316,29 @@ const expectValue = defineTool({
   },
 });
 
+const expectList = defineTool({
+  schema: {
+    name: 'browser_expect_list_visible',
+    title: 'Expect list visible',
+    description: 'Expect list is visible on the page, ensures items is present in the element in the exact order',
+    inputSchema: z.object({
+      listRole: z.string().describe('Aria role of the list element as in the snapshot'),
+      listAccessibleName: z.string().optional().describe('Accessible name of the list element as in the snapshot'),
+      itemRole: z.string().describe('Aria role of the list items as in the snapshot, should all be the same'),
+      items: z.array(z.string().describe('Text to look for in the list item, can be either from accessible name of self / nested text content')),
+    }),
+  },
+
+  handle: async (context, params) => {
+    const template = `- ${params.listRole}:
+${params.items.map(item => `  - ${params.itemRole}: ${yamlEscapeValueIfNeeded(item)}`).join('\n')}`;
+    return await context.runActionAndWait({
+      method: 'expectAria',
+      template,
+    });
+  },
+});
+
 export default [
   snapshot,
   click,
@@ -327,4 +351,5 @@ export default [
   expectVisible,
   expectVisibleText,
   expectValue,
+  expectList,
 ] as ToolDefinition<any>[];

packages/playwright-core/src/utils.ts

@@ -31,6 +31,7 @@ export * from './utils/isomorphic/stringUtils';
 export * from './utils/isomorphic/time';
 export * from './utils/isomorphic/timeoutRunner';
 export * from './utils/isomorphic/urlMatch';
+export * from './utils/isomorphic/yaml';
 
 export * from './server/utils/ascii';
 export * from './server/utils/comparators';

packages/playwright-core/src/utils/isomorphic/yaml.ts

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function yamlEscapeKeyIfNeeded(str: string): string {
+  if (!yamlStringNeedsQuotes(str))
+    return str;
+  return `'` + str.replace(/'/g, `''`) + `'`;
+}
+
+export function yamlEscapeValueIfNeeded(str: string): string {
+  if (!yamlStringNeedsQuotes(str))
+    return str;
+  return '"' + str.replace(/[\\"\x00-\x1f\x7f-\x9f]/g, c => {
+    switch (c) {
+      case '\\':
+        return '\\\\';
+      case '"':
+        return '\\"';
+      case '\b':
+        return '\\b';
+      case '\f':
+        return '\\f';
+      case '\n':
+        return '\\n';
+      case '\r':
+        return '\\r';
+      case '\t':
+        return '\\t';
+      default:
+        const code = c.charCodeAt(0);
+        return '\\x' + code.toString(16).padStart(2, '0');
+    }
+  }) + '"';
+}
+
+function yamlStringNeedsQuotes(str: string): boolean {
+  if (str.length === 0)
+    return true;
+
+  // Strings with leading or trailing whitespace need quotes
+  if (/^\s|\s$/.test(str))
+    return true;
+
+  // Strings containing control characters need quotes
+  if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/.test(str))
+    return true;
+
+  // Strings starting with '-' need quotes
+  if (/^-/.test(str))
+    return true;
+
+  // Strings containing ':' or '\n' followed by a space or at the end need quotes
+  if (/[\n:](\s|$)/.test(str))
+    return true;
+
+  // Strings containing '#' preceded by a space need quotes (comment indicator)
+  if (/\s#/.test(str))
+    return true;
+
+  // Strings that contain line breaks need quotes
+  if (/[\n\r]/.test(str))
+    return true;
+
+  // Strings starting with indicator characters or quotes need quotes
+  if (/^[&*\],?!>|@"'#%]/.test(str))
+    return true;
+
+  // Strings containing special characters that could cause ambiguity
+  if (/[{}`]/.test(str))
+    return true;
+
+  // YAML array starts with [
+  if (/^\[/.test(str))
+    return true;
+
+  // Non-string types recognized by YAML
+  if (!isNaN(Number(str)) || ['y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off', 'null'].includes(str.toLowerCase()))
+    return true;
+
+  return false;
+}