chore: iterate on perform

Author: pavelfeldman

docs/src/api/class-browser.md

@@ -267,6 +267,9 @@ await browser.CloseAsync();
 ### option: Browser.newContext.storageStatePath = %%-csharp-java-context-option-storage-state-path-%%
 * since: v1.9
 
+### option: Browser.newContext.agent = %%-js-context-option-agent-%%
+* since: v1.58
+
 ## async method: Browser.newPage
 * since: v1.8
 - returns: <[Page]>

docs/src/api/class-page.md

@@ -2024,6 +2024,38 @@ Name of the function on the window object
 
 Callback function which will be called in Playwright's context.
 
+## async method: Page.extract
+* since: v1.58
+* langs: js
+- returns: <[any]>
+
+Extract information from the page using the agentic loop, return it in a given Zod format.
+
+**Usage**
+
+```js
+await page.extract('List of items in the cart', z.object({
+  title: z.string().describe('Item title to extract'),
+  price: z.string().describe('Item price to extract'),
+}).array());
+```
+
+### param: Page.extract.query
+* since: v1.58
+- `query` <[string]>
+
+Task to perform using agentic loop.
+
+### param: Page.extract.schema
+* since: v1.58
+- `schema` <[z.ZodSchema]>
+
+### option: Page.extract.maxTurns
+* since: v1.58
+- `maxTurns` <[int]>
+
+Maximum number of agentic steps to take while extracting the information.
+
 ## async method: Page.fill
 * since: v1.8
 * discouraged: Use locator-based [`method: Locator.fill`] instead. Read more about [locators](../locators.md).
@@ -2997,6 +3029,38 @@ Whether or not to generate tagged (accessible) PDF. Defaults to `false`.
 
 Whether or not to embed the document outline into the PDF. Defaults to `false`.
 
+## async method: Page.perform
+* since: v1.58
+* langs: js
+
+Perform action using agentic loop.
+
+**Usage**
+
+```js
+await page.perform('Click submit button');
+```
+
+### param: Page.perform.task
+* since: v1.58
+- `task` <[string]>
+
+Task to perform using agentic loop.
+
+### option: Page.perform.key
+* since: v1.58
+- `key` <[string]>
+
+All the agentic actions are converted to the Playwright calls and are cached.
+By default, they are cached globally with the `task` as a key. This option allows controlling the cache key explicitly.
+
+### option: Page.perform.maxTurns
+* since: v1.58
+- `maxTurns` <[int]>
+
+Maximum number of agentic steps to take while performing this action.
+
+
 ## async method: Page.press
 * since: v1.8
 * discouraged: Use locator-based [`method: Locator.press`] instead. Read more about [locators](../locators.md).

docs/src/api/params.md

@@ -370,6 +370,16 @@ It makes the execution of the tests non-deterministic.
 Emulates consistent window screen size available inside web page via `window.screen`. Is only used when the
 [`option: viewport`] is set.
 
+## js-context-option-agent
+* langs: js
+- `agent` <[Object]>
+  - `provider` <[string]> LLM provider to use
+  - `model` <[string]> Model identifier within provider
+  - `cacheDir` ?<[string]> Cache folder to use/generate code for performed actions into. Cache is not used if not specified (default).
+  - `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defauls to 'auto'
+
+Agent settings for [`method: Page.perform`] and [`method: Page.extract`].
+
 ## fetch-param-url
 - `url` <[string]>
 

docs/src/test-api/class-testoptions.md

@@ -46,6 +46,10 @@ export default defineConfig({
 });
 ```
 
+## property: TestOptions.agent = %%-js-context-option-agent-%%
+* since: v1.58
+
+
 ## property: TestOptions.baseURL = %%-context-option-baseURL-%%
 * since: v1.10
 

package-lock.json

@@ -57,7 +57,7 @@
     "@eslint/compat": "^1.3.2",
     "@eslint/eslintrc": "^3.3.1",
     "@eslint/js": "^9.34.0",
-    "@lowire/loop": "^0.0.4",
+    "@lowire/loop": "^0.0.6",
     "@modelcontextprotocol/sdk": "^1.17.5",
     "@octokit/graphql-schema": "^15.26.0",
     "@stylistic/eslint-plugin": "^5.2.3",

package.json

@@ -27,6 +27,11 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector
   state?: 'visible'|'attached';
 };
 
+// @ts-ignore this will be any if zod is not installed
+type ZodTypeAny = import('zod').ZodTypeAny;
+// @ts-ignore this will be any if zod is not installed
+type ZodInfer<T extends ZodTypeAny> = import('zod').infer<T>;
+
 /**
  * Page provides methods to interact with a single tab in a [Browser](https://playwright.dev/docs/api/class-browser),
  * or an [extension background page](https://developer.chrome.com/extensions/background_pages) in Chromium. One
@@ -1013,6 +1018,24 @@ export interface Page {
      */
     behavior?: 'wait'|'ignoreErrors'|'default'
   }): Promise<void>;
+
+  /**
+   * Extract information from the page using the agentic loop, return it in a given Zod format.
+   *
+   * **Usage**
+   *
+   * ```js
+   * await page.extract('List of items in the cart', z.object({
+   *   title: z.string().describe('Item title to extract'),
+   *   price: z.string().describe('Item price to extract'),
+   * }).array());
+   * ```
+   *
+   * @param query Task to perform using agentic loop.
+   * @param schema
+   * @param options
+   */
+  extract<Schema extends ZodTypeAny>(query: string, schema: Schema): Promise<ZodInfer<Schema>>;
   /**
    * Emitted when the page closes.
    */
@@ -3796,6 +3819,31 @@ export interface Page {
     width?: string|number;
   }): Promise<Buffer>;
 
+  /**
+   * Perform action using agentic loop.
+   *
+   * **Usage**
+   *
+   * ```js
+   * await page.perform('Click submit button');
+   * ```
+   *
+   * @param task Task to perform using agentic loop.
+   * @param options
+   */
+  perform(task: string, options?: {
+    /**
+     * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally
+     * with the `task` as a key. This option allows controlling the cache key explicitly.
+     */
+    key?: string;
+
+    /**
+     * Maximum number of agentic steps to take while performing this action.
+     */
+    maxTurns?: number;
+  }): Promise<void>;
+
   /**
    * **NOTE** Use locator-based [locator.press(key[, options])](https://playwright.dev/docs/api/class-locator#locator-press)
    * instead. Read more about [locators](https://playwright.dev/docs/locators).
@@ -22033,6 +22081,32 @@ export interface BrowserContextOptions {
    */
   acceptDownloads?: boolean;
 
+  /**
+   * Agent settings for [page.perform(task[, options])](https://playwright.dev/docs/api/class-page#page-perform) and
+   * [page.extract(query, schema[, options])](https://playwright.dev/docs/api/class-page#page-extract).
+   */
+  agent?: {
+    /**
+     * LLM provider to use
+     */
+    provider: string;
+
+    /**
+     * Model identifier within provider
+     */
+    model: string;
+
+    /**
+     * Cache folder to use/generate code for performed actions into. Cache is not used if not specified (default).
+     */
+    cacheDir?: string;
+
+    /**
+     * Cache control, defauls to 'auto'
+     */
+    cacheMode?: 'force'|'ignore'|'auto';
+  };
+
   /**
    * When using [page.goto(url[, options])](https://playwright.dev/docs/api/class-page#page-goto),
    * [page.route(url, handler[, options])](https://playwright.dev/docs/api/class-page#page-route),

packages/playwright-client/types/types.d.ts

@@ -17,6 +17,7 @@
 import type { BrowserContext } from './browserContext';
 import type { APIRequestContext } from './fetch';
 import type { StackFrame } from '@protocol/channels';
+import type { Page } from './page';
 
 // Instrumentation can mutate the data, for example change apiName or stepId.
 export interface ApiCallData {
@@ -35,6 +36,7 @@ export interface ClientInstrumentation {
   onApiCallBegin(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any> }): void;
   onApiCallEnd(apiCall: ApiCallData): void;
   onWillPause(options: { keepTestTimeout: boolean }): void;
+  onPage(page: Page): void;
 
   runAfterCreateBrowserContext(context: BrowserContext): Promise<void>;
   runAfterCreateRequestContext(context: APIRequestContext): Promise<void>;
@@ -46,7 +48,7 @@ export interface ClientInstrumentationListener {
   onApiCallBegin?(apiCall: ApiCallData, channel: { type: string, method: string, params?: Record<string, any>  }): void;
   onApiCallEnd?(apiCall: ApiCallData): void;
   onWillPause?(options: { keepTestTimeout: boolean }): void;
-
+  onPage?(page: Page): void;
   runAfterCreateBrowserContext?(context: BrowserContext): Promise<void>;
   runAfterCreateRequestContext?(context: APIRequestContext): Promise<void>;
   runBeforeCloseBrowserContext?(context: BrowserContext): Promise<void>;

packages/playwright-core/src/client/clientInstrumentation.ts

@@ -54,6 +54,7 @@ import type * as api from '../../types/types';
 import type { ByRoleOptions } from '../utils/isomorphic/locatorUtils';
 import type { URLMatch } from '../utils/isomorphic/urlMatch';
 import type * as channels from '@protocol/channels';
+import type z from 'zod';
 
 type PDFOptions = Omit<channels.PagePdfParams, 'width' | 'height' | 'margin'> & {
   width?: string | number,
@@ -116,6 +117,7 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
 
   constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.PageInitializer) {
     super(parent, type, guid, initializer);
+    this._instrumentation.onPage(this);
     this._browserContext = parent as unknown as BrowserContext;
     this._timeoutSettings = new TimeoutSettings(this._platform, this._browserContext._timeoutSettings);
 
@@ -844,6 +846,14 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
     return result.pdf;
   }
 
+  async perform(task: string, options: { key?: string, maxTurns?: number } = {}): Promise<void> {
+    throw new Error('Not implemented in playwright-core');
+  }
+
+  extract<Schema extends z.ZodTypeAny>(query: string, schema: Schema, options: { maxTurns?: number } = {}): Promise<z.infer<Schema>> {
+    throw new Error('Not implemented in playwright-core');
+  }
+
   async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {
     return await this._channel.snapshotForAI({ timeout: this._timeoutSettings.timeout(options), track: options.track });
   }

packages/playwright-core/src/client/page.ts

@@ -602,6 +602,12 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({
   serviceWorkers: tOptional(tEnum(['allow', 'block'])),
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
+  agent: tOptional(tObject({
+    provider: tString,
+    model: tString,
+    cacheDir: tOptional(tString),
+    cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+  })),
   userDataDir: tString,
   slowMo: tOptional(tFloat),
 });
@@ -694,6 +700,12 @@ scheme.BrowserNewContextParams = tObject({
   serviceWorkers: tOptional(tEnum(['allow', 'block'])),
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
+  agent: tOptional(tObject({
+    provider: tString,
+    model: tString,
+    cacheDir: tOptional(tString),
+    cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+  })),
   proxy: tOptional(tObject({
     server: tString,
     bypass: tOptional(tString),
@@ -765,6 +777,12 @@ scheme.BrowserNewContextForReuseParams = tObject({
   serviceWorkers: tOptional(tEnum(['allow', 'block'])),
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
+  agent: tOptional(tObject({
+    provider: tString,
+    model: tString,
+    cacheDir: tOptional(tString),
+    cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+  })),
   proxy: tOptional(tObject({
     server: tString,
     bypass: tOptional(tString),
@@ -881,6 +899,12 @@ scheme.BrowserContextInitializer = tObject({
     serviceWorkers: tOptional(tEnum(['allow', 'block'])),
     selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
     testIdAttributeName: tOptional(tString),
+    agent: tOptional(tObject({
+      provider: tString,
+      model: tString,
+      cacheDir: tOptional(tString),
+      cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+    })),
   }),
 });
 scheme.BrowserContextBindingCallEvent = tObject({
@@ -2770,6 +2794,12 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({
   serviceWorkers: tOptional(tEnum(['allow', 'block'])),
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
+  agent: tOptional(tObject({
+    provider: tString,
+    model: tString,
+    cacheDir: tOptional(tString),
+    cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+  })),
   pkg: tOptional(tString),
   args: tOptional(tArray(tString)),
   proxy: tOptional(tObject({