chore: perform-specific tools

Author: pavelfeldman

packages/playwright-core/src/client/DEPS.list

@@ -1,3 +1,4 @@
 [*]
 ../protocol/
 ../utils/isomorphic
+../mcpBundle.ts

packages/playwright-core/src/client/page.ts

@@ -41,6 +41,7 @@ import { urlMatches, urlMatchesEqual } from '../utils/isomorphic/urlMatch';
 import { LongStandingScope } from '../utils/isomorphic/manualPromise';
 import { isObject, isRegExp, isString } from '../utils/isomorphic/rtti';
 import { ConsoleMessage } from './consoleMessage';
+import { zodToJsonSchema } from '../mcpBundle';
 
 import type { BrowserContext } from './browserContext';
 import type { Clock } from './clock';
@@ -847,11 +848,12 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
   }
 
   async perform(task: string, options: { key?: string, maxTurns?: number } = {}): Promise<void> {
-    throw new Error('Not implemented in playwright-core');
+    await this._channel.perform({ task, ...options });
   }
 
-  extract<Schema extends z.ZodTypeAny>(query: string, schema: Schema, options: { maxTurns?: number } = {}): Promise<z.infer<Schema>> {
-    throw new Error('Not implemented in playwright-core');
+  async extract<Schema extends z.ZodTypeAny>(query: string, schema: Schema, options: { maxTurns?: number } = {}): Promise<z.infer<Schema>> {
+    const { result } = await this._channel.extract({ query, schema: zodToJsonSchema(schema), ...options });
+    return result;
   }
 
   async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {

packages/playwright-core/src/protocol/validator.ts

@@ -1506,6 +1506,20 @@ scheme.PageUpdateSubscriptionParams = tObject({
   enabled: tBoolean,
 });
 scheme.PageUpdateSubscriptionResult = tOptional(tObject({}));
+scheme.PagePerformParams = tObject({
+  task: tString,
+  key: tOptional(tString),
+  maxTurns: tOptional(tInt),
+});
+scheme.PagePerformResult = tOptional(tObject({}));
+scheme.PageExtractParams = tObject({
+  query: tString,
+  schema: tAny,
+  maxTurns: tOptional(tInt),
+});
+scheme.PageExtractResult = tObject({
+  result: tAny,
+});
 scheme.FrameInitializer = tObject({
   url: tString,
   name: tString,

packages/playwright-core/src/server/agent/DEPS.list

@@ -0,0 +1,6 @@
+[*]
+../browserContext.ts
+../page.ts
+../progress.ts
+../../mcpBundle.ts
+../../utilsBundle.ts

packages/playwright-core/src/server/agent/actionRunner.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { waitForCompletion, wrapResult } from './utils';
+
+import type { Progress } from '../progress';
+import type { Page } from '../page';
+import type { Action } from './actions';
+
+export async function runActionFromTool(progress: Progress, page: Page, action: Action) {
+  await waitForCompletion(progress, page, () => runAction(progress, page, action));
+  return await wrapResult(progress, page, action);
+}
+
+export async function runAction(progress: Progress, page: Page, action: Action) {
+  const frame = page.mainFrame();
+  switch (action.method) {
+    case 'click':
+      await frame.click(progress, action.selector, { ...action.options, ...strictTrue });
+      break;
+    case 'drag':
+      await frame.dragAndDrop(progress, action.sourceSelector, action.targetSelector, { ...strictTrue });
+      break;
+    case 'hover':
+      await frame.hover(progress, action.selector, { ...action.options, ...strictTrue });
+      break;
+    case 'selectOption':
+      await frame.selectOption(progress, action.selector, [], action.values.map(a => ({ value: a })), { ...strictTrue });
+      break;
+    case 'pressKey':
+      await page.keyboard.press(progress, action.key);
+      break;
+    case 'pressSequentially':
+      await frame.type(progress, action.selector, action.text, { ...strictTrue });
+      if (action.submit)
+        await page.keyboard.press(progress, 'Enter');
+      break;
+    case 'fill':
+      await frame.fill(progress, action.selector, action.text, { ...strictTrue });
+      if (action.submit)
+        await page.keyboard.press(progress, 'Enter');
+      break;
+  }
+}
+
+const strictTrue =  { strict: true };

packages/playwright-core/src/server/agent/actions.ts

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as channels from '@protocol/channels';
+
+export type ClickAction = {
+  method: 'click';
+  selector: string;
+  options: Pick<channels.FrameClickParams, 'button' | 'clickCount' | 'modifiers'>;
+};
+
+export type DragAction = {
+  method: 'drag';
+  sourceSelector: string;
+  targetSelector: string;
+};
+
+export type HoverAction = {
+  method: 'hover';
+  selector: string;
+  options: Pick<channels.FrameHoverParams, 'modifiers'>;
+};
+
+export type SelectOptionAction = {
+  method: 'selectOption';
+  selector: string;
+  values: string[];
+};
+
+export type PressAction = {
+  method: 'pressKey';
+  key: string;
+};
+
+export type PressSequentiallyAction = {
+  method: 'pressSequentially';
+  selector: string;
+  text: string;
+  submit?: boolean;
+};
+
+export type FillAction = {
+  method: 'fill';
+  selector: string;
+  text: string;
+  submit?: boolean;
+};
+
+export type Action = ClickAction | DragAction | HoverAction | SelectOptionAction | PressAction | PressSequentiallyAction | FillAction;

packages/playwright-core/src/server/agent/agent.ts

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { toolsForLoop } from './backend';
+import { debug } from '../../utilsBundle';
+import { Loop } from '../../mcpBundle';
+
+import type { Progress } from '../progress';
+import type * as channels from '@protocol/channels';
+import type { Page } from '../page';
+import type * as loopTypes from '@lowire/loop';
+
+export async function pagePerform(progress: Progress, page: Page, options: channels.PagePerformParams): Promise<void> {
+  const resultSchema = {
+    type: 'object' as const,
+    properties: {
+      actions: { type: 'array' as const, items: { type: 'object' as const } },
+    },
+    required: ['actions']
+  };
+  const task = `
+### Instructions
+- Perform the following actions on the page.
+- Return performed actions.
+
+### Task
+${options.task}
+  `;
+  const actions = await perform(progress, page, task, resultSchema, options);
+  // eslint-disable-next-line no-console
+  console.log(actions);
+}
+
+export async function pageExtract(progress: Progress, page: Page, options: channels.PageExtractParams) {
+  const task = `
+### Instructions
+Extract the following information from the page. Do not perform any actions, just extract the information.
+
+### Query
+${options.query}`;
+  return await perform(progress, page, task, options.schema, options);
+}
+
+async function perform(progress: Progress, page: Page, userTask: string, resultSchema: loopTypes.Schema, options: { maxTurns?: number } = {}): Promise<any> {
+  const context = page.browserContext;
+  if (!context._options.agent)
+    throw new Error(`page.perform() and page.extract() require the agent to be set on the browser context`);
+
+  const { full } = await page.snapshotForAI(progress);
+  const { tools, callTool } = toolsForLoop(page);
+
+  const loop = new Loop(context._options.agent.provider as any, {
+    model: context._options.agent.model,
+    summarize: true,
+    debug,
+    callTool,
+    tools,
+    ...options
+  });
+
+  const task = `${userTask}
+
+### Page snapshot
+${full}
+`;
+
+  return await loop.run(task, {
+    resultSchema
+  });
+}

packages/playwright-core/src/server/agent/backend.ts

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import toolDefinitions from './tools';
+import { ProgressController } from '../progress';
+import { zodToJsonSchema } from '../../mcpBundle';
+
+import type * as loopTypes from '@lowire/loop';
+import type { Page } from '../page';
+
+export function toolsForLoop(page: Page): { tools: loopTypes.Tool[], callTool: loopTypes.ToolCallback } {
+  const tools = toolDefinitions.map(tool => {
+    const result: loopTypes.Tool = {
+      name: tool.schema.name,
+      description: tool.schema.description,
+      inputSchema: zodToJsonSchema(tool.schema.inputSchema) as loopTypes.Schema,
+    };
+    return result;
+  });
+
+  const callTool: loopTypes.ToolCallback = async params => {
+    const tool = toolDefinitions.find(t => t.schema.name === params.name);
+    if (!tool) {
+      return {
+        content: [{ type: 'text',
+          text: `Tool ${params.name} not found. Available tools: ${toolDefinitions.map(t => t.schema.name)}`
+        }],
+        isError: true,
+      };
+    }
+
+    const progressController = new ProgressController();
+    return await progressController.run(async progress => {
+      return await tool.handle(progress, page, params.arguments);
+    });
+  };
+
+  return {
+    tools,
+    callTool,
+  };
+}