chore: add perform secrets, form tool

docs/src/api/params.md

@@ -373,10 +373,11 @@ Emulates consistent window screen size available inside web page via `window.scr
 ## js-context-option-agent
 * langs: js
 - `agent` <[Object]>
-  - `provider` <[string]> LLM provider to use
-  - `model` <[string]> Model identifier within provider
+  - `provider` <[string]> LLM provider to use.
+  - `model` <[string]> Model identifier within provider.
   - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
-  - `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defauls to 'auto'
+  - `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defaults to 'auto'.
+  - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.
 
 Agent settings for [`method: Page.perform`] and [`method: Page.extract`].
 

packages/playwright-client/types/types.d.ts

@@ -22087,12 +22087,12 @@ export interface BrowserContextOptions {
    */
   agent?: {
     /**
-     * LLM provider to use
+     * LLM provider to use.
      */
     provider: string;
 
     /**
-     * Model identifier within provider
+     * Model identifier within provider.
      */
     model: string;
 
@@ -22102,9 +22102,14 @@ export interface BrowserContextOptions {
     cacheFile?: string;
 
     /**
-     * Cache control, defauls to 'auto'
+     * Cache control, defaults to 'auto'.
      */
     cacheMode?: 'force'|'ignore'|'auto';
+
+    /**
+     * Secrets to hide from the LLM.
+     */
+    secrets?: { [key: string]: string; };
   };
 
   /**

packages/playwright-core/src/client/browserContext.ts

@@ -558,6 +558,7 @@ export async function prepareBrowserContextParams(platform: Platform, options: B
     network.validateHeaders(options.extraHTTPHeaders);
   const contextParams: channels.BrowserNewContextParams = {
     ...options,
+    agent: toAgentProtocol(options.agent),
     viewport: options.viewport === null ? undefined : options.viewport,
     noDefaultViewport: options.viewport === null,
     extraHTTPHeaders: options.extraHTTPHeaders ? headersObjectToArray(options.extraHTTPHeaders) : undefined,
@@ -581,6 +582,13 @@ export async function prepareBrowserContextParams(platform: Platform, options: B
   return contextParams;
 }
 
+function toAgentProtocol(agent?: BrowserContextOptions['agent']): channels.BrowserNewContextParams['agent'] {
+  if (!agent)
+    return undefined;
+  const secrets = agent.secrets ? Object.entries(agent.secrets).map(([name, value]) => ({ name, value })) : undefined;
+  return { ...agent, secrets };
+}
+
 function toAcceptDownloadsProtocol(acceptDownloads?: boolean) {
   if (acceptDownloads === undefined)
     return undefined;

packages/playwright-core/src/client/types.ts

@@ -58,7 +58,10 @@ export type ClientCertificate = {
   passphrase?: string;
 };
 
-export type BrowserContextOptions = Omit<channels.BrowserNewContextOptions, 'viewport' | 'noDefaultViewport' | 'extraHTTPHeaders' | 'clientCertificates' | 'storageState' | 'recordHar' | 'colorScheme' | 'reducedMotion' | 'forcedColors' | 'acceptDownloads' | 'contrast'> & {
+export type AgentOptions = Omit<Exclude<channels.BrowserNewContextOptions['agent'], undefined>, 'secrets'> & { secrets?: Record<string, string> };
+
+export type BrowserContextOptions = Omit<channels.BrowserNewContextOptions, 'viewport' | 'noDefaultViewport' | 'extraHTTPHeaders' | 'clientCertificates' | 'storageState' | 'recordHar' | 'colorScheme' | 'reducedMotion' | 'forcedColors' | 'acceptDownloads' | 'contrast' | 'agent'> & {
+  agent?: AgentOptions;
   viewport?: Size | null;
   extraHTTPHeaders?: Headers;
   logger?: Logger;

packages/playwright-core/src/protocol/validator.ts

@@ -607,6 +607,7 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({
     model: tString,
     cacheFile: tOptional(tString),
     cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+    secrets: tOptional(tArray(tType('NameValue'))),
   })),
   userDataDir: tString,
   slowMo: tOptional(tFloat),
@@ -705,6 +706,7 @@ scheme.BrowserNewContextParams = tObject({
     model: tString,
     cacheFile: tOptional(tString),
     cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+    secrets: tOptional(tArray(tType('NameValue'))),
   })),
   proxy: tOptional(tObject({
     server: tString,
@@ -782,6 +784,7 @@ scheme.BrowserNewContextForReuseParams = tObject({
     model: tString,
     cacheFile: tOptional(tString),
     cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+    secrets: tOptional(tArray(tType('NameValue'))),
   })),
   proxy: tOptional(tObject({
     server: tString,
@@ -904,6 +907,7 @@ scheme.BrowserContextInitializer = tObject({
       model: tString,
       cacheFile: tOptional(tString),
       cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+      secrets: tOptional(tArray(tType('NameValue'))),
     })),
   }),
 });
@@ -2813,6 +2817,7 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({
     model: tString,
     cacheFile: tOptional(tString),
     cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])),
+    secrets: tOptional(tArray(tType('NameValue'))),
   })),
   pkg: tOptional(tString),
   args: tOptional(tArray(tString)),

packages/playwright-core/src/server/agent/actionRunner.ts

@@ -17,8 +17,9 @@
 import type * as actions from './actions';
 import type { Page } from '../page';
 import type { Progress } from '../progress';
+import type { NameValue } from '@protocol/channels';
 
-export async function runAction(progress: Progress, page: Page, action: actions.Action) {
+export async function runAction(progress: Progress, page: Page, action: actions.Action, secrets: NameValue[]) {
   const frame = page.mainFrame();
   switch (action.method) {
     case 'click':
@@ -31,21 +32,31 @@ export async function runAction(progress: Progress, page: Page, action: actions.
       await frame.hover(progress, action.selector, { ...action.options, ...strictTrue });
       break;
     case 'selectOption':
-      await frame.selectOption(progress, action.selector, [], action.values.map(a => ({ value: a })), { ...strictTrue });
+      await frame.selectOption(progress, action.selector, [], action.labels.map(a => ({ label: a })), { ...strictTrue });
       break;
     case 'pressKey':
       await page.keyboard.press(progress, action.key);
       break;
-    case 'pressSequentially':
-      await frame.type(progress, action.selector, action.text, { ...strictTrue });
+    case 'pressSequentially': {
+      const secret = secrets?.find(s => s.name === action.text)?.value ?? action.text;
+      await frame.type(progress, action.selector, secret, { ...strictTrue });
       if (action.submit)
         await page.keyboard.press(progress, 'Enter');
       break;
-    case 'fill':
-      await frame.fill(progress, action.selector, action.text, { ...strictTrue });
+    }
+    case 'fill': {
+      const secret = secrets?.find(s => s.name === action.text)?.value ?? action.text;
+      await frame.fill(progress, action.selector, secret, { ...strictTrue });
       if (action.submit)
         await page.keyboard.press(progress, 'Enter');
       break;
+    }
+    case 'setChecked':
+      if (action.checked)
+        await frame.check(progress, action.selector, { ...strictTrue });
+      else
+        await frame.uncheck(progress, action.selector, { ...strictTrue });
+      break;
   }
 }
 

packages/playwright-core/src/server/agent/actions.ts

@@ -37,7 +37,7 @@ export type HoverAction = {
 export type SelectOptionAction = {
   method: 'selectOption';
   selector: string;
-  values: string[];
+  labels: string[];
 };
 
 export type PressAction = {
@@ -59,4 +59,10 @@ export type FillAction = {
   submit?: boolean;
 };
 
-export type Action = ClickAction | DragAction | HoverAction | SelectOptionAction | PressAction | PressSequentiallyAction | FillAction;
+export type SetChecked = {
+  method: 'setChecked';
+  selector: string;
+  checked: boolean;
+};
+
+export type Action = ClickAction | DragAction | HoverAction | SelectOptionAction | PressAction | PressSequentiallyAction | FillAction | SetChecked;

packages/playwright-core/src/server/agent/agent.ts

@@ -85,26 +85,25 @@ type CachedActions = Record<string, actions.Action[]>;
 const allCaches = new Map<string, CachedActions>();
 
 async function cachedPerform(context: Context, options: channels.PagePerformParams): Promise<boolean> {
-  const agentSettings = context.page.browserContext._options.agent;
-  if (!agentSettings?.cacheFile || agentSettings.cacheMode === 'ignore')
+  if (!context.options?.cacheFile || context.options.cacheMode === 'ignore')
     return false;
 
-  const cache = await cachedActions(agentSettings.cacheFile);
+  const cache = await cachedActions(context.options.cacheFile);
   const cacheKey = options.key ?? options.task;
   const actions = cache[cacheKey];
   if (!actions) {
-    if (agentSettings.cacheMode === 'force')
+    if (context.options.cacheMode === 'force')
       throw new Error(`No cached actions for key "${cacheKey}", but cache mode is set to "force"`);
     return false;
   }
 
   for (const action of actions)
-    await runAction(context.progress, context.page, action);
+    await runAction(context.progress, context.page, action, context.options.secrets ?? []);
   return true;
 }
 
 async function updateCache(context: Context, options: channels.PagePerformParams) {
-  const cacheFile = context.page.browserContext._options.agent?.cacheFile;
+  const cacheFile = context.options?.cacheFile;
   if (!cacheFile)
     return;
   const cache = await cachedActions(cacheFile);

packages/playwright-core/src/server/agent/context.ts

@@ -22,20 +22,33 @@ import type * as loopTypes from '@lowire/loop';
 import type * as actions from './actions';
 import type { Page } from '../page';
 import type { Progress } from '../progress';
+import type { BrowserContextOptions } from '../types';
+
+type AgentOptions = BrowserContextOptions['agent'];
 
 export class Context {
+  readonly options: AgentOptions;
   readonly progress: Progress;
   readonly page: Page;
   readonly actions: actions.Action[] = [];
 
   constructor(progress: Progress, page: Page) {
     this.progress = progress;
     this.page = page;
+    this.options = page.browserContext._options.agent;
+  }
+
+  async runActionAndWait(action: actions.Action) {
+    return await this.runActionsAndWait([action]);
   }
 
-  async runAction(action: actions.Action) {
-    await this.waitForCompletion(() => runAction(this.progress, this.page, action));
-    this.actions.push(action);
+  async runActionsAndWait(action: actions.Action[]) {
+    await this.waitForCompletion(async () => {
+      for (const a of action) {
+        await runAction(this.progress, this.page, a, this.options?.secrets ?? []);
+        this.actions.push(a);
+      }
+    });
     return await this.snapshotResult();
   }
 
@@ -71,7 +84,9 @@ export class Context {
   }
 
   async snapshotResult(): Promise<loopTypes.ToolResult> {
-    const { full } = await this.page.snapshotForAI(this.progress);
+    let { full } = await this.page.snapshotForAI(this.progress);
+    full = this._redactText(full);
+
     const text = [`# Page snapshot\n${full}`];
 
     return {
@@ -94,4 +109,18 @@ export class Context {
       }
     }));
   }
+
+  private _redactText(text: string): string {
+    const secrets = this.options?.secrets;
+    if (!secrets)
+      return text;
+
+    const redactText = (text: string) => {
+      for (const { name, value } of secrets)
+        text = text.replaceAll(value, `<secret>${name}</secret>`);
+      return text;
+    };
+
+    return redactText(text);
+  }
 }