From a9c21415620664355be892c13e42fbf05b10bc91 Mon Sep 17 00:00:00 2001 From: DongYun Kang Date: Fri, 6 Nov 2026 14:38:24 +0900 Subject: [PATCH 1/2] perf: decode and encode PNGs with @napi-rs/image e2e, @e2e-dev/mobile, and @e2e-dev/decision read and write PNGs with @napi-rs/image instead of fast-png and pngjs. Decoding is 8 to 15 times faster, encoding 5 to 8 times, and a toHaveScreenshot capture 4 to 12 times. - Chromium's 8-bit RGB captures are widened to RGBA by the addon, not by a JS loop. Other color types go through a short layout step. - e2e loads the addon when the first PNG is read or written, so --version, --help, and a run with no screenshot never need its native binding. A missing binding fails the call with IMAGE_CODEC_UNAVAILABLE. - pixelmatch stays: the addon has no diff API. Co-Authored-By: Claude Sonnet 5.5 --- .changeset/napi-rs-image.md | 7 + apps/testbed/package.json | 3 +- apps/testbed/tests/screenshot.e2e.ts | 35 ++- docs/reference/errors.mdx | 1 + packages/decision/package.json | 3 +- packages/decision/src/overlay.ts | 18 +- packages/decision/src/png.ts | 68 ++++++ .../decision/tests/executor-vision.test.ts | 18 +- packages/decision/tests/helpers.ts | 6 +- packages/decision/tests/overlay.test.ts | 37 +++- packages/decision/tests/png-helpers.ts | 44 ++++ packages/e2e/package.json | 4 +- packages/e2e/src/agent/pixels.ts | 7 +- packages/e2e/src/expect/screenshot.ts | 5 +- packages/e2e/src/internal/image.ts | 153 ++++++++++--- .../integration/screenshot-assertions.test.ts | 26 ++- .../integration/screenshot-codec.test.ts | 82 +++++++ .../integration/typescript-loader.test.ts | 17 ++ packages/e2e/tests/unit/agent-pixels.test.ts | 19 +- packages/e2e/tests/unit/image-codec.test.ts | 72 ++++++ packages/e2e/tests/unit/image.test.ts | 178 ++++++++++++++- .../e2e/tests/unit/package-imports.test.ts | 9 +- packages/mobile/package.json | 3 +- packages/mobile/src/png.ts | 82 ++++++- packages/mobile/tests/helpers/png.ts | 60 ++++- packages/mobile/tests/unit/png.test.ts | 28 ++- packages/web/package.json | 3 +- packages/web/tests/helpers/png.ts | 14 +- pnpm-lock.yaml | 206 +++++++++++++----- pnpm-workspace.yaml | 2 - 30 files changed, 1004 insertions(+), 206 deletions(-) create mode 100644 .changeset/napi-rs-image.md create mode 100644 packages/decision/src/png.ts create mode 100644 packages/decision/tests/png-helpers.ts create mode 100644 packages/e2e/tests/integration/screenshot-codec.test.ts create mode 100644 packages/e2e/tests/unit/image-codec.test.ts diff --git a/.changeset/napi-rs-image.md b/.changeset/napi-rs-image.md new file mode 100644 index 0000000000..fa4500cb23 --- /dev/null +++ b/.changeset/napi-rs-image.md @@ -0,0 +1,7 @@ +--- +"e2e": patch +"@e2e-dev/decision": patch +"@e2e-dev/mobile": patch +--- + +PNG decoding and encoding now run on `@napi-rs/image`, a native addon with a prebuilt binary per platform, in place of `fast-png` and `pngjs`: decoding is 8 to 15 times faster, encoding 5 to 8 times, and a `toHaveScreenshot` capture 4 to 12 times. Stored screenshots decode to the same pixels as before. A screenshot the runner writes (a new baseline, or one rewritten because the screen changed) has the same pixels in different bytes: a Chromium capture comes out about 15% larger and a device screenshot about 20% smaller. The binary adds about 15 to 21 MB to an install, depending on the platform, and the package publishes none for Linux on ppc64, s390x, or riscv64. `e2e` loads it when the first screenshot is read or written, so `--version` and a run that takes none never need it, and a missing binding fails that call with `IMAGE_CODEC_UNAVAILABLE`; `@e2e-dev/mobile` and `@e2e-dev/decision` load it when they are imported. Gray PNGs with 1, 2, or 4 bits per sample now decode, and a 16-bit gray or RGB PNG with a transparent color now compares pixels to it at 16 bits, so a pixel that only rounds to it at 8 bits stays opaque. diff --git a/apps/testbed/package.json b/apps/testbed/package.json index 5a8550e6d1..25debed470 100644 --- a/apps/testbed/package.json +++ b/apps/testbed/package.json @@ -26,10 +26,9 @@ }, "devDependencies": { "@ai-sdk/devtools": "1.0.66", + "@napi-rs/image": "1.16.0", "@types/node": "26.6.6", - "@types/pngjs": "6.0.5", "ai": "7.0.163", - "pngjs": "7.0.0", "typescript": "7.0.2", "zod": "4.6.7" } diff --git a/apps/testbed/tests/screenshot.e2e.ts b/apps/testbed/tests/screenshot.e2e.ts index c4fab3390a..090111233f 100644 --- a/apps/testbed/tests/screenshot.e2e.ts +++ b/apps/testbed/tests/screenshot.e2e.ts @@ -2,7 +2,7 @@ * Adapted from microsoft/playwright tests/page/page-screenshot.spec.ts at v1.64.1. * Copyright Microsoft Corporation. Licensed under the Apache License 2.0 (see ../app/playwright/LICENSE). * Rewritten against the e2e public API: `page.screenshot()` is `app.screenshot()`, read back with - * pngjs, and a `mask` is the `toHaveScreenshot` option, compared against the same viewport with + * @napi-rs/image, and a `mask` is the `toHaveScreenshot` option, compared against the same viewport with * boxes of the mask color painted over the masked elements. Playwright's golden images belong to * its own renderer setup, so a golden comparison becomes a check of the pixels it shows. */ @@ -51,7 +51,7 @@ import { copyFileSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; import path from 'node:path'; import { test, type Browser, type WebApp } from '@e2e-dev/web'; import { expect } from 'e2e'; -import { PNG } from 'pngjs'; +import { JsColorType, Transformer } from '@napi-rs/image'; import { content, CROSS, PREFIX } from './helpers.ts'; const stored = new Set(); @@ -63,13 +63,30 @@ interface MaskTarget { readonly index: number; } -/** The PNG at `file`, decoded. */ -function readPng(file: string): PNG { - return PNG.sync.read(readFileSync(file)); +/** A decoded screenshot: 8-bit RGBA, rows top to bottom. */ +interface Rgba { + readonly width: number; + readonly height: number; + readonly data: Uint8Array; +} + +/** The RGB or RGBA PNG at `file`, decoded to RGBA. */ +function readPng(file: string): Rgba { + const image = new Transformer(readFileSync(file)); + const { width, height, colorType } = image.metadataSync(); + const raw = image.rawPixelsSync(); + if (colorType === JsColorType.Rgba8) return { width, height, data: raw }; + if (colorType !== JsColorType.Rgb8) throw new Error(`${file} is neither an RGB nor an RGBA PNG: color type ${colorType}`); + const data = new Uint8Array(width * height * 4); + for (let from = 0, to = 0; from < raw.length; from += 3, to += 4) { + data.set(raw.subarray(from, from + 3), to); + data[to + 3] = 255; + } + return { width, height, data }; } /** The red, green, and blue of the pixel at (`x`, `y`). */ -function rgb(png: PNG, x: number, y: number): [number, number, number] { +function rgb(png: Rgba, x: number, y: number): [number, number, number] { const offset = (y * png.width + x) * 4; return [png.data[offset]!, png.data[offset + 1]!, png.data[offset + 2]!]; } @@ -80,12 +97,12 @@ function near(actual: readonly number[], expected: readonly number[], slack = 3) } /** The pixels of `box` cut out of `png`, rounded out to whole pixels. */ -function crop(png: PNG, box: { x: number; y: number; width: number; height: number }): Buffer { +function crop(png: Rgba, box: { x: number; y: number; width: number; height: number }): Buffer { const left = Math.floor(box.x); const top = Math.floor(box.y); const right = Math.ceil(box.x + box.width); const bottom = Math.ceil(box.y + box.height); - const rows: Buffer[] = []; + const rows: Uint8Array[] = []; for (let y = top; y < bottom; y++) rows.push(png.data.subarray((y * png.width + left) * 4, (y * png.width + right) * 4)); return Buffer.concat(rows); } @@ -256,7 +273,7 @@ test.describe('page screenshot', { requires: ['browser'], tags: ['playwright'] } } /** Whether the line of `step` shows in `screenshot`: the canvas sits at the body margin, 8 pixels in. */ - function hasLine(screenshot: PNG, step: number): boolean { + function hasLine(screenshot: Rgba, step: number): boolean { const y = 8 + step * 100; return [y - 1, y].some((row) => row >= 8 && rgb(screenshot, 8 + 150, row)[0] < 200); } diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index e741699d3e..32669278f7 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -174,6 +174,7 @@ These appear in terminal output and in the report as plain string codes. | `WORKER_ENTRY_MISSING` | The worker entry module could not be resolved. Packaging fault. | infrastructure | 3 | | `NODE_UNSUPPORTED` | The runtime cannot run e2e's TypeScript loader: a Node.js outside `^22.22.3 \|\| >=24.8.0`, Bun (no `module.registerHooks`), or Deno. The CLI checks this before it starts and exits 2 with the same sentence. The loader checks again where it registers its hooks, for a host that runs e2e's runner without the CLI | infrastructure | 3 | | `TYPESCRIPT_COMPILER_UNAVAILABLE` | oxc, which compiles e2e's TypeScript, could not be loaded; the message gives the cause. Most often its native bindings for this platform (`@oxc-transform/binding--`, `@oxc-parser/binding--`) are not installed. npm leaves them out with `--omit=optional`, or when `package-lock.json` was written on another platform (npm/cli#4828). Install again without `--omit=optional`, or delete `package-lock.json` and `node_modules` and run `npm install` | infrastructure | 3 | +| `IMAGE_CODEC_UNAVAILABLE` | `@napi-rs/image`, which reads and writes the PNGs of screenshots, could not be loaded; the message gives the cause. It is loaded when the first screenshot is read or written, so a run that takes none never needs it. Most often its native binding for this platform (`@napi-rs/image--*`) is not installed. npm leaves it out with `--omit=optional`, or when `package-lock.json` was written on another platform (npm/cli#4828). Install again without `--omit=optional`, or delete `package-lock.json` and `node_modules` and run `npm install`. The package publishes no binding for Linux on ppc64, s390x, or riscv64, where `toHaveScreenshot` and the agent's screenshots cannot work | infrastructure | 3 | | `ENGINE_FAILURE` | Any engine fault with no more specific mapping | infrastructure | 3 | | `WORKER_PROTOCOL` | A worker reported the wrong unit, an attempt overlapped a live attempt, `openHost` tried to acquire an engine instance another host owns, or `host.openSession` tried to reuse a closed host or overlap an opening, live, or closing session | infrastructure | 3 | | `SESSION_KEY_INVALID` | A worker received a session key that is not 32 bytes. Packaging or IPC fault. | infrastructure | 3 | diff --git a/packages/decision/package.json b/packages/decision/package.json index e6c088d763..65417a96ba 100644 --- a/packages/decision/package.json +++ b/packages/decision/package.json @@ -31,7 +31,7 @@ "test": "vitest run" }, "dependencies": { - "pngjs": "7.0.0", + "@napi-rs/image": "1.16.0", "zod": "4.6.7" }, "peerDependencies": { @@ -43,7 +43,6 @@ "@ai-sdk/typesafe-ai": "3.0.34", "@e2e-dev/web": "workspace:*", "@types/node": "26.6.6", - "@types/pngjs": "6.0.5", "ai": "7.0.163", "ajv": "8.20.0", "ajv-formats": "3.0.1", diff --git a/packages/decision/src/overlay.ts b/packages/decision/src/overlay.ts index 02c139a377..2648f47141 100644 --- a/packages/decision/src/overlay.ts +++ b/packages/decision/src/overlay.ts @@ -1,6 +1,6 @@ -import { PNG } from 'pngjs'; import type { ExecutorPixels } from 'e2e'; import { fnv1a } from './hash.ts'; +import { decodeRgba, encodeRgba, type Rgba } from './png.ts'; /** Pixels as the decision model receives them: the runner's capture without its masking count. */ export type Screenshot = Omit; /** A viewport point in CSS pixels. */ @@ -54,7 +54,8 @@ export interface Zoom { /** * The `box` by `box` CSS-pixel region around `center`, kept inside the * screenshot and enlarged `factor` times (nearest neighbor), for a finer - * closer look at a drawn control. Undefined when the pixels cannot be decoded. + * closer look at a drawn control. Undefined when the pixels cannot be + * decoded or the region comes out under one pixel, which no PNG can hold. */ export function zoomAround(screenshot: Screenshot, center: Point, box: number, factor: number): Zoom | undefined { const png = decode(screenshot); @@ -68,7 +69,8 @@ export function zoomAround(screenshot: Screenshot, center: Point, box: number, f y: Math.round(Math.max(0, Math.min(cssHeight - side, center.y - side / 2))), }; const size = Math.round(side * factor); - const out = new PNG({ width: size, height: size }); + if (!Number.isFinite(size) || size < 1) return undefined; + const out = { width: size, height: size, data: new Uint8Array(size * size * 4) }; for (let y = 0; y < size; y += 1) { for (let x = 0; x < size; x += 1) { const sx = Math.min(png.width - 1, Math.floor((origin.x + x / factor) * scale)); @@ -81,7 +83,7 @@ export function zoomAround(screenshot: Screenshot, center: Point, box: number, f out.data[to + 3] = 255; } } - return { screenshot: { mediaType: 'image/png', data: new Uint8Array(PNG.sync.write(out)), width: size, height: size, scale: 1 }, origin, factor }; + return { screenshot: { mediaType: 'image/png', data: encodeRgba(out), width: size, height: size, scale: 1 }, origin, factor }; } /** * A hash of the `box` by `box` CSS-pixel region around `center`: what a tap @@ -120,17 +122,17 @@ export function withGrid(screenshot: Screenshot, grid: Grid): Screenshot { } } } - return { ...screenshot, data: new Uint8Array(PNG.sync.write(png)) }; + return { ...screenshot, data: encodeRgba(png) }; } -function decode(screenshot: Screenshot): PNG | undefined { +function decode(screenshot: Screenshot): Rgba | undefined { try { - return PNG.sync.read(Buffer.from(screenshot.data)); + return decodeRgba(screenshot.data); } catch { return undefined; } } const GRID: readonly [number, number, number] = [255, 0, 255]; -function paint(png: PNG, x: number, y: number, color: readonly [number, number, number]): void { +function paint(png: Rgba, x: number, y: number, color: readonly [number, number, number]): void { if (x < 0 || y < 0 || x >= png.width || y >= png.height) return; const offset = (y * png.width + x) * 4; png.data[offset] = color[0]; diff --git a/packages/decision/src/png.ts b/packages/decision/src/png.ts new file mode 100644 index 0000000000..8f33291c3d --- /dev/null +++ b/packages/decision/src/png.ts @@ -0,0 +1,68 @@ +import { endianness } from 'node:os'; +import { FilterType, JsColorType, Transformer } from '@napi-rs/image'; +/** An 8-bit RGBA image, rows top to bottom. */ +export interface Rgba { + readonly width: number; + readonly height: number; + readonly data: Uint8Array; +} +const LITTLE_ENDIAN = endianness() === 'LE'; +/** Channels per pixel, and whether each is a 16-bit sample, for every color type a decode reads: 8-bit RGB is widened by the addon before that. */ +const LAYOUTS: Readonly>> = { + [JsColorType.L8]: { channels: 1, wide: false }, + [JsColorType.La8]: { channels: 2, wide: false }, + [JsColorType.Rgba8]: { channels: 4, wide: false }, + [JsColorType.L16]: { channels: 1, wide: true }, + [JsColorType.La16]: { channels: 2, wide: true }, + [JsColorType.Rgb16]: { channels: 3, wide: true }, + [JsColorType.Rgba16]: { channels: 4, wide: true }, +}; +/** + * Decodes PNG bytes to 8-bit RGBA, whatever the color type, bit depth, or + * palette; throws on bytes that are not a PNG. An 8-bit RGB screenshot is + * widened by the addon: opacity 1 keeps every sample and adds the opaque + * alpha channel, several times faster than `spread` does it. + */ +export function decodeRgba(bytes: Uint8Array): Rgba { + const decoder = new Transformer(bytes); + const { width, height, format, colorType } = decoder.metadataSync(); + if (format !== 'png') throw new Error(`not a PNG: the bytes are ${format}`); + const rgb = colorType === JsColorType.Rgb8; + const raw = (rgb ? decoder.opacity(1) : decoder).rawPixelsSync(); + const layout = LAYOUTS[rgb ? JsColorType.Rgba8 : colorType]; + if (layout === undefined) throw new Error(`unsupported PNG color type ${colorType}`); + const { channels, wide } = layout; + if (raw.length !== width * height * channels * (wide ? 2 : 1)) throw new Error('the decoded PNG pixels do not fill its size'); + const samples = new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength); + if (channels === 4 && !wide) return { width, height, data: samples }; + return { width, height, data: spread(wide ? narrow(samples) : samples, channels, width * height) }; +} +/** + * Encodes an image as PNG bytes, each row adaptively filtered: the addon's + * default writes no filter, which makes screenshots markedly larger. + */ +export function encodeRgba(image: Rgba): Uint8Array { + const png = Transformer.fromRgbaPixels(image.data, image.width, image.height).pngSync({ filterType: FilterType.Adaptive }); + return new Uint8Array(png.buffer, png.byteOffset, png.byteLength); +} +/** 16-bit samples in native byte order as 8-bit samples, rounded. */ +function narrow(bytes: Uint8Array): Uint8Array { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const samples = new Uint8Array(bytes.length / 2); + for (let at = 0; at < samples.length; at += 1) samples[at] = Math.round((view.getUint16(at * 2, LITTLE_ENDIAN) * 255) / 65535); + return samples; +} +/** 8-bit samples of `channels` per pixel (gray, gray and alpha, RGB) as RGBA; a gray or RGB image is opaque. */ +function spread(samples: Uint8Array, channels: number, pixels: number): Uint8Array { + const out = new Uint8Array(pixels * 4); + const gray = channels <= 2; + const alpha = channels === 2 || channels === 4; + for (let pixel = 0, from = 0, to = 0; pixel < pixels; pixel += 1, from += channels, to += 4) { + const r = samples[from]!; + out[to] = r; + out[to + 1] = gray ? r : samples[from + 1]!; + out[to + 2] = gray ? r : samples[from + 2]!; + out[to + 3] = alpha ? samples[from + channels - 1]! : 255; + } + return out; +} diff --git a/packages/decision/tests/executor-vision.test.ts b/packages/decision/tests/executor-vision.test.ts index 60866c3f7f..91be63694d 100644 --- a/packages/decision/tests/executor-vision.test.ts +++ b/packages/decision/tests/executor-vision.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { decisionExecutor } from '../src/index.ts'; -import { PNG } from 'pngjs'; import { BUTTONS, context, invalidConfig, scriptedDecision, scriptedOutputs, whitePixels } from './helpers.ts'; +import { pngOf, rgbaOf } from './png-helpers.ts'; describe('vision', () => { @@ -121,7 +121,7 @@ describe('vision details', () => { const fixture = context({ tree: BUTTONS, observation: { pixels: whitePixels() }, model: text.model }); await decisionExecutor({ model, textModel: text.model, vision: true }).runStep(fixture.ctx); const sent = (index: number) => { - const png = PNG.sync.read(Buffer.from(requests[index]?.files[0]?.data as Uint8Array)); + const png = rgbaOf(requests[index]?.files[0]?.data as Uint8Array); return { png, at: (x: number, y: number) => [...png.data.subarray((y * png.width + x) * 4, (y * png.width + x) * 4 + 3)] }; }; const full = sent(1); @@ -191,10 +191,9 @@ describe('vision details', () => { it('replays a landed point for the same control on the same page, and relocates one that missed', async () => { /** Two screenshots that differ only inside the tapped region. */ const frame = (dot: boolean) => { - const png = new PNG({ width: 800, height: 600 }); - png.data.fill(255); - if (dot) for (let i = 0; i < 4; i += 1) png.data[(100 * 800 + 100) * 4 + i] = 0; - return { data: new Uint8Array(PNG.sync.write(png)), mediaType: 'image/png' as const, width: 800, height: 600, scale: 1, maskedRegionCount: 0 }; + const rgba = new Uint8Array(800 * 600 * 4).fill(255); + if (dot) for (let i = 0; i < 4; i += 1) rgba[(100 * 800 + 100) * 4 + i] = 0; + return { data: pngOf(800, 600, rgba), mediaType: 'image/png' as const, width: 800, height: 600, scale: 1, maskedRegionCount: 0 }; }; const { model, requests } = scriptedDecision((id, keys, call) => { if (id === 'operation') return { choice: call < 12 ? 'tap_at' : 'blocked' }; @@ -229,10 +228,9 @@ describe('vision details', () => { let look = 0; fixture.observe.mockImplementation(async () => { look += 1; - const png = new PNG({ width: 800, height: 600 }); - png.data.fill(255); - png.data[look] = 0; - return { revision: String(look), text: 'x', truncated: false, viewport: { width: 800, height: 600 }, tree: BUTTONS, pixels: { data: new Uint8Array(PNG.sync.write(png)), mediaType: 'image/png' as const, width: 800, height: 600, scale: 1, maskedRegionCount: 0 } }; + const rgba = new Uint8Array(800 * 600 * 4).fill(255); + rgba[look] = 0; + return { revision: String(look), text: 'x', truncated: false, viewport: { width: 800, height: 600 }, tree: BUTTONS, pixels: { data: pngOf(800, 600, rgba), mediaType: 'image/png' as const, width: 800, height: 600, scale: 1, maskedRegionCount: 0 } }; }); const verdict = await decisionExecutor({ model, textModel: text.model, vision: true }).runStep(fixture.ctx); expect(verdict).toMatchObject({ status: 'blocked', summary: 'The decision model cannot make progress.' }); diff --git a/packages/decision/tests/helpers.ts b/packages/decision/tests/helpers.ts index 8615249e8a..ba0acaee5e 100644 --- a/packages/decision/tests/helpers.ts +++ b/packages/decision/tests/helpers.ts @@ -2,7 +2,7 @@ import type { LanguageModel } from 'ai'; import type { DecisionExecutorOptions } from '../src/index.ts'; import type { ExecutorActions, ExecutorModelCall, ExecutorNode, ExecutorObservation, ExecutorPixels, JsonValue, StepExecutorContext, StepTurn } from 'e2e'; import { expect, vi } from 'vitest'; -import { PNG } from 'pngjs'; +import { whitePng } from './png-helpers.ts'; /** One scripted answer: the choice, its distribution, and the reported confidence. */ export interface ScriptedAnswer { readonly choice: string; @@ -187,7 +187,5 @@ export const FIELD: ExecutorNode = { id: 'root', children: [ ]}; /** A decodable white screenshot of the fixture viewport. */ export function whitePixels(): ExecutorPixels { - const png = new PNG({ width: 800, height: 600 }); - png.data.fill(255); - return { data: new Uint8Array(PNG.sync.write(png)), mediaType: 'image/png', width: 800, height: 600, scale: 1, maskedRegionCount: 0 }; + return { data: whitePng(800, 600), mediaType: 'image/png', width: 800, height: 600, scale: 1, maskedRegionCount: 0 }; } diff --git a/packages/decision/tests/overlay.test.ts b/packages/decision/tests/overlay.test.ts index 53d4a22ac9..81613f78ac 100644 --- a/packages/decision/tests/overlay.test.ts +++ b/packages/decision/tests/overlay.test.ts @@ -1,18 +1,21 @@ import { describe, expect, it } from 'vitest'; -import { PNG } from 'pngjs'; import { gridFor, pointOf, regionHash, withGrid, zoomAround } from '../src/overlay.ts'; +import { pngOf, rawPng, rgbaOf, whitePng } from './png-helpers.ts'; /** A white PNG of the given size as the executor's screenshot. */ function white(width: number, height: number, scale = 1) { - const png = new PNG({ width, height }); - png.data.fill(255); - return { mediaType: 'image/png' as const, data: new Uint8Array(PNG.sync.write(png)), width, height, scale }; + return { mediaType: 'image/png' as const, data: whitePng(width, height), width, height, scale }; } function pixel(data: Uint8Array, x: number, y: number): number[] { - const png = PNG.sync.read(Buffer.from(data)); + const png = rgbaOf(data); const offset = (y * png.width + x) * 4; return [...png.data.subarray(offset, offset + 3)]; } +/** A 4 by 4 screenshot whose every pixel has the same `samples`, written as the color type and bit depth say. */ +function filled(depth: number, colorType: number, samples: readonly number[]) { + const row = Array.from({ length: 4 }, () => samples).flat(); + return { mediaType: 'image/png' as const, data: rawPng(4, 4, depth, colorType, Array.from({ length: 4 }, () => row)), width: 4, height: 4, scale: 1 }; +} describe('gridFor', () => { it('fits at most ten columns and rows near the target cell size', () => { expect(gridFor(1280, 720, 80)).toEqual({ columns: 10, rows: 9, cellWidth: 128, cellHeight: 80 }); @@ -44,6 +47,23 @@ describe('withGrid', () => { expect(withGrid(screenshot, { columns: 1, rows: 1, cellWidth: 1, cellHeight: 1 })).toBe(screenshot); }); }); +describe('every PNG color type', () => { + const shots = [ + ['RGB', filled(8, 2, [10, 20, 30]), [10, 20, 30]], + ['gray', filled(8, 0, [200]), [200, 200, 200]], + ['gray with alpha', filled(8, 4, [200, 128]), [200, 200, 200]], + ['16-bit RGB', filled(16, 2, [255, 255, 128, 0, 1, 1]), [255, 128, 1]], + ['16-bit RGBA', filled(16, 6, [0, 0, 255, 255, 0, 0, 255, 255]), [0, 255, 0]], + ] as const; + it.each(shots)('reads a %s screenshot into the grid and the zoom', (_label, shot, rgb) => { + const grid = withGrid(shot, { columns: 1, rows: 1, cellWidth: 4, cellHeight: 4 }); + expect(pixel(grid.data, 0, 0)).toEqual([255, 0, 255]); + expect(pixel(grid.data, 1, 1)).toEqual(rgb); + const zoom = zoomAround(shot, { x: 2, y: 2 }, 4, 1); + expect(zoom?.screenshot).toMatchObject({ width: 4, height: 4 }); + expect(pixel(zoom?.screenshot.data ?? new Uint8Array(), 1, 1)).toEqual(rgb); + }); +}); describe('zoomAround', () => { it('crops a box around the point, kept inside the image, and enlarges it', () => { const zoom = zoomAround(white(400, 300), { x: 390, y: 10 }, 100, 2); @@ -59,14 +79,17 @@ describe('zoomAround', () => { it('returns nothing for pixels it cannot decode', () => { expect(zoomAround({ mediaType: 'image/png', data: new Uint8Array([1]), width: 1, height: 1, scale: 1 }, { x: 0, y: 0 }, 10, 2)).toBeUndefined(); }); + it('returns nothing for a region under one pixel, which no PNG can hold', () => { + expect(zoomAround(white(1, 1, 4), { x: 0, y: 0 }, 10, 1)).toBeUndefined(); + }); }); describe('regionHash', () => { it('changes only when the region around the point changes', () => { const base = white(400, 300); const dot = white(400, 300); - const png = PNG.sync.read(Buffer.from(dot.data)); + const png = rgbaOf(dot.data); png.data[(10 * 400 + 10) * 4] = 0; - const dotted = { ...dot, data: new Uint8Array(PNG.sync.write(png)) }; + const dotted = { ...dot, data: pngOf(400, 300, png.data) }; expect(regionHash(base, { x: 20, y: 20 }, 50)).not.toBe(regionHash(dotted, { x: 20, y: 20 }, 50)); expect(regionHash(base, { x: 350, y: 250 }, 50)).toBe(regionHash(dotted, { x: 350, y: 250 }, 50)); expect(regionHash(undefined, { x: 0, y: 0 }, 50)).toBeUndefined(); diff --git a/packages/decision/tests/png-helpers.ts b/packages/decision/tests/png-helpers.ts new file mode 100644 index 0000000000..97f81be575 --- /dev/null +++ b/packages/decision/tests/png-helpers.ts @@ -0,0 +1,44 @@ +import { crc32, deflateSync } from 'node:zlib'; +import { JsColorType, Transformer } from '@napi-rs/image'; +/** PNG bytes of 8-bit RGBA pixels. */ +export function pngOf(width: number, height: number, rgba: Uint8Array): Uint8Array { + return new Uint8Array(Transformer.fromRgbaPixels(rgba, width, height).pngSync()); +} +/** A white, opaque RGBA PNG. */ +export function whitePng(width: number, height: number): Uint8Array { + return pngOf(width, height, new Uint8Array(width * height * 4).fill(255)); +} +/** The size and RGBA pixels of a PNG the package wrote, which is always 8-bit RGBA. */ +export function rgbaOf(bytes: Uint8Array): { width: number; height: number; data: Uint8Array } { + const image = new Transformer(bytes); + const { width, height, colorType } = image.metadataSync(); + if (colorType !== JsColorType.Rgba8) throw new Error(`expected an 8-bit RGBA PNG, got color type ${colorType}`); + return { width, height, data: image.rawPixelsSync() }; +} +/** One PNG chunk: its length, type, data, and the CRC of type and data. */ +function chunk(type: string, data: Uint8Array = new Uint8Array()): Buffer { + const body = Buffer.concat([Buffer.from(type, 'latin1'), data]); + const out = Buffer.alloc(body.length + 8); + out.writeUInt32BE(data.length, 0); + body.copy(out, 4); + out.writeUInt32BE(crc32(body), body.length + 4); + return out; +} +/** + * A PNG written straight from the format, so a test can hand the package a + * color type no encoder of its own would write: `rows` hold each scanline's + * sample bytes as the color type and `depth` pack them, with no filter. + */ +export function rawPng(width: number, height: number, depth: number, colorType: number, rows: readonly (readonly number[])[]): Uint8Array { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = depth; + header[9] = colorType; + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + chunk('IHDR', header), + chunk('IDAT', deflateSync(Buffer.concat(rows.map((row) => Buffer.from([0, ...row]))))), + chunk('IEND'), + ]); +} diff --git a/packages/e2e/package.json b/packages/e2e/package.json index 7368b1b516..a25ad1f273 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -96,10 +96,10 @@ "@clack/prompts": "1.8.1", "@jridgewell/remapping": "2.3.5", "@modelcontextprotocol/server": "2.4.0", + "@napi-rs/image": "1.16.0", "@oxc-project/runtime": "0.157.0", "@vitest/expect": "5.1.0", "commander": "15.0.0", - "fast-png": "8.0.0", "get-tsconfig": "4.14.3", "oxc-parser": "0.157.0", "oxc-transform": "0.157.0", @@ -145,11 +145,9 @@ "@modelcontextprotocol/client": "2.4.0", "@openrouter/ai-sdk-provider": "3.1.0", "@types/node": "26.6.6", - "@types/pngjs": "6.0.5", "ai": "7.0.163", "ajv": "8.20.0", "ajv-formats": "3.0.1", - "pngjs": "7.0.0", "typescript": "7.0.2", "vitest": "5.1.0" }, diff --git a/packages/e2e/src/agent/pixels.ts b/packages/e2e/src/agent/pixels.ts index 72b15fa62b..8be8595018 100644 --- a/packages/e2e/src/agent/pixels.ts +++ b/packages/e2e/src/agent/pixels.ts @@ -10,6 +10,7 @@ */ import type { ObservationPixels } from '../engine/surface.ts'; +import { InfrastructureError } from '../internal/errors.ts'; import { decodePng, encodePng, resampleImage, type RgbaImage } from '../internal/image.ts'; /** @@ -32,7 +33,8 @@ export function sizeForModel

(pixels: P): P { * Resamples the image so its longer side is at most `maxLongSide`, with a * box filter over the source pixels each target pixel covers. Returns the * input untouched when it already fits or cannot be decoded; whatever else - * the caller's pixels carry (a masked-region count) rides along unchanged. + * the caller's pixels carry (a masked-region count) rides along unchanged. A + * codec that cannot load throws: that is the install's fault, not the image's. */ export function downscalePixels

(pixels: P, maxLongSide: number): P { const longSide = Math.max(pixels.width, pixels.height); @@ -43,7 +45,8 @@ export function downscalePixels

(pixels: P, maxLongS let source: RgbaImage; try { source = decodePng(pixels.data); - } catch { + } catch (cause) { + if (cause instanceof InfrastructureError) throw cause; return pixels; } const data = encodePng(resampleImage(source, width, height)); diff --git a/packages/e2e/src/expect/screenshot.ts b/packages/e2e/src/expect/screenshot.ts index 2877661099..7823f16a9c 100644 --- a/packages/e2e/src/expect/screenshot.ts +++ b/packages/e2e/src/expect/screenshot.ts @@ -9,7 +9,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; import type { LocatorExpression } from '../engine/surface.ts'; import { writeFileAtomic } from '../internal/atomic-write.ts'; -import { TestError } from '../internal/errors.ts'; +import { InfrastructureError, TestError } from '../internal/errors.ts'; import { compareImages, decodePng, encodePng, type ImageComparison, type RgbaImage } from '../internal/image.ts'; import { pollReads, type Deadline, type ReadWindow } from '../internal/time.ts'; import { describeExpression } from '../locator/expression.ts'; @@ -205,11 +205,12 @@ async function expectDifferent(run: Run, expected: RgbaImage | undefined, deadli }); } -/** Reads a stored screenshot; one that is not a PNG fails the call. */ +/** Reads a stored screenshot; one that is not a PNG fails the call, and a codec that cannot load fails it as itself. */ function readStored(api: string, stored: Stored): RgbaImage { try { return decodePng(readFileSync(stored.file)); } catch (cause) { + if (cause instanceof InfrastructureError) throw cause; throw new TestError('INVALID_ARGUMENT', `${api} could not read the stored screenshot ${stored.shown} as a PNG; delete it or run with --update-snapshots`, { cause }); } } diff --git a/packages/e2e/src/internal/image.ts b/packages/e2e/src/internal/image.ts index 787461d2aa..c2cf5a500a 100644 --- a/packages/e2e/src/internal/image.ts +++ b/packages/e2e/src/internal/image.ts @@ -1,7 +1,12 @@ /** RGBA images in memory: PNG decoding and encoding, resampling, cropping, painting boxes, and comparing two images. */ -import { convertIndexedToRgb, decode, encode, type DecodedPng } from 'fast-png'; +import { createRequire } from 'node:module'; +import { endianness } from 'node:os'; +import type * as NapiImage from '@napi-rs/image'; import pixelmatch from 'pixelmatch'; +import { InfrastructureError } from './errors.ts'; + +const ownRequire = createRequire(import.meta.url); /** An RGBA image, 8 bits per channel, rows top to bottom. */ export interface RgbaImage { @@ -18,50 +23,130 @@ export interface ImageBox { readonly height: number; } -/** Decodes PNG bytes to 8-bit RGBA: any color type, at 8 or 16 bits or with a palette; throws on anything else. */ +const LITTLE_ENDIAN = endianness() === 'LE'; + +/** `@napi-rs/image`, loaded on the first PNG decoded or encoded. */ +let addon: typeof NapiImage | undefined; + +/** + * `@napi-rs/image`, loaded when the first PNG is decoded or encoded rather + * than when e2e loads, so a command or a test that touches no screenshot + * (`--version`, `--help`, `doctor`) still runs without its native binding. + * npm leaves that binding out with `--omit=optional`, and from a lockfile + * written on another platform (npm/cli#4828); the package publishes none for + * Linux on ppc64, s390x, or riscv64. The error names the binding and the + * way back. + */ +function napi(): typeof NapiImage { + if (addon !== undefined) return addon; + try { + addon = ownRequire('@napi-rs/image') as typeof NapiImage; + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + const platform = `${process.platform}-${process.arch}`; + // napi-rs's loader says so when no platform package is installed. + const fix = message.includes('Cannot find native binding') + ? `its native binding for ${platform} (@napi-rs/image-${platform}*) is not installed. npm skips it with --omit=optional, or when package-lock.json was written on another platform (npm/cli#4828): install again without --omit=optional, or delete package-lock.json and node_modules and run npm install. The package has no binding for Linux on ppc64, s390x, or riscv64` + : `it could not be loaded (${message.split('\n', 1)[0]})`; + throw new InfrastructureError('IMAGE_CODEC_UNAVAILABLE', `e2e reads and writes screenshots with @napi-rs/image, and ${fix}`, { cause }); + } + return addon; +} + +/** Channels per pixel, and whether each is a 16-bit sample, for every color type `toRgba` is handed: 8-bit RGB is widened by the addon before that. */ +function layoutOf(colorType: NapiImage.JsColorType): { readonly channels: number; readonly wide: boolean } | undefined { + const { JsColorType } = napi(); + switch (colorType) { + case JsColorType.L8: + return { channels: 1, wide: false }; + case JsColorType.La8: + return { channels: 2, wide: false }; + case JsColorType.Rgba8: + return { channels: 4, wide: false }; + case JsColorType.L16: + return { channels: 1, wide: true }; + case JsColorType.La16: + return { channels: 2, wide: true }; + case JsColorType.Rgb16: + return { channels: 3, wide: true }; + case JsColorType.Rgba16: + return { channels: 4, wide: true }; + default: + return undefined; + } +} + +/** + * Decodes PNG bytes to 8-bit RGBA: any color type, at 1 to 16 bits or with a + * palette; throws on anything else. A Chromium capture is 8-bit RGB, and the + * addon widens that itself: opacity 1 keeps every sample and adds the opaque + * alpha channel, several times faster than `spread` does it. + */ export function decodePng(bytes: Uint8Array): RgbaImage { - const png = decode(bytes); - return { width: png.width, height: png.height, data: toRgba(png) }; + const { JsColorType, Transformer } = napi(); + const decoder = new Transformer(bytes); + const { width, height, format, colorType } = decoder.metadataSync(); + if (format !== 'png') throw new Error(`not a PNG: the bytes are ${format}`); + const rgb = colorType === JsColorType.Rgb8; + const raw = (rgb ? decoder.opacity(1) : decoder).rawPixelsSync(); + return { width, height, data: toRgba(raw, rgb ? JsColorType.Rgba8 : colorType, width * height) }; } -/** Encodes an image as PNG bytes. */ +/** + * Encodes an image as PNG bytes, each row adaptively filtered: the addon's + * default writes no filter, which makes screenshots markedly larger. The + * addon also aborts the whole process on a pixel buffer longer than the + * image, so a buffer of the wrong length throws here instead of reaching it. + */ export function encodePng(image: RgbaImage): Uint8Array { - return encode({ width: image.width, height: image.height, data: image.data, channels: 4, depth: 8 }); + if (image.data.length !== image.width * image.height * 4) { + throw new Error(`a ${image.width}x${image.height} image is ${image.width * image.height * 4} bytes of RGBA, not ${image.data.length}`); + } + const { FilterType, Transformer } = napi(); + return plain(Transformer.fromRgbaPixels(image.data, image.width, image.height).pngSync({ filterType: FilterType.Adaptive })); +} + +/** A native buffer's bytes as a plain `Uint8Array`, whose `slice` copies where a `Buffer`'s shares memory. */ +function plain(buffer: Buffer): Uint8Array { + return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); } /** - * A decoded PNG's samples as 8-bit RGBA: a palette expanded, gray spread to - * three channels, a missing alpha opaque, and 16-bit samples scaled to 8 - * bits with rounding. A gray or RGB image's transparent color, when it - * declares one, comes out with alpha 0. + * The decoder's pixels as 8-bit RGBA. The decoder has already expanded a + * palette, sub-byte samples, and a transparent color, so what is left is a + * layout: one to four channels at 8 or 16 bits, the 16-bit samples in native + * byte order. Gray spreads to three channels, a missing alpha is opaque, and + * 16-bit samples scale to 8 bits with rounding. */ -function toRgba(png: DecodedPng): Uint8Array { - const pixels = png.width * png.height; - if (png.palette !== undefined) { - const expanded = convertIndexedToRgb(png); - if (expanded.length === pixels * 4) return expanded; - return spread(expanded, 3, pixels, undefined); - } - if (png.depth !== 8 && png.depth !== 16) throw new Error(`unsupported PNG: ${png.depth}-bit samples without a palette`); - if (png.depth === 8 && png.channels === 4) return png.data instanceof Uint8Array ? png.data : Uint8Array.from(png.data); - const scale = png.depth === 16 ? (sample: number): number => Math.round((sample * 255) / 65535) : (sample: number): number => sample; - const samples = png.depth === 16 ? Uint8Array.from(png.data, scale) : png.data; - const key = png.channels === 1 || png.channels === 3 ? png.transparency : undefined; - return spread(samples, png.channels, pixels, key === undefined ? undefined : Array.from(key, scale)); +function toRgba(raw: Buffer, colorType: NapiImage.JsColorType, pixels: number): Uint8Array { + const layout = layoutOf(colorType); + if (layout === undefined) throw new Error(`unsupported PNG color type ${colorType}`); + const { channels, wide } = layout; + if (raw.length !== pixels * channels * (wide ? 2 : 1)) throw new Error('the decoded PNG pixels do not fill its size'); + const bytes = plain(raw); + if (channels === 4 && !wide) return bytes; + return spread(wide ? narrow(bytes) : bytes, channels, pixels); +} + +/** 16-bit samples in native byte order as 8-bit samples, rounded. */ +function narrow(bytes: Uint8Array): Uint8Array { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const samples = new Uint8Array(bytes.length / 2); + for (let at = 0; at < samples.length; at += 1) samples[at] = Math.round((view.getUint16(at * 2, LITTLE_ENDIAN) * 255) / 65535); + return samples; } -/** Samples of `channels` per pixel as RGBA; a pixel equal to `key` is transparent. */ -function spread(samples: ArrayLike, channels: number, pixels: number, key: readonly number[] | undefined): Uint8Array { +/** 8-bit samples of `channels` per pixel (gray, gray and alpha, RGB) as RGBA. */ +function spread(samples: Uint8Array, channels: number, pixels: number): Uint8Array { const out = new Uint8Array(pixels * 4); - for (let pixel = 0; pixel < pixels; pixel += 1) { - const at = pixel * channels; - const gray = channels <= 2; - const r = samples[at]!; - const g = gray ? r : samples[at + 1]!; - const b = gray ? r : samples[at + 2]!; - const alpha = channels === 2 || channels === 4 ? samples[at + channels - 1]! : 255; - const keyed = key !== undefined && (gray ? r === key[0] : r === key[0] && g === key[1] && b === key[2]); - out.set([r, g, b, keyed ? 0 : alpha], pixel * 4); + const gray = channels <= 2; + const alpha = channels === 2 || channels === 4; + for (let pixel = 0, from = 0, to = 0; pixel < pixels; pixel += 1, from += channels, to += 4) { + const r = samples[from]!; + out[to] = r; + out[to + 1] = gray ? r : samples[from + 1]!; + out[to + 2] = gray ? r : samples[from + 2]!; + out[to + 3] = alpha ? samples[from + channels - 1]! : 255; } return out; } diff --git a/packages/e2e/tests/integration/screenshot-assertions.test.ts b/packages/e2e/tests/integration/screenshot-assertions.test.ts index e776b546c9..8faa55eeb6 100644 --- a/packages/e2e/tests/integration/screenshot-assertions.test.ts +++ b/packages/e2e/tests/integration/screenshot-assertions.test.ts @@ -7,7 +7,7 @@ import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import path from 'node:path'; -import { PNG } from 'pngjs'; +import { JsColorType, Transformer } from '@napi-rs/image'; import { describe, expect, it } from 'vitest'; import type { EngineObserveOptions, ObservationPixels, SemanticNode } from '../../src/engine/index.ts'; import type { E2EConfig } from '../../src/index.ts'; @@ -36,20 +36,20 @@ const TREE: SemanticNode = { function screenPng(background: Rgb, button: Rgb, scale = 1): ObservationPixels { const width = WIDTH * scale; const height = HEIGHT * scale; - const png = new PNG({ width, height }); + const rgba = new Uint8Array(width * height * 4); for (let y = 0; y < height; y += 1) { for (let x = 0; x < width; x += 1) { const inButton = x >= BUTTON.x * scale && x < (BUTTON.x + BUTTON.width) * scale && y >= BUTTON.y * scale && y < (BUTTON.y + BUTTON.height) * scale; const color = inButton ? button : background; const at = (y * width + x) * 4; - png.data[at] = color[0]; - png.data[at + 1] = color[1]; - png.data[at + 2] = color[2]; - png.data[at + 3] = 255; + rgba[at] = color[0]; + rgba[at + 1] = color[1]; + rgba[at + 2] = color[2]; + rgba[at + 3] = 255; } } - return { data: new Uint8Array(PNG.sync.write(png)), mediaType: 'image/png', width, height, scale }; + return { data: new Uint8Array(Transformer.fromRgbaPixels(rgba, width, height).pngSync()), mediaType: 'image/png', width, height, scale }; } const WHITE: Rgb = [255, 255, 255]; @@ -97,8 +97,12 @@ function attemptOf(outcome: RunOutcome, title: string) { return result.attempts.at(-1)!; } -function readPng(file: string): PNG { - return PNG.sync.read(readFileSync(file)); +/** The PNG at `file` as RGBA, which is how the runner writes every screenshot it stores. */ +function readPng(file: string): { width: number; height: number; data: Uint8Array } { + const image = new Transformer(readFileSync(file)); + const { width, height, colorType } = image.metadataSync(); + if (colorType !== JsColorType.Rgba8) throw new Error(`${file} is not an 8-bit RGBA PNG: color type ${colorType}`); + return { width, height, data: image.rawPixelsSync() }; } const SCREEN_TEST = `import { test, expect } from 'e2e'; @@ -412,9 +416,7 @@ test('threshold', async ({ app, screen }) => { const { project } = await runProject({ 'tests/edges.e2e.ts': test }, options); try { const directory = path.join(project.dir, 'tests', 'edges.e2e.ts-snapshots'); - const small = new PNG({ width: 10, height: 10 }); - small.data.fill(255); - writeFileSync(path.join(directory, `small${SUFFIX}.png`), PNG.sync.write(small)); + writeFileSync(path.join(directory, `small${SUFFIX}.png`), Transformer.fromRgbaPixels(new Uint8Array(10 * 10 * 4).fill(255), 10, 10).pngSync()); screen = screenPng(WHITE, [0, 0, 250]); const outcome = await runExisting(project, options); diff --git a/packages/e2e/tests/integration/screenshot-codec.test.ts b/packages/e2e/tests/integration/screenshot-codec.test.ts new file mode 100644 index 0000000000..9f3d2fac98 --- /dev/null +++ b/packages/e2e/tests/integration/screenshot-codec.test.ts @@ -0,0 +1,82 @@ +/** + * `toHaveScreenshot` in a process where `@napi-rs/image` cannot load, as after + * `npm install --omit=optional`: the call fails as the codec's absence, with a + * screenshot stored or not, not as an unreadable baseline or a screen that + * never settled, and nothing is written. + */ + +import { existsSync } from 'node:fs'; +import { registerHooks } from 'node:module'; +import path from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import type { E2EConfig } from '../../src/index.ts'; +import { createFakeEngine, FAKE_APP, FAKE_APP_URL } from '../helpers/fake-engine.ts'; +import { runProject, type RunOutcome } from '../helpers/run-project.ts'; + +// This file imports nothing that loads the addon, so no earlier test in the process has. +const hooks = registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === '@napi-rs/image') throw new Error('Cannot find native binding. npm has a bug related to optional dependencies'); + return nextResolve(specifier, context); + }, +}); +afterAll(() => hooks.deregister()); + +const SCREEN_TEST = `import { test, expect } from 'e2e'; + +test('home', async ({ app, screen }) => { + await app.open('/'); + await expect(screen).toHaveScreenshot('home.png', { timeout: 1000 }); +}); +`; + +const SNAPSHOTS = 'tests/home.e2e.ts-snapshots'; +const STORED = `${SNAPSHOTS}/home-fake-${process.platform}.png`; + +/** A screen whose bytes are no PNG: a decode that got as far as reading them would fail differently. */ +function config(): E2EConfig { + const pixels = { data: new Uint8Array([1, 2, 3]), mediaType: 'image/png', width: 1, height: 1, scale: 1 } as const; + const fake = createFakeEngine({ fixtures: true, pixels: () => ({ pixels }) }); + return { targets: [{ name: 'fake', platform: 'web', engine: fake.engine, app: FAKE_APP }] } as E2EConfig; +} + +function attemptError(outcome: RunOutcome) { + const result = outcome.report.run.results.find((candidate) => candidate.titlePath.at(-1) === 'home'); + if (result === undefined) throw new Error('no reported result titled home'); + return result.attempts.at(-1)!.error; +} + +describe('toHaveScreenshot without the @napi-rs/image binding', () => { + it( + 'fails as IMAGE_CODEC_UNAVAILABLE, not as an unreadable stored screenshot', + async () => { + const { outcome, project } = await runProject( + { 'tests/home.e2e.ts': SCREEN_TEST, [STORED]: 'a screenshot some other machine took' }, + { appUrl: FAKE_APP_URL, config: config() }, + ); + try { + const error = attemptError(outcome); + expect(error?.code).toBe('IMAGE_CODEC_UNAVAILABLE'); + expect(error?.message).toContain(`@napi-rs/image-${process.platform}-${process.arch}`); + expect(error?.message).not.toContain('could not read the stored screenshot'); + } finally { + project.cleanup(); + } + }, + 60_000, + ); + + it( + 'fails the capture as IMAGE_CODEC_UNAVAILABLE when nothing is stored, and writes no screenshot', + async () => { + const { outcome, project } = await runProject({ 'tests/home.e2e.ts': SCREEN_TEST }, { appUrl: FAKE_APP_URL, config: config() }); + try { + expect(attemptError(outcome)?.code).toBe('IMAGE_CODEC_UNAVAILABLE'); + expect(existsSync(path.join(project.dir, SNAPSHOTS))).toBe(false); + } finally { + project.cleanup(); + } + }, + 60_000, + ); +}); diff --git a/packages/e2e/tests/integration/typescript-loader.test.ts b/packages/e2e/tests/integration/typescript-loader.test.ts index f12ca8c1c2..7185ff24b7 100644 --- a/packages/e2e/tests/integration/typescript-loader.test.ts +++ b/packages/e2e/tests/integration/typescript-loader.test.ts @@ -293,6 +293,23 @@ describe('the TypeScript loader', () => { expect(output).toContain(`@oxc-transform/binding-${process.platform}-${process.arch}`); }); + it('runs --version, --help, and a test that takes no screenshot without @napi-rs/image', async () => { + createProject({ type: 'module' }); + const hook = "import { registerHooks } from 'node:module';\nregisterHooks({ resolve(specifier, context, nextResolve) {\n if (specifier === '@napi-rs/image') throw new Error('Cannot find native binding. npm has a bug related to optional dependencies');\n return nextResolve(specifier, context);\n} });\n"; + write({ + 'no-image.mjs': hook, + 'tests/plain.e2e.ts': "import { expect, test } from 'e2e';\n\ntest('runs', () => {\n expect(1 as number).toBe(1);\n});\n", + }); + const env = { NODE_OPTIONS: `--import ${pathToFileURL(path.join(dir, 'no-image.mjs')).href}` }; + for (const flag of ['--version', '--help']) { + const { stdout } = await execFileAsync(process.execPath, [CLI, flag], { cwd: dir, env: { ...process.env, ...env } }); + expect(stdout.length).toBeGreaterThan(0); + } + const { code, stdout } = await runCli(['tests/plain.e2e.ts'], env); + expect(stdout).toContain('1 passed'); + expect(code).toBe(0); + }); + it.each([ ['an auto-accessor', { 'lib/accessor.ts': 'export class A {\n accessor w = 5;\n}\n' }, '../lib/accessor', 'lib/accessor.ts:2:3: `accessor` class fields (auto-accessors) are not supported'], [ diff --git a/packages/e2e/tests/unit/agent-pixels.test.ts b/packages/e2e/tests/unit/agent-pixels.test.ts index 50ff3652ea..806849710d 100644 --- a/packages/e2e/tests/unit/agent-pixels.test.ts +++ b/packages/e2e/tests/unit/agent-pixels.test.ts @@ -1,28 +1,31 @@ /** Sizing a screenshot for the model: the cap, the scale arithmetic, and masks that stay black. */ -import { PNG } from 'pngjs'; +import { JsColorType, Transformer } from '@napi-rs/image'; import { describe, expect, it } from 'vitest'; import type { ObservationPixels } from '../../src/engine/surface.ts'; import { downscalePixels } from '../../src/agent/pixels.ts'; /** A width by height PNG: white, with a black box at the given rect. */ function picture(width: number, height: number, box: { x: number; y: number; w: number; h: number }): ObservationPixels { - const png = new PNG({ width, height }); + const rgba = new Uint8Array(width * height * 4); for (let y = 0; y < height; y += 1) { for (let x = 0; x < width; x += 1) { const at = (y * width + x) * 4; const inBox = x >= box.x && x < box.x + box.w && y >= box.y && y < box.y + box.h; - png.data[at] = png.data[at + 1] = png.data[at + 2] = inBox ? 0 : 255; - png.data[at + 3] = 255; + rgba[at] = rgba[at + 1] = rgba[at + 2] = inBox ? 0 : 255; + rgba[at + 3] = 255; } } - return { data: new Uint8Array(PNG.sync.write(png)), mediaType: 'image/png', width, height, scale: 1 }; + return { data: new Uint8Array(Transformer.fromRgbaPixels(rgba, width, height).pngSync()), mediaType: 'image/png', width, height, scale: 1 }; } +/** The color of one pixel of a downscaled screenshot, which the runner writes as RGBA. */ function pixelAt(pixels: ObservationPixels, x: number, y: number): number[] { - const png = PNG.sync.read(Buffer.from(pixels.data)); - const at = (y * png.width + x) * 4; - return [...png.data.subarray(at, at + 3)]; + const image = new Transformer(pixels.data); + const { width, colorType } = image.metadataSync(); + if (colorType !== JsColorType.Rgba8) throw new Error(`expected an 8-bit RGBA PNG, got color type ${colorType}`); + const at = (y * width + x) * 4; + return [...image.rawPixelsSync().subarray(at, at + 3)]; } describe('downscalePixels', () => { diff --git a/packages/e2e/tests/unit/image-codec.test.ts b/packages/e2e/tests/unit/image-codec.test.ts new file mode 100644 index 0000000000..48eb315e13 --- /dev/null +++ b/packages/e2e/tests/unit/image-codec.test.ts @@ -0,0 +1,72 @@ +/** `@napi-rs/image` loads when the first PNG is decoded or encoded, and a binding that is missing fails that call, and only that, as IMAGE_CODEC_UNAVAILABLE. */ + +import { registerHooks } from 'node:module'; +import { afterAll, describe, expect, it } from 'vitest'; +import { downscalePixels } from '../../src/agent/pixels.ts'; +import { InfrastructureError } from '../../src/internal/errors.ts'; +import { clipBox, compareImages, cropImage, decodePng, encodePng, resampleImage, type RgbaImage } from '../../src/internal/image.ts'; + +// What the loader reports when npm left the binding out. This file imports nothing that loads the addon, so no earlier test in the process has. +let loaderFailure = new Error('Cannot find native binding. npm has a bug related to optional dependencies'); +const hooks = registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier === '@napi-rs/image') throw loaderFailure; + return nextResolve(specifier, context); + }, +}); +afterAll(() => hooks.deregister()); + +/** The error `run` throws; the test fails when it throws none. */ +function failure(run: () => unknown): InfrastructureError { + try { + run(); + } catch (cause) { + if (cause instanceof InfrastructureError) return cause; + throw cause; + } + throw new Error('expected a throw'); +} + +/** A width by height image of one gray level, built without the addon. */ +function gray(width: number, height: number, level: number): RgbaImage { + const data = new Uint8Array(width * height * 4); + for (let at = 0; at < data.length; at += 4) data.set([level, level, level, 255], at); + return { width, height, data }; +} + +const BINDING = `@napi-rs/image-${process.platform}-${process.arch}`; +const exact = { threshold: 0, maxDiffPixels: undefined, maxDiffPixelRatio: undefined }; + +describe('without the @napi-rs/image binding', () => { + it('fails a decode and an encode as IMAGE_CODEC_UNAVAILABLE, naming the binding and the way back', () => { + for (const read of [() => decodePng(new Uint8Array(8)), () => encodePng(gray(1, 1, 0))]) { + const error = failure(read); + expect(error.code).toBe('IMAGE_CODEC_UNAVAILABLE'); + expect(error.category).toBe('infrastructure'); + expect(error.message).toContain(BINDING); + expect(error.message).toContain('npm/cli#4828'); + expect(error.cause).toBe(loaderFailure); + } + }); + + it('gives the loader error for a codec that fails to load for another reason', () => { + loaderFailure = new Error('dlopen failed: bad ELF header\n at Module._extensions'); + const error = failure(() => decodePng(new Uint8Array(8))); + expect(error.message).toContain('it could not be loaded (dlopen failed: bad ELF header)'); + expect(error.message).not.toContain('--omit=optional'); + }); + + it('fails a model-sized screenshot instead of passing the capture on', () => { + const pixels = { data: new Uint8Array(8), mediaType: 'image/png', width: 2000, height: 1000, scale: 1 } as const; + expect(failure(() => downscalePixels(pixels, 768)).code).toBe('IMAGE_CODEC_UNAVAILABLE'); + expect(downscalePixels(pixels, 2000)).toBe(pixels); + }); + + it('still runs the image operations that read no PNG', () => { + const image = gray(8, 8, 90); + expect(compareImages(image, gray(8, 8, 90), exact).kind).toBe('match'); + expect(compareImages(image, gray(8, 8, 200), exact).kind).toBe('pixels'); + expect(resampleImage(image, 4, 4).data.length).toBe(4 * 4 * 4); + expect(cropImage(image, clipBox(image, { x: 2, y: 2, width: 3, height: 3 })!).width).toBe(3); + }); +}); diff --git a/packages/e2e/tests/unit/image.test.ts b/packages/e2e/tests/unit/image.test.ts index 2e3e32fd93..d764c571ef 100644 --- a/packages/e2e/tests/unit/image.test.ts +++ b/packages/e2e/tests/unit/image.test.ts @@ -1,7 +1,7 @@ /** The in-memory image operations `toHaveScreenshot` is built on: clipping, cropping, painting, the PNG round trip, and the comparison's tolerance. */ -import { encode } from 'fast-png'; -import { PNG } from 'pngjs'; +import { crc32, deflateSync } from 'node:zlib'; +import { Transformer } from '@napi-rs/image'; import { describe, expect, it } from 'vitest'; import { clipBox, compareImages, cropImage, decodePng, encodePng, fillBoxes, resampleImage, type RgbaImage } from '../../src/internal/image.ts'; @@ -14,6 +14,53 @@ function gray(width: number, height: number, level: number): RgbaImage { const exact = { threshold: 0, maxDiffPixels: undefined, maxDiffPixelRatio: undefined }; +/** One PNG chunk: its length, type, data, and the CRC of type and data. */ +function chunk(type: string, data: Uint8Array = new Uint8Array()): Buffer { + const body = Buffer.concat([Buffer.from(type, 'latin1'), data]); + const out = Buffer.alloc(body.length + 8); + out.writeUInt32BE(data.length, 0); + body.copy(out, 4); + out.writeUInt32BE(crc32(body), body.length + 4); + return out; +} + +/** Samples as 16-bit big-endian bytes, the order a PNG stores them in. */ +function sixteen(...samples: number[]): number[] { + return samples.flatMap((sample) => [sample >> 8, sample & 255]); +} + +/** + * A PNG written straight from the format: `rows` hold each scanline's sample + * bytes, packed as the color type and `depth` say, with no filter. It does + * not go through the library under test, so it can say what an encoder never would. + */ +function png(spec: { + width: number; + height: number; + depth: number; + colorType: number; + rows: readonly (readonly number[])[]; + palette?: readonly (readonly number[])[]; + transparency?: readonly number[]; +}): Uint8Array { + const header = Buffer.alloc(13); + header.writeUInt32BE(spec.width, 0); + header.writeUInt32BE(spec.height, 4); + header[8] = spec.depth; + header[9] = spec.colorType; + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + chunk('IHDR', header), + ...(spec.palette === undefined ? [] : [chunk('PLTE', Buffer.from(spec.palette.flat()))]), + ...(spec.transparency === undefined ? [] : [chunk('tRNS', Buffer.from(spec.transparency))]), + chunk('IDAT', deflateSync(Buffer.concat(spec.rows.map((row) => Buffer.from([0, ...row]))))), + chunk('IEND'), + ]); +} + +/** Red, green, and blue: the palette the indexed cases share. */ +const PALETTE = [[255, 0, 0], [0, 255, 0], [0, 0, 255]]; + describe('image', () => { it('round-trips through PNG', () => { const image = gray(3, 2, 120); @@ -21,15 +68,124 @@ describe('image', () => { }); it.each([ - ['gray', { channels: 1, depth: 8, data: Uint8Array.from([0, 90, 180, 255, 30, 60]) }], - ['gray with alpha', { channels: 2, depth: 8, data: Uint8Array.from([0, 255, 90, 128, 180, 0, 255, 255, 30, 10, 60, 200]) }], - ['RGB', { channels: 3, depth: 8, data: Uint8Array.from([255, 0, 0, 0, 255, 0, 0, 0, 255, 10, 20, 30, 40, 50, 60, 70, 80, 90]) }], - ['16-bit RGBA', { channels: 4, depth: 16, data: Uint16Array.from([65535, 0, 0, 65535, 0, 32768, 0, 65535, 0, 0, 65535, 0, 4096, 8192, 12288, 65535, 1, 2, 3, 4, 60000, 50000, 40000, 30000]) }], - ['a palette', { channels: 1, depth: 8, data: Uint8Array.from([0, 1, 2, 1, 0, 2]), palette: [[255, 0, 0], [0, 255, 0], [0, 0, 255]] }], - ] as const)('decodes %s to the RGBA pngjs reads from the same file', (_label, image) => { - const bytes = encode({ width: 3, height: 2, ...image } as Parameters[0]); - const oracle = PNG.sync.read(Buffer.from(bytes)); - expect([...decodePng(bytes).data]).toEqual([...oracle.data]); + ['gray', png({ width: 3, height: 2, depth: 8, colorType: 0, rows: [[0, 90, 180], [255, 30, 60]] }), [ + 0, 0, 0, 255, 90, 90, 90, 255, 180, 180, 180, 255, + 255, 255, 255, 255, 30, 30, 30, 255, 60, 60, 60, 255, + ]], + ['gray with alpha', png({ width: 3, height: 2, depth: 8, colorType: 4, rows: [[0, 255, 90, 128, 180, 0], [255, 255, 30, 10, 60, 200]] }), [ + 0, 0, 0, 255, 90, 90, 90, 128, 180, 180, 180, 0, + 255, 255, 255, 255, 30, 30, 30, 10, 60, 60, 60, 200, + ]], + ['RGB', png({ width: 3, height: 2, depth: 8, colorType: 2, rows: [[255, 0, 0, 0, 255, 0, 0, 0, 255], [10, 20, 30, 40, 50, 60, 70, 80, 90]] }), [ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, + 10, 20, 30, 255, 40, 50, 60, 255, 70, 80, 90, 255, + ]], + ['RGBA', png({ width: 3, height: 2, depth: 8, colorType: 6, rows: [[255, 0, 0, 255, 0, 255, 0, 128, 0, 0, 255, 0], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]] }), [ + 255, 0, 0, 255, 0, 255, 0, 128, 0, 0, 255, 0, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + ]], + ['16-bit gray', png({ width: 3, height: 2, depth: 16, colorType: 0, rows: [sixteen(0, 32768, 65535), sixteen(1, 257, 60000)] }), [ + 0, 0, 0, 255, 128, 128, 128, 255, 255, 255, 255, 255, + 0, 0, 0, 255, 1, 1, 1, 255, 233, 233, 233, 255, + ]], + ['16-bit RGBA', png({ + width: 3, height: 2, depth: 16, colorType: 6, + rows: [sixteen(65535, 0, 0, 65535, 0, 32768, 0, 65535, 0, 0, 65535, 0), sixteen(4096, 8192, 12288, 65535, 1, 2, 3, 4, 60000, 50000, 40000, 30000)], + }), [ + 255, 0, 0, 255, 0, 128, 0, 255, 0, 0, 255, 0, + 16, 32, 48, 255, 0, 0, 0, 0, 233, 195, 156, 117, + ]], + ['a palette', png({ width: 3, height: 2, depth: 8, colorType: 3, palette: PALETTE, rows: [[0, 1, 2], [1, 0, 2]] }), [ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, + 0, 255, 0, 255, 255, 0, 0, 255, 0, 0, 255, 255, + ]], + ['a palette with alpha', png({ width: 3, height: 2, depth: 8, colorType: 3, palette: PALETTE, transparency: [255, 128, 0], rows: [[0, 1, 2], [1, 0, 2]] }), [ + 255, 0, 0, 255, 0, 255, 0, 128, 0, 0, 255, 0, + 0, 255, 0, 128, 255, 0, 0, 255, 0, 0, 255, 0, + ]], + ['a 2-bit palette', png({ width: 3, height: 2, depth: 2, colorType: 3, palette: PALETTE, rows: [[0b00_01_10_00], [0b01_00_10_00]] }), [ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, + 0, 255, 0, 255, 255, 0, 0, 255, 0, 0, 255, 255, + ]], + ['1-bit gray', png({ width: 3, height: 2, depth: 1, colorType: 0, rows: [[0b010_00000], [0b101_00000]] }), [ + 0, 0, 0, 255, 255, 255, 255, 255, 0, 0, 0, 255, + 255, 255, 255, 255, 0, 0, 0, 255, 255, 255, 255, 255, + ]], + ['4-bit gray', png({ width: 3, height: 2, depth: 4, colorType: 0, rows: [[0x0f, 0x80], [0x37, 0x00]] }), [ + 0, 0, 0, 255, 255, 255, 255, 255, 136, 136, 136, 255, + 51, 51, 51, 255, 119, 119, 119, 255, 0, 0, 0, 255, + ]], + ['gray with a transparent color', png({ width: 3, height: 2, depth: 8, colorType: 0, transparency: [0, 90], rows: [[0, 90, 180], [255, 90, 60]] }), [ + 0, 0, 0, 255, 90, 90, 90, 0, 180, 180, 180, 255, + 255, 255, 255, 255, 90, 90, 90, 0, 60, 60, 60, 255, + ]], + ['RGB with a transparent color', png({ width: 3, height: 2, depth: 8, colorType: 2, transparency: [0, 0, 0, 255, 0, 0], rows: [[255, 0, 0, 0, 255, 0, 0, 0, 255], [10, 20, 30, 0, 255, 0, 70, 80, 90]] }), [ + 255, 0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 255, + 10, 20, 30, 255, 0, 255, 0, 0, 70, 80, 90, 255, + ]], + // Samples chosen where scaling 16 bits to 8 rounds both ways: 128 and 255 down, 129 and 32768 up. + ['16-bit RGB', png({ + width: 3, height: 2, depth: 16, colorType: 2, + rows: [sixteen(0, 65535, 32768, 257, 128, 129, 1000, 40000, 65000), sixteen(65535, 0, 0, 12345, 54321, 22222, 255, 256, 65534)], + }), [ + 0, 255, 128, 255, 1, 0, 1, 255, 4, 156, 253, 255, + 255, 0, 0, 255, 48, 211, 86, 255, 1, 1, 255, 255, + ]], + ['16-bit gray with alpha', png({ + width: 3, height: 2, depth: 16, colorType: 4, + rows: [sixteen(0, 65535, 32768, 32768, 65535, 0), sixteen(1000, 129, 257, 128, 40000, 65000)], + }), [ + 0, 0, 0, 255, 128, 128, 128, 128, 255, 255, 255, 0, + 4, 4, 4, 1, 1, 1, 1, 0, 156, 156, 156, 253, + ]], + // Every sample here scales to 0 at 8 bits, so only the 16-bit comparison tells (1, 2, 3) from the colors beside it. + ['16-bit RGB with a transparent color', png({ + width: 3, height: 2, depth: 16, colorType: 2, transparency: sixteen(1, 2, 3), + rows: [sixteen(1, 2, 3, 4, 5, 6, 7, 8, 9), sixteen(1, 2, 3, 1, 2, 4, 9, 9, 9)], + }), [ + 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 255, + 0, 0, 0, 0, 0, 0, 0, 255, 0, 0, 0, 255, + ]], + ] as const)('decodes %s to 8-bit RGBA', (_label, bytes, rgba) => { + const decoded = decodePng(bytes); + expect([decoded.width, decoded.height]).toEqual([3, 2]); + expect([...decoded.data]).toEqual(rgba); + }); + + it('keeps every sample of an odd-sized 8-bit RGB image when it adds the alpha channel', () => { + const width = 37; + const height = 11; + const samples = Array.from({ length: width * height * 3 }, (_, at) => (at * 2654435761) >>> 24); + const rows = Array.from({ length: height }, (_, y) => samples.slice(y * width * 3, (y + 1) * width * 3)); + const expected = Array.from({ length: width * height }, (_, pixel) => [...samples.slice(pixel * 3, pixel * 3 + 3), 255]).flat(); + const decoded = decodePng(png({ width, height, depth: 8, colorType: 2, rows })); + expect([decoded.width, decoded.height]).toEqual([width, height]); + expect([...decoded.data]).toEqual(expected); + }); + + it('encodes 8-bit RGBA PNG and returns decoded pixels a painted copy does not share', () => { + const encoded = encodePng(gray(4, 4, 0)); + expect([encoded[24], encoded[25]]).toEqual([8, 6]); + const decoded = decodePng(encoded); + const painted = fillBoxes(decoded, [{ x: 0, y: 0, width: 2, height: 2 }], [255, 0, 0]); + expect([...painted.data.subarray(0, 4)]).toEqual([255, 0, 0, 255]); + expect([...decoded.data.subarray(0, 4)]).toEqual([0, 0, 0, 255]); + }); + + it('throws on bytes that are not a PNG, a PNG cut short, or one with a corrupt chunk', () => { + const valid = png({ width: 2, height: 2, depth: 8, colorType: 2, rows: [[9, 9, 9, 9, 9, 9], [9, 9, 9, 9, 9, 9]] }); + const corrupt = Uint8Array.from(valid); + corrupt[20] = 255; + const jpeg = new Uint8Array(Transformer.fromRgbaPixels(new Uint8Array(16).fill(255), 2, 2).jpegSync()); + for (const bytes of [new Uint8Array(), new Uint8Array([1, 2, 3]), valid.subarray(0, valid.length - 10), valid.subarray(0, 40), corrupt, jpeg]) { + expect(() => decodePng(bytes)).toThrow(); + } + expect(() => decodePng(jpeg)).toThrow('not a PNG'); + }); + + it('refuses pixels that do not fill the image, where the addon would abort the process', () => { + expect(() => encodePng({ width: 2, height: 2, data: new Uint8Array(17) })).toThrow('not 17'); + expect(() => encodePng({ width: 2, height: 2, data: new Uint8Array(15) })).toThrow('not 15'); }); it('clips a box to the image on whole pixels, and has none for a box outside it', () => { diff --git a/packages/e2e/tests/unit/package-imports.test.ts b/packages/e2e/tests/unit/package-imports.test.ts index 673c8a4fab..80582b0985 100644 --- a/packages/e2e/tests/unit/package-imports.test.ts +++ b/packages/e2e/tests/unit/package-imports.test.ts @@ -183,9 +183,12 @@ function startupPackages(dir: string, entry: string): Map { /** What a process loads for an entry before it knows what it will do, and the packages that serve only some of what it does. */ const STARTUP: readonly { readonly dir: string; readonly entry: string; readonly lazy: readonly string[] }[] = [ // `e2e run`, `list`, and `--help` need none of the MCP server, the prompts, or the schemas of `mcp`, `init`, `login`, and `explore`. - { dir: 'e2e', entry: 'cli/index.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod'] }, - { dir: 'e2e', entry: 'runner.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod', 'commander'] }, - { dir: 'e2e', entry: 'run/worker/entry.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod', 'commander'] }, + // None needs the image codec's native binding either: the first PNG decoded or encoded loads it, and a missing one fails only that call. + { dir: 'e2e', entry: 'cli/index.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod', '@napi-rs/image'] }, + { dir: 'e2e', entry: 'runner.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod', 'commander', '@napi-rs/image'] }, + { dir: 'e2e', entry: 'run/worker/entry.ts', lazy: ['@modelcontextprotocol/server', '@clack/prompts', 'zod', 'commander', '@napi-rs/image'] }, + // Every test file imports this entry. + { dir: 'e2e', entry: 'index.ts', lazy: ['@napi-rs/image'] }, // A config imports these in every process, and only a worker that runs an ACP step or leases a Limrun device calls them. { dir: 'acp', entry: 'index.ts', lazy: ['@agentclientprotocol/sdk', '@modelcontextprotocol/server'] }, { dir: 'limrun', entry: 'index.ts', lazy: ['@limrun/api'] }, diff --git a/packages/mobile/package.json b/packages/mobile/package.json index 7e515eb000..f26ca9eaea 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -63,8 +63,8 @@ "test:unit": "vitest run tests/unit" }, "dependencies": { + "@napi-rs/image": "1.16.0", "agent-device": "0.21.36", - "pngjs": "7.0.0", "zod": "4.6.7" }, "peerDependencies": { @@ -78,7 +78,6 @@ }, "devDependencies": { "@types/node": "26.6.6", - "@types/pngjs": "6.0.5", "ai": "7.0.163", "e2e": "workspace:*", "typescript": "7.0.2", diff --git a/packages/mobile/src/png.ts b/packages/mobile/src/png.ts index b6537a3564..184f5164d9 100644 --- a/packages/mobile/src/png.ts +++ b/packages/mobile/src/png.ts @@ -1,11 +1,32 @@ -/** Screenshot masking policy; PNG parsing and encoding belong to pngjs. */ -import { PNG } from 'pngjs'; +/** Screenshot masking policy: a PNG decoded with @napi-rs/image, its secure regions painted black, encoded again; the layout code turns the addon's pixels into 8-bit RGBA. */ +import { endianness } from 'node:os'; +import { FilterType, JsColorType, Transformer } from '@napi-rs/image'; import type { Rect } from './support.ts'; +const LITTLE_ENDIAN = endianness() === 'LE'; + +/** Channels per pixel, and whether each is a 16-bit sample, for every color type a decode reads: 8-bit RGB is widened by the addon before that. */ +const LAYOUTS: Readonly>> = { + [JsColorType.L8]: { channels: 1, wide: false }, + [JsColorType.La8]: { channels: 2, wide: false }, + [JsColorType.Rgba8]: { channels: 4, wide: false }, + [JsColorType.L16]: { channels: 1, wide: true }, + [JsColorType.La16]: { channels: 2, wide: true }, + [JsColorType.Rgb16]: { channels: 3, wide: true }, + [JsColorType.Rgba16]: { channels: 4, wide: true }, +}; + +/** An 8-bit RGBA image, rows top to bottom. */ +interface Rgba { + readonly width: number; + readonly height: number; + readonly data: Uint8Array; +} + /** Paints each requested rectangle opaque black, rounding outward and clipping to the image. */ export function maskPng(bytes: Uint8Array, rects: readonly Rect[]): Uint8Array { if (rects.length === 0) return bytes; - const image = PNG.sync.read(Buffer.from(bytes)); + const image = decodeRgba(bytes); for (const rect of rects) { const x0 = Math.max(0, Math.floor(rect.x)); const y0 = Math.max(0, Math.floor(rect.y)); @@ -21,5 +42,58 @@ export function maskPng(bytes: Uint8Array, rects: readonly Rect[]): Uint8Array { } } } - return PNG.sync.write(image); + return encodeRgba(image); +} + +/** + * Decodes PNG bytes to 8-bit RGBA, whatever the color type, bit depth, or + * palette; throws on bytes that are not a PNG. An 8-bit RGB screenshot is + * widened by the addon: opacity 1 keeps every sample and adds the opaque + * alpha channel, several times faster than `spread` does it. + */ +function decodeRgba(bytes: Uint8Array): Rgba { + const decoder = new Transformer(bytes); + const { width, height, format, colorType } = decoder.metadataSync(); + if (format !== 'png') throw new Error(`not a PNG: the bytes are ${format}`); + const rgb = colorType === JsColorType.Rgb8; + const raw = (rgb ? decoder.opacity(1) : decoder).rawPixelsSync(); + const layout = LAYOUTS[rgb ? JsColorType.Rgba8 : colorType]; + if (layout === undefined) throw new Error(`unsupported PNG color type ${colorType}`); + const { channels, wide } = layout; + if (raw.length !== width * height * channels * (wide ? 2 : 1)) throw new Error('the decoded PNG pixels do not fill its size'); + const samples = new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength); + if (channels === 4 && !wide) return { width, height, data: samples }; + return { width, height, data: spread(wide ? narrow(samples) : samples, channels, width * height) }; +} + +/** + * Encodes an image as PNG bytes, each row adaptively filtered: the addon's + * default writes no filter, which makes screenshots markedly larger. + */ +function encodeRgba(image: Rgba): Uint8Array { + const png = Transformer.fromRgbaPixels(image.data, image.width, image.height).pngSync({ filterType: FilterType.Adaptive }); + return new Uint8Array(png.buffer, png.byteOffset, png.byteLength); +} + +/** 16-bit samples in native byte order as 8-bit samples, rounded. */ +function narrow(bytes: Uint8Array): Uint8Array { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const samples = new Uint8Array(bytes.length / 2); + for (let at = 0; at < samples.length; at += 1) samples[at] = Math.round((view.getUint16(at * 2, LITTLE_ENDIAN) * 255) / 65535); + return samples; +} + +/** 8-bit samples of `channels` per pixel (gray, gray and alpha, RGB) as RGBA; a gray or RGB image is opaque. */ +function spread(samples: Uint8Array, channels: number, pixels: number): Uint8Array { + const out = new Uint8Array(pixels * 4); + const gray = channels <= 2; + const alpha = channels === 2 || channels === 4; + for (let pixel = 0, from = 0, to = 0; pixel < pixels; pixel += 1, from += channels, to += 4) { + const r = samples[from]!; + out[to] = r; + out[to + 1] = gray ? r : samples[from + 1]!; + out[to + 2] = gray ? r : samples[from + 2]!; + out[to + 3] = alpha ? samples[from + channels - 1]! : 255; + } + return out; } diff --git a/packages/mobile/tests/helpers/png.ts b/packages/mobile/tests/helpers/png.ts index df01f245e2..a2bcc92247 100644 --- a/packages/mobile/tests/helpers/png.ts +++ b/packages/mobile/tests/helpers/png.ts @@ -1,4 +1,5 @@ -import { PNG } from 'pngjs'; +import { crc32, deflateSync } from 'node:zlib'; +import { FilterType, JsColorType, Transformer } from '@napi-rs/image'; export interface DecodedPng { readonly width: number; @@ -7,18 +8,53 @@ export interface DecodedPng { readonly pixels: Uint8Array; } -/** Reads assertion pixels through the library, independently of the masking policy. */ +/** Reads assertion pixels through the library, independently of the masking policy: RGB and RGBA PNGs, both as RGBA. */ export function decodePng(bytes: Uint8Array): DecodedPng { - const image = PNG.sync.read(Buffer.from(bytes)); - return { width: image.width, height: image.height, channels: 4, pixels: image.data }; + const image = new Transformer(bytes); + const { width, height, colorType } = image.metadataSync(); + const raw = image.rawPixelsSync(); + if (colorType === JsColorType.Rgba8) return { width, height, channels: 4, pixels: raw }; + if (colorType !== JsColorType.Rgb8) throw new Error(`expected an RGB or RGBA PNG, got color type ${colorType}`); + const pixels = new Uint8Array(width * height * 4); + for (let from = 0, to = 0; from < raw.length; from += 3, to += 4) { + pixels.set(raw.subarray(from, from + 3), to); + pixels[to + 3] = 255; + } + return { width, height, channels: 4, pixels }; } -/** Encodes a synthetic screenshot in RGB or RGBA, with a chosen PNG row filter. */ -export function encodePng(image: DecodedPng, filterType = -1): Uint8Array { - return PNG.sync.write({ width: image.width, height: image.height, data: Buffer.from(image.pixels) } as PNG, { - inputColorType: image.channels === 3 ? 2 : 6, - colorType: image.channels === 3 ? 2 : 6, - inputHasAlpha: image.channels === 4, - filterType, - }); +/** One PNG chunk: its length, type, data, and the CRC of type and data. */ +function chunk(type: string, data: Uint8Array = new Uint8Array()): Buffer { + const body = Buffer.concat([Buffer.from(type, 'latin1'), data]); + const out = Buffer.alloc(body.length + 8); + out.writeUInt32BE(data.length, 0); + body.copy(out, 4); + out.writeUInt32BE(crc32(body), body.length + 4); + return out; +} + +/** + * A PNG written straight from the format, so a test can hand the code under + * test a color type no encoder of its own would write: `rows` hold each + * scanline's sample bytes as the color type and `depth` pack them, with no filter. + */ +export function rawPng(width: number, height: number, depth: number, colorType: number, rows: readonly (readonly number[])[]): Uint8Array { + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = depth; + header[9] = colorType; + return Buffer.concat([ + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + chunk('IHDR', header), + chunk('IDAT', deflateSync(Buffer.concat(rows.map((row) => Buffer.from([0, ...row]))))), + chunk('IEND'), + ]); +} + +/** Encodes a synthetic screenshot in RGB or RGBA. An RGBA one takes a chosen PNG row filter; an RGB one is written unfiltered. */ +export function encodePng(image: DecodedPng, filterType: FilterType = FilterType.Adaptive): Uint8Array { + if (image.channels === 4) return new Uint8Array(Transformer.fromRgbaPixels(image.pixels, image.width, image.height).pngSync({ filterType })); + const stride = image.width * 3; + return rawPng(image.width, image.height, 8, 2, Array.from({ length: image.height }, (_, row) => [...image.pixels.subarray(row * stride, (row + 1) * stride)])); } diff --git a/packages/mobile/tests/unit/png.test.ts b/packages/mobile/tests/unit/png.test.ts index 53007978c5..8c2b512572 100644 --- a/packages/mobile/tests/unit/png.test.ts +++ b/packages/mobile/tests/unit/png.test.ts @@ -1,6 +1,7 @@ +import { FilterType, Transformer } from '@napi-rs/image'; import { describe, expect, it } from 'vitest'; import { maskPng } from '../../src/png.ts'; -import { decodePng, encodePng, type DecodedPng } from '../helpers/png.ts'; +import { decodePng, encodePng, rawPng, type DecodedPng } from '../helpers/png.ts'; function solid(width: number, height: number, channels: 3 | 4, value: number): DecodedPng { const pixels = new Uint8Array(width * height * channels).fill(value); @@ -9,7 +10,7 @@ function solid(width: number, height: number, channels: 3 | 4, value: number): D } describe('screenshot masking', () => { - it.each([0, 4])('masks and clips RGBA screenshots encoded with filter %i', (filter) => { + it.each([FilterType.NoFilter, FilterType.Paeth])('masks and clips RGBA screenshots encoded with filter %i', (filter) => { const masked = decodePng(maskPng(encodePng(solid(10, 10, 4, 255), filter), [{ x: 2, y: 2, width: 3, height: 2 }, { x: 8, y: 8, width: 50, height: 50 }])); const at = (x: number, y: number) => [...masked.pixels.subarray((y * 10 + x) * 4, (y * 10 + x) * 4 + 4)]; expect(at(2, 2)).toEqual([0, 0, 0, 255]); @@ -20,6 +21,27 @@ describe('screenshot masking', () => { expect(at(7, 7)).toEqual([255, 255, 255, 200]); }); + /** A 4 by 4 screenshot whose every pixel has the same `samples`, as the color type and bit depth pack them. */ + function filled(depth: number, colorType: number, samples: readonly number[]): Uint8Array { + const row = Array.from({ length: 4 }, () => samples).flat(); + return rawPng(4, 4, depth, colorType, Array.from({ length: 4 }, () => row)); + } + + it.each([ + ['gray', filled(8, 0, [200]), [200, 200, 200, 255]], + ['gray with alpha', filled(8, 4, [200, 128]), [200, 200, 200, 128]], + ['RGB', filled(8, 2, [10, 20, 30]), [10, 20, 30, 255]], + ['16-bit RGB', filled(16, 2, [255, 255, 128, 0, 1, 1]), [255, 128, 1, 255]], + ['16-bit RGBA', filled(16, 6, [0, 0, 255, 255, 0, 0, 128, 0]), [0, 255, 0, 128]], + ] as const)('masks a %s screenshot and writes the rest of it as RGBA', (_label, bytes, color) => { + const masked = decodePng(maskPng(bytes, [{ x: 0, y: 0, width: 2, height: 2 }])); + const at = (x: number, y: number) => [...masked.pixels.subarray((y * 4 + x) * 4, (y * 4 + x) * 4 + 4)]; + expect(at(0, 0)).toEqual([0, 0, 0, 255]); + expect(at(1, 1)).toEqual([0, 0, 0, 255]); + expect(at(2, 0)).toEqual(color); + expect(at(3, 3)).toEqual(color); + }); + it('returns the bytes untouched when there is nothing to mask, and refuses other formats', () => { const bytes = encodePng(solid(2, 2, 3, 9)); expect(maskPng(bytes, [])).toBe(bytes); @@ -29,6 +51,8 @@ describe('screenshot masking', () => { const corrupt = Uint8Array.from(bytes); corrupt[20] = 255; expect(() => maskPng(corrupt, rects)).toThrow(); + const jpeg = new Uint8Array(Transformer.fromRgbaPixels(new Uint8Array(16).fill(255), 2, 2).jpegSync()); + expect(() => maskPng(jpeg, rects)).toThrow('not a PNG'); const masked = decodePng(maskPng(bytes, rects)); expect([...masked.pixels.subarray(0, 4)]).toEqual([0, 0, 0, 255]); expect([...masked.pixels.subarray(4, 8)]).toEqual([9, 9, 9, 255]); diff --git a/packages/web/package.json b/packages/web/package.json index 35cfebfae4..2ad4a51112 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -61,10 +61,9 @@ "e2e": ">=0.22.1 <1" }, "devDependencies": { + "@napi-rs/image": "1.16.0", "@types/node": "26.6.6", - "@types/pngjs": "6.0.5", "e2e": "workspace:*", - "pngjs": "7.0.0", "typescript": "7.0.2", "vitest": "5.1.0" }, diff --git a/packages/web/tests/helpers/png.ts b/packages/web/tests/helpers/png.ts index 4c9e873410..31d51f386d 100644 --- a/packages/web/tests/helpers/png.ts +++ b/packages/web/tests/helpers/png.ts @@ -1,14 +1,18 @@ -import { PNG } from 'pngjs'; +import { JsColorType, Transformer } from '@napi-rs/image'; -/** Decodes screenshot pixels for assertions. */ +/** Decodes screenshot pixels for assertions: an RGB or an RGBA PNG, the two a browser writes. */ export function decodePng(bytes: Uint8Array) { - const { width, height, data } = PNG.sync.read(Buffer.from(bytes)); + const image = new Transformer(bytes); + const { width, height, colorType } = image.metadataSync(); + const data = image.rawPixelsSync(); + const channels = colorType === JsColorType.Rgba8 ? 4 : colorType === JsColorType.Rgb8 ? 3 : undefined; + if (channels === undefined) throw new Error(`expected an RGB or RGBA PNG, got color type ${colorType}`); return { width, height, pixelAt(x: number, y: number): readonly [number, number, number, number] { - const at = (y * width + x) * 4; - return [data[at]!, data[at + 1]!, data[at + 2]!, data[at + 3]!]; + const at = (y * width + x) * channels; + return [data[at]!, data[at + 1]!, data[at + 2]!, channels === 4 ? data[at + 3]! : 255]; }, }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67febff4cb..2fc5b5431e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -108,8 +108,6 @@ settings: overrides: brace-expansion@<1.1.21: 1.1.21 fast-uri@>=3.0.0 <3.1.8: 3.1.8 - fflate@>=0.7.0 <0.7.5: 0.7.5 - fflate@>=0.8.0 <0.8.3: 0.8.3 js-yaml@<3.15.2: 3.15.2 js-yaml@>=4.0.0 <4.3.2: 4.3.2 nanoid@<3.3.18: 3.3.18 @@ -254,18 +252,15 @@ importers: '@ai-sdk/devtools': specifier: 1.0.66 version: 1.0.66(zod@4.6.7) + '@napi-rs/image': + specifier: 1.16.0 + version: 1.16.0 '@types/node': specifier: 26.6.6 version: 26.6.6 - '@types/pngjs': - specifier: 6.0.5 - version: 6.0.5 ai: specifier: 7.0.163 version: 7.0.163(zod@4.6.7) - pngjs: - specifier: 7.0.0 - version: 7.0.0 typescript: specifier: 7.0.2 version: 7.0.2 @@ -400,9 +395,9 @@ importers: packages/decision: dependencies: - pngjs: - specifier: 7.0.0 - version: 7.0.0 + '@napi-rs/image': + specifier: 1.16.0 + version: 1.16.0 zod: specifier: 4.6.7 version: 4.6.7 @@ -419,9 +414,6 @@ importers: '@types/node': specifier: 26.6.6 version: 26.6.6 - '@types/pngjs': - specifier: 6.0.5 - version: 6.0.5 ai: specifier: 7.0.163 version: 7.0.163(zod@4.6.7) @@ -455,6 +447,9 @@ importers: '@modelcontextprotocol/server': specifier: 2.4.0 version: 2.4.0 + '@napi-rs/image': + specifier: 1.16.0 + version: 1.16.0 '@oxc-project/runtime': specifier: 0.157.0 version: 0.157.0 @@ -464,9 +459,6 @@ importers: commander: specifier: 15.0.0 version: 15.0.0 - fast-png: - specifier: 8.0.0 - version: 8.0.0 get-tsconfig: specifier: 4.14.3 version: 4.14.3 @@ -513,9 +505,6 @@ importers: '@types/node': specifier: 26.6.6 version: 26.6.6 - '@types/pngjs': - specifier: 6.0.5 - version: 6.0.5 ai: specifier: 7.0.163 version: 7.0.163(zod@4.6.7) @@ -525,9 +514,6 @@ importers: ajv-formats: specifier: 3.0.1 version: 3.0.1(ajv@8.20.0) - pngjs: - specifier: 7.0.0 - version: 7.0.0 typescript: specifier: 7.0.2 version: 7.0.2 @@ -634,12 +620,12 @@ importers: packages/mobile: dependencies: + '@napi-rs/image': + specifier: 1.16.0 + version: 1.16.0 agent-device: specifier: 0.21.36 version: 0.21.36(ai@7.0.163(zod@4.6.7)) - pngjs: - specifier: 7.0.0 - version: 7.0.0 zod: specifier: 4.6.7 version: 4.6.7 @@ -647,9 +633,6 @@ importers: '@types/node': specifier: 26.6.6 version: 26.6.6 - '@types/pngjs': - specifier: 6.0.5 - version: 6.0.5 ai: specifier: 7.0.163 version: 7.0.163(zod@4.6.7) @@ -715,18 +698,15 @@ importers: specifier: 1.64.1 version: 1.64.1 devDependencies: + '@napi-rs/image': + specifier: 1.16.0 + version: 1.16.0 '@types/node': specifier: 26.6.6 version: 26.6.6 - '@types/pngjs': - specifier: 6.0.5 - version: 6.0.5 e2e: specifier: workspace:* version: link:../e2e - pngjs: - specifier: 7.0.0 - version: 7.0.0 typescript: specifier: 7.0.2 version: 7.0.2 @@ -1847,6 +1827,86 @@ packages: resolution: {integrity: sha512-rgR9yxSaAJ+bcsi/XroT/S2AfPiIWloXAnrdfYSCms9ivVju0ishu/wW0J21Ho6+ZOR1r7RyeRbjiRuF87ix5Q==} engines: {node: '>=20'} + '@napi-rs/image-android-arm64@1.16.0': + resolution: {integrity: sha512-pwn/41RoGaE0gHlJBvEIikYXASeiHhs1lyo5+fWZZSXsS9hpJt0zQqWiSr7cCdCwLdg84pQwPbqd32Kp+scLSw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/image-darwin-arm64@1.16.0': + resolution: {integrity: sha512-JJesjwlmWA9m0iN4y0GvHpfZkgAviU4Yhk+qvV+D44obYPisqAhJdMCQMY1/u8Yj/PblijimCMY05fMRvCmrdQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/image-darwin-x64@1.16.0': + resolution: {integrity: sha512-EV/SHAnsIoadlJdyeSVtHo2RE+N6BAWTK6S3y52vvCF8/s3z38fe9o1tbk40xxPpcSXdu+gh6ASBeFHic2/NOQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/image-freebsd-x64@1.16.0': + resolution: {integrity: sha512-2Q2XcPqeFfr5U8oL9zwZgbnfJLvFrA85+XPaOgjaUiIhEFfRKDgp5l7R0jDFZvymCw0ljMKtVzUD+Id2LUQasg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/image-linux-arm-gnueabihf@1.16.0': + resolution: {integrity: sha512-qOXhoEXK66U396sBtL1wDvho4yooW6EPse0Ue23kRHlRHbR6SiDKz6mKcEoB1MrQzZGeKOYRh2F9rk0iHlsPiw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/image-linux-arm64-gnu@1.16.0': + resolution: {integrity: sha512-S0j/r3ciH+WW/NnsOaLwhol0wly0x0YtljZnfQ4o1YCfkVrcAPMQGgGmPyjKIG/TLQ8tqPd6liuUL4Nms017lA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/image-linux-arm64-musl@1.16.0': + resolution: {integrity: sha512-ILcos5cDYUQw6h2aty8BBlm5B4l4I5y52i0tSSU44HBM8tmGE6sUU1DREPfMx1NLhB2HX4xu1Y9DNQq7tIXhpQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/image-linux-x64-gnu@1.16.0': + resolution: {integrity: sha512-R2LnMJzHxztTYyySZnyWIiZmDxgK7pq8DWYT1jAuryCXcGmhUhM14BgEnQKj5h3zvDVbtf1czm6DSD9f8PDl4g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/image-linux-x64-musl@1.16.0': + resolution: {integrity: sha512-UXcwFSIEEHia8/JhJO6H3uehQmTrsG8UK8tpD1qpzi8WHxUVgLJyuoVS3zKm8cmqlYq0No1cyqU6h2OnArShOw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/image-win32-arm64-msvc@1.16.0': + resolution: {integrity: sha512-C1lmWAajIqkiYyVb0aBz0svCWIbzFAT6YpNoQ/QC/QsyNaQ602WSLSoq4ixnNhBPQrlro/Va3n/KANv4iqp4GA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/image-win32-ia32-msvc@1.16.0': + resolution: {integrity: sha512-j6OMycTmw47pY4m3yw4FnurkikzA8r36xgtbHqxsXWg1TGdY2uQq37Or56USNjLIeA6Oyv4az5YXc7L1oVv03w==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/image-win32-x64-msvc@1.16.0': + resolution: {integrity: sha512-jdmbjZhvR1T++hTZpi5CYASA+2Js9tuUQGtQ0fVtrtbGt82sGkYrrax8gAVNsgyvEu9/B2yxbB/4mlJcXkwt0Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/image@1.16.0': + resolution: {integrity: sha512-kciiEskvG7m6t0+VwJ/dUT5Zn79mKr1/RAAmjvG8TfK6aG9dI1lZqohtt8QGSQjdhRkyYJ85YKW686gQGFgzWw==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@1.2.3': resolution: {integrity: sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q==} engines: {node: ^20.19.0 || ^22.13.0 || >=23.5.0} @@ -2580,9 +2640,6 @@ packages: '@types/node@26.6.6': resolution: {integrity: sha512-MzQj+8Kx3XI40OmUAE6Qoh58ZMNstNNMwn2nfd3qWsfe7nDNSqJNzes4njhz6AHwrSHOffSLwXU6yb474Q4aog==} - '@types/pngjs@6.0.5': - resolution: {integrity: sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==} - '@types/react-dom@19.3.0': resolution: {integrity: sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q==} peerDependencies: @@ -3421,9 +3478,6 @@ packages: fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - fast-png@8.0.0: - resolution: {integrity: sha512-gCysNasJ8KEMgfdYIKd/wTDo6ENK1PWT0RJO7O+0pgmuHPw2O6tA1WvdxFRJoLf9V8yFYpG0FA1YgI8X97OhJA==} - fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -3456,9 +3510,6 @@ packages: fetch-nodeshim@0.4.10: resolution: {integrity: sha512-m6I8ALe4L4XpdETy7MJZWs6L1IVMbjs99bwbpIKphxX+0CTns4IKDWJY0LWfr4YsFjfg+z1TjzTMU8lKl8rG0w==} - fflate@0.8.3: - resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} - fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -3595,9 +3646,6 @@ packages: invariant@2.2.4: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} - iobuffer@6.0.1: - resolution: {integrity: sha512-SZWYkWNfjIXIBYSDpXDYIgshqtbOPsi4lviawAEceR1Kqk+sHDlcQjWrzNQsii80AyBY0q5c8HCTNjqo74ul+Q==} - is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -6516,6 +6564,57 @@ snapshots: '@modelcontextprotocol/core': 2.4.0 zod: 4.6.7 + '@napi-rs/image-android-arm64@1.16.0': + optional: true + + '@napi-rs/image-darwin-arm64@1.16.0': + optional: true + + '@napi-rs/image-darwin-x64@1.16.0': + optional: true + + '@napi-rs/image-freebsd-x64@1.16.0': + optional: true + + '@napi-rs/image-linux-arm-gnueabihf@1.16.0': + optional: true + + '@napi-rs/image-linux-arm64-gnu@1.16.0': + optional: true + + '@napi-rs/image-linux-arm64-musl@1.16.0': + optional: true + + '@napi-rs/image-linux-x64-gnu@1.16.0': + optional: true + + '@napi-rs/image-linux-x64-musl@1.16.0': + optional: true + + '@napi-rs/image-win32-arm64-msvc@1.16.0': + optional: true + + '@napi-rs/image-win32-ia32-msvc@1.16.0': + optional: true + + '@napi-rs/image-win32-x64-msvc@1.16.0': + optional: true + + '@napi-rs/image@1.16.0': + optionalDependencies: + '@napi-rs/image-android-arm64': 1.16.0 + '@napi-rs/image-darwin-arm64': 1.16.0 + '@napi-rs/image-darwin-x64': 1.16.0 + '@napi-rs/image-freebsd-x64': 1.16.0 + '@napi-rs/image-linux-arm-gnueabihf': 1.16.0 + '@napi-rs/image-linux-arm64-gnu': 1.16.0 + '@napi-rs/image-linux-arm64-musl': 1.16.0 + '@napi-rs/image-linux-x64-gnu': 1.16.0 + '@napi-rs/image-linux-x64-musl': 1.16.0 + '@napi-rs/image-win32-arm64-msvc': 1.16.0 + '@napi-rs/image-win32-ia32-msvc': 1.16.0 + '@napi-rs/image-win32-x64-msvc': 1.16.0 + '@napi-rs/wasm-runtime@1.2.3(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -7001,10 +7100,6 @@ snapshots: dependencies: undici-types: 8.9.0 - '@types/pngjs@6.0.5': - dependencies: - '@types/node': 26.6.6 - '@types/react-dom@19.3.0(@types/react@19.3.0)': dependencies: '@types/react': 19.3.0 @@ -7841,11 +7936,6 @@ snapshots: fast-deep-equal@3.1.3: {} - fast-png@8.0.0: - dependencies: - fflate: 0.8.3 - iobuffer: 6.0.1 - fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -7870,8 +7960,6 @@ snapshots: fetch-nodeshim@0.4.10: {} - fflate@0.8.3: {} - fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -8031,8 +8119,6 @@ snapshots: dependencies: loose-envify: 1.4.0 - iobuffer@6.0.1: {} - is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5478454acd..4fb23b94c4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,8 +21,6 @@ allowBuilds: overrides: 'brace-expansion@<1.1.21': 1.1.21 'fast-uri@>=3.0.0 <3.1.8': 3.1.8 - 'fflate@>=0.7.0 <0.7.5': 0.7.5 - 'fflate@>=0.8.0 <0.8.3': 0.8.3 'js-yaml@<3.15.2': 3.15.2 'js-yaml@>=4.0.0 <4.3.2': 4.3.2 'nanoid@<3.3.18': 3.3.18 From 6672d6c0b6c602378e63e58386a1e040e5912b17 Mon Sep 17 00:00:00 2001 From: DongYun Kang Date: Fri, 6 Nov 2026 14:47:53 +0900 Subject: [PATCH 2/2] test: cover 2-bit gray and palette PNGs, check the format in PNG test helpers - image.test.ts gains a 2-bit gray row and decision's overlay test a palette row, so every color type decodePng claims is exercised. - The six test helpers that decode with Transformer now throw unless the bytes are a PNG, as pngjs did, so a JPEG cannot pass for one. - The IMAGE_CODEC_UNAVAILABLE row in errors.mdx names the agent screenshots that need the addon: those the runner resizes, over 768 px. Co-Authored-By: Claude Sonnet 5.5 --- apps/testbed/tests/screenshot.e2e.ts | 3 ++- docs/reference/errors.mdx | 2 +- packages/decision/tests/overlay.test.ts | 7 ++++--- packages/decision/tests/png-helpers.ts | 9 ++++++--- .../e2e/tests/integration/screenshot-assertions.test.ts | 3 ++- packages/e2e/tests/unit/agent-pixels.test.ts | 3 ++- packages/e2e/tests/unit/image.test.ts | 4 ++++ packages/mobile/tests/helpers/png.ts | 3 ++- packages/web/tests/helpers/png.ts | 3 ++- 9 files changed, 25 insertions(+), 12 deletions(-) diff --git a/apps/testbed/tests/screenshot.e2e.ts b/apps/testbed/tests/screenshot.e2e.ts index 090111233f..9864bf0f79 100644 --- a/apps/testbed/tests/screenshot.e2e.ts +++ b/apps/testbed/tests/screenshot.e2e.ts @@ -73,7 +73,8 @@ interface Rgba { /** The RGB or RGBA PNG at `file`, decoded to RGBA. */ function readPng(file: string): Rgba { const image = new Transformer(readFileSync(file)); - const { width, height, colorType } = image.metadataSync(); + const { width, height, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`${file} is not a PNG: the bytes are ${format}`); const raw = image.rawPixelsSync(); if (colorType === JsColorType.Rgba8) return { width, height, data: raw }; if (colorType !== JsColorType.Rgb8) throw new Error(`${file} is neither an RGB nor an RGBA PNG: color type ${colorType}`); diff --git a/docs/reference/errors.mdx b/docs/reference/errors.mdx index 32669278f7..29f29e2d3e 100644 --- a/docs/reference/errors.mdx +++ b/docs/reference/errors.mdx @@ -174,7 +174,7 @@ These appear in terminal output and in the report as plain string codes. | `WORKER_ENTRY_MISSING` | The worker entry module could not be resolved. Packaging fault. | infrastructure | 3 | | `NODE_UNSUPPORTED` | The runtime cannot run e2e's TypeScript loader: a Node.js outside `^22.22.3 \|\| >=24.8.0`, Bun (no `module.registerHooks`), or Deno. The CLI checks this before it starts and exits 2 with the same sentence. The loader checks again where it registers its hooks, for a host that runs e2e's runner without the CLI | infrastructure | 3 | | `TYPESCRIPT_COMPILER_UNAVAILABLE` | oxc, which compiles e2e's TypeScript, could not be loaded; the message gives the cause. Most often its native bindings for this platform (`@oxc-transform/binding--`, `@oxc-parser/binding--`) are not installed. npm leaves them out with `--omit=optional`, or when `package-lock.json` was written on another platform (npm/cli#4828). Install again without `--omit=optional`, or delete `package-lock.json` and `node_modules` and run `npm install` | infrastructure | 3 | -| `IMAGE_CODEC_UNAVAILABLE` | `@napi-rs/image`, which reads and writes the PNGs of screenshots, could not be loaded; the message gives the cause. It is loaded when the first screenshot is read or written, so a run that takes none never needs it. Most often its native binding for this platform (`@napi-rs/image--*`) is not installed. npm leaves it out with `--omit=optional`, or when `package-lock.json` was written on another platform (npm/cli#4828). Install again without `--omit=optional`, or delete `package-lock.json` and `node_modules` and run `npm install`. The package publishes no binding for Linux on ppc64, s390x, or riscv64, where `toHaveScreenshot` and the agent's screenshots cannot work | infrastructure | 3 | +| `IMAGE_CODEC_UNAVAILABLE` | `@napi-rs/image`, which reads and writes the PNGs of screenshots, could not be loaded; the message gives the cause. It is loaded when the first screenshot is read or written, so a run that takes none never needs it. Most often its native binding for this platform (`@napi-rs/image--*`) is not installed. npm leaves it out with `--omit=optional`, or when `package-lock.json` was written on another platform (npm/cli#4828). Install again without `--omit=optional`, or delete `package-lock.json` and `node_modules` and run `npm install`. The package publishes no binding for Linux on ppc64, s390x, or riscv64, where `toHaveScreenshot` cannot work, and neither can an agent screenshot the runner has to resize (one with a side over 768 px) | infrastructure | 3 | | `ENGINE_FAILURE` | Any engine fault with no more specific mapping | infrastructure | 3 | | `WORKER_PROTOCOL` | A worker reported the wrong unit, an attempt overlapped a live attempt, `openHost` tried to acquire an engine instance another host owns, or `host.openSession` tried to reuse a closed host or overlap an opening, live, or closing session | infrastructure | 3 | | `SESSION_KEY_INVALID` | A worker received a session key that is not 32 bytes. Packaging or IPC fault. | infrastructure | 3 | diff --git a/packages/decision/tests/overlay.test.ts b/packages/decision/tests/overlay.test.ts index 81613f78ac..e99b88921f 100644 --- a/packages/decision/tests/overlay.test.ts +++ b/packages/decision/tests/overlay.test.ts @@ -11,10 +11,10 @@ function pixel(data: Uint8Array, x: number, y: number): number[] { const offset = (y * png.width + x) * 4; return [...png.data.subarray(offset, offset + 3)]; } -/** A 4 by 4 screenshot whose every pixel has the same `samples`, written as the color type and bit depth say. */ -function filled(depth: number, colorType: number, samples: readonly number[]) { +/** A 4 by 4 screenshot whose every pixel has the same `samples`, written as the color type and bit depth say; an indexed one names its `palette`. */ +function filled(depth: number, colorType: number, samples: readonly number[], palette?: readonly (readonly number[])[]) { const row = Array.from({ length: 4 }, () => samples).flat(); - return { mediaType: 'image/png' as const, data: rawPng(4, 4, depth, colorType, Array.from({ length: 4 }, () => row)), width: 4, height: 4, scale: 1 }; + return { mediaType: 'image/png' as const, data: rawPng(4, 4, depth, colorType, Array.from({ length: 4 }, () => row), palette), width: 4, height: 4, scale: 1 }; } describe('gridFor', () => { it('fits at most ten columns and rows near the target cell size', () => { @@ -50,6 +50,7 @@ describe('withGrid', () => { describe('every PNG color type', () => { const shots = [ ['RGB', filled(8, 2, [10, 20, 30]), [10, 20, 30]], + ['palette', filled(8, 3, [1], [[255, 0, 0], [10, 20, 30]]), [10, 20, 30]], ['gray', filled(8, 0, [200]), [200, 200, 200]], ['gray with alpha', filled(8, 4, [200, 128]), [200, 200, 200]], ['16-bit RGB', filled(16, 2, [255, 255, 128, 0, 1, 1]), [255, 128, 1]], diff --git a/packages/decision/tests/png-helpers.ts b/packages/decision/tests/png-helpers.ts index 97f81be575..c5a39db4f7 100644 --- a/packages/decision/tests/png-helpers.ts +++ b/packages/decision/tests/png-helpers.ts @@ -11,7 +11,8 @@ export function whitePng(width: number, height: number): Uint8Array { /** The size and RGBA pixels of a PNG the package wrote, which is always 8-bit RGBA. */ export function rgbaOf(bytes: Uint8Array): { width: number; height: number; data: Uint8Array } { const image = new Transformer(bytes); - const { width, height, colorType } = image.metadataSync(); + const { width, height, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`expected a PNG, got ${format}`); if (colorType !== JsColorType.Rgba8) throw new Error(`expected an 8-bit RGBA PNG, got color type ${colorType}`); return { width, height, data: image.rawPixelsSync() }; } @@ -27,9 +28,10 @@ function chunk(type: string, data: Uint8Array = new Uint8Array()): Buffer { /** * A PNG written straight from the format, so a test can hand the package a * color type no encoder of its own would write: `rows` hold each scanline's - * sample bytes as the color type and `depth` pack them, with no filter. + * sample bytes as the color type and `depth` pack them, with no filter; an + * indexed one names its `palette`. */ -export function rawPng(width: number, height: number, depth: number, colorType: number, rows: readonly (readonly number[])[]): Uint8Array { +export function rawPng(width: number, height: number, depth: number, colorType: number, rows: readonly (readonly number[])[], palette?: readonly (readonly number[])[]): Uint8Array { const header = Buffer.alloc(13); header.writeUInt32BE(width, 0); header.writeUInt32BE(height, 4); @@ -38,6 +40,7 @@ export function rawPng(width: number, height: number, depth: number, colorType: return Buffer.concat([ Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk('IHDR', header), + ...(palette === undefined ? [] : [chunk('PLTE', Buffer.from(palette.flat()))]), chunk('IDAT', deflateSync(Buffer.concat(rows.map((row) => Buffer.from([0, ...row]))))), chunk('IEND'), ]); diff --git a/packages/e2e/tests/integration/screenshot-assertions.test.ts b/packages/e2e/tests/integration/screenshot-assertions.test.ts index 8faa55eeb6..73eab9dbaf 100644 --- a/packages/e2e/tests/integration/screenshot-assertions.test.ts +++ b/packages/e2e/tests/integration/screenshot-assertions.test.ts @@ -100,7 +100,8 @@ function attemptOf(outcome: RunOutcome, title: string) { /** The PNG at `file` as RGBA, which is how the runner writes every screenshot it stores. */ function readPng(file: string): { width: number; height: number; data: Uint8Array } { const image = new Transformer(readFileSync(file)); - const { width, height, colorType } = image.metadataSync(); + const { width, height, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`${file} is not a PNG: the bytes are ${format}`); if (colorType !== JsColorType.Rgba8) throw new Error(`${file} is not an 8-bit RGBA PNG: color type ${colorType}`); return { width, height, data: image.rawPixelsSync() }; } diff --git a/packages/e2e/tests/unit/agent-pixels.test.ts b/packages/e2e/tests/unit/agent-pixels.test.ts index 806849710d..c1c52e2ed5 100644 --- a/packages/e2e/tests/unit/agent-pixels.test.ts +++ b/packages/e2e/tests/unit/agent-pixels.test.ts @@ -22,7 +22,8 @@ function picture(width: number, height: number, box: { x: number; y: number; w: /** The color of one pixel of a downscaled screenshot, which the runner writes as RGBA. */ function pixelAt(pixels: ObservationPixels, x: number, y: number): number[] { const image = new Transformer(pixels.data); - const { width, colorType } = image.metadataSync(); + const { width, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`expected a PNG, got ${format}`); if (colorType !== JsColorType.Rgba8) throw new Error(`expected an 8-bit RGBA PNG, got color type ${colorType}`); const at = (y * width + x) * 4; return [...image.rawPixelsSync().subarray(at, at + 3)]; diff --git a/packages/e2e/tests/unit/image.test.ts b/packages/e2e/tests/unit/image.test.ts index d764c571ef..0165010512 100644 --- a/packages/e2e/tests/unit/image.test.ts +++ b/packages/e2e/tests/unit/image.test.ts @@ -111,6 +111,10 @@ describe('image', () => { 0, 0, 0, 255, 255, 255, 255, 255, 0, 0, 0, 255, 255, 255, 255, 255, 0, 0, 0, 255, 255, 255, 255, 255, ]], + ['2-bit gray', png({ width: 3, height: 2, depth: 2, colorType: 0, rows: [[0b00_01_10_00], [0b11_10_01_00]] }), [ + 0, 0, 0, 255, 85, 85, 85, 255, 170, 170, 170, 255, + 255, 255, 255, 255, 170, 170, 170, 255, 85, 85, 85, 255, + ]], ['4-bit gray', png({ width: 3, height: 2, depth: 4, colorType: 0, rows: [[0x0f, 0x80], [0x37, 0x00]] }), [ 0, 0, 0, 255, 255, 255, 255, 255, 136, 136, 136, 255, 51, 51, 51, 255, 119, 119, 119, 255, 0, 0, 0, 255, diff --git a/packages/mobile/tests/helpers/png.ts b/packages/mobile/tests/helpers/png.ts index a2bcc92247..aa0062b191 100644 --- a/packages/mobile/tests/helpers/png.ts +++ b/packages/mobile/tests/helpers/png.ts @@ -11,7 +11,8 @@ export interface DecodedPng { /** Reads assertion pixels through the library, independently of the masking policy: RGB and RGBA PNGs, both as RGBA. */ export function decodePng(bytes: Uint8Array): DecodedPng { const image = new Transformer(bytes); - const { width, height, colorType } = image.metadataSync(); + const { width, height, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`expected a PNG, got ${format}`); const raw = image.rawPixelsSync(); if (colorType === JsColorType.Rgba8) return { width, height, channels: 4, pixels: raw }; if (colorType !== JsColorType.Rgb8) throw new Error(`expected an RGB or RGBA PNG, got color type ${colorType}`); diff --git a/packages/web/tests/helpers/png.ts b/packages/web/tests/helpers/png.ts index 31d51f386d..cbcc889d65 100644 --- a/packages/web/tests/helpers/png.ts +++ b/packages/web/tests/helpers/png.ts @@ -3,7 +3,8 @@ import { JsColorType, Transformer } from '@napi-rs/image'; /** Decodes screenshot pixels for assertions: an RGB or an RGBA PNG, the two a browser writes. */ export function decodePng(bytes: Uint8Array) { const image = new Transformer(bytes); - const { width, height, colorType } = image.metadataSync(); + const { width, height, format, colorType } = image.metadataSync(); + if (format !== 'png') throw new Error(`expected a PNG, got ${format}`); const data = image.rawPixelsSync(); const channels = colorType === JsColorType.Rgba8 ? 4 : colorType === JsColorType.Rgb8 ? 3 : undefined; if (channels === undefined) throw new Error(`expected an RGB or RGBA PNG, got color type ${colorType}`);