feat(fe): add canvas support with web-gl (#198)

Author: dos1in

docs/API-Reference.md

@@ -25,6 +25,7 @@ Vue 作为底层的渲染框架，自然会与小程序的语法存在一定的
 | ------------------ |
 | block              |
 | button             |
+| canvas             |
 | checkbox           |
 | checkbox-group     |
 | cover-image        |
@@ -160,6 +161,9 @@ DMPApp.init(context, { apiNamespaces: ["myapp"] })
 | 界面 - 滚动   | pageScrollTo                     | ✓       | ✓   | ✓       | ✓   |
 | 界面 - 菜单   | getMenuButtonBoundingClientRect  | ✓       | ✓   | ✓       | ✓   |
 | 界面 - 动画   | createAnimation                  | ✓       | ✓   | ✓       | ✓   |
+| 界面 - Canvas | createCanvasContext              | ✓       | ✓   | ✓       | ✓   |
+|               | createOffscreenCanvas            | ✓       | ✓   | ✓       | ✓   |
+|               | canvasToTempFilePath             | ✓       | ✓   | ✓       | ✓   |
 | WXML          | createSelectorQuery              | ✓       | ✓   | ✓       | ✓   |
 |               | createIntersectionObserver       | ✓       | ✓   | ✓       | ✓   |
 | 网络          | request                          | ✓       | ✓   | ✓       | ✓   |

fe/packages/compiler/src/common/compatibility-reference.js

@@ -4,6 +4,7 @@
 const supportedBuiltinComponents = [
 	"block",
 	"button",
+	"canvas",
 	"checkbox",
 	"checkbox-group",
 	"cover-image",
@@ -65,6 +66,9 @@ const supportedWxApis = [
 	"pageScrollTo",
 	"getMenuButtonBoundingClientRect",
 	"createAnimation",
+	"createCanvasContext",
+	"createOffscreenCanvas",
+	"canvasToTempFilePath",
 	"createSelectorQuery",
 	"createIntersectionObserver",
 	"request",

fe/packages/render/__tests__/runtime.spec.js

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { JSDOM } from 'jsdom'
 import { createApp, h, nextTick } from 'vue'
 
@@ -122,4 +122,95 @@ describe('runtime template components', () => {
 
 		app.unmount()
 	})
+
+	it('returns a serializable canvas node from selector node fields', async () => {
+		runtime.ensureElementReady = async element => element
+		runtime.canvasNodes.clear()
+
+		const canvas = document.createElement('canvas')
+		canvas.setAttribute('type', 'webgl')
+		canvas.getBoundingClientRect = vi.fn(() => ({
+			left: 0,
+			top: 0,
+			right: 300,
+			bottom: 300,
+			width: 300,
+			height: 300,
+		}))
+		document.body.append(canvas)
+
+		const result = await runtime.parseElement(canvas, {
+			node: true,
+			size: true,
+		})
+
+		expect(result.node.__diminaNodeType).toBe('dimina-canvas-node')
+		expect(result.node.type).toBe('webgl')
+		expect(result.node.width).toBe(300)
+		expect(result.node.height).toBe(300)
+		expect(canvas.width).toBe(300)
+		expect(canvas.height).toBe(300)
+		expect(runtime.canvasNodes.has(result.node.nodeId)).toBe(true)
+
+		canvas.width = 600
+		canvas.height = 600
+		const nextResult = await runtime.parseElement(canvas, {
+			node: true,
+		})
+		expect(nextResult.node.width).toBe(600)
+		expect(nextResult.node.height).toBe(600)
+		expect(canvas.width).toBe(600)
+		expect(canvas.height).toBe(600)
+	})
+
+	it('replays canvas node webgl operations against the real context', () => {
+		runtime.canvasNodes.clear()
+		runtime.canvasResources.clear()
+
+		const shader = { kind: 'shader' }
+		const gl = {
+			VERTEX_SHADER: 0x8B31,
+			createShader: vi.fn(() => shader),
+			shaderSource: vi.fn(),
+			compileShader: vi.fn(),
+			viewport: vi.fn(),
+		}
+		const canvas = document.createElement('canvas')
+		canvas.getContext = vi.fn(() => gl)
+		runtime.canvasNodes.set('canvas_1', {
+			canvas,
+			contexts: new Map(),
+		})
+
+		runtime.canvasNodeFlush({
+			bridgeId: 'bridge_1',
+			params: {
+				nodeId: 'canvas_1',
+				operations: [
+					{ op: 'setCanvasProperty', prop: 'width', value: 600 },
+					{ op: 'getContext', contextId: 'ctx_1', contextType: 'webgl' },
+					{ op: 'contextCall', contextId: 'ctx_1', method: 'viewport', args: [0, 0, 300, 150] },
+					{ op: 'contextCall', contextId: 'ctx_1', method: 'createShader', args: [0x8B31], resultId: 'shader_1' },
+					{
+						op: 'contextCall',
+						contextId: 'ctx_1',
+						method: 'shaderSource',
+						args: [{ __canvasResourceId: 'shader_1' }, 'void main() {}'],
+					},
+					{
+						op: 'contextCall',
+						contextId: 'ctx_1',
+						method: 'compileShader',
+						args: [{ __canvasResourceId: 'shader_1' }],
+					},
+				],
+			},
+		})
+
+		expect(canvas.width).toBe(600)
+		expect(canvas.getContext).toHaveBeenCalledWith('webgl', undefined)
+		expect(gl.viewport).toHaveBeenCalledWith(0, 0, 300, 150)
+		expect(gl.shaderSource).toHaveBeenCalledWith(shader, 'void main() {}')
+		expect(gl.compileShader).toHaveBeenCalledWith(shader)
+	})
 })

fe/packages/render/src/core/runtime.js

@@ -59,6 +59,23 @@ const VUE_RUNTIME_HELPERS = {
 	_withDirectives: withDirectives,
 }
 
+const CANVAS_NODE_TYPE = 'dimina-canvas-node'
+const TYPED_ARRAY_CTORS = {
+	Int8Array,
+	Uint8Array,
+	Uint8ClampedArray,
+	Int16Array,
+	Uint16Array,
+	Int32Array,
+	Uint32Array,
+	Float32Array,
+	Float64Array,
+}
+
+function isCanvasElement(element) {
+	return element?.tagName?.toLowerCase() === 'canvas'
+}
+
 class Runtime {
 	constructor() {
 		this.app = null
@@ -69,6 +86,9 @@ class Runtime {
 		this.initializedModules = new Set()
 		this.preInitUpdates = new Map()
 		this.intersectionObservers = new Map()
+		this.canvasNodes = new Map()
+		this.canvasResources = new Map()
+		this.canvasRafIds = new Map()
 		// 追踪"mC 已发出但 service 侧 created 尚未完成"的组件 setup
 		// key: moduleId, value: Promise（created 完成时 resolve）
 		this._pendingSetups = new Map()
@@ -711,6 +731,209 @@ class Runtime {
 		return true
 	}
 
+	getCanvasNodeId(canvas) {
+		if (!canvas.__diminaCanvasNodeId) {
+			Object.defineProperty(canvas, '__diminaCanvasNodeId', {
+				value: `canvas_${uuid()}`,
+				configurable: true,
+			})
+		}
+		return canvas.__diminaCanvasNodeId
+	}
+
+	registerCanvasNode(canvas, type = canvas.getAttribute?.('type') || '2d') {
+		const nodeId = this.getCanvasNodeId(canvas)
+		const isNewNode = !this.canvasNodes.has(nodeId)
+		const rect = canvas.getBoundingClientRect?.()
+		const width = Math.round(rect?.width || 0)
+		const height = Math.round(rect?.height || 0)
+		if (isNewNode && width > 0 && height > 0) {
+			if (canvas.width !== width) {
+				canvas.width = width
+			}
+			if (canvas.height !== height) {
+				canvas.height = height
+			}
+		}
+
+		if (isNewNode) {
+			this.canvasNodes.set(nodeId, {
+				canvas,
+				contexts: new Map(),
+			})
+		}
+		return {
+			__diminaNodeType: CANVAS_NODE_TYPE,
+			nodeId,
+			type,
+			width: canvas.width || width || 300,
+			height: canvas.height || height || 150,
+		}
+	}
+
+	createOffscreenCanvas({ params }) {
+		const { nodeId, width = 300, height = 150, type = '2d' } = params
+		const canvas = document.createElement('canvas')
+		canvas.width = width
+		canvas.height = height
+		this.canvasNodes.set(nodeId, {
+			canvas,
+			type,
+			contexts: new Map(),
+		})
+	}
+
+	resolveCanvasArg(value) {
+		if (value === null || value === undefined) {
+			return value
+		}
+
+		if (Array.isArray(value)) {
+			return value.map(item => this.resolveCanvasArg(item))
+		}
+
+		if (typeof value !== 'object') {
+			return value
+		}
+
+		if (value.__canvasResourceId) {
+			return this.canvasResources.get(value.__canvasResourceId)
+		}
+
+		if (value.__canvasNodeId) {
+			return this.canvasNodes.get(value.__canvasNodeId)?.canvas
+		}
+
+		if (value.__canvasTypedArray) {
+			const Ctor = TYPED_ARRAY_CTORS[value.__canvasTypedArray]
+			if (Ctor) {
+				return new Ctor(value.data || [])
+			}
+			if (value.__canvasTypedArray === 'DataView') {
+				return new DataView(new Uint8Array(value.data || []).buffer)
+			}
+		}
+
+		if (value.__canvasArrayBuffer) {
+			return new Uint8Array(value.data || []).buffer
+		}
+
+		const result = {}
+		for (const [key, item] of Object.entries(value)) {
+			result[key] = this.resolveCanvasArg(item)
+		}
+		return result
+	}
+
+	getCanvasResource(id) {
+		return this.canvasResources.get(id)
+	}
+
+	setCanvasResource(id, value) {
+		if (id) {
+			this.canvasResources.set(id, value)
+		}
+	}
+
+	getCanvasImage(imageId) {
+		let image = this.getCanvasResource(imageId)
+		if (!image) {
+			image = new Image()
+			this.setCanvasResource(imageId, image)
+		}
+		return image
+	}
+
+	executeCanvasOperation(node, operation, bridgeId) {
+		switch (operation.op) {
+			case 'setCanvasProperty':
+				node.canvas[operation.prop] = operation.value
+				break
+			case 'getContext': {
+				const context = node.canvas.getContext(operation.contextType, this.resolveCanvasArg(operation.attributes))
+				node.contexts.set(operation.contextId, context)
+				this.setCanvasResource(operation.contextId, context)
+				break
+			}
+			case 'contextSetProperty': {
+				const context = this.getCanvasResource(operation.contextId)
+				if (context) {
+					context[operation.prop] = this.resolveCanvasArg(operation.value)
+				}
+				break
+			}
+			case 'contextCall': {
+				const context = this.getCanvasResource(operation.contextId)
+				const method = context?.[operation.method]
+				if (typeof method === 'function') {
+					const result = method.apply(context, (operation.args || []).map(arg => this.resolveCanvasArg(arg)))
+					this.setCanvasResource(operation.resultId, result)
+				}
+				break
+			}
+			case 'resourceCall': {
+				const resource = this.getCanvasResource(operation.resourceId)
+				const method = resource?.[operation.method]
+				if (typeof method === 'function') {
+					const result = method.apply(resource, (operation.args || []).map(arg => this.resolveCanvasArg(arg)))
+					this.setCanvasResource(operation.resultId, result)
+				}
+				break
+			}
+			case 'createImage':
+				this.getCanvasImage(operation.imageId)
+				break
+			case 'imageSetSrc': {
+				const image = this.getCanvasImage(operation.imageId)
+				image.onload = () => {
+					this.triggerCallback(bridgeId, operation.onload, {
+						width: image.width,
+						height: image.height,
+					})
+				}
+				image.onerror = () => {
+					this.triggerCallback(bridgeId, operation.onerror, {
+						errMsg: `createImage:fail ${operation.src}`,
+					})
+				}
+				image.src = operation.src
+				break
+			}
+			default:
+				console.warn('[system]', '[render]', `Unsupported canvas node operation: ${operation.op}`)
+		}
+	}
+
+	canvasNodeFlush({ bridgeId, params }) {
+		const node = this.canvasNodes.get(params.nodeId)
+		if (!node) {
+			console.warn('[system]', '[render]', `canvas node ${params.nodeId} not found`)
+			return
+		}
+
+		for (const operation of params.operations || []) {
+			this.executeCanvasOperation(node, operation, bridgeId)
+		}
+	}
+
+	canvasNodeRequestAnimationFrame({ bridgeId, params }) {
+		const key = `${params.nodeId}:${params.requestId}`
+		const frameId = requestAnimationFrame((timestamp) => {
+			this.canvasRafIds.delete(key)
+			this.triggerCallback(bridgeId, params.callback, timestamp)
+		})
+		this.canvasRafIds.set(key, frameId)
+	}
+
+	canvasNodeCancelAnimationFrame({ params }) {
+		const key = `${params.nodeId}:${params.requestId}`
+		const frameId = this.canvasRafIds.get(key)
+		if (frameId !== undefined) {
+			cancelAnimationFrame(frameId)
+			this.canvasRafIds.delete(key)
+		}
+	}
+
 	async selectorQuery(opts) {
 		const { bridgeId, params: { tasks, success } } = opts
 
@@ -886,9 +1109,11 @@ class Runtime {
 			data.computedStyle = styles
 		}
 
-		// TODO: 支持获取 Canvas 和 ScrollViewContext
-		// if (fields.node) {
-		// }
+		if (fields.node) {
+			data.node = isCanvasElement(targetElement)
+				? this.registerCanvasNode(targetElement)
+				: null
+		}
 		// TODO: 支持获取 VideoContext、CanvasContext、LivePlayerContext、EditorContext和 MapContext
 		// if (fields.context) {
 		// }