Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions src/launch/mcp/capture.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// runtime screenshots are downscaled to at most this width to keep payloads small
const CAPTURE_MAX_WIDTH = 800;

/**
* Size of a runtime screenshot of a canvas: its drawing buffer, downscaled to at most
* {@link CAPTURE_MAX_WIDTH} wide.
*
* @param width - The canvas drawing buffer width (device pixels).
* @param height - The canvas drawing buffer height (device pixels).
* @returns The screenshot size.
*/
const captureSize = (width: number, height: number) => {
if (width <= CAPTURE_MAX_WIDTH) {
return { width, height };
}
return { width: CAPTURE_MAX_WIDTH, height: Math.round(height * (CAPTURE_MAX_WIDTH / width)) };
};

/**
* Convert a point on a runtime screenshot to CSS pixels from the canvas top-left, the space
* input events are dispatched in. The screenshot is taken from the drawing buffer, so it differs
* from the CSS size whenever the device pixel ratio isn't 1 or the canvas was downscaled.
*
* @param x - The x coordinate on the screenshot.
* @param y - The y coordinate on the screenshot.
* @param image - The screenshot size.
* @param image.width - The screenshot width.
* @param image.height - The screenshot height.
* @param css - The canvas CSS size.
* @param css.width - The canvas CSS width.
* @param css.height - The canvas CSS height.
* @returns The point in CSS pixels.
*/
const captureToCss = (
x: number,
y: number,
image: { width: number; height: number },
css: { width: number; height: number }
) => ({
x: image.width ? (x * css.width) / image.width : x,
y: image.height ? (y * css.height) / image.height : y
});

export { CAPTURE_MAX_WIDTH, captureSize, captureToCss };
44 changes: 28 additions & 16 deletions src/launch/mcp/runtime.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { mcp, PROTOCOL_VERSION } from '@/editor/mcp/connection';

import { captureSize, captureToCss } from './capture';

/**
* MCP "runtime" peer for the launch page (ported from the former Chrome extension's
* launch content script). Captures console output while an MCP session owns the page,
Expand Down Expand Up @@ -141,16 +143,8 @@ mcp.method(
app.off('frameend', onEnd);
try {
const canvas = device.canvas;
const width = canvas.width;
const height = canvas.height;

const maxWidth = 800;
let dstWidth = width;
let dstHeight = height;
if (width > maxWidth) {
dstWidth = maxWidth;
dstHeight = Math.round(height * (maxWidth / width));
}
const { width: dstWidth, height: dstHeight } = captureSize(canvas.width, canvas.height);
const rect = canvas.getBoundingClientRect();

const dstCanvas = document.createElement('canvas');
dstCanvas.width = dstWidth;
Expand All @@ -165,6 +159,12 @@ mcp.method(
mimeType: 'image/webp',
width: dstWidth,
height: dstHeight,
// input is dispatched in CSS pixels, which differ from the image's
// whenever the device pixel ratio isn't 1 or the image was downscaled
cssWidth: rect.width,
cssHeight: rect.height,
scaleX: dstWidth ? rect.width / dstWidth : 1,
scaleY: dstHeight ? rect.height / dstHeight : 1,
deviceType: device.deviceType
}
});
Expand Down Expand Up @@ -464,6 +464,16 @@ mcp.method('runtime:input', async (payload: any = {}) => {
}
const betweenMs = Number.isFinite(payload.betweenMs) ? payload.betweenMs : 0;

// mouse/touch coordinates are CSS pixels from the canvas top-left, unless the caller
// read them off a capture_runtime image
const toCss = (ev: any) => {
if (payload.space !== 'capture') {
return { x: ev.x, y: ev.y };
}
const rect = canvas.getBoundingClientRect();
return captureToCss(ev.x || 0, ev.y || 0, captureSize(canvas.width, canvas.height), rect);
};

let dispatched = 0;
for (const ev of events) {
if (ev.type === 'key') {
Expand All @@ -485,19 +495,21 @@ mcp.method('runtime:input', async (payload: any = {}) => {
dispatched++;
}
} else if (ev.type === 'mouse') {
const { x, y } = toCss(ev);
if (ev.action === 'click') {
dispatchMouse('mousedown', canvas, ev.x, ev.y, ev.button);
dispatchMouse('mouseup', canvas, ev.x, ev.y, ev.button);
dispatchMouse('mousedown', canvas, x, y, ev.button);
dispatchMouse('mouseup', canvas, x, y, ev.button);
} else {
dispatchMouse(`mouse${ev.action}`, canvas, ev.x, ev.y, ev.button);
dispatchMouse(`mouse${ev.action}`, canvas, x, y, ev.button);
}
dispatched++;
} else if (ev.type === 'touch') {
const { x, y } = toCss(ev);
if (ev.action === 'tap') {
dispatchTouch('touchstart', canvas, ev.x, ev.y, ev.id);
dispatchTouch('touchend', canvas, ev.x, ev.y, ev.id);
dispatchTouch('touchstart', canvas, x, y, ev.id);
dispatchTouch('touchend', canvas, x, y, ev.id);
} else {
dispatchTouch(`touch${ev.action}`, canvas, ev.x, ev.y, ev.id);
dispatchTouch(`touch${ev.action}`, canvas, x, y, ev.id);
}
dispatched++;
}
Expand Down
48 changes: 48 additions & 0 deletions test/launch/mcp/capture.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import { expect } from 'chai';
import { describe, it } from 'mocha';

import { captureSize, captureToCss } from '../../../src/launch/mcp/capture';

describe('captureSize', () => {
it('keeps a drawing buffer up to 800 wide', () => {
expect(captureSize(800, 600)).to.deep.equal({ width: 800, height: 600 });
expect(captureSize(640, 480)).to.deep.equal({ width: 640, height: 480 });
});

it('downscales a wider drawing buffer to 800 wide', () => {
expect(captureSize(1600, 900)).to.deep.equal({ width: 800, height: 450 });
expect(captureSize(1001, 333)).to.deep.equal({ width: 800, height: 266 });
});
});

describe('captureToCss', () => {
it('is the identity when the image matches the CSS size', () => {
const size = { width: 640, height: 480 };
expect(captureToCss(320, 240, size, size)).to.deep.equal({ x: 320, y: 240 });
});

it('scales up a canvas wider than 800 CSS px', () => {
// DPR 1, 1600x900 canvas: the centre shows at 400x225 in the 800x450 image
const image = captureSize(1600, 900);
expect(captureToCss(400, 225, image, { width: 1600, height: 900 })).to.deep.equal({ x: 800, y: 450 });
});

it('scales down when the device pixel ratio is above 1', () => {
// DPR 2, 300x200 CSS canvas: 600x400 drawing buffer, captured at full size
const image = captureSize(600, 400);
expect(captureToCss(300, 200, image, { width: 300, height: 200 })).to.deep.equal({ x: 150, y: 100 });
});

it('handles a high device pixel ratio together with the downscale', () => {
// DPR 2, 1000x500 CSS canvas: 2000x1000 drawing buffer, captured at 800x400
const image = captureSize(2000, 1000);
expect(captureToCss(400, 200, image, { width: 1000, height: 500 })).to.deep.equal({ x: 500, y: 250 });
});

it('leaves the point alone for an empty image', () => {
expect(captureToCss(10, 20, { width: 0, height: 0 }, { width: 100, height: 100 })).to.deep.equal({
x: 10,
y: 20
});
});
});
Loading