408 lines
16 KiB
TypeScript
408 lines
16 KiB
TypeScript
import { createServer } from "node:http";
|
|
import { expect, test, type Page } from "@playwright/test";
|
|
import { parsePartialJSON } from "@tanstack/ai";
|
|
import recording from "../apps/product/fixtures/recorded-neutral-gemini.json" with { type: "json" };
|
|
import {
|
|
nativeIncrementalConversationResponse,
|
|
nativeImageStream,
|
|
neutralTurn,
|
|
} from "../apps/product/fixtures/native-conversation-stream";
|
|
import { ModelTurnSchema } from "../apps/product/standard-schemas/narrative-turn.schema";
|
|
import type { PersonaId } from "../apps/product/standard-schemas/persona.schema";
|
|
|
|
const recordedTurn = ModelTurnSchema.parse(
|
|
JSON.parse(recording.chunks.map((c) => c.delta).join("")),
|
|
);
|
|
const personas: PersonaId[] = [
|
|
"samantha69",
|
|
"nelly",
|
|
"xixi",
|
|
"hisoka",
|
|
"cybelle",
|
|
"galatee",
|
|
"polaire",
|
|
];
|
|
const cases = [
|
|
...personas.map((personaId) => ({ personaId, scenario: "recorded" })),
|
|
{ personaId: "nelly" as const, scenario: "image" },
|
|
{ personaId: "nelly" as const, scenario: "rewrite" },
|
|
{ personaId: "nelly" as const, scenario: "invalid" },
|
|
{ personaId: "storytellurx_gateway" as const, scenario: "handoff" },
|
|
{ personaId: "nelly" as const, scenario: "motion" },
|
|
{ personaId: "nelly" as const, scenario: "scroll-away" },
|
|
{ personaId: "storytellurx_gateway" as const, scenario: "system" },
|
|
{ personaId: "storytellurx_pause" as const, scenario: "system" },
|
|
{ personaId: "nelly" as const, scenario: "french" },
|
|
{ personaId: "nelly" as const, scenario: "six-long-suggestions" },
|
|
{ personaId: "nelly" as const, scenario: "late-state" },
|
|
{ personaId: "nelly" as const, scenario: "image-wide" },
|
|
{ personaId: "nelly" as const, scenario: "cancel-navigation" },
|
|
];
|
|
|
|
declare global {
|
|
interface Window {
|
|
auditIds?: WeakMap<Element, number>;
|
|
auditNextId?: number;
|
|
}
|
|
}
|
|
async function snapshot(page: Page, stage: string) {
|
|
return page.evaluate((stage) => {
|
|
const ids = (window.auditIds ??= new WeakMap<Element, number>());
|
|
const viewport = document.querySelector<HTMLElement>('[role="region"]:has([role="log"])');
|
|
const shell = document.querySelector('[data-testid="public-shell"]');
|
|
const scale = shell ? Number(getComputedStyle(shell).scale) || 1 : 1;
|
|
const measure = (element: Element | null | undefined) => {
|
|
if (!element) return null;
|
|
if (!ids.has(element)) ids.set(element, (window.auditNextId = (window.auditNextId ?? 0) + 1));
|
|
const r = element.getBoundingClientRect();
|
|
const css = getComputedStyle(element);
|
|
const round = (n: number) => Math.round(n * 100) / 100;
|
|
return {
|
|
node: ids.get(element),
|
|
x: round(r.x),
|
|
y: round(r.y),
|
|
width: round(r.width),
|
|
height: round(r.height),
|
|
contentY: viewport
|
|
? round((r.y - viewport.getBoundingClientRect().y) / scale + viewport.scrollTop)
|
|
: null,
|
|
position: css.position,
|
|
order: css.order,
|
|
font: css.fontFamily,
|
|
paddingTop: css.paddingTop,
|
|
paddingBottom: css.paddingBottom,
|
|
};
|
|
};
|
|
const last = Array.from(document.querySelectorAll('[data-role="assistant"]')).at(-1);
|
|
const paragraphs = Array.from(last?.querySelectorAll("p") ?? []);
|
|
const fragment = paragraphs.find((p) => p.textContent.startsWith("Rows of wooden"));
|
|
const rail = document.querySelector("form [data-suggestion]")?.parentElement;
|
|
const status = document.querySelector("[data-status-bar]");
|
|
const composer = document.querySelector("form");
|
|
return {
|
|
stage,
|
|
viewport: measure(viewport),
|
|
status: measure(status),
|
|
statusDirection: status?.getAttribute("data-scroll-direction"),
|
|
composer: measure(composer),
|
|
input: measure(composer?.querySelector("textarea")),
|
|
rail: measure(rail),
|
|
cards: Array.from(document.querySelectorAll("[data-suggestion]")).map(measure),
|
|
receipts: Array.from(document.querySelectorAll("[data-turn-receipt]")).map((receipt) => {
|
|
const trigger = receipt.querySelector("button[aria-expanded]");
|
|
return {
|
|
box: measure(receipt),
|
|
trigger: measure(trigger),
|
|
expanded: trigger?.getAttribute("aria-expanded"),
|
|
};
|
|
}),
|
|
constrainedAuxiliary: Array.from(
|
|
document.querySelectorAll(
|
|
'[data-turn-receipt], [data-turn-receipt] *, [data-role="user"] > div',
|
|
),
|
|
).flatMap((element) => {
|
|
const css = getComputedStyle(element);
|
|
return css.overflowY === "auto" || css.overflowY === "scroll" || css.maxBlockSize !== "none"
|
|
? [{ tag: element.tagName, overflowY: css.overflowY, maxBlockSize: css.maxBlockSize }]
|
|
: [];
|
|
}),
|
|
message: measure(last),
|
|
fragment: measure(fragment),
|
|
fragmentText: fragment?.textContent,
|
|
user: measure(Array.from(document.querySelectorAll('[data-role="user"]')).at(-1)),
|
|
notes: paragraphs
|
|
.filter(
|
|
(p) =>
|
|
!p.textContent.startsWith("Rows of wooden") &&
|
|
!p.textContent.startsWith("A tall arched"),
|
|
)
|
|
.map(measure),
|
|
images: Array.from(last?.querySelectorAll("img") ?? []).map(measure),
|
|
busy: document.querySelector('[role="log"]')?.getAttribute("aria-busy"),
|
|
alert: measure(document.querySelector('[role="alert"]')),
|
|
scrollTop: viewport?.scrollTop,
|
|
scrollHeight: viewport?.scrollHeight,
|
|
clientHeight: viewport?.clientHeight,
|
|
autoScrolling: document
|
|
.querySelector("[data-autoscrolling]")
|
|
?.getAttribute("data-autoscrolling"),
|
|
inset: document
|
|
.querySelector<HTMLElement>('[data-testid="public-shell"]')
|
|
?.style.getPropertyValue("--composer-height"),
|
|
activeElement: document.activeElement?.tagName,
|
|
};
|
|
}, stage);
|
|
}
|
|
|
|
for (const { personaId, scenario } of cases) {
|
|
test(`chunk geometry ${personaId} ${scenario}`, async ({ page, context }, info) => {
|
|
test.skip(
|
|
process.env["STX_LAYOUT_AUDIT"] !== "1",
|
|
"Opt-in DOM diagnostic with targeted natural-layout assertions; not blanket UX approval.",
|
|
);
|
|
test.setTimeout(60000);
|
|
if (scenario === "motion") await page.emulateMedia({ reducedMotion: "no-preference" });
|
|
let turn = recordedTurn;
|
|
if (scenario === "image" || scenario === "image-wide")
|
|
turn = {
|
|
...turn,
|
|
statePatch: { ...turn.statePatch, modality: "text-image" },
|
|
visualPrompt: "A neutral square.",
|
|
};
|
|
if (scenario === "system")
|
|
turn = {
|
|
...neutralTurn(personaId, 0.5, 1, { suggestionCount: 2 }),
|
|
fragments: recordedTurn.fragments,
|
|
};
|
|
if (scenario === "six-long-suggestions")
|
|
turn = {
|
|
...turn,
|
|
suggestions: Array.from({ length: 6 }, (_, index) => ({
|
|
displayText: `Read shelf ${index + 1} beside the tall window overlooking the quiet garden`,
|
|
promptValue: `Describe the books on shelf ${index + 1} beside the window.`,
|
|
style: "default" as const,
|
|
})),
|
|
};
|
|
if (scenario === "rewrite")
|
|
turn = {
|
|
...turn,
|
|
rewrite: {
|
|
text: "Please describe the quiet library and the view from its tall arched window in a few simple sentences.",
|
|
explanation: "Expanded the neutral request for clarity.",
|
|
},
|
|
};
|
|
if (scenario === "handoff")
|
|
turn = { ...turn, statePatch: { ...turn.statePatch, personaId: "nelly" } };
|
|
if (scenario === "scroll-away")
|
|
turn = { ...turn, fragments: turn.fragments.map((text) => (text + " ").repeat(5)) };
|
|
const exactRecording =
|
|
scenario === "recorded" || scenario === "motion" || scenario === "invalid";
|
|
const { statePatch, ...beforeState } = turn;
|
|
const serialized = JSON.stringify(
|
|
scenario === "late-state" ? { ...beforeState, statePatch } : turn,
|
|
);
|
|
const deltas = exactRecording
|
|
? recording.chunks.map((c) => c.delta)
|
|
: serialized.match(/[\s\S]{1,120}/g)!;
|
|
if (scenario === "invalid")
|
|
deltas[deltas.length - 1] = deltas.at(-1)!.replace(/}\s*$/, ',"unexpected":true}');
|
|
const gates = deltas.map(() => Promise.withResolvers<void>());
|
|
const start = Promise.withResolvers<void>();
|
|
const imageGate = Promise.withResolvers<void>();
|
|
const controllers = new Set<AbortController>();
|
|
const unexpected: string[] = [];
|
|
let requests = 0;
|
|
let emitted = 0;
|
|
let imageRequests = 0;
|
|
let cancelled = false;
|
|
const server = createServer((_request, outgoing) => {
|
|
const controller = new AbortController();
|
|
controllers.add(controller);
|
|
outgoing.on("close", () => {
|
|
if (!outgoing.writableEnded) controller.abort();
|
|
});
|
|
void (async () => {
|
|
requests++;
|
|
await start.promise;
|
|
const response = nativeIncrementalConversationResponse(
|
|
turn,
|
|
deltas.map((delta, i) => ({
|
|
delta,
|
|
get after() {
|
|
emitted = i + 1;
|
|
return gates[i]!.promise;
|
|
},
|
|
})),
|
|
controller,
|
|
);
|
|
outgoing.writeHead(200, {
|
|
"Content-Type": "text/event-stream",
|
|
"Access-Control-Allow-Origin": "*",
|
|
});
|
|
for await (const chunk of response.body!) outgoing.write(chunk);
|
|
outgoing.end();
|
|
})().catch(() => outgoing.destroy());
|
|
});
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
const address = server.address();
|
|
if (!address || typeof address === "string") throw new Error("Missing fixture port.");
|
|
const samples = [];
|
|
try {
|
|
await context.addCookies([
|
|
{
|
|
name: "locale",
|
|
value: scenario === "french" ? "fr_FR" : "en_GB",
|
|
url: "http://127.0.0.1:3000",
|
|
},
|
|
]);
|
|
await context.route("**/*", (route) => {
|
|
const url = new URL(route.request().url());
|
|
if (
|
|
(url.hostname === "127.0.0.1" || url.hostname === "localhost") &&
|
|
!url.pathname.startsWith("/api/ai/")
|
|
)
|
|
return route.continue();
|
|
unexpected.push(url.origin + url.pathname);
|
|
return route.abort();
|
|
});
|
|
await page.route("**/api/ai/chat", (route) =>
|
|
route.continue({ url: `http://127.0.0.1:${address.port}/fixture` }),
|
|
);
|
|
await page.route("**/api/ai/image", async (route) => {
|
|
imageRequests++;
|
|
await imageGate.promise;
|
|
const imageUrl =
|
|
scenario === "image-wide"
|
|
? "data:image/svg+xml," +
|
|
encodeURIComponent(
|
|
'<svg xmlns="http://www.w3.org/2000/svg" width="320" height="120"><rect width="320" height="120" fill="#ddd"/></svg>',
|
|
)
|
|
: undefined;
|
|
await route.fulfill({
|
|
contentType: "text/event-stream",
|
|
body: await nativeImageStream("success", imageUrl),
|
|
});
|
|
});
|
|
await page.goto(`/chat?persona-id=${personaId}&model-id=gemini&session-id=audit-${scenario}`);
|
|
const input = page.getByRole("textbox");
|
|
await expect(input).toBeEnabled();
|
|
await page.evaluate(() => document.fonts.ready);
|
|
await page.waitForTimeout(600);
|
|
samples.push(await snapshot(page, "initial"));
|
|
await input.fill("Describe the library.");
|
|
await expect(input).toBeFocused();
|
|
samples.push(await snapshot(page, "draft-focused"));
|
|
await input.press("Enter");
|
|
await expect.poll(() => requests).toBe(1);
|
|
await page.waitForTimeout(250);
|
|
samples.push(await snapshot(page, "submitted-before-first-byte"));
|
|
start.resolve();
|
|
for (let i = 0; i < deltas.length; i++) {
|
|
await expect.poll(() => emitted).toBe(i + 1);
|
|
await page.evaluate(
|
|
() =>
|
|
new Promise<void>((resolve) =>
|
|
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
|
|
),
|
|
);
|
|
samples.push(await snapshot(page, `chunk-${i + 1}-frame`));
|
|
await page.waitForTimeout(250);
|
|
samples.push(await snapshot(page, `chunk-${i + 1}-settled`));
|
|
if (scenario === "cancel-navigation" && i === 2) {
|
|
await page.goto("/start");
|
|
await expect
|
|
.poll(() => Array.from(controllers).some((controller) => controller.signal.aborted))
|
|
.toBe(true);
|
|
samples.push(await snapshot(page, "cancelled-navigation"));
|
|
cancelled = true;
|
|
break;
|
|
}
|
|
if (scenario === "scroll-away" && i === 9) {
|
|
await page.getByRole("region", { name: "Messages", exact: true }).hover();
|
|
await page.mouse.wheel(0, -200);
|
|
await page.waitForTimeout(650);
|
|
samples.push(await snapshot(page, "manual-scroll-away"));
|
|
}
|
|
gates[i]!.resolve();
|
|
}
|
|
if (!cancelled) {
|
|
await expect(input).toHaveJSProperty("readOnly", false);
|
|
await page.waitForTimeout(650);
|
|
samples.push(await snapshot(page, "complete"));
|
|
}
|
|
if (scenario === "image" || scenario === "image-wide") {
|
|
await expect.poll(() => imageRequests).toBe(1);
|
|
imageGate.resolve();
|
|
await expect(page.locator('[data-role="assistant"]').last().locator("img")).toBeVisible();
|
|
await page.waitForTimeout(650);
|
|
samples.push(await snapshot(page, "image-loaded"));
|
|
}
|
|
expect(unexpected).toEqual([]);
|
|
expect(requests).toBe(1);
|
|
await info.attach("chunk-geometry", {
|
|
body: JSON.stringify(
|
|
{
|
|
personaId,
|
|
scenario,
|
|
viewport: info.project.name,
|
|
exactRecording,
|
|
provider: recording.provenance,
|
|
pacing: "manual barriers; two animation frames + 250ms; completion 650ms",
|
|
chunks: deltas.map((delta, index) => {
|
|
const partial: unknown = parsePartialJSON(deltas.slice(0, index + 1).join(""));
|
|
return { index: index + 1, delta, partial };
|
|
}),
|
|
samples,
|
|
unexpectedRequests: unexpected,
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
contentType: "application/json",
|
|
});
|
|
const activeSamples = samples.filter((sample) => sample.stage !== "cancelled-navigation");
|
|
const range = (values: number[]) => Math.max(...values) - Math.min(...values);
|
|
// Natural composer/title changes remain measured above, not prohibited by fixed reservations.
|
|
for (const sample of activeSamples) {
|
|
expect(
|
|
sample.constrainedAuxiliary,
|
|
`${sample.stage}: no nested constrained notes/rewrite scroller`,
|
|
).toEqual([]);
|
|
for (const receipt of sample.receipts) {
|
|
expect(receipt.expanded, `${sample.stage}: notes start collapsed`).toBe("false");
|
|
expect(receipt.box).not.toBeNull();
|
|
expect(receipt.trigger).not.toBeNull();
|
|
expect(
|
|
Math.abs((receipt.box?.height ?? 0) - (receipt.trigger?.height ?? 0)),
|
|
"closed receipt occupies only its trigger",
|
|
).toBeLessThanOrEqual(1);
|
|
}
|
|
}
|
|
if (scenario !== "handoff" && scenario !== "scroll-away")
|
|
expect(
|
|
activeSamples.every((sample) => !sample.statusDirection),
|
|
"automatic scrolling must not change header direction",
|
|
).toBe(true);
|
|
const fragments = activeSamples.flatMap((sample) =>
|
|
sample.fragment ? [sample.fragment] : [],
|
|
);
|
|
if (fragments.length) {
|
|
expect(
|
|
new Set(fragments.map((fragment) => fragment.node)).size,
|
|
"fragment DOM identity",
|
|
).toBe(1);
|
|
if (scenario !== "rewrite" && scenario !== "handoff")
|
|
expect(
|
|
range(
|
|
fragments.flatMap((fragment) =>
|
|
fragment.contentY !== null ? [fragment.contentY] : [],
|
|
),
|
|
),
|
|
"late fields must not insert above prose",
|
|
).toBeLessThanOrEqual(1);
|
|
}
|
|
if (scenario === "scroll-away") {
|
|
const readingIndex = activeSamples.findIndex(
|
|
(sample) => sample.stage === "manual-scroll-away",
|
|
);
|
|
const reading = activeSamples
|
|
.slice(readingIndex)
|
|
.flatMap((sample) => (sample.fragment ? [sample.fragment.y] : []));
|
|
expect(range(reading), "reader anchor during late notes").toBeLessThanOrEqual(1);
|
|
} else if (!cancelled) {
|
|
await expect(input).toBeFocused();
|
|
}
|
|
if (scenario === "invalid") {
|
|
await expect(page.getByRole("alert")).toBeVisible();
|
|
await expect(page.getByText(turn.fragments[0]!, { exact: true })).toHaveCount(0);
|
|
}
|
|
} finally {
|
|
start.resolve();
|
|
imageGate.resolve();
|
|
gates.forEach((gate) => gate.resolve());
|
|
controllers.forEach((controller) => controller.abort());
|
|
server.closeAllConnections();
|
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
}
|
|
});
|
|
}
|