583 lines
21 KiB
TypeScript
583 lines
21 KiB
TypeScript
import { expect, test, type Locator } from "@playwright/test";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { z } from "zod";
|
|
import {
|
|
nativeConversationStream,
|
|
nativeImageStream,
|
|
neutralTurn,
|
|
} from "../../src/apps/product/fixtures/native-conversation-stream";
|
|
import {
|
|
PersonaIdSchema,
|
|
type PersonaId,
|
|
} from "../../src/apps/product/standard-schemas/persona.schema";
|
|
import { demoPrompts, scenarioTurn } from "./viewport-scenario";
|
|
import { startLiveCapture } from "./live-capture";
|
|
import { phaseCaption } from "./phase-captions";
|
|
|
|
test("record a clean session and every main user control", async ({ page }) => {
|
|
const directory = "reports/viewport-demo";
|
|
await mkdir(directory, { recursive: true });
|
|
const started = Date.now();
|
|
const stopCapture = await startLiveCapture(page, directory);
|
|
const clips: { id: string; caption: string; startMs: number; endMs: number; persona?: string }[] =
|
|
[];
|
|
const errors: string[] = [];
|
|
page.on("pageerror", (error) => errors.push(error.message));
|
|
const moment = async (
|
|
id: string,
|
|
caption: string,
|
|
action: () => Promise<unknown>,
|
|
hold = 1000,
|
|
) => {
|
|
const startMs = Date.now() - started;
|
|
await action();
|
|
const actionOnly = [
|
|
"request-image",
|
|
"image-reveal",
|
|
"replay-image",
|
|
"boundary-request",
|
|
"rewrite-cut",
|
|
].includes(id);
|
|
const words = phaseCaption(id, caption)
|
|
.filter((segment) => segment.clips.includes(id))
|
|
.map((segment) => segment.text)
|
|
.join(" ")
|
|
.trim()
|
|
.split(/\s+/).length;
|
|
const startupMinimum = ["start", "language", "participation", "choose-persona"].includes(id)
|
|
? 2500
|
|
: ["introduction", "warnings"].includes(id)
|
|
? 2200
|
|
: undefined;
|
|
const minimum =
|
|
startupMinimum ??
|
|
([
|
|
"introduction",
|
|
"warnings",
|
|
"thoughts",
|
|
"image-details",
|
|
"inspect-request",
|
|
"rewrite",
|
|
"about",
|
|
].includes(id)
|
|
? 5000
|
|
: 3000);
|
|
const tutorialTime = /^tour-[0-3]$/.test(id) ? 1600 : 2500;
|
|
const readingTime = actionOnly
|
|
? 0
|
|
: id.startsWith("tour-")
|
|
? tutorialTime
|
|
: (startupMinimum ?? Math.max(minimum, (words / 2.5) * 1000 + 800));
|
|
await page.waitForTimeout(Math.max(hold, readingTime - (Date.now() - started - startMs)));
|
|
clips.push({ id, caption, startMs, endMs: Date.now() - started });
|
|
console.log(`DEMO ${id}`);
|
|
};
|
|
const input = page.getByRole("textbox");
|
|
const send = async (text: string) => {
|
|
await input.fill(text);
|
|
await page.waitForTimeout(1200);
|
|
await input.press("Enter");
|
|
};
|
|
const menu = async () => {
|
|
await page.getByRole("button", { name: "Open menu", exact: true }).click();
|
|
};
|
|
const closeMenu = async () => {
|
|
await page.getByRole("button", { name: "Close", exact: true }).click();
|
|
};
|
|
const revealSuggestion = async (suggestion: Locator) => {
|
|
// Keyboard modality lets Motion's focus animation unzip Polaire's reply.
|
|
await page.keyboard.press("Tab");
|
|
await suggestion.focus();
|
|
await expect(suggestion).toBeFocused();
|
|
await expect
|
|
.poll(() => suggestion.evaluate((node) => node.matches(":focus-visible")))
|
|
.toBe(true);
|
|
await expect(suggestion).toHaveJSProperty("disabled", false);
|
|
await page.waitForTimeout(2500);
|
|
};
|
|
let requests = 0;
|
|
const replies = await Promise.all(
|
|
Array.from({ length: 7 }, (_, index) =>
|
|
nativeConversationStream(scenarioTurn(Math.min(index, 4))),
|
|
),
|
|
);
|
|
const personaReplies = new Map<PersonaId, string>(
|
|
await Promise.all(
|
|
PersonaIdSchema.options
|
|
.filter((persona) => persona !== "polaire")
|
|
.map(async (persona) => {
|
|
const turn = neutralTurn(persona, 0.5, 1, {
|
|
modality: "text-image",
|
|
visualPrompt: "An abstract painted close-up of two adults sharing a kiss",
|
|
});
|
|
turn.statePatch.shortTitleSummary = "A slow invitation";
|
|
return [persona, await nativeConversationStream(turn)] as const;
|
|
}),
|
|
),
|
|
);
|
|
const image = z
|
|
.string()
|
|
.regex(/^data:image\/webp;base64,[A-Za-z0-9+/]+=*$/)
|
|
.parse((await readFile(`${directory}/Untitled-1`, "utf8")).trim());
|
|
await page.route("**/api/ai/chat", async (route) => {
|
|
const index = requests++;
|
|
const currentPersona = PersonaIdSchema.parse(
|
|
new URL(page.url()).searchParams.get("persona-id"),
|
|
);
|
|
const body = index < 7 ? replies[index]! : personaReplies.get(currentPersona);
|
|
if (!body) throw new Error(`Missing demo reply for ${currentPersona}`);
|
|
await new Promise((resolve) => setTimeout(resolve, index === 5 ? 5000 : 450));
|
|
// The sixth request demonstrates the user's native cancellation control.
|
|
if (index === 5)
|
|
await route.fulfill({ contentType: "text/event-stream", body }).catch(() => undefined);
|
|
else await route.fulfill({ contentType: "text/event-stream", body });
|
|
});
|
|
await page.route("**/api/ai/image", async (route) => {
|
|
await new Promise((resolve) => setTimeout(resolve, 1100));
|
|
await route.fulfill({
|
|
contentType: "text/event-stream",
|
|
body: await nativeImageStream("success", image),
|
|
});
|
|
});
|
|
await page.goto("/start");
|
|
expect(await page.evaluate(() => innerWidth)).toBe(390);
|
|
await expect(page.locator("[data-start-ready]")).toHaveAttribute("data-start-ready", "true", {
|
|
timeout: 20000,
|
|
});
|
|
expect(
|
|
await page.evaluate(
|
|
() =>
|
|
Object.keys(sessionStorage).filter((key) => key.startsWith("stx:onboarding:v2:")).length,
|
|
),
|
|
).toBe(0);
|
|
await moment(
|
|
"credits",
|
|
"Meet the project team.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "Crédits", exact: true }).click();
|
|
},
|
|
500,
|
|
);
|
|
await page.getByRole("button", { name: "Crédits", exact: true }).click();
|
|
await moment("start", "Start a new story.", () => Promise.resolve());
|
|
await page.getByRole("link", { name: "Créer une nouvelle histoire", exact: true }).click();
|
|
await moment("language", "Select a language.", async () => {
|
|
await page.getByRole("radio", { name: "English", exact: true }).check();
|
|
});
|
|
await page.getByRole("button", { name: "Continue", exact: true }).click();
|
|
await moment("introduction", "Read about the experience.", async () => {
|
|
await expect(
|
|
page.getByRole("heading", { name: "An encounter with StoryTellurX" }),
|
|
).toBeVisible();
|
|
});
|
|
await page.getByRole("button", { name: "Continue", exact: true }).click();
|
|
await moment(
|
|
"warnings",
|
|
"Read the safety information.",
|
|
async () => {
|
|
await page
|
|
.getByRole("checkbox", { name: "I confirm that I am at least 18 years old." })
|
|
.check();
|
|
await page
|
|
.getByRole("checkbox", { name: "I have read and understood these warnings." })
|
|
.check();
|
|
},
|
|
0,
|
|
);
|
|
await page.getByRole("button", { name: "Continue", exact: true }).click();
|
|
await moment("participation", "Confirm your choice.", async () => {
|
|
await page
|
|
.getByRole("checkbox", { name: "Yes, I want to take part in this experience." })
|
|
.check();
|
|
});
|
|
await page.getByRole("button", { name: "Choose a world", exact: true }).click();
|
|
await moment("choose-persona", "Select a character.", async () => {
|
|
await expect(page.locator("[data-persona-tile]")).toHaveCount(8);
|
|
});
|
|
await page.locator('[data-persona-tile="polaire"]').click();
|
|
await expect(input).toBeVisible({ timeout: 20000 });
|
|
const sessionId = new URL(page.url()).searchParams.get("session-id");
|
|
expect(sessionId).toBeTruthy();
|
|
await expect(page.getByRole("button", { name: "Next", exact: true })).toBeVisible();
|
|
for (let i = 0; i < 9; i++) {
|
|
const next = page.getByRole("button", { name: "Next", exact: true });
|
|
const finish = page.getByRole("button", { name: "Finish", exact: true });
|
|
await moment(
|
|
`tour-${i}`,
|
|
"Explore the controls.",
|
|
async () => {
|
|
if (i !== 4) return;
|
|
await expect(page.locator("#tour-step-title-suggestions")).toBeVisible();
|
|
const suggestions = page.locator("[data-suggestion-slot] [data-suggestion]");
|
|
await expect(suggestions).toHaveCount(2);
|
|
for (const suggestion of await suggestions.all()) {
|
|
await revealSuggestion(suggestion);
|
|
}
|
|
},
|
|
0,
|
|
);
|
|
if (await finish.isVisible()) {
|
|
await finish.click();
|
|
break;
|
|
}
|
|
await next.click();
|
|
}
|
|
await moment("tour-finish", "Use the controls at your own pace.", () => Promise.resolve());
|
|
await moment("send-message", "Write your message.", () => send(demoPrompts.opening), 1600);
|
|
await moment(
|
|
"suggestion",
|
|
"Use a suggested reply.",
|
|
async () => {
|
|
const suggestion = page.getByRole("button", {
|
|
name: "Ask what Polaire wants",
|
|
exact: true,
|
|
});
|
|
await revealSuggestion(suggestion);
|
|
await suggestion.click();
|
|
await expect(input).toHaveValue("What would you like us to do next?");
|
|
await input.focus();
|
|
await page.waitForTimeout(1800);
|
|
await input.press("Enter");
|
|
},
|
|
1500,
|
|
);
|
|
await moment(
|
|
"thoughts",
|
|
"Read the reason for the reply.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "Show my thoughts", exact: true }).last().click();
|
|
},
|
|
1600,
|
|
);
|
|
await page.getByRole("button", { name: "Show my thoughts", exact: true }).last().click();
|
|
await moment("request-image", "Ask for an image.", () => send(demoPrompts.image), 0);
|
|
await moment(
|
|
"image-reveal",
|
|
"View the image.",
|
|
async () => {
|
|
await expect(page.locator("[data-image-reveal]")).toBeVisible({ timeout: 15000 });
|
|
await expect(page.locator('[data-reveal-renderer="webgpu"]')).toBeVisible();
|
|
await expect(page.locator("[data-image-reveal]")).toHaveCount(0, { timeout: 10000 });
|
|
},
|
|
0,
|
|
);
|
|
await moment(
|
|
"replay-image",
|
|
"Find the image replay control.",
|
|
async () => {
|
|
const image = page.getByRole("button", { name: /^Replay image reveal:/ }).last();
|
|
await image.scrollIntoViewIfNeeded();
|
|
await page.keyboard.press("Tab");
|
|
await image.focus();
|
|
await expect(image).toBeFocused();
|
|
await expect(page.locator("[data-image-reveal]")).toHaveCount(0);
|
|
},
|
|
1800,
|
|
);
|
|
await moment(
|
|
"gallery",
|
|
"View your saved images.",
|
|
async () => {
|
|
await menu();
|
|
await page.locator("[data-gallery-image]").scrollIntoViewIfNeeded();
|
|
},
|
|
1200,
|
|
);
|
|
await moment(
|
|
"image-details",
|
|
"Check the image details.",
|
|
() => page.getByLabel("Image 1 details", { exact: true }).click(),
|
|
1400,
|
|
);
|
|
await page.getByLabel("Image 1 details", { exact: true }).click();
|
|
await moment(
|
|
"model",
|
|
"Select another model.",
|
|
async () => {
|
|
await page.getByRole("combobox", { name: "Model", exact: true }).click();
|
|
await page.locator('[role="option"][data-select-value="hermes"]').click();
|
|
},
|
|
800,
|
|
);
|
|
if (!(await page.getByRole("button", { name: "Close", exact: true }).isVisible())) await menu();
|
|
await moment(
|
|
"sound",
|
|
"Turn the sound on or off.",
|
|
() => page.locator("[data-sound-toggle]").click(),
|
|
700,
|
|
);
|
|
await moment(
|
|
"language-switch",
|
|
"Change the language.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "Français", exact: true }).click();
|
|
await page.waitForTimeout(450);
|
|
await page.getByRole("button", { name: "English", exact: true }).click();
|
|
},
|
|
500,
|
|
);
|
|
await closeMenu();
|
|
await moment(
|
|
"boundary-request",
|
|
"The character does not agree.",
|
|
() => send(demoPrompts.boundary),
|
|
0,
|
|
);
|
|
await moment(
|
|
"rewrite-cut",
|
|
"The character has a choice.",
|
|
async () => {
|
|
await expect(page.locator("[data-rewrite-cut]")).toBeVisible();
|
|
await expect(page.locator('[data-rewrite-cut-renderer="webgpu"]')).toBeVisible();
|
|
await expect(page.locator("[data-interface-tutorial]")).toHaveCount(0);
|
|
await page.screenshot({ path: `${directory}/rewrite-cut.png` });
|
|
await expect(page.locator("[data-rewrite-cut]")).toHaveCount(0, { timeout: 5000 });
|
|
},
|
|
0,
|
|
);
|
|
await moment(
|
|
"rewrite",
|
|
"The character changes your request.",
|
|
async () => {
|
|
await expect(page.locator("[data-persona-intervention]")).toContainText("accept or refuse");
|
|
const tutorial = page
|
|
.getByRole("dialog")
|
|
.filter({ has: page.locator("[data-interface-tutorial]") });
|
|
await expect(tutorial).toBeVisible();
|
|
await expect(page.locator("[data-rewrite-cut]")).toHaveCount(0);
|
|
await page.waitForTimeout(2200);
|
|
await tutorial.getByRole("button", { name: "Finish", exact: true }).click();
|
|
await page.locator("[data-persona-intervention]").scrollIntoViewIfNeeded();
|
|
},
|
|
2200,
|
|
);
|
|
await moment(
|
|
"consensual-reply",
|
|
"Ask before you move closer.",
|
|
() => send(demoPrompts.repair),
|
|
1400,
|
|
);
|
|
await moment(
|
|
"interrupt-response",
|
|
"Interrupt a reply.",
|
|
async () => {
|
|
await send("Tell me more about the garden.");
|
|
await expect.poll(() => requests).toBe(6);
|
|
await page.getByRole("button", { name: "Stop response", exact: true }).click();
|
|
await expect(page.getByRole("button", { name: "Send message", exact: true })).toBeVisible();
|
|
},
|
|
1000,
|
|
);
|
|
await moment(
|
|
"retry-response",
|
|
"Try the interrupted reply again.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "Try again.", exact: true }).click();
|
|
await expect(page.getByRole("button", { name: "Try again.", exact: true })).toHaveCount(0);
|
|
await expect(
|
|
page.getByText("That is better, Alex. Yes, you may sit a little closer.", { exact: true }),
|
|
).toHaveCount(2, { timeout: 10000 });
|
|
await expect(page.getByRole("button", { name: "Send message", exact: true })).toBeVisible();
|
|
},
|
|
1200,
|
|
);
|
|
await moment("pause", "Stop the scene.", () => page.locator("[data-panic-stop]").click(), 1400);
|
|
await moment(
|
|
"resume",
|
|
"Continue at your own pace.",
|
|
() => page.getByRole("button", { name: "Resume at my pace", exact: true }).click(),
|
|
850,
|
|
);
|
|
await moment(
|
|
"inspect-request",
|
|
"Check the request and response.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "//LLM_REQUEST_PAYLOAD", exact: true }).last().click();
|
|
await page.getByRole("tab", { name: "Raw JSON", exact: true }).click();
|
|
},
|
|
1400,
|
|
);
|
|
await page.getByRole("button", { name: "Close", exact: true }).click();
|
|
await moment(
|
|
"end-session",
|
|
"End the session when you choose.",
|
|
async () => {
|
|
await page.locator("[data-panic-stop]").click();
|
|
await page.getByRole("button", { name: "End session", exact: true }).click();
|
|
},
|
|
1000,
|
|
);
|
|
await moment(
|
|
"export-report",
|
|
"Save a copy of the exchange.",
|
|
async () => {
|
|
const download = page.waitForEvent("download");
|
|
await page.getByRole("button", { name: "Download a copy (optional)", exact: true }).click();
|
|
await (await download).saveAs(`${directory}/session-report.json`);
|
|
},
|
|
700,
|
|
);
|
|
const returnHome = page.getByRole("button", { name: "Return home", exact: true });
|
|
await returnHome.focus();
|
|
await returnHome.press("Enter");
|
|
await moment(
|
|
"history",
|
|
"Open a saved story.",
|
|
async () => {
|
|
await menu();
|
|
await page.getByRole("link", { name: "History", exact: true }).click();
|
|
await expect(page.getByRole("link", { name: /A slow invitation/ })).toBeVisible();
|
|
},
|
|
1200,
|
|
);
|
|
await moment(
|
|
"export-history",
|
|
"Save the story on your device.",
|
|
async () => {
|
|
const download = page.waitForEvent("download");
|
|
await page
|
|
.getByRole("button", { name: /^EXPORT / })
|
|
.first()
|
|
.click();
|
|
await (await download).saveAs(`${directory}/saved-story.json`);
|
|
},
|
|
700,
|
|
);
|
|
await moment(
|
|
"resume-history",
|
|
"Continue your saved story.",
|
|
() => page.getByRole("link", { name: /A slow invitation/ }).click(),
|
|
1000,
|
|
);
|
|
await expect(page.locator("[data-image-reveal]")).toHaveCount(0);
|
|
await moment(
|
|
"review-tutorial",
|
|
"Open the guide again.",
|
|
async () => {
|
|
await menu();
|
|
await page.getByRole("button", { name: "Review interface tutorial", exact: true }).click();
|
|
await expect(page.getByRole("button", { name: "Close tutorial", exact: true })).toBeVisible();
|
|
},
|
|
400,
|
|
);
|
|
await page.getByRole("button", { name: "Close tutorial", exact: true }).click();
|
|
await moment(
|
|
"about",
|
|
"Read about the project.",
|
|
async () => {
|
|
await menu();
|
|
await page.getByRole("link", { name: "About", exact: true }).click();
|
|
},
|
|
1200,
|
|
);
|
|
await page.goBack();
|
|
await expect(input).toBeVisible();
|
|
await moment(
|
|
"review-setup",
|
|
"Review your setup.",
|
|
async () => {
|
|
await menu();
|
|
await page.getByRole("link", { name: "Review setup", exact: true }).click();
|
|
},
|
|
1200,
|
|
);
|
|
await page.goBack();
|
|
await expect(input).toBeVisible();
|
|
// One live browser page and one saved conversation continue through all contexts.
|
|
for (const persona of PersonaIdSchema.options.filter((id) => id !== "polaire")) {
|
|
await menu();
|
|
await page.getByRole("combobox", { name: "Context", exact: true }).click();
|
|
await page.locator(`[role="option"][data-select-value="${persona}"]`).click();
|
|
await expect(page.locator("[data-persona-chat]")).toHaveAttribute("data-persona-chat", persona);
|
|
if (await page.getByRole("button", { name: "Close", exact: true }).isVisible())
|
|
await closeMenu();
|
|
await expect(page.locator("[data-menu-backdrop]")).toHaveCount(0);
|
|
await page.waitForTimeout(300);
|
|
// Existing images correctly keep their original persona. Generate a new turn in this world.
|
|
await send(demoPrompts.image);
|
|
await expect(page.locator("[data-image-reveal]")).toBeVisible({ timeout: 20000 });
|
|
const card = await page.locator(".image-reveal__picture").boundingBox();
|
|
if (!card) throw new Error(`Missing reveal card for ${persona}`);
|
|
// Hold the card as soon as it mounts, before waiting for its GPU reveal.
|
|
await page.mouse.move(card.x + card.width / 2, card.y + card.height / 2);
|
|
await page.mouse.down();
|
|
await page.mouse.up();
|
|
await expect(page.locator('[data-image-reveal="reveal"]')).toBeVisible({ timeout: 20000 });
|
|
await expect(page.locator("[data-image-reveal]")).toHaveAttribute(
|
|
"data-reveal-persona",
|
|
persona,
|
|
);
|
|
await expect(page.locator('[data-reveal-renderer="webgpu"]')).toBeVisible();
|
|
await page.waitForTimeout(200);
|
|
const startMs = Date.now() - started;
|
|
await page.waitForTimeout(1600);
|
|
await expect(page.locator('[data-image-reveal="reveal"]')).toBeVisible();
|
|
clips.push({
|
|
id: `persona-${persona}`,
|
|
caption: `Explore ${persona.replace("storytellurx_", "").toUpperCase()}.`,
|
|
startMs,
|
|
endMs: startMs + 1600,
|
|
persona,
|
|
});
|
|
console.log(`DEMO persona-${persona}`);
|
|
await page.keyboard.press("Escape");
|
|
await expect(page.locator("[data-image-reveal]")).toHaveCount(0, { timeout: 5000 });
|
|
expect(new URL(page.url()).searchParams.get("session-id")).toBe(sessionId);
|
|
}
|
|
// Record history cleanup after the montage; the edit places these controls before it.
|
|
await menu();
|
|
await page.getByRole("link", { name: "History", exact: true }).click();
|
|
await moment(
|
|
"delete-story",
|
|
"Delete a saved story.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "DELETE A slow invitation", exact: true }).click();
|
|
await expect(page.getByRole("link", { name: /A slow invitation/ })).toHaveCount(0);
|
|
},
|
|
1000,
|
|
);
|
|
await menu();
|
|
await page.getByRole("link", { name: "Personas", exact: true }).click();
|
|
await page.getByRole("link", { name: "Create new story", exact: true }).click();
|
|
await expect(page.locator('[data-persona-tile="nelly"]')).toBeVisible({ timeout: 20000 });
|
|
await page.locator('[data-persona-tile="nelly"]').click();
|
|
await expect(input).toBeVisible();
|
|
const closeTour = page.getByRole("button", { name: "Close tutorial", exact: true });
|
|
if (await closeTour.isVisible()) await closeTour.click();
|
|
await menu();
|
|
await page.getByRole("link", { name: "History", exact: true }).click();
|
|
await moment(
|
|
"delete-history",
|
|
"Delete all saved stories.",
|
|
async () => {
|
|
await page.getByRole("button", { name: "DELETE ALL HISTORY", exact: true }).click();
|
|
await expect(
|
|
page.getByRole("button", { name: "DELETE ALL HISTORY", exact: true }),
|
|
).toHaveCount(0);
|
|
},
|
|
1000,
|
|
);
|
|
const finished = Date.now();
|
|
const recording = await stopCapture();
|
|
await page.close();
|
|
await writeFile(
|
|
`${directory}/capture.json`,
|
|
JSON.stringify(
|
|
{
|
|
started,
|
|
finished,
|
|
...recording,
|
|
clips,
|
|
requests,
|
|
sessionId,
|
|
errors,
|
|
source:
|
|
"Real UI interactions from empty browser storage; scripted provider replies and the supplied reports/viewport-demo/Untitled-1 image.",
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
expect(requests).toBe(7 + personaReplies.size);
|
|
expect(errors).toEqual([]);
|
|
});
|