From 7be2097ab4e6aa3e12145138bcd069424777063c Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 13:50:19 +0000 Subject: [PATCH 1/9] Group systemic playwright flakes Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/flaky-reporter.ts | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/playwright/flaky-reporter.ts b/playwright/flaky-reporter.ts index 770633decb7..f816d7651e8 100644 --- a/playwright/flaky-reporter.ts +++ b/playwright/flaky-reporter.ts @@ -24,18 +24,40 @@ type PaginationLinks = { first?: string; }; +// We see quite a few test flakes which are caused by the app exploding +// so we have some magic strings we check the logs for to better track the flake with its cause +const SPECIAL_CASES = { + "ChunkLoadError": "ChunkLoadError", + "Unreachable code should not be executed": "Rust crypto panic", + "Out of bounds memory access": "Rust crypto memory error", +}; + class FlakyReporter implements Reporter { private flakes = new Map(); public onTestEnd(test: TestCase): void { // Ignores flakes on Dendrite and Pinecone as they have their own flakes we do not track if (["Dendrite", "Pinecone"].includes(test.parent.project()?.name)) return; - const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`; + let failures = [`${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`]; if (test.outcome() === "flaky") { - if (!this.flakes.has(title)) { - this.flakes.set(title, []); + const timedOutRuns = test.results.filter((result) => result.status === "timedOut"); + const pageLogs = timedOutRuns.flatMap((result) => + result.attachments.filter((attachment) => attachment.name.startsWith("page-")), + ); + // If a test failed due to a systemic fault then the test is not flaky, the app is, record it as such. + const specialCases = Object.keys(SPECIAL_CASES).filter((log) => + pageLogs.some((attachment) => attachment.name.startsWith("page-") && attachment.body.includes(log)), + ); + if (specialCases.length > 0) { + failures = specialCases.map((specialCase) => SPECIAL_CASES[specialCase]); + } + + for (const title of failures) { + if (!this.flakes.has(title)) { + this.flakes.set(title, []); + } + this.flakes.get(title).push(test); } - this.flakes.get(title).push(test); } } From 029bd92b28b85ae9e4591378c75bbb76cebd1a37 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 14:16:39 +0000 Subject: [PATCH 2/9] Fix more flaky tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../e2e/audio-player/audio-player.spec.ts | 21 ++++++++-------- .../e2e/knock/create-knock-room.spec.ts | 1 + playwright/e2e/knock/knock-into-room.spec.ts | 1 + playwright/e2e/messages/messages.spec.ts | 6 +++++ .../pinned-messages/pinned-messages.spec.ts | 8 +++---- .../spaces/threads-activity-centre/index.ts | 2 ++ playwright/pages/client.ts | 24 ++++++++++++------- 7 files changed, 41 insertions(+), 22 deletions(-) diff --git a/playwright/e2e/audio-player/audio-player.spec.ts b/playwright/e2e/audio-player/audio-player.spec.ts index a6d920dcb84..bb766d6b88b 100644 --- a/playwright/e2e/audio-player/audio-player.spec.ts +++ b/playwright/e2e/audio-player/audio-player.spec.ts @@ -13,6 +13,14 @@ import { SettingLevel } from "../../../src/settings/SettingLevel"; import { Layout } from "../../../src/settings/enums/Layout"; import { ElementAppPage } from "../../pages/ElementAppPage"; +// Find and click "Reply" button +const clickButtonReply = async (tile: Locator) => { + await expect(async () => { + await tile.hover(); + await tile.getByRole("button", { name: "Reply", exact: true }).click(); + }).toPass(); +}; + test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.use({ displayName: "Hanako", @@ -222,8 +230,7 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { // Find and click "Reply" button on MessageActionBar const tile = page.locator(".mx_EventTile_last"); - await tile.hover(); - await tile.getByRole("button", { name: "Reply", exact: true }).click(); + await clickButtonReply(tile); // Reply to the player with another audio file await uploadFile(page, "playwright/sample-files/1sec.ogg"); @@ -251,18 +258,12 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { const tile = page.locator(".mx_EventTile_last"); - // Find and click "Reply" button - const clickButtonReply = async () => { - await tile.hover(); - await tile.getByRole("button", { name: "Reply", exact: true }).click(); - }; - await uploadFile(page, "playwright/sample-files/upload-first.ogg"); // Assert that the audio player is rendered await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible(); - await clickButtonReply(); + await clickButtonReply(tile); // Reply to the player with another audio file await uploadFile(page, "playwright/sample-files/upload-second.ogg"); @@ -270,7 +271,7 @@ test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { // Assert that the audio player is rendered await expect(page.locator(".mx_EventTile_last .mx_AudioPlayer_container")).toBeVisible(); - await clickButtonReply(); + await clickButtonReply(tile); // Reply to the player with yet another audio file to create a reply chain await uploadFile(page, "playwright/sample-files/upload-third.ogg"); diff --git a/playwright/e2e/knock/create-knock-room.spec.ts b/playwright/e2e/knock/create-knock-room.spec.ts index 29733481ddc..e21b30a3c29 100644 --- a/playwright/e2e/knock/create-knock-room.spec.ts +++ b/playwright/e2e/knock/create-knock-room.spec.ts @@ -81,6 +81,7 @@ test.describe("Create Knock Room", () => { const spotlightDialog = await app.openSpotlight(); await spotlightDialog.filter(Filter.PublicRooms); + await spotlightDialog.search("Cyber"); await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity"); }); }); diff --git a/playwright/e2e/knock/knock-into-room.spec.ts b/playwright/e2e/knock/knock-into-room.spec.ts index 9c2f1ee76be..be6619697de 100644 --- a/playwright/e2e/knock/knock-into-room.spec.ts +++ b/playwright/e2e/knock/knock-into-room.spec.ts @@ -284,6 +284,7 @@ test.describe("Knock Into Room", () => { const spotlightDialog = await app.openSpotlight(); await spotlightDialog.filter(Filter.PublicRooms); + await spotlightDialog.search("Cyber"); await expect(spotlightDialog.results.nth(0)).toContainText("Cybersecurity"); await spotlightDialog.results.nth(0).click(); diff --git a/playwright/e2e/messages/messages.spec.ts b/playwright/e2e/messages/messages.spec.ts index 03c93d2620f..0ff8ba4b82f 100644 --- a/playwright/e2e/messages/messages.spec.ts +++ b/playwright/e2e/messages/messages.spec.ts @@ -133,6 +133,12 @@ test.describe("Message rendering", () => { const msgTile = await sendMessage(page, "مرحبا بالعالم!"); await expect(msgTile).toMatchScreenshot(`basic-message-rtl-${direction}displayname.png`, { mask: [page.locator(".mx_MessageTimestamp")], + // Hide the jump to bottom button in the timeline to avoid flakiness + css: ` + .mx_JumpToBottomButton { + display: none !important; + } + `, }); }); diff --git a/playwright/e2e/pinned-messages/pinned-messages.spec.ts b/playwright/e2e/pinned-messages/pinned-messages.spec.ts index bb72c026107..de954fb8d43 100644 --- a/playwright/e2e/pinned-messages/pinned-messages.spec.ts +++ b/playwright/e2e/pinned-messages/pinned-messages.spec.ts @@ -35,10 +35,10 @@ test.describe("Pinned messages", () => { mask: [tile.locator(".mx_MessageTimestamp")], // Hide the jump to bottom button in the timeline to avoid flakiness css: ` - .mx_JumpToBottomButton { - display: none !important; - } - `, + .mx_JumpToBottomButton { + display: none !important; + } + `, }); }, ); diff --git a/playwright/e2e/spaces/threads-activity-centre/index.ts b/playwright/e2e/spaces/threads-activity-centre/index.ts index d3d3cb352b4..c0834731633 100644 --- a/playwright/e2e/spaces/threads-activity-centre/index.ts +++ b/playwright/e2e/spaces/threads-activity-centre/index.ts @@ -38,11 +38,13 @@ export const test = base.extend<{ room1Name: "Room 1", room1: async ({ room1Name: name, app, user, bot }, use) => { const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); + await bot.waitForRoom(roomId); await use({ name, roomId }); }, room2Name: "Room 2", room2: async ({ room2Name: name, app, user, bot }, use) => { const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); + await bot.waitForRoom(roomId); await use({ name, roomId }); }, msg: async ({ page, app, util }, use) => { diff --git a/playwright/pages/client.ts b/playwright/pages/client.ts index 362915ce71f..09973dd555d 100644 --- a/playwright/pages/client.ts +++ b/playwright/pages/client.ts @@ -171,15 +171,9 @@ export class Client { }); } - /** - * Create a room with given options. - * @param options the options to apply when creating the room - * @return the ID of the newly created room - */ - public async createRoom(options: ICreateRoomOpts): Promise { + public async waitForRoom(roomId: string): Promise { const client = await this.prepareClient(); - return await client.evaluate(async (cli, options) => { - const { room_id: roomId } = await cli.createRoom(options); + return await client.evaluate(async (cli, roomId) => { if (!cli.getRoom(roomId)) { await new Promise((resolve) => { const onRoom = (room: Room) => { @@ -191,8 +185,22 @@ export class Client { cli.on(window.matrixcs.ClientEvent.Room, onRoom); }); } + }, roomId); + } + + /** + * Create a room with given options. + * @param options the options to apply when creating the room + * @return the ID of the newly created room + */ + public async createRoom(options: ICreateRoomOpts): Promise { + const client = await this.prepareClient(); + const roomId = await client.evaluate(async (cli, options) => { + const { room_id: roomId } = await cli.createRoom(options); return roomId; }, options); + await this.awaitRoomMembership(roomId); + return roomId; } /** From e387e3a57001735080318c4959dbdcdd19958797 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 14:45:09 +0000 Subject: [PATCH 3/9] Fix another flake Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/timeline/timeline.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/playwright/e2e/timeline/timeline.spec.ts b/playwright/e2e/timeline/timeline.spec.ts index 885c15d90f4..3de0f7c0f29 100644 --- a/playwright/e2e/timeline/timeline.spec.ts +++ b/playwright/e2e/timeline/timeline.spec.ts @@ -1195,6 +1195,7 @@ test.describe("Timeline", () => { }); await sendImage(app.client, room.roomId, NEW_AVATAR); + await app.timeline.scrollToBottom(); await expect(page.locator(".mx_MImageBody").first()).toBeVisible(); // Exclude timestamp and read marker from snapshot From 6af4c34e4809986d7ad53036947d977b1b926f81 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 15:09:31 +0000 Subject: [PATCH 4/9] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/settings/encryption-user-tab/index.ts | 3 +-- playwright/e2e/settings/encryption-user-tab/recovery.spec.ts | 2 +- playwright/e2e/spaces/spaces.spec.ts | 4 ++-- playwright/pages/ElementAppPage.ts | 4 ---- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/playwright/e2e/settings/encryption-user-tab/index.ts b/playwright/e2e/settings/encryption-user-tab/index.ts index f8adbb2e336..9fc8eecb712 100644 --- a/playwright/e2e/settings/encryption-user-tab/index.ts +++ b/playwright/e2e/settings/encryption-user-tab/index.ts @@ -89,8 +89,7 @@ class Helpers { await expect(dialog.getByText(title, { exact: true })).toBeVisible(); await expect(dialog).toMatchScreenshot(screenshot); - const handle = await this.page.evaluateHandle(() => navigator.clipboard.readText()); - const clipboardContent = await handle.jsonValue(); + const clipboardContent = await this.app.getClipboard(); await dialog.getByRole("textbox").fill(clipboardContent); await dialog.getByRole("button", { name: confirmButtonLabel }).click(); await expect(dialog).toMatchScreenshot("default-recovery.png"); diff --git a/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts b/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts index e6812cd450b..f856da8064c 100644 --- a/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts +++ b/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts @@ -53,7 +53,7 @@ test.describe("Recovery section in Encryption tab", () => { test( "should change the recovery key", - { tag: "@screenshot" }, + { tag: ["@screenshot", "@no-webkit"] }, async ({ page, app, homeserver, credentials, util, context }) => { await verifySession(app, "new passphrase"); const dialog = await util.openEncryptionTab(); diff --git a/playwright/e2e/spaces/spaces.spec.ts b/playwright/e2e/spaces/spaces.spec.ts index 5acb3a672fe..37e5606cc13 100644 --- a/playwright/e2e/spaces/spaces.spec.ts +++ b/playwright/e2e/spaces/spaces.spec.ts @@ -84,7 +84,7 @@ test.describe("Spaces", () => { // Copy matrix.to link await page.getByRole("button", { name: "Share invite link" }).click(); - expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`); + expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#lets-have-a-riot:${user.homeServer}`); // Go to space home await page.getByRole("button", { name: "Go to my first room" }).click(); @@ -177,7 +177,7 @@ test.describe("Spaces", () => { const shareDialog = page.locator(".mx_SpacePublicShare"); // Copy link first await shareDialog.getByRole("button", { name: "Share invite link" }).click(); - expect(await app.getClipboardText()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`); + expect(await app.getClipboard()).toEqual(`https://matrix.to/#/#space:${user.homeServer}`); // Start Matrix invite flow await shareDialog.getByRole("button", { name: "Invite people" }).click(); diff --git a/playwright/pages/ElementAppPage.ts b/playwright/pages/ElementAppPage.ts index 98d0bf30fba..d530c75b54e 100644 --- a/playwright/pages/ElementAppPage.ts +++ b/playwright/pages/ElementAppPage.ts @@ -158,10 +158,6 @@ export class ElementAppPage { return button.click(); } - public async getClipboardText(): Promise { - return this.page.evaluate("navigator.clipboard.readText()"); - } - public async openSpotlight(): Promise { const spotlight = new Spotlight(this.page); await spotlight.open(); From feb8494d7cb4a0e3889d3929f79f52c6a4fa63f1 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 15:35:07 +0000 Subject: [PATCH 5/9] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .../encryption-user-tab/recovery.spec.ts | 2 +- .../e2e/spaces/threads-activity-centre/index.ts | 4 ++-- playwright/pages/client.ts | 17 ----------------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts b/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts index f856da8064c..9d614a8df34 100644 --- a/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts +++ b/playwright/e2e/settings/encryption-user-tab/recovery.spec.ts @@ -81,7 +81,7 @@ test.describe("Recovery section in Encryption tab", () => { }, ); - test("should setup the recovery key", { tag: "@screenshot" }, async ({ page, app, util }) => { + test("should setup the recovery key", { tag: ["@screenshot", "@no-webkit"] }, async ({ page, app, util }) => { await verifySession(app, "new passphrase"); await util.removeSecretStorageDefaultKeyId(); diff --git a/playwright/e2e/spaces/threads-activity-centre/index.ts b/playwright/e2e/spaces/threads-activity-centre/index.ts index c0834731633..7da6974a92c 100644 --- a/playwright/e2e/spaces/threads-activity-centre/index.ts +++ b/playwright/e2e/spaces/threads-activity-centre/index.ts @@ -38,13 +38,13 @@ export const test = base.extend<{ room1Name: "Room 1", room1: async ({ room1Name: name, app, user, bot }, use) => { const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); - await bot.waitForRoom(roomId); + await bot.awaitRoomMembership(roomId); await use({ name, roomId }); }, room2Name: "Room 2", room2: async ({ room2Name: name, app, user, bot }, use) => { const roomId = await app.client.createRoom({ name, invite: [bot.credentials.userId] }); - await bot.waitForRoom(roomId); + await bot.awaitRoomMembership(roomId); await use({ name, roomId }); }, msg: async ({ page, app, util }, use) => { diff --git a/playwright/pages/client.ts b/playwright/pages/client.ts index 09973dd555d..7d3ce113e8a 100644 --- a/playwright/pages/client.ts +++ b/playwright/pages/client.ts @@ -171,23 +171,6 @@ export class Client { }); } - public async waitForRoom(roomId: string): Promise { - const client = await this.prepareClient(); - return await client.evaluate(async (cli, roomId) => { - if (!cli.getRoom(roomId)) { - await new Promise((resolve) => { - const onRoom = (room: Room) => { - if (room.roomId === roomId) { - cli.off(window.matrixcs.ClientEvent.Room, onRoom); - resolve(); - } - }; - cli.on(window.matrixcs.ClientEvent.Room, onRoom); - }); - } - }, roomId); - } - /** * Create a room with given options. * @param options the options to apply when creating the room From 30b64ca880585ae372afb11e7af2549814269965 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 16:18:02 +0000 Subject: [PATCH 6/9] delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/pages/client.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/playwright/pages/client.ts b/playwright/pages/client.ts index 7d3ce113e8a..611a6cef190 100644 --- a/playwright/pages/client.ts +++ b/playwright/pages/client.ts @@ -15,7 +15,6 @@ import type { ICreateRoomOpts, ISendEventResponse, MatrixClient, - Room, MatrixEvent, ReceiptType, IRoomDirectoryOptions, From 2b4f66a727287a1bb856175fd37f29849ede1242 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 16:37:30 +0000 Subject: [PATCH 7/9] Fix more flakes Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/messages/messages.spec.ts | 78 +++++++++++++++--------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/playwright/e2e/messages/messages.spec.ts b/playwright/e2e/messages/messages.spec.ts index 0ff8ba4b82f..0cfbdaaadb3 100644 --- a/playwright/e2e/messages/messages.spec.ts +++ b/playwright/e2e/messages/messages.spec.ts @@ -58,6 +58,16 @@ async function editMessage(page: Page, message: Locator, newMsg: string): Promis await editComposer.press("Enter"); } +const screenshotOptions = (page: Page) => ({ + mask: [page.locator(".mx_MessageTimestamp")], + // Hide the jump to bottom button in the timeline to avoid flakiness + css: ` + .mx_JumpToBottomButton { + display: none !important; + } + `, +}); + test.describe("Message rendering", () => { [ { direction: "ltr", displayName: "Quentin" }, @@ -79,9 +89,10 @@ test.describe("Message rendering", () => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "Hello, world!"); - await expect(msgTile).toMatchScreenshot(`basic-message-ltr-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - }); + await expect(msgTile).toMatchScreenshot( + `basic-message-ltr-${direction}displayname.png`, + screenshotOptions(page), + ); }, ); @@ -89,14 +100,20 @@ test.describe("Message rendering", () => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me lays an egg"); - await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`); + await expect(msgTile).toMatchScreenshot( + `emote-ltr-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render an LTR rich text emote", async ({ page, user, app, room }) => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me lays a *free range* egg"); - await expect(msgTile).toMatchScreenshot(`emote-rich-ltr-${direction}displayname.png`); + await expect(msgTile).toMatchScreenshot( + `emote-rich-ltr-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render an edited LTR message", async ({ page, user, app, room }) => { @@ -106,9 +123,10 @@ test.describe("Message rendering", () => { await editMessage(page, msgTile, "Hello, universe!"); - await expect(msgTile).toMatchScreenshot(`edited-message-ltr-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - }); + await expect(msgTile).toMatchScreenshot( + `edited-message-ltr-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render a reply of a LTR message", async ({ page, user, app, room }) => { @@ -122,38 +140,40 @@ test.describe("Message rendering", () => { ]); await replyMessage(page, msgTile, "response to multiline message"); - await expect(msgTile).toMatchScreenshot(`reply-message-ltr-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - }); + await expect(msgTile).toMatchScreenshot( + `reply-message-ltr-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render a basic RTL text message", async ({ page, user, app, room }) => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "مرحبا بالعالم!"); - await expect(msgTile).toMatchScreenshot(`basic-message-rtl-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - // Hide the jump to bottom button in the timeline to avoid flakiness - css: ` - .mx_JumpToBottomButton { - display: none !important; - } - `, - }); + await expect(msgTile).toMatchScreenshot( + `basic-message-rtl-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render an RTL emote", async ({ page, user, app, room }) => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me يضع بيضة"); - await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`); + await expect(msgTile).toMatchScreenshot( + `emote-rtl-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render a richtext RTL emote", async ({ page, user, app, room }) => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me أضع بيضة *حرة النطاق*"); - await expect(msgTile).toMatchScreenshot(`emote-rich-rtl-${direction}displayname.png`); + await expect(msgTile).toMatchScreenshot( + `emote-rich-rtl-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render an edited RTL message", async ({ page, user, app, room }) => { @@ -163,9 +183,10 @@ test.describe("Message rendering", () => { await editMessage(page, msgTile, "مرحبا بالكون!"); - await expect(msgTile).toMatchScreenshot(`edited-message-rtl-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - }); + await expect(msgTile).toMatchScreenshot( + `edited-message-rtl-${direction}displayname.png`, + screenshotOptions(page), + ); }); test("should render a reply of a RTL message", async ({ page, user, app, room }) => { @@ -179,9 +200,10 @@ test.describe("Message rendering", () => { ]); await replyMessage(page, msgTile, "مرحبا بالعالم!"); - await expect(msgTile).toMatchScreenshot(`reply-message-trl-${direction}displayname.png`, { - mask: [page.locator(".mx_MessageTimestamp")], - }); + await expect(msgTile).toMatchScreenshot( + `reply-message-trl-${direction}displayname.png`, + screenshotOptions(page), + ); }); }); }); From b4805d0913acb4c0a0ebfb5976fcf3401634ffb7 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 15 Jan 2025 17:10:36 +0000 Subject: [PATCH 8/9] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/messages/messages.spec.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/playwright/e2e/messages/messages.spec.ts b/playwright/e2e/messages/messages.spec.ts index 0cfbdaaadb3..5185be43c95 100644 --- a/playwright/e2e/messages/messages.spec.ts +++ b/playwright/e2e/messages/messages.spec.ts @@ -58,8 +58,8 @@ async function editMessage(page: Page, message: Locator, newMsg: string): Promis await editComposer.press("Enter"); } -const screenshotOptions = (page: Page) => ({ - mask: [page.locator(".mx_MessageTimestamp")], +const screenshotOptions = (page?: Page) => ({ + mask: page ? [page.locator(".mx_MessageTimestamp")] : undefined, // Hide the jump to bottom button in the timeline to avoid flakiness css: ` .mx_JumpToBottomButton { @@ -100,10 +100,7 @@ test.describe("Message rendering", () => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me lays an egg"); - await expect(msgTile).toMatchScreenshot( - `emote-ltr-${direction}displayname.png`, - screenshotOptions(page), - ); + await expect(msgTile).toMatchScreenshot(`emote-ltr-${direction}displayname.png`, screenshotOptions()); }); test("should render an LTR rich text emote", async ({ page, user, app, room }) => { @@ -112,7 +109,7 @@ test.describe("Message rendering", () => { const msgTile = await sendMessage(page, "/me lays a *free range* egg"); await expect(msgTile).toMatchScreenshot( `emote-rich-ltr-${direction}displayname.png`, - screenshotOptions(page), + screenshotOptions(), ); }); @@ -160,10 +157,7 @@ test.describe("Message rendering", () => { await page.goto(`#/room/${room.roomId}`); const msgTile = await sendMessage(page, "/me يضع بيضة"); - await expect(msgTile).toMatchScreenshot( - `emote-rtl-${direction}displayname.png`, - screenshotOptions(page), - ); + await expect(msgTile).toMatchScreenshot(`emote-rtl-${direction}displayname.png`, screenshotOptions()); }); test("should render a richtext RTL emote", async ({ page, user, app, room }) => { @@ -172,7 +166,7 @@ test.describe("Message rendering", () => { const msgTile = await sendMessage(page, "/me أضع بيضة *حرة النطاق*"); await expect(msgTile).toMatchScreenshot( `emote-rich-rtl-${direction}displayname.png`, - screenshotOptions(page), + screenshotOptions(), ); }); From 95fce28a548d29b89bf4ef3eac79ba46c74d51a3 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 16 Jan 2025 10:02:20 +0000 Subject: [PATCH 9/9] Fix skip tests being wrong Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/services.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/playwright/services.ts b/playwright/services.ts index c15d63bd025..a501bf61384 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -155,9 +155,13 @@ export const test = base.extend({ { scope: "worker" }, ], - context: async ({ homeserverType, synapseConfig, logger, context, request, homeserver }, use, testInfo) => { + context: async ( + { homeserverType, synapseConfig, logger, context, request, _homeserver, homeserver }, + use, + testInfo, + ) => { testInfo.skip( - !(homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0, + !(_homeserver instanceof SynapseContainer) && Object.keys(synapseConfig).length > 0, `Test specifies Synapse config options so is unsupported with ${homeserverType}`, ); homeserver.setRequest(request);