Add Playwright tests for settings toggles (#30318)

* Add playwright tests

* import pages/ remove duplicate create-room

* Update screenshots

* Fix accessibility for devtools

* Disable region test

* Fixup headers

* remove extra test

* Fix permissions dialog

* fixup tests

* update snapshot

* Update jest tests

* Clear up playwright tests

* update widget screenshot

* Fix wrong snaps from using wrong compound version

* Revert mistaken s/checkbox/switch/

playwright/e2e/settings/notifications/notifications-settings-2-tab.spec.ts

@@ -0,0 +1,28 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test, expect } from "../../../element-web-test";
+import { SettingLevel } from "../../../../src/settings/SettingLevel";
+
+test.describe("Notifications 2 tab", () => {
+    test.use({
+        displayName: "Alice",
+    });
+
+    test("should display notification settings", { tag: "@screenshot" }, async ({ page, app, user, axe }) => {
+        await app.settings.setValue("feature_notification_settings2", null, SettingLevel.DEVICE, true);
+        await page.setViewportSize({ width: 1024, height: 2000 });
+        const settings = await app.settings.openUserSettings("Notifications");
+
+        axe.disableRules("color-contrast"); // XXX: Inheriting colour contrast issues from room view.
+        await expect(axe).toHaveNoViolations();
+        await expect(settings).toMatchScreenshot("standard-notifications-2-settings.png", {
+            // Mask the mxid.
+            mask: [settings.locator("#mx_NotificationSettings2_MentionCheckbox span")],
+        });
+    });
+});

playwright/e2e/settings/notifications/notifications-settings-tab.spec.ts

@@ -0,0 +1,25 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test, expect } from "../../../element-web-test";
+
+test.describe("Notifications tab", () => {
+    test.use({
+        displayName: "Alice",
+    });
+
+    test("should display notification settings", { tag: "@screenshot" }, async ({ page, app, user, axe }) => {
+        await page.setViewportSize({ width: 1024, height: 1400 });
+        const settings = await app.settings.openUserSettings("Notifications");
+        await settings.getByLabel("Enable notifications for this account").check();
+        await settings.getByLabel("Enable notifications for this device").check();
+
+        axe.disableRules("color-contrast"); // XXX: Inheriting colour contrast issues from room view.
+        await expect(axe).toHaveNoViolations();
+        await expect(settings).toMatchScreenshot("standard-notification-settings.png");
+    });
+});

playwright/e2e/settings/room-settings/roles-permissions-room-settings-tab.spec.ts

@@ -8,7 +8,7 @@
 
 import { type Locator } from "@playwright/test";
 
-import { test, expect } from "../../element-web-test";
+import { test, expect } from "../../../element-web-test";
 
 test.describe("Roles & Permissions room settings tab", () => {
     const roomName = "Test room";

playwright/e2e/settings/room-settings/room-security-tab.spec.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2025 New Vector Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+import { type Locator } from "@playwright/test";
+
+import { test, expect } from "../../../element-web-test";
+
+test.describe("Roles & Permissions room settings tab", () => {
+    const roomName = "Test room";
+
+    test.use({
+        displayName: "Alice",
+    });
+
+    let settings: Locator;
+
+    test.beforeEach(async ({ user, app }) => {
+        await app.client.createRoom({ name: roomName });
+        await app.viewRoomByName(roomName);
+        settings = await app.settings.openRoomSettings("Security & Privacy");
+    });
+
+    test(
+        "should be able to toggle on encryption in a room",
+        { tag: "@screenshot" },
+        async ({ page, app, user, axe }) => {
+            await page.setViewportSize({ width: 1024, height: 1400 });
+            const encryptedToggle = settings.getByLabel("Encrypted");
+            await encryptedToggle.click();
+
+            // Accept the dialog.
+            await page.getByRole("button", { name: "Ok " }).click();
+
+            await expect(encryptedToggle).toBeChecked();
+            await expect(encryptedToggle).toBeDisabled();
+
+            await settings.getByLabel("Only send messages to verified users.").check();
+
+            axe.disableRules("color-contrast"); // XXX: Inheriting colour contrast issues from room view.
+            await expect(axe).toHaveNoViolations();
+            await expect(settings).toMatchScreenshot("room-security-settings.png");
+        },
+    );
+});

playwright/e2e/settings/room-settings/room-video-tab.spec.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2025 New Vector Ltd.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+ * Please see LICENSE files in the repository root for full details.
+ */
+
+import { type Locator } from "@playwright/test";
+
+import { test, expect } from "../../../element-web-test";
+import { SettingLevel } from "../../../../src/settings/SettingLevel";
+
+test.describe("Voice & Video room settings tab", () => {
+    const roomName = "Test room";
+
+    test.use({
+        displayName: "Alice",
+    });
+
+    let settings: Locator;
+
+    test.beforeEach(async ({ user, app, page }) => {
+        // Execute client actions before setting, as the setting will force a reload.
+        await app.client.createRoom({ name: roomName });
+        await app.settings.setValue("feature_group_calls", null, SettingLevel.DEVICE, true);
+        await app.viewRoomByName(roomName);
+        settings = await app.settings.openRoomSettings("Voice & Video");
+    });
+
+    test(
+        "should be able to toggle on Element Call in the room",
+        { tag: "@screenshot" },
+        async ({ page, app, user, axe }) => {
+            await page.setViewportSize({ width: 1024, height: 1400 });
+            const callToggle = settings.getByLabel("Enable Element Call as an additional calling option in this room");
+            await callToggle.check();
+            axe.disableRules("color-contrast"); // XXX: Inheriting colour contrast issues from room view.
+            await expect(axe).toHaveNoViolations();
+            await expect(settings).toMatchScreenshot("room-video-settings.png");
+        },
+    );
+});

playwright/e2e/settings/security-user-settings-tab.spec.ts

@@ -41,6 +41,18 @@ test.describe("Security user settings tab", () => {
             });
         });
 
+        test("should render the security tab", { tag: "@screenshot" }, async ({ app, page, user }) => {
+            await page.setViewportSize({ width: 1024, height: 1400 });
+            const tab = await app.settings.openUserSettings("Security");
+            await expect(tab).toMatchScreenshot("security-settings-tab.png", {
+                mask: [
+                    // Contains IM name.
+                    tab.locator("#mx_SetIntegrationManager_BodyText"),
+                    tab.locator("#mx_SetIntegrationManager_ManagerName"),
+                ],
+            });
+        });
+
         test("should be able to set an ID server", async ({ app, context, user, page }) => {
             const tab = await app.settings.openUserSettings("Security");