Browser automation test with Playwright

.github/workflows/nightly-tests.yml

@@ -10,6 +10,11 @@ on:
   workflow_dispatch:
 
 jobs:
+  nightly-e2e:
+    # Cannot check the existence of secrets, so limiting to repository name to prevent all forks to run nightly.
+    # See: https://github.com/actions/runner/issues/520
+    if: ${{ github.repository == 'Stormbase/django-otp-webauthn' }}
+    uses: ./.github/workflows/playwright-tests.yml
   nightly-test:
     # Cannot check the existence of secrets, so limiting to repository name to prevent all forks to run nightly.
     # See: https://github.com/actions/runner/issues/520
@@ -38,7 +43,7 @@ jobs:
 
       - name: Compile translations
         run: |
-          /bin/sh ./compile_translations.sh
+          /bin/sh ./scripts/compile_translations.sh
 
       - name: Run 'future' tests
         id: test

.github/workflows/playwright-tests.yml

@@ -0,0 +1,49 @@
+name: Playwright Tests
+on:
+  workflow_call:
+  workflow_dispatch:
+
+env:
+  FORCE_COLOR: "1" # Make tools pretty.
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  PIP_NO_PYTHON_VERSION_WARNING: "1"
+
+jobs:
+  playwright:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.13"
+          cache: "pip"
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+      - name: Install additional system packages
+        run: |
+          sudo apt-get update -y
+          # Need `gettext` to compile translations
+          sudo apt-get install gettext
+      - name: Install dependencies
+        run: |
+          yarn install --frozen-lockfile
+      - name: Compile JavaScript
+        run: |
+          yarn build
+      - name: Install dependencies
+        run: |
+          python -m pip install .[testing,playwright]
+      - name: Compile translations
+        run: /bin/sh ./scripts/compile_translations.sh
+      - name: Run E2E tests
+        run: |
+          /bin/sh ./scripts/run_e2e_testsuite.sh
+      - name: Upload Playwright traces
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-traces
+          path: test-results/
+          retention-days: 3

.github/workflows/publish-test.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Compile translations
         run: |
-          /bin/sh ./compile_translations.sh
+          /bin/sh ./scripts/compile_translations.sh
 
       - name: Build package distributions
         run: python -m build

.github/workflows/publish.yml

@@ -42,7 +42,7 @@ jobs:
 
       - name: Compile translations
         run: |
-          /bin/sh ./compile_translations.sh
+          /bin/sh ./scripts/compile_translations.sh
 
       - name: Build package distributions
         run: python -m build

.github/workflows/test.yml

@@ -60,7 +60,7 @@ jobs:
           python -m pip install .[testing] tox coverage
 
       - name: Compile translations
-        run: /bin/sh ./compile_translations.sh
+        run: /bin/sh ./scripts/compile_translations.sh
 
         # This step runs only for jobs NOT in the include matrix
       - name: Run tox targets for Python ${{ matrix.python }}
@@ -77,3 +77,7 @@ jobs:
         uses: codecov/codecov-action@v4
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  e2e_tests:
+    name: End-to-End Tests
+    uses: ./.github/workflows/playwright-tests.yml

client/src/auth.ts

@@ -1,4 +1,4 @@
-import { State, Config } from "./types";
+import { State, Config, StatusEnum } from "./types";
 import { getConfig } from "./utils";
 import {
   browserSupportsWebAuthn,
@@ -220,6 +220,7 @@ import {
         );
         await setPasskeyVerifyState({
           buttonDisabled: true,
+          statusEnum: StatusEnum.BUSY,
           buttonLabel: gettext("Verifying..."),
         });
 
@@ -238,6 +239,7 @@ import {
             buttonDisabled: false,
             buttonLabel,
             requestFocus: true,
+            statusEnum: StatusEnum.GET_OPTIONS_FAILED,
             status: gettext(
               "Verification failed. Could not retrieve parameters from the server.",
             ),
@@ -271,6 +273,7 @@ import {
                   buttonDisabled: false,
                   buttonLabel,
                   requestFocus: true,
+                  statusEnum: StatusEnum.ABORTED,
                   status: gettext("Verification aborted."),
                 });
                 break;
@@ -279,6 +282,7 @@ import {
                   buttonDisabled: false,
                   buttonLabel,
                   requestFocus: true,
+                  statusEnum: StatusEnum.NOT_ALLOWED_OR_ABORTED,
                   status: gettext("Verification canceled or not allowed."),
                 });
                 break;
@@ -287,6 +291,7 @@ import {
                   buttonDisabled: false,
                   buttonLabel,
                   requestFocus: true,
+                  statusEnum: StatusEnum.UNKNOWN_ERROR,
                   status: gettext(
                     "Verification failed. An unknown error occurred.",
                   ),
@@ -308,6 +313,7 @@ import {
 
         await setPasskeyVerifyState({
           buttonDisabled: true,
+          statusEnum: StatusEnum.BUSY,
           buttonLabel: gettext("Finishing verification..."),
         });
 
@@ -340,6 +346,7 @@ import {
             buttonDisabled: false,
             buttonLabel,
             requestFocus: true,
+            statusEnum: StatusEnum.SERVER_ERROR,
             status: gettext(
               "Verification failed. An unknown server error occurred.",
             ),
@@ -366,6 +373,7 @@ import {
           await setPasskeyVerifyState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SERVER_ERROR,
             requestFocus: true,
             status: msg,
           });
@@ -386,6 +394,7 @@ import {
           await setPasskeyVerifyState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SUCCESS,
             status: gettext("Verification successful!"),
           });
 
@@ -411,6 +420,7 @@ import {
           await setPasskeyVerifyState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SERVER_ERROR,
             requestFocus: true,
             status: msg,
           });
@@ -432,6 +442,12 @@ import {
       passkeyAuthButton.disabled = state.buttonDisabled;
       passkeyAuthButton.textContent = state.buttonLabel;
 
+      if (state.statusEnum) {
+        passkeyStatusText.setAttribute("data-status-enum", state.statusEnum);
+      } else {
+        passkeyStatusText.removeAttribute("data-status-enum");
+      }
+
       if (passkeyStatusText) {
         // If there is a status message, we want to make sure screen readers
         // announce it to the user for clarity as to what is happening.

client/src/register.ts

@@ -1,4 +1,4 @@
-import { State, Config } from "./types";
+import { State, Config, StatusEnum } from "./types";
 import { getConfig } from "./utils";
 import {
   browserSupportsWebAuthn,
@@ -60,6 +60,7 @@ import {
             buttonDisabled: false,
             buttonLabel,
             requestFocus: true,
+            statusEnum: StatusEnum.GET_OPTIONS_FAILED,
             status: gettext(
               "Registration failed. Unable to fetch registration options from server.",
             ),
@@ -89,6 +90,7 @@ import {
                 setPasskeyRegisterState({
                   buttonDisabled: false,
                   buttonLabel,
+                  statusEnum: StatusEnum.ABORTED,
                   status: gettext("Registration aborted."),
                   requestFocus: true,
                 });
@@ -97,6 +99,7 @@ import {
                 setPasskeyRegisterState({
                   buttonDisabled: false,
                   buttonLabel,
+                  statusEnum: StatusEnum.STATE_ERROR,
                   status: gettext(
                     "Registration failed. You most likely already have a Passkey registered for this site.",
                   ),
@@ -107,6 +110,7 @@ import {
                 setPasskeyRegisterState({
                   buttonDisabled: false,
                   buttonLabel,
+                  statusEnum: StatusEnum.NOT_ALLOWED_OR_ABORTED,
                   status: gettext("Registration aborted or not allowed."),
                   requestFocus: true,
                 });
@@ -115,6 +119,7 @@ import {
                 setPasskeyRegisterState({
                   buttonDisabled: false,
                   buttonLabel,
+                  statusEnum: StatusEnum.SECURITY_ERROR,
                   status: gettext(
                     "Registration failed. A technical problem occurred that prevents you from registering a Passkey for this site.",
                   ),
@@ -125,6 +130,7 @@ import {
                 setPasskeyRegisterState({
                   buttonDisabled: false,
                   buttonLabel,
+                  statusEnum: StatusEnum.UNKNOWN_ERROR,
                   status: gettext(
                     "Registration failed. An unknown error occurred.",
                   ),
@@ -146,6 +152,7 @@ import {
 
         setPasskeyRegisterState({
           buttonDisabled: true,
+          statusEnum: StatusEnum.BUSY,
           buttonLabel: gettext("Finishing registration..."),
         });
 
@@ -164,6 +171,7 @@ import {
           setPasskeyRegisterState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SERVER_ERROR,
             status: gettext(
               "Registration failed. The server was unable to verify this passkey.",
             ),
@@ -189,6 +197,7 @@ import {
           setPasskeyRegisterState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SERVER_ERROR,
             status: gettext("Registration failed. A server error occurred."),
             requestFocus: true,
           });
@@ -211,6 +220,7 @@ import {
           setPasskeyRegisterState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SUCCESS,
             status: gettext("Registration successful!"),
             requestFocus: true,
           });
@@ -230,6 +240,7 @@ import {
           setPasskeyRegisterState({
             buttonDisabled: false,
             buttonLabel,
+            statusEnum: StatusEnum.SERVER_ERROR,
             status: msg,
             requestFocus: true,
           });
@@ -258,6 +269,12 @@ import {
         passkeyRegisterButton.disabled = state.buttonDisabled;
         passkeyRegisterButton.textContent = state.buttonLabel;
 
+        if (state.statusEnum) {
+          passkeyStatusText.setAttribute("data-status-enum", state.statusEnum);
+        } else {
+          passkeyStatusText.removeAttribute("data-status-enum");
+        }
+
         if (passkeyStatusText) {
           if (state.status) {
             // If there is a status message, we want to make sure screen readers

client/src/types.ts

@@ -1,8 +1,21 @@
+export enum StatusEnum {
+  UNKNOWN_ERROR = "unknown-error",
+  STATE_ERROR = "state-error",
+  SECURITY_ERROR = "security-error",
+  GET_OPTIONS_FAILED = "get-options-failed",
+  ABORTED = "aborted",
+  NOT_ALLOWED_OR_ABORTED = "not-allowed-or-aborted",
+  SERVER_ERROR = "server-error",
+  SUCCESS = "success",
+  BUSY = "busy",
+}
+
 export type State = {
   buttonDisabled: boolean;
   buttonLabel: string;
   /** Text to display in the status field. */
   status?: string;
+  statusEnum?: StatusEnum;
   /** Request the focus be returned to the button. */
   requestFocus?: boolean;
 };

pyproject.toml

@@ -55,6 +55,10 @@ testing =  [
   "pytest-mock>=3.14,<4",
   "jsonschema ~= 4.23",
 ]
+playwright = [
+  "playwright>=1.49,<2",
+  "pytest-playwright>=0.5.2,<1",
+]
 
 [project.urls]
 # TODO: documentation link
@@ -88,8 +92,7 @@ include = [
   "sandbox/",
   "client/",
   "tests/",
-  "update_translations.sh",
-  "compile_translations.sh",
+  "scripts/",
   "README.md",
   "LICENSE",
   "CHANGELOG.md",
@@ -131,6 +134,7 @@ known-first-party = ["django_otp_webauthn", "sandbox", "tests"]
 pythonpath = "sandbox"
 DJANGO_SETTINGS_MODULE = "settings"
 testpaths = "tests/"
+norecursedirs = ["tests/e2e"]
 addopts = "--reuse-db"
 
 [tool.coverage.run]