feat: e2e tests for apps (#4171)

* feat: e2e tests for apps

* chore: fix gh workflow

.github/scripts/start-tipi.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Make runtipi directory
+
+mkdir -p runtipi/
+
+# Download the cli
+
+cd runtipi/
+
+wget https://github.com/runtipi/runtipi/releases/latest/download/runtipi-cli-linux-x86_64.tar.gz
+
+# Extract cli
+
+tar -xvf runtipi-cli-linux-x86_64.tar.gz
+
+# Rename cli
+
+mv runtipi-cli-linux-x86_64 runtipi-cli
+
+# Delete archive
+
+rm -rf runtipi-cli-linux-x86_64.tar.gz
+
+# Make cli executable
+
+chmod +x runtipi-cli
+
+# Start runtipi
+
+./runtipi-cli start

.github/workflows/app-e2e.yml

@@ -0,0 +1,37 @@
+name: App e2e test
+on:
+  workflow_dispatch:
+    inputs:
+      app:
+        description: 'App name'
+        required: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Install dependencies
+        run: npm install -g pnpm && pnpm install
+
+      - name: Install Playwright Browsers
+        run: pnpm exec playwright install --with-deps
+
+      - name: Start runtipi
+        run: ./.github/scripts/start-tipi.sh
+
+      - name: Run Playwright tests
+        run: APP_NAME=${{ github.event.inputs.app }} pnpm exec playwright test
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/

.github/workflows/renovate-ci.yml

@@ -37,7 +37,7 @@ jobs:
             if [[ $changed_file == *"docker-compose.yml"* ]]; then
               app_name=$(echo $changed_file | cut -d'/' -f 2)
               echo "App name: $app_name"
-              ./.github/workflows/renovate-app-version.sh $app_name
+              ./.github/scripts/renovate-app-version.sh $app_name
             fi
           done
 
@@ -46,7 +46,7 @@ jobs:
         with:
           author_name: Tipi CI
           author_email: ci@runtipi.io
-          message: "Update app version [ready]"
+          message: 'Update app version [ready]'
           commit: --no-verify
           push: origin HEAD:${{ github.head_ref }}
 

.gitignore

@@ -4,3 +4,9 @@ github.secrets
 .DS_Store
 .nova
 .vscode/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+/runtipi
+app.png

e2e/fixtures/fixtures.ts

@@ -0,0 +1,39 @@
+import { Page, expect } from '@playwright/test';
+
+export const signUpUser = async (page: Page) => {
+  // Skip insecure config thing
+  await page.addLocatorHandler(page.getByText('Insecure configuration'), async () => {
+    await page.getByRole('button', { name: 'Close' }).click();
+  });
+
+  // Go to register page
+  await page.goto('http://127.0.0.1/register');
+
+  // Sign up
+  await page.getByPlaceholder('you@example.com').click();
+  await page.getByPlaceholder('you@example.com').fill('tester@test.com');
+
+  await page.getByPlaceholder('Enter your password', { exact: true }).fill('password');
+  await page.getByPlaceholder('Confirm your password').fill('password');
+
+  await page.getByRole('button', { name: 'Register' }).click();
+
+  await expect(page.getByRole('heading', { name: 'Thanks for using Runtipi' })).toBeVisible();
+  await page.getByRole('button', { name: 'Save and enter' }).click();
+};
+
+export const loginUser = async (page: Page) => {
+  // Skip insecure config thing
+  await page.addLocatorHandler(page.getByText('Insecure configuration'), async () => {
+    await page.getByRole('button', { name: 'Close' }).click();
+  });
+
+  // Login user
+  await page.goto('http://127.0.0.1/login');
+
+  await page.getByPlaceholder('you@example.com').fill('tester@test.com');
+  await page.getByPlaceholder('Your password').fill('password');
+  await page.getByRole('button', { name: 'Login' }).click();
+
+  await expect(page.getByRole('heading', { name: 'Dashboard' })).toBeVisible();
+};

e2e/helpers/helpers.ts

@@ -0,0 +1,10 @@
+import { chromium } from '@playwright/test';
+import { signUpUser } from '../fixtures/fixtures';
+
+async function globalSetup() {
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  await signUpUser(page);
+}
+
+export default globalSetup;

e2e/install.spec.ts

@@ -0,0 +1,42 @@
+import { test, expect } from '@playwright/test';
+import { loginUser } from './fixtures/fixtures';
+
+test('app should install', async ({ page }) => {
+  // Set timeout
+  test.setTimeout(70000);
+
+  // Sign in
+  await loginUser(page);
+
+  // Get app from env and go to the app page
+  await page.goto(`http://127.0.0.1/apps/${process.env.APP_NAME || ''}`);
+
+  // Click install button
+  await page.getByRole('button', { name: 'Install' }).click();
+
+  // Click install button in modal
+  await page.getByRole('button', { name: 'Install' }).click();
+
+  // App should install
+  await expect(page.getByText('Running')).toBeVisible({ timeout: 65000 });
+});
+
+test('app should...work', async ({ page }) => {
+  // Sign in
+  await loginUser(page);
+
+  // Get app from env and go to the app page
+  await page.goto(`http://127.0.0.1/apps/${process.env.APP_NAME || ''}`);
+
+  // Click open button
+  await page.getByRole('button', { name: 'Open' }).click();
+
+  // Open the app
+  await page.getByRole('menuitem', { name: /127.0.0.1/i }).click();
+
+  // Wait for page redirect
+  await expect(page.title()).not.toBe(new RegExp('/' + process.env.APP_NAME + '/i'));
+
+  // Take screenshot
+  await page.screenshot({ fullPage: true });
+});

package.json

@@ -23,6 +23,7 @@
     "@commitlint/cli": "^19.3.0",
     "@commitlint/config-conventional": "^19.2.2",
     "@commitlint/cz-commitlint": "^19.2.0",
+    "@playwright/test": "^1.45.2",
     "@types/jest": "^28.1.6",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^20.14.11",

playwright.config.ts

@@ -0,0 +1,31 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  // Sign up to tipi before running
+  globalSetup: require.resolve('./e2e/helpers/helpers'),
+  // Run tests in files in parallel
+  testDir: './e2e',
+  // Fail the build on CI if you accidentally left test.only in the source code.
+  fullyParallel: false,
+  // Retry on CI only
+  forbidOnly: !!process.env.CI,
+  // Opt out of parallel tests on CI.
+  retries: process.env.CI ? 2 : 0,
+  // Reporter to use. See https://playwright.dev/docs/test-reporter
+  workers: process.env.CI ? 1 : undefined,
+  // Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions.
+  reporter: 'html',
+  use: {
+    // Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer
+    trace: 'on-first-retry',
+    // Enable video
+    video: 'on',
+  },
+  // No need for multiple browsers to just take a screenshot
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});