From 95b2bb25d4175ff676c5a3d4e5ff0ef214f7b306 Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Tue, 21 Mar 2023 23:18:44 +0900 Subject: Add visual diffing in CI --- .eslintrc.json | 347 ++++++++++++++++++++++++++++++------- .github/workflows/playwright.yml | 109 ++++++++++++ .gitignore | 5 + package-lock.json | 74 +++++++- package.json | 1 + playwright.config.js | 109 ++++++++++++ test/data/html/test-document2.html | 229 ++++++++++++------------ test/playwright/visual.spec.js | 120 +++++++++++++ 8 files changed, 823 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/playwright.yml create mode 100644 playwright.config.js create mode 100644 test/playwright/visual.spec.js diff --git a/.eslintrc.json b/.eslintrc.json index 756acbda..b5328f41 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,7 +5,7 @@ "plugin:jsonc/recommended-with-json" ], "parserOptions": { - "ecmaVersion": 8, + "ecmaVersion": 9, "sourceType": "script", "ecmaFeatures": { "globalReturn": false, @@ -14,7 +14,7 @@ }, "env": { "browser": true, - "es2017": true, + "es2018": true, "webextensions": true }, "plugins": [ @@ -27,12 +27,24 @@ "/ext/lib/" ], "rules": { - "arrow-parens": ["error", "always"], - "comma-dangle": ["error", "never"], - "curly": ["error", "all"], + "arrow-parens": [ + "error", + "always" + ], + "comma-dangle": [ + "error", + "never" + ], + "curly": [ + "error", + "all" + ], "dot-notation": "error", "eqeqeq": "error", - "func-names": ["error", "always"], + "func-names": [ + "error", + "always" + ], "guard-for-in": "error", "new-parens": "error", "no-case-declarations": "error", @@ -41,57 +53,184 @@ "no-global-assign": "error", "no-param-reassign": "off", "no-prototype-builtins": "error", - "no-shadow": ["error", {"builtinGlobals": false}], + "no-shadow": [ + "error", + { + "builtinGlobals": false + } + ], "no-undef": "error", "no-undefined": "error", - "no-underscore-dangle": ["error", {"allowAfterThis": true, "allowAfterSuper": false, "allowAfterThisConstructor": false}], + "no-underscore-dangle": [ + "error", + { + "allowAfterThis": true, + "allowAfterSuper": false, + "allowAfterThisConstructor": false + } + ], "no-unexpected-multiline": "error", "no-unneeded-ternary": "error", - "no-unused-vars": ["error", {"vars": "local", "args": "after-used", "argsIgnorePattern": "^_", "caughtErrors": "none"}], + "no-unused-vars": [ + "error", + { + "vars": "local", + "args": "after-used", + "argsIgnorePattern": "^_", + "caughtErrors": "none" + } + ], "no-unused-expressions": "error", "no-var": "error", - "prefer-const": ["error", {"destructuring": "all"}], - "quote-props": ["error", "consistent"], - "quotes": ["error", "single", "avoid-escape"], + "prefer-const": [ + "error", + { + "destructuring": "all" + } + ], + "quote-props": [ + "error", + "consistent" + ], + "quotes": [ + "error", + "single", + "avoid-escape" + ], "require-atomic-updates": "off", "semi": "error", - "wrap-iife": ["error", "inside"], - - "brace-style": ["error", "1tbs", {"allowSingleLine": true}], - "indent": ["error", 4, {"SwitchCase": 1, "MemberExpression": 1, "flatTernaryExpressions": true, "ignoredNodes": ["ConditionalExpression"]}], + "wrap-iife": [ + "error", + "inside" + ], + "brace-style": [ + "error", + "1tbs", + { + "allowSingleLine": true + } + ], + "indent": [ + "error", + 4, + { + "SwitchCase": 1, + "MemberExpression": 1, + "flatTernaryExpressions": true, + "ignoredNodes": [ + "ConditionalExpression" + ] + } + ], "object-curly-newline": "error", - "padded-blocks": ["error", "never"], - - "array-bracket-spacing": ["error", "never"], - "arrow-spacing": ["error", {"before": true, "after": true}], - "block-spacing": ["error", "always"], - "comma-spacing": ["error", {"before": false, "after": true}], - "computed-property-spacing": ["error", "never"], - "func-call-spacing": ["error", "never"], - "function-paren-newline": ["error", "multiline-arguments"], - "generator-star-spacing": ["error", "before"], - "key-spacing": ["error", {"beforeColon": false, "afterColon": true, "mode": "strict"}], - "keyword-spacing": ["error", {"before": true, "after": true}], + "padded-blocks": [ + "error", + "never" + ], + "array-bracket-spacing": [ + "error", + "never" + ], + "arrow-spacing": [ + "error", + { + "before": true, + "after": true + } + ], + "block-spacing": [ + "error", + "always" + ], + "comma-spacing": [ + "error", + { + "before": false, + "after": true + } + ], + "computed-property-spacing": [ + "error", + "never" + ], + "func-call-spacing": [ + "error", + "never" + ], + "function-paren-newline": [ + "error", + "multiline-arguments" + ], + "generator-star-spacing": [ + "error", + "before" + ], + "key-spacing": [ + "error", + { + "beforeColon": false, + "afterColon": true, + "mode": "strict" + } + ], + "keyword-spacing": [ + "error", + { + "before": true, + "after": true + } + ], "no-trailing-spaces": "error", "no-whitespace-before-property": "error", - "object-curly-spacing": ["error", "never"], - "rest-spread-spacing": ["error", "never"], - "semi-spacing": ["error", {"before": false, "after": true}], - "space-before-function-paren": ["error", { - "anonymous": "never", - "named": "never", - "asyncArrow": "always" - }], - "space-in-parens": ["error", "never"], + "object-curly-spacing": [ + "error", + "never" + ], + "rest-spread-spacing": [ + "error", + "never" + ], + "semi-spacing": [ + "error", + { + "before": false, + "after": true + } + ], + "space-before-function-paren": [ + "error", + { + "anonymous": "never", + "named": "never", + "asyncArrow": "always" + } + ], + "space-in-parens": [ + "error", + "never" + ], "space-unary-ops": "error", - "spaced-comment": ["error", "always"], - "switch-colon-spacing": ["error", {"after": true, "before": false}], - "template-curly-spacing": ["error", "never"], - "template-tag-spacing": ["error", "never"], - + "spaced-comment": [ + "error", + "always" + ], + "switch-colon-spacing": [ + "error", + { + "after": true, + "before": false + } + ], + "template-curly-spacing": [ + "error", + "never" + ], + "template-tag-spacing": [ + "error", + "never" + ], "no-unsanitized/method": "error", "no-unsanitized/property": "error", - "jsdoc/check-access": "error", "jsdoc/check-alignment": "error", "jsdoc/check-line-alignment": "error", @@ -103,11 +242,17 @@ "jsdoc/empty-tags": "error", "jsdoc/implements-on-classes": "error", "jsdoc/multiline-blocks": "error", - "jsdoc/newline-after-description": ["error", "never"], + "jsdoc/newline-after-description": [ + "error", + "never" + ], "jsdoc/no-bad-blocks": "error", "jsdoc/no-multi-asterisks": "error", "jsdoc/require-asterisk-prefix": "error", - "jsdoc/require-hyphen-before-param-description": ["error", "never"], + "jsdoc/require-hyphen-before-param-description": [ + "error", + "never" + ], "jsdoc/require-jsdoc": "off", "jsdoc/require-param": "error", "jsdoc/require-param-description": "error", @@ -126,17 +271,51 @@ "jsdoc/require-yields-check": "error", "jsdoc/tag-lines": "error", "jsdoc/valid-types": "error", - - "jsonc/indent": ["error", 4], - "jsonc/array-bracket-newline": ["error", "consistent"], - "jsonc/array-bracket-spacing": ["error", "never"], - "jsonc/array-element-newline": ["error", "consistent"], - "jsonc/comma-style": ["error", "last"], - "jsonc/key-spacing": ["error", {"beforeColon": false, "afterColon": true, "mode": "strict"}], + "jsonc/indent": [ + "error", + 4 + ], + "jsonc/array-bracket-newline": [ + "error", + "consistent" + ], + "jsonc/array-bracket-spacing": [ + "error", + "never" + ], + "jsonc/array-element-newline": [ + "error", + "consistent" + ], + "jsonc/comma-style": [ + "error", + "last" + ], + "jsonc/key-spacing": [ + "error", + { + "beforeColon": false, + "afterColon": true, + "mode": "strict" + } + ], "jsonc/no-octal-escape": "error", - "jsonc/object-curly-newline": ["error", {"consistent": true}], - "jsonc/object-curly-spacing": ["error", "never"], - "jsonc/object-property-newline": ["error", {"allowAllPropertiesOnSameLine": true}] + "jsonc/object-curly-newline": [ + "error", + { + "consistent": true + } + ], + "jsonc/object-curly-spacing": [ + "error", + "never" + ], + "jsonc/object-property-newline": [ + "error", + { + "allowAllPropertiesOnSameLine": true + } + ] }, "overrides": [ { @@ -152,7 +331,10 @@ "test/data/translator-test-results.json" ], "rules": { - "jsonc/indent": ["error", 2] + "jsonc/indent": [ + "error", + 2 + ] } }, { @@ -160,8 +342,12 @@ "test/data/dictionaries/valid-dictionary1/term_bank_1.json" ], "rules": { - "jsonc/array-element-newline": ["off"], - "jsonc/object-property-newline": ["off"] + "jsonc/array-element-newline": [ + "off" + ], + "jsonc/object-property-newline": [ + "off" + ] } }, { @@ -188,7 +374,9 @@ } }, { - "files": ["ext/**/*.js"], + "files": [ + "ext/**/*.js" + ], "excludedFiles": [ "ext/js/core.js", "ext/js/accessibility/google-docs.js", @@ -215,7 +403,9 @@ } }, { - "files": ["ext/**/*.js"], + "files": [ + "ext/**/*.js" + ], "excludedFiles": [ "ext/js/core.js", "ext/js/accessibility/google-docs.js", @@ -227,7 +417,9 @@ } }, { - "files": ["ext/js/yomichan.js"], + "files": [ + "ext/js/yomichan.js" + ], "globals": { "chrome": "writable" } @@ -237,7 +429,9 @@ "test/**/*.js", "dev/**/*.js" ], - "excludedFiles": ["test/data/html/*.js"], + "excludedFiles": [ + "test/data/html/*.js" + ], "parserOptions": { "ecmaVersion": 8, "sourceType": "module" @@ -249,6 +443,35 @@ "webextensions": false } }, + { + "files": [ + "playwright.config.js" + ], + "env": { + "browser": false, + "es2017": true, + "node": true, + "webextensions": false + }, + "rules": { + "no-undefined": "off" + } + }, + { + "files": [ + "visual.spec.js" + ], + "env": { + "browser": false, + "es2017": true, + "node": true, + "webextensions": false + }, + "rules": { + "no-undefined": "off", + "no-empty-pattern": "off" + } + }, { "files": [ "ext/js/core.js", diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000..5ee786ef --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,109 @@ +name: Playwright Tests +on: + push: + branches: [master] + pull_request: +permissions: + contents: read +jobs: + playwright: + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: write + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + cache: "npm" + node-version-file: ".node-version" + + - name: Install dependencies + run: npm ci + + - name: Cache playwright browsers + id: cache-playwright + uses: actions/cache@v3 + with: + path: | + ~/.cache/ms-playwright + key: cache-playwright-${{ hashFiles('package-lock.json') }} # playwright version is included in package-lock, so this serves as a reasonable cache key + + - if: ${{ steps.cache-playwright.outputs.cache-hit != 'true' }} + name: Install Playwright Browsers + run: npx playwright install --with-deps chromium + + - name: Grab latest dictionaries from dictionaries branch + uses: actions/checkout@v3 + with: + ref: dictionaries + path: dictionaries + + - name: Grab latest screenshots from master branch + uses: dawidd6/action-download-artifact@5e780fc7bbd0cac69fc73271ed86edf5dcb72d67 # pin@v2 + continue-on-error: true + id: download-screenshots + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + name: playwright-screenshots + branch: master + workflow: playwright.yml + workflow_conclusion: success + path: test/playwright/__screenshots__/ + + - name: "[PR] Generate new screenshots & compare against master" + id: playwright + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "PLAYWRIGHT_OUTPUT<<$EOF" >> $GITHUB_OUTPUT + npx playwright test 2>&1 | tee $GITHUB_OUTPUT || true + echo "$EOF" >> $GITHUB_OUTPUT + echo "NUM_FAILED=$(grep -c 'Screenshot comparison failed' $GITHUB_OUTPUT)" >> $GITHUB_OUTPUT + continue-on-error: true + if: github.event_name == 'pull_request' && steps.download-screenshots.outcome != 'failure' + + - name: "[Push] Generate new authoritative screenshots for master" + id: playwright-master + run: npx playwright test -u + if: github.event_name == 'push' + + - uses: actions/upload-artifact@v3 + with: + name: playwright-screenshots + path: test/playwright/__screenshots__/ + + - uses: actions/upload-artifact@v3 + with: + name: playwright-report + path: playwright-report/ + + - name: "[Couldn't download screenshots] Comment results on PR" + uses: mshick/add-pr-comment@a65df5f64fc741e91c59b8359a4bc56e57aaf5b1 # pin@v2 + if: github.event_name == 'pull_request' && steps.download-screenshots.outcome == 'failure' + with: + message: | + :heavy_exclamation_mark: Could not fetch screenshots from master branch, so had nothing to make a visual comparison against; please check the "download-screenshots" step in the workflow run and rerun it before merging. + + - name: "[Success] Comment results on PR" + uses: mshick/add-pr-comment@a65df5f64fc741e91c59b8359a4bc56e57aaf5b1 # pin@v2 + if: github.event_name == 'pull_request' && steps.download-screenshots.outcome != 'failure' && steps.playwright.outputs.NUM_FAILED == 0 + with: + message: | + :heavy_check_mark: No visual differences introduced by this PR. + [View Playwright Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) (note: open the "playwright-report" artifact) + + - name: "[Failure] Comment results on PR" + uses: mshick/add-pr-comment@a65df5f64fc741e91c59b8359a4bc56e57aaf5b1 # pin@v2 + if: github.event_name == 'pull_request' && steps.download-screenshots.outcome != 'failure' && steps.playwright.outputs.NUM_FAILED != 0 + with: + message: | + :warning: {{ steps.playwright.outputs.NUM_FAILED }} visual differences introduced by this PR; please validate if they are desirable. +
+ Playwright Test Results + 
+            ${{ steps.playwright.outputs.PLAYWRIGHT_OUTPUT }}
+
+
+ [View Playwright Report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) (note: open the "playwright-report" artifact) diff --git a/.gitignore b/.gitignore index ab07d570..d4e5da07 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ node_modules/ builds/ .DS_Store +dictionaries/ +/test-results/ +/playwright-report/ +/playwright/.cache/ +/test/playwright/__screenshots__/ diff --git a/package-lock.json b/package-lock.json index 5dd54bc6..4878437a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { - "name": "yomichan", + "name": "yomitan", "version": "0.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "yomichan", + "name": "yomitan", "version": "0.0.0", "hasInstallScript": true, "license": "GPL-3.0-or-later", "devDependencies": { + "@playwright/test": "^1.31.2", "ajv": "^8.11.0", "browserify": "^17.0.0", "css": "^3.0.0", @@ -332,6 +333,25 @@ "node": ">= 8" } }, + "node_modules/@playwright/test": { + "version": "1.31.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.31.2.tgz", + "integrity": "sha512-BYVutxDI4JeZKV1+ups6dt5WiqKhjBtIYowyZIJ3kBDmJgsuPKsqqKNIMFbUePLSCmp2cZu+BDL427RcNKTRYw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "playwright-core": "1.31.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, "node_modules/@pnpm/network.ca-file": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@pnpm/network.ca-file/-/network.ca-file-1.0.1.tgz", @@ -3474,6 +3494,20 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -6409,6 +6443,18 @@ "integrity": "sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ==", "dev": true }, + "node_modules/playwright-core": { + "version": "1.31.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.31.2.tgz", + "integrity": "sha512-a1dFgCNQw4vCsG7bnojZjDnPewZcw7tZUNFN0ZkcLYKj+mPmXvg4MpaaKZ5SgqPsOmqIf2YsVRkgqiRDxD+fDQ==", + "dev": true, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/postcss": { "version": "8.4.18", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", @@ -9389,6 +9435,17 @@ "fastq": "^1.6.0" } }, + "@playwright/test": { + "version": "1.31.2", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.31.2.tgz", + "integrity": "sha512-BYVutxDI4JeZKV1+ups6dt5WiqKhjBtIYowyZIJ3kBDmJgsuPKsqqKNIMFbUePLSCmp2cZu+BDL427RcNKTRYw==", + "dev": true, + "requires": { + "@types/node": "*", + "fsevents": "2.3.2", + "playwright-core": "1.31.2" + } + }, "@pnpm/network.ca-file": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@pnpm/network.ca-file/-/network.ca-file-1.0.1.tgz", @@ -11834,6 +11891,13 @@ "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -14049,6 +14113,12 @@ "integrity": "sha512-mMMOwSKrmyl+Y12Ri2xhH1lbzQxwwpuru9VjyJpgFIH4asSj88F2csdMwN6+M5g1Ll4rmsYghHLQJw81tgZ7LQ==", "dev": true }, + "playwright-core": { + "version": "1.31.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.31.2.tgz", + "integrity": "sha512-a1dFgCNQw4vCsG7bnojZjDnPewZcw7tZUNFN0ZkcLYKj+mPmXvg4MpaaKZ5SgqPsOmqIf2YsVRkgqiRDxD+fDQ==", + "dev": true + }, "postcss": { "version": "8.4.18", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", diff --git a/package.json b/package.json index 6ec3eecf..2de7bcef 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "sourceDir": "ext" }, "devDependencies": { + "@playwright/test": "^1.31.2", "ajv": "^8.11.0", "browserify": "^17.0.0", "css": "^3.0.0", diff --git a/playwright.config.js b/playwright.config.js new file mode 100644 index 00000000..0f15ff59 --- /dev/null +++ b/playwright.config.js @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +// @ts-check +const {defineConfig, devices} = require('@playwright/test'); + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * @see https://playwright.dev/docs/test-configuration + */ +module.exports = defineConfig({ + testDir: './test/playwright', + snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', + /* Maximum time one test can run for. */ + timeout: 60 * 60 * 1000, + expect: { + /** + * Maximum time expect() should wait for the condition to be met. + * For example in `await expect(locator).toHaveText();` + */ + timeout: 5000 + }, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */ + actionTimeout: 60 * 1000, + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry' + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: {...devices['Desktop Chrome']} + } + + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] }, + // }, + + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { channel: 'chrome' }, + // }, + ] + + /* Folder for test artifacts such as screenshots, videos, traces, etc. */ + // outputDir: 'test-results/', + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // port: 3000, + // }, +}); + diff --git a/test/data/html/test-document2.html b/test/data/html/test-document2.html index fb3adb59..399c3bd8 100644 --- a/test/data/html/test-document2.html +++ b/test/data/html/test-document2.html @@ -1,34 +1,40 @@ - - - - Yomichan Manual Tests - - - - - + + + + + Yomichan Manual Tests + + + + + +

Yomichan Manual Tests

@@ -36,56 +42,62 @@ Standard content. -
-
- ありがとう -
-
- Toggle fullscreen +
+
+
+ ありがとう +
+
-
+
Content inside of an open shadow DOM. -
+
Content inside of a closed shadow DOM. -
+
<iframe> element. - + <iframe> element inside of an open shadow DOM. -
+
@@ -93,7 +105,7 @@ <iframe> element inside of a closed shadow DOM. -
+
@@ -101,22 +113,26 @@ <iframe> element with data URL. - + <iframe> element with blob URL. - + <iframe> element with srcdoc. - + - <iframe> element with srcdoc and sandbox="allow-same-origin allow-scripts". - + <iframe> element with srcdoc and + sandbox="allow-same-origin allow-scripts". + @@ -124,41 +140,39 @@ <iframe> element with srcdoc and sandbox="allow-scripts".
This element is expected to not work. - + SVG <img>. - + SVG <object>. - + SVG <embed>. - + SVG <iframe>. - + SVG <svg>. - - + + dominant-baseline: hanging;"> ありがとう @@ -166,42 +180,43 @@ + diff --git a/test/playwright/visual.spec.js b/test/playwright/visual.spec.js new file mode 100644 index 00000000..acb12e97 --- /dev/null +++ b/test/playwright/visual.spec.js @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2023 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +const path = require('path'); +const {test: base, chromium} = require('@playwright/test'); +const root = path.join(__dirname, '..', '..'); + +export const test = base.extend({ + context: async ({ }, use) => { + const pathToExtension = path.join(root, 'ext'); + const context = await chromium.launchPersistentContext('', { + // headless: false, + args: [ + '--headless=new', + `--disable-extensions-except=${pathToExtension}`, + `--load-extension=${pathToExtension}` + ] + }); + await use(context); + await context.close(); + }, + extensionId: async ({context}, use) => { + let [background] = context.serviceWorkers(); + if (!background) { + background = await context.waitForEvent('serviceworker'); + } + + const extensionId = background.url().split('/')[2]; + await use(extensionId); + } +}); +const expect = test.expect; + +test('visual', async ({context, page, extensionId}) => { + // wait for the on-install welcome.html tab to load, which becomes the foreground tab + const welcome = await context.waitForEvent('page'); + welcome.close(); // close the welcome tab so our main tab becomes the foreground tab -- otherwise, the screenshot can hang + + // open settings + await page.goto(`chrome-extension://${extensionId}/settings.html`); + + await expect(page.locator('id=dictionaries')).toBeVisible(); + + // get the locator for the disk usage indicator so we can later mask it out of the screenshot + const storage_locator = page.locator('.storage-use-finite >> xpath=..'); + + // take a simple screenshot of the settings page + await expect.soft(page).toHaveScreenshot('settings-fresh.png', {mask: [storage_locator]}); + + // load in jmdict_english.zip + await page.locator('input[id="dictionary-import-file-input"]').setInputFiles(path.join(root, 'dictionaries/jmdict_english.zip')); + await expect(page.locator('id=dictionaries')).toHaveText('Dictionaries (1 installed, 1 enabled)', {timeout: 5 * 60 * 1000}); + + // take a screenshot of the settings page with jmdict loaded + await expect.soft(page).toHaveScreenshot('settings-jmdict-loaded.png', {mask: [storage_locator]}); + + const screenshot = async (doc_number, test_number, el, offset) => { + const test_name = 'doc' + doc_number + '-test' + test_number; + + const box = await el.boundingBox(); + + // find the popup frame if it exists + let popup_frame = page.frames().find((f) => f.url().includes('popup.html')); + + // otherwise prepare for it to be attached + let frame_attached; + if (popup_frame === undefined) { + frame_attached = page.waitForEvent('frameattached'); + } + await page.mouse.move(box.x + offset.x, box.y + offset.y, {steps: 10}); // hover over the test + if (popup_frame === undefined) { + popup_frame = await frame_attached; // wait for popup to be attached + } + try { + await (await popup_frame.frameElement()).waitForElementState('visible', {timeout: 500}); // some tests don't have a popup, so don't fail if it's not there; TODO: check if the popup is expected to be there + } catch (error) { + console.log(test_name + ' has no popup'); + } + + await page.bringToFront(); // bring the page to the foreground so the screenshot doesn't hang; for some reason the frames result in page being in the background + await expect.soft(page).toHaveScreenshot(test_name + '.png'); + + await page.mouse.click(0, 0); // click away so popup disappears + await (await popup_frame.frameElement()).waitForElementState('hidden'); // wait for popup to disappear + }; + + // Load test-document1.html + await page.goto('file://' + path.join(root, 'test/data/html/test-document1.html')); + await page.setViewportSize({width: 1000, height: 1800}); + await page.keyboard.down('Shift'); + let i = 1; + for (const el of await page.locator('div > *:nth-child(1)').elementHandles()) { + await screenshot(1, i, el, {x: 6, y: 6}); + i++; + } + + // Load test-document2.html + await page.goto('file://' + path.join(root, 'test/data/html/test-document2.html')); + await page.setViewportSize({width: 1000, height: 4500}); + await page.keyboard.down('Shift'); + i = 1; + for (const el of await page.locator('.hovertarget').elementHandles()) { + await screenshot(2, i, el, {x: 15, y: 15}); + i++; + } +}); -- cgit v1.2.3