ci: implement screenshot diffing (#2571)

This commit is contained in:
Niklas von Hertzen 2021-07-13 18:48:18 +08:00 committed by GitHub
parent 171585491d
commit e29af58661
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 338 additions and 1 deletions

View File

@ -309,3 +309,43 @@ jobs:
FOLDER: docs
SINGLE_COMMIT: true
CLEAN: true
diff-reftests:
runs-on: ubuntu-latest
name: Diff reftest screenshots
needs: browser-test
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: actions/setup-node@v1
with:
node-version: 12
- name: Cache node modules
uses: actions/cache@v2
env:
cache-name: cache-node-modules
with:
# npm cache files are stored in `~/.npm` on Linux/macOS
path: ~/.npm
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Npm install
run: npm ci
- name: Checkout current snapshots
run: git checkout origin/gh-pages results
- name: Download test results
uses: actions/download-artifact@v2
with:
name: reftest-results
path: tmp/reftests
- name: Run diff
run: npm run reftests-diff
continue-on-error: true
- name: Upload diff
uses: actions/upload-artifact@v2
with:
name: snapshot-diffs
path: tmp/snapshot-diffs

264
package-lock.json generated
View File

@ -5,7 +5,7 @@
"requires": true,
"packages": {
"": {
"version": "1.0.0",
"version": "1.1.0",
"license": "MIT",
"dependencies": {
"css-line-break": "1.1.1"
@ -23,6 +23,7 @@
"@types/express": "^4.17.13",
"@types/glob": "^7.1.1",
"@types/jest": "^26.0.24",
"@types/jest-image-snapshot": "^4.3.1",
"@types/karma": "^6.3.0",
"@types/mocha": "^8.2.3",
"@types/node": "^16.3.1",
@ -49,6 +50,7 @@
"glob": "7.1.3",
"html2canvas-proxy": "1.0.1",
"jest": "^27.0.6",
"jest-image-snapshot": "^4.5.1",
"jquery": "^3.5.1",
"js-polyfills": "^0.1.42",
"karma": "^6.3.2",
@ -3758,6 +3760,17 @@
"pretty-format": "^26.0.0"
}
},
"node_modules/@types/jest-image-snapshot": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/@types/jest-image-snapshot/-/jest-image-snapshot-4.3.1.tgz",
"integrity": "sha512-WDdUruGF14C53axe/mNDgQP2YIhtcwXrwmmVP8eOGyfNTVD+FbxWjWR7RTU+lzEy4K6V6+z7nkVDm/auI/r3xQ==",
"dev": true,
"dependencies": {
"@types/jest": "*",
"@types/pixelmatch": "*",
"ssim.js": "^3.1.1"
}
},
"node_modules/@types/jest/node_modules/@jest/types": {
"version": "26.6.2",
"resolved": "https://registry.npmjs.org/@jest/types/-/types-26.6.2.tgz",
@ -3962,6 +3975,15 @@
"integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
"dev": true
},
"node_modules/@types/pixelmatch": {
"version": "5.2.4",
"resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.4.tgz",
"integrity": "sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/platform": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/@types/platform/-/platform-1.3.4.tgz",
@ -12886,6 +12908,12 @@
"node": ">=8"
}
},
"node_modules/glur": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz",
"integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=",
"dev": true
},
"node_modules/graceful-fs": {
"version": "4.2.6",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
@ -12978,6 +13006,18 @@
"node": ">= 0.4.0"
}
},
"node_modules/has-ansi": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
"integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
"dev": true,
"dependencies": {
"ansi-regex": "^2.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@ -15313,6 +15353,105 @@
"node": ">=8.0"
}
},
"node_modules/jest-image-snapshot": {
"version": "4.5.1",
"resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-4.5.1.tgz",
"integrity": "sha512-0YkgupgkkCx0wIZkxvqs/oNiUT0X0d2WTpUhaAp+Dy6CpqBUZMRTIZo4KR1f+dqmx6WXrLCvecjnHLIsLkI+gQ==",
"dev": true,
"dependencies": {
"chalk": "^1.1.3",
"get-stdin": "^5.0.1",
"glur": "^1.1.2",
"lodash": "^4.17.4",
"mkdirp": "^0.5.1",
"pixelmatch": "^5.1.0",
"pngjs": "^3.4.0",
"rimraf": "^2.6.2",
"ssim.js": "^3.1.1"
},
"engines": {
"node": ">= 10.14.2"
},
"peerDependencies": {
"jest": ">=20 <=27"
}
},
"node_modules/jest-image-snapshot/node_modules/ansi-styles": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/jest-image-snapshot/node_modules/chalk": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"dev": true,
"dependencies": {
"ansi-styles": "^2.2.1",
"escape-string-regexp": "^1.0.2",
"has-ansi": "^2.0.0",
"strip-ansi": "^3.0.0",
"supports-color": "^2.0.0"
},
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/jest-image-snapshot/node_modules/get-stdin": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz",
"integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=",
"dev": true,
"engines": {
"node": ">=0.12.0"
}
},
"node_modules/jest-image-snapshot/node_modules/pixelmatch": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
"integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
"dev": true,
"dependencies": {
"pngjs": "^4.0.1"
},
"bin": {
"pixelmatch": "bin/pixelmatch"
}
},
"node_modules/jest-image-snapshot/node_modules/pixelmatch/node_modules/pngjs": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
"integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
"dev": true,
"engines": {
"node": ">=8.0.0"
}
},
"node_modules/jest-image-snapshot/node_modules/rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"dev": true,
"dependencies": {
"glob": "^7.1.3"
},
"bin": {
"rimraf": "bin.js"
}
},
"node_modules/jest-image-snapshot/node_modules/supports-color": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
"dev": true,
"engines": {
"node": ">=0.8.0"
}
},
"node_modules/jest-jasmine2": {
"version": "27.0.6",
"resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-27.0.6.tgz",
@ -22207,6 +22346,12 @@
"tweetnacl": "~0.14.0"
}
},
"node_modules/ssim.js": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
"integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
"dev": true
},
"node_modules/ssri": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz",
@ -28773,6 +28918,17 @@
}
}
},
"@types/jest-image-snapshot": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/@types/jest-image-snapshot/-/jest-image-snapshot-4.3.1.tgz",
"integrity": "sha512-WDdUruGF14C53axe/mNDgQP2YIhtcwXrwmmVP8eOGyfNTVD+FbxWjWR7RTU+lzEy4K6V6+z7nkVDm/auI/r3xQ==",
"dev": true,
"requires": {
"@types/jest": "*",
"@types/pixelmatch": "*",
"ssim.js": "^3.1.1"
}
},
"@types/json-schema": {
"version": "7.0.8",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.8.tgz",
@ -28825,6 +28981,15 @@
"integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
"dev": true
},
"@types/pixelmatch": {
"version": "5.2.4",
"resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.4.tgz",
"integrity": "sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/platform": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/@types/platform/-/platform-1.3.4.tgz",
@ -36075,6 +36240,12 @@
}
}
},
"glur": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz",
"integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=",
"dev": true
},
"graceful-fs": {
"version": "4.2.6",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
@ -36145,6 +36316,15 @@
"function-bind": "^1.1.1"
}
},
"has-ansi": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
"integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
"dev": true,
"requires": {
"ansi-regex": "^2.0.0"
}
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@ -37990,6 +38170,82 @@
}
}
},
"jest-image-snapshot": {
"version": "4.5.1",
"resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-4.5.1.tgz",
"integrity": "sha512-0YkgupgkkCx0wIZkxvqs/oNiUT0X0d2WTpUhaAp+Dy6CpqBUZMRTIZo4KR1f+dqmx6WXrLCvecjnHLIsLkI+gQ==",
"dev": true,
"requires": {
"chalk": "^1.1.3",
"get-stdin": "^5.0.1",
"glur": "^1.1.2",
"lodash": "^4.17.4",
"mkdirp": "^0.5.1",
"pixelmatch": "^5.1.0",
"pngjs": "^3.4.0",
"rimraf": "^2.6.2",
"ssim.js": "^3.1.1"
},
"dependencies": {
"ansi-styles": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
"dev": true
},
"chalk": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"dev": true,
"requires": {
"ansi-styles": "^2.2.1",
"escape-string-regexp": "^1.0.2",
"has-ansi": "^2.0.0",
"strip-ansi": "^3.0.0",
"supports-color": "^2.0.0"
}
},
"get-stdin": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz",
"integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=",
"dev": true
},
"pixelmatch": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
"integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
"dev": true,
"requires": {
"pngjs": "^4.0.1"
},
"dependencies": {
"pngjs": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
"integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
"dev": true
}
}
},
"rimraf": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"dev": true,
"requires": {
"glob": "^7.1.3"
}
},
"supports-color": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
"dev": true
}
}
},
"jest-jasmine2": {
"version": "27.0.6",
"resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-27.0.6.tgz",
@ -43377,6 +43633,12 @@
"tweetnacl": "~0.14.0"
}
},
"ssim.js": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
"integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
"dev": true
},
"ssri": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz",

View File

@ -35,6 +35,7 @@
"@types/express": "^4.17.13",
"@types/glob": "^7.1.1",
"@types/jest": "^26.0.24",
"@types/jest-image-snapshot": "^4.3.1",
"@types/karma": "^6.3.0",
"@types/mocha": "^8.2.3",
"@types/node": "^16.3.1",
@ -61,6 +62,7 @@
"glob": "7.1.3",
"html2canvas-proxy": "1.0.1",
"jest": "^27.0.6",
"jest-image-snapshot": "^4.5.1",
"jquery": "^3.5.1",
"js-polyfills": "^0.1.42",
"karma": "^6.3.2",
@ -107,6 +109,7 @@
"lint": "eslint src/**/*.ts --max-warnings 0",
"test": "npm run lint && npm run unittest && npm run karma",
"unittest": "jest",
"reftests-diff": "mkdirp tmp/snapshots && jest --roots=tests --testMatch=**/reftest-diff.ts",
"karma": "ts-node tests/karma",
"watch": "rollup -c rollup.config.ts -w",
"watch:unittest": "mocha --require ts-node/register --watch-extensions ts -w src/**/__tests__/*.ts",

32
tests/reftest-diff.ts Normal file
View File

@ -0,0 +1,32 @@
import {sync} from 'glob';
import {resolve, basename} from 'path';
import {existsSync, promises} from 'fs';
import {toMatchImageSnapshot} from 'jest-image-snapshot';
const resultsDir = resolve(__dirname, '../results');
const customSnapshotsDir = resolve(__dirname, '../tmp/snapshots');
const customDiffDir = resolve(__dirname, '../tmp/snapshot-diffs');
expect.extend({toMatchImageSnapshot});
describe('Image diff', () => {
const files: string[] = sync('../tmp/reftests/**/*.png', {
cwd: __dirname,
root: resolve(__dirname, '../../')
}).filter((path) => existsSync(resolve(resultsDir, basename(path))));
it.each(files.map((path) => basename(path)))('%s', async (filename) => {
const previous = resolve(resultsDir, filename);
const previousSnap = resolve(customSnapshotsDir, `${filename}-snap.png`);
await promises.copyFile(previous, previousSnap);
const updated = resolve(__dirname, '../tmp/reftests/', filename);
const buffer = await promises.readFile(updated);
// @ts-ignore
expect(buffer).toMatchImageSnapshot({
customSnapshotsDir,
customSnapshotIdentifier: () => filename,
customDiffDir
});
});
});