ci: implement screenshot diffing (#2571)

.github/workflows/ci.yml

@@ -309,3 +309,43 @@ jobs:
           FOLDER: docs
           SINGLE_COMMIT: true
           CLEAN: true
+  diff-reftests:
+    runs-on: ubuntu-latest
+    name: Diff reftest screenshots
+    needs: browser-test
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 12
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Npm install
+        run: npm ci
+      - name: Checkout current snapshots
+        run: git checkout origin/gh-pages results
+      - name: Download test results
+        uses: actions/download-artifact@v2
+        with:
+          name: reftest-results
+          path: tmp/reftests
+      - name: Run diff
+        run: npm run reftests-diff
+        continue-on-error: true
+      - name: Upload diff
+        uses: actions/upload-artifact@v2
+        with:
+          name: snapshot-diffs
+          path: tmp/snapshot-diffs

package-lock.json

@@ -5,7 +5,7 @@
   "requires": true,
   "packages": {
     "": {
-      "version": "1.0.0",
+      "version": "1.1.0",
       "license": "MIT",
       "dependencies": {
         "css-line-break": "1.1.1"
@@ -23,6 +23,7 @@
         "@types/express": "^4.17.13",
         "@types/glob": "^7.1.1",
         "@types/jest": "^26.0.24",
+        "@types/jest-image-snapshot": "^4.3.1",
         "@types/karma": "^6.3.0",
         "@types/mocha": "^8.2.3",
         "@types/node": "^16.3.1",
@@ -49,6 +50,7 @@
         "glob": "7.1.3",
         "html2canvas-proxy": "1.0.1",
         "jest": "^27.0.6",
+        "jest-image-snapshot": "^4.5.1",
         "jquery": "^3.5.1",
         "js-polyfills": "^0.1.42",
         "karma": "^6.3.2",
@@ -3758,6 +3760,17 @@
         "pretty-format": "^26.0.0"
       }
     },
+    "node_modules/@types/jest-image-snapshot": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/@types/jest-image-snapshot/-/jest-image-snapshot-4.3.1.tgz",
+      "integrity": "sha512-WDdUruGF14C53axe/mNDgQP2YIhtcwXrwmmVP8eOGyfNTVD+FbxWjWR7RTU+lzEy4K6V6+z7nkVDm/auI/r3xQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/jest": "*",
+        "@types/pixelmatch": "*",
+        "ssim.js": "^3.1.1"
+      }
+    },
     "node_modules/@types/jest/node_modules/@jest/types": {
       "version": "26.6.2",
       "resolved": "https://registry.npmjs.org/@jest/types/-/types-26.6.2.tgz",
@@ -3962,6 +3975,15 @@
       "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
       "dev": true
     },
+    "node_modules/@types/pixelmatch": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.4.tgz",
+      "integrity": "sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/platform": {
       "version": "1.3.4",
       "resolved": "https://registry.npmjs.org/@types/platform/-/platform-1.3.4.tgz",
@@ -12886,6 +12908,12 @@
         "node": ">=8"
       }
     },
+    "node_modules/glur": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz",
+      "integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=",
+      "dev": true
+    },
     "node_modules/graceful-fs": {
       "version": "4.2.6",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
@@ -12978,6 +13006,18 @@
         "node": ">= 0.4.0"
       }
     },
+    "node_modules/has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dev": true,
+      "dependencies": {
+        "ansi-regex": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -15313,6 +15353,105 @@
         "node": ">=8.0"
       }
     },
+    "node_modules/jest-image-snapshot": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-4.5.1.tgz",
+      "integrity": "sha512-0YkgupgkkCx0wIZkxvqs/oNiUT0X0d2WTpUhaAp+Dy6CpqBUZMRTIZo4KR1f+dqmx6WXrLCvecjnHLIsLkI+gQ==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "^1.1.3",
+        "get-stdin": "^5.0.1",
+        "glur": "^1.1.2",
+        "lodash": "^4.17.4",
+        "mkdirp": "^0.5.1",
+        "pixelmatch": "^5.1.0",
+        "pngjs": "^3.4.0",
+        "rimraf": "^2.6.2",
+        "ssim.js": "^3.1.1"
+      },
+      "engines": {
+        "node": ">= 10.14.2"
+      },
+      "peerDependencies": {
+        "jest": ">=20 <=27"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/ansi-styles": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/chalk": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+      "dev": true,
+      "dependencies": {
+        "ansi-styles": "^2.2.1",
+        "escape-string-regexp": "^1.0.2",
+        "has-ansi": "^2.0.0",
+        "strip-ansi": "^3.0.0",
+        "supports-color": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/get-stdin": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz",
+      "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.12.0"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/pixelmatch": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
+      "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
+      "dev": true,
+      "dependencies": {
+        "pngjs": "^4.0.1"
+      },
+      "bin": {
+        "pixelmatch": "bin/pixelmatch"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/pixelmatch/node_modules/pngjs": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
+      "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
+      "dev": true,
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/rimraf": {
+      "version": "2.7.1",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
+      "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
+      "dev": true,
+      "dependencies": {
+        "glob": "^7.1.3"
+      },
+      "bin": {
+        "rimraf": "bin.js"
+      }
+    },
+    "node_modules/jest-image-snapshot/node_modules/supports-color": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+      "dev": true,
+      "engines": {
+        "node": ">=0.8.0"
+      }
+    },
     "node_modules/jest-jasmine2": {
       "version": "27.0.6",
       "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-27.0.6.tgz",
@@ -22207,6 +22346,12 @@
         "tweetnacl": "~0.14.0"
       }
     },
+    "node_modules/ssim.js": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
+      "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
+      "dev": true
+    },
     "node_modules/ssri": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz",
@@ -28773,6 +28918,17 @@
         }
       }
     },
+    "@types/jest-image-snapshot": {
+      "version": "4.3.1",
+      "resolved": "https://registry.npmjs.org/@types/jest-image-snapshot/-/jest-image-snapshot-4.3.1.tgz",
+      "integrity": "sha512-WDdUruGF14C53axe/mNDgQP2YIhtcwXrwmmVP8eOGyfNTVD+FbxWjWR7RTU+lzEy4K6V6+z7nkVDm/auI/r3xQ==",
+      "dev": true,
+      "requires": {
+        "@types/jest": "*",
+        "@types/pixelmatch": "*",
+        "ssim.js": "^3.1.1"
+      }
+    },
     "@types/json-schema": {
       "version": "7.0.8",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.8.tgz",
@@ -28825,6 +28981,15 @@
       "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
       "dev": true
     },
+    "@types/pixelmatch": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.4.tgz",
+      "integrity": "sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/platform": {
       "version": "1.3.4",
       "resolved": "https://registry.npmjs.org/@types/platform/-/platform-1.3.4.tgz",
@@ -36075,6 +36240,12 @@
         }
       }
     },
+    "glur": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz",
+      "integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=",
+      "dev": true
+    },
     "graceful-fs": {
       "version": "4.2.6",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz",
@@ -36145,6 +36316,15 @@
         "function-bind": "^1.1.1"
       }
     },
+    "has-ansi": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
+      "dev": true,
+      "requires": {
+        "ansi-regex": "^2.0.0"
+      }
+    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -37990,6 +38170,82 @@
         }
       }
     },
+    "jest-image-snapshot": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-4.5.1.tgz",
+      "integrity": "sha512-0YkgupgkkCx0wIZkxvqs/oNiUT0X0d2WTpUhaAp+Dy6CpqBUZMRTIZo4KR1f+dqmx6WXrLCvecjnHLIsLkI+gQ==",
+      "dev": true,
+      "requires": {
+        "chalk": "^1.1.3",
+        "get-stdin": "^5.0.1",
+        "glur": "^1.1.2",
+        "lodash": "^4.17.4",
+        "mkdirp": "^0.5.1",
+        "pixelmatch": "^5.1.0",
+        "pngjs": "^3.4.0",
+        "rimraf": "^2.6.2",
+        "ssim.js": "^3.1.1"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "2.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+          "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+          "dev": true
+        },
+        "chalk": {
+          "version": "1.1.3",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^2.2.1",
+            "escape-string-regexp": "^1.0.2",
+            "has-ansi": "^2.0.0",
+            "strip-ansi": "^3.0.0",
+            "supports-color": "^2.0.0"
+          }
+        },
+        "get-stdin": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz",
+          "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=",
+          "dev": true
+        },
+        "pixelmatch": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
+          "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
+          "dev": true,
+          "requires": {
+            "pngjs": "^4.0.1"
+          },
+          "dependencies": {
+            "pngjs": {
+              "version": "4.0.1",
+              "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
+              "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
+              "dev": true
+            }
+          }
+        },
+        "rimraf": {
+          "version": "2.7.1",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
+          "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
+          "dev": true,
+          "requires": {
+            "glob": "^7.1.3"
+          }
+        },
+        "supports-color": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+          "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+          "dev": true
+        }
+      }
+    },
     "jest-jasmine2": {
       "version": "27.0.6",
       "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-27.0.6.tgz",
@@ -43377,6 +43633,12 @@
         "tweetnacl": "~0.14.0"
       }
     },
+    "ssim.js": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/ssim.js/-/ssim.js-3.5.0.tgz",
+      "integrity": "sha512-Aj6Jl2z6oDmgYFFbQqK7fght19bXdOxY7Tj03nF+03M9gCBAjeIiO8/PlEGMfKDwYpw4q6iBqVq2YuREorGg/g==",
+      "dev": true
+    },
     "ssri": {
       "version": "6.0.2",
       "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz",

package.json

@@ -35,6 +35,7 @@
     "@types/express": "^4.17.13",
     "@types/glob": "^7.1.1",
     "@types/jest": "^26.0.24",
+    "@types/jest-image-snapshot": "^4.3.1",
     "@types/karma": "^6.3.0",
     "@types/mocha": "^8.2.3",
     "@types/node": "^16.3.1",
@@ -61,6 +62,7 @@
     "glob": "7.1.3",
     "html2canvas-proxy": "1.0.1",
     "jest": "^27.0.6",
+    "jest-image-snapshot": "^4.5.1",
     "jquery": "^3.5.1",
     "js-polyfills": "^0.1.42",
     "karma": "^6.3.2",
@@ -107,6 +109,7 @@
     "lint": "eslint src/**/*.ts --max-warnings 0",
     "test": "npm run lint && npm run unittest && npm run karma",
     "unittest": "jest",
+    "reftests-diff": "mkdirp tmp/snapshots && jest --roots=tests --testMatch=**/reftest-diff.ts",
     "karma": "ts-node tests/karma",
     "watch": "rollup -c rollup.config.ts -w",
     "watch:unittest": "mocha --require ts-node/register --watch-extensions ts -w src/**/__tests__/*.ts",

tests/reftest-diff.ts

@@ -0,0 +1,32 @@
+import {sync} from 'glob';
+import {resolve, basename} from 'path';
+import {existsSync, promises} from 'fs';
+import {toMatchImageSnapshot} from 'jest-image-snapshot';
+
+const resultsDir = resolve(__dirname, '../results');
+const customSnapshotsDir = resolve(__dirname, '../tmp/snapshots');
+const customDiffDir = resolve(__dirname, '../tmp/snapshot-diffs');
+
+expect.extend({toMatchImageSnapshot});
+
+describe('Image diff', () => {
+    const files: string[] = sync('../tmp/reftests/**/*.png', {
+        cwd: __dirname,
+        root: resolve(__dirname, '../../')
+    }).filter((path) => existsSync(resolve(resultsDir, basename(path))));
+
+    it.each(files.map((path) => basename(path)))('%s', async (filename) => {
+        const previous = resolve(resultsDir, filename);
+        const previousSnap = resolve(customSnapshotsDir, `${filename}-snap.png`);
+        await promises.copyFile(previous, previousSnap);
+        const updated = resolve(__dirname, '../tmp/reftests/', filename);
+        const buffer = await promises.readFile(updated);
+
+        // @ts-ignore
+        expect(buffer).toMatchImageSnapshot({
+            customSnapshotsDir,
+            customSnapshotIdentifier: () => filename,
+            customDiffDir
+        });
+    });
+});