From 773084235c88f2d3c6af8708ee879bea5f104a52 Mon Sep 17 00:00:00 2001 From: MoyuScript Date: Wed, 17 Sep 2014 00:11:47 +0300 Subject: [PATCH] Simplify webdriver tests --- Gruntfile.js | 5 +- package.json | 3 +- tests/certificate.pem.enc | 43 ------- tests/selenium.js | 258 ++++++++++++++------------------------ 4 files changed, 101 insertions(+), 208 deletions(-) delete mode 100644 tests/certificate.pem.enc diff --git a/Gruntfile.js b/Gruntfile.js index 4e74383..d9c5385 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -157,7 +157,10 @@ module.exports = function(grunt) { var selenium = require("./tests/selenium.js"); var done = this.async(); var browsers = (browser) ? [grunt.config.get(this.name + "." + browser)] : _.values(grunt.config.get(this.name)); - selenium.tests(browsers, test).onValue(done); + selenium.tests(browsers, test).finally(function() { + console.log("Done"); + done(); + }); }); grunt.loadNpmTasks('grunt-contrib-watch'); diff --git a/package.json b/package.json index 1f11499..08d371b 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "url": "https://github.com/niklasvh/html2canvas/issues" }, "devDependencies": { - "baconjs": "^0.7.11", "base64-arraybuffer": ">= 0.1.0", + "bluebird": "^2.3.2", "grunt": "^0.4.5", "grunt-contrib-concat": "*", "grunt-contrib-connect": "0.7.1", @@ -30,6 +30,7 @@ "grunt-contrib-uglify": "*", "grunt-contrib-watch": "~0.5.1", "grunt-execute": "^0.2.2", + "humanize-duration": "^2.0.1", "lodash": "^2.4.1", "png-js": ">= 0.1.1", "wd": "^0.2.21" diff --git a/tests/certificate.pem.enc b/tests/certificate.pem.enc deleted file mode 100644 index b97a212..0000000 --- a/tests/certificate.pem.enc +++ /dev/null @@ -1,43 +0,0 @@ -U2FsdGVkX1/1OhTi++Zu8+GzAWDR9xD8SFu4sB/1mvbeKt7eF3z/t217AJii/H52 -FYMoPSnwFzEvXuiS/EIpVzRYugcfc4qSkOA2PE726V1b6UQoxbcJnHK4UW2464Yb -a8dw+saN7KFpzaY0+L09QzIGqVMvtYQG2dTQk3dfAZ4Y/hduoGuCDReyRP7DbIJw -WD933bja/p1HL91AnNo7lIGD529nXugBdlCAEeYGkeAfa7/Or+68sw7fhki3MQJw -xWbo/oND/Fr4WzlGDeTQMTVfEVGYmT8K+aNtcKrgAp3SDQf37zuMnxVaKREmlXOF -IRKaoR4jzDEgiwlsMZ7CvASwKDZDckFKjg3v/yKLjg+NKg2eET+VD+3CvTai5PHQ -xRzPqkmyh+gh3Op2ZpTCtqnlgx7CfobAXjL4BOtu8aT6LZsoS/mMmqfbX49SDBw3 -wVXFuhPPGsXDJD2F39kG5vX6LeygOGPMOPbzqimUftWlxcUJhb4mGKV5qcLz+WzH -17E2fv6kEwMRTfjGjUR9zMR8JjBBu0GcKGyiW6MqQwjF+BXvWoIRuwv99x2cpOjN -fAHk9Y9IgJlzWmrTpg1i5sqTNKgmQaKqarJqXwvuq86kJuCMkYr0SxleUlAmbzpC -w25JAMcNqabQLDaTJfV6V5EWAh3sHh+lAcL+QKs2Ddx5K9RwwCnNXtIhatECsu0n -oNDeWcFtaLmGVF/zXpkCkKBg0EsEcFIXZM662TaGCqukIktq61/UsuR7hVQdPXyH -/4rJTITQAXbi+FaQXzrrxKrikU7Rdovn6az5YNXBXh0pMQ9FrahxhYhDrpzpCsNc -STL2JN50u3X28ZgfRs7rB5fY9Xr5pERIAFSrNzFPCHzlStoYNSphJP8lO/XjNkcn -CTOzZrzxVpzqqzXlmArCfx3QtVo+WUUUW9hdOBgKLl19Wt0x5ZvYDPm6ODht9BjD -wfbmkRvdP9CTICBR17l2tPiGcu+4paTKBF6W6aSxQU7Z48uZP19/4cuOiITbcYnt -4wHoNk+M5G9SjLSsOlfmYNMcl0D57AOcbyUUTMZ2Wu0zc3/StJ9aotzEcFSxA1Gs -wPZr/2iqeEynJmUQeFjs01JSwxwYlLF+HOStdn6MEbsW4ftxsXO9Mr2NGAOTaO1/ -Y8pyMCRJKOH6kRvHPpzza/stIZQQZk+PMRzS3PgMXsnJoF7cuanrWXJTgiNT6B37 -y/ARNr3KtZTGOjc5gE+k32hPuqPi5tVKPrFVUrG2l/87Gqjs4JqNDTjTHBlUIViT -y87uB2f8/8/V50lXf+RYGPKaJ+jzbyaUxp6QRZdqJk8CEZQnrGwpUEzBVMREd+Ow -jn99f/Isn5Vadsdmt/aMy7+TnLVyjPqucIo8dUdkWdfLKF1Q7M06rYgu9MCrsbgi -hGp1/YjqSmYxWZuSUZHvd4ps2x9YI6wuwdWXAH/7+L0IVUghpSNoARrsmi/D7Rg5 -4zF6eCOqH13z1luMRg1ZMpRyVQcI/Yu3IbyepnuIeYIqPklDJ6jl3kIdubEtXZTV -1XMlTvZjFuRKLeUXsF/ecTmlob+rc06YvOu91vwpxSMkpYUIc/Yt4tTWey1AbmTG -t4pf8q0cVXrVRuXf8HFEEqQlXumJGWi/WCxAawj35znEIquQDu9ERLgSemdTFH2z -ncoJseApHD9Gf2UpM4Vva+XdI9wrlYURr83/gOxenCgMq7wCWH87yuCG1IsOryPx -1DbES6axy6P+XBQJTDHK560zKSNF+3VVBGEeTQRMAnQs/IHlwWu7Af0DTmUAEO2Z -S1R+Ex2HWMM3i8/Sr8P1JDkm0u7gn2wkfI0xEIf5B10md1iniKb0GovMlfMGJTTo -+mytGTp3XUZl8lXDpbLTR9nqfpB+hL5Qi30Dv08CAQdvX9OzOEQz5T//t13wAKlo -bsbzwIuLhyYbua+wz/3uMS6QAbNNVCgIm/J1rqlwLRmzvSTJyhraamjd2UuwwIau -fgdIA5bYqdagzbUuIk8msItfUrAbhxkyBElVb5RgLVJsUbBQTHW3anrbsTxxMrcR -9jnZZV2Is2ahbnmcqqIjr7fxSqTCtwpyBxRkW8Bh+Sfu2R8x+DLcJ+ID25fbxWto -QrDSGEifpW/5bjGx4lM0j8IxoWqOl0600hlzGsD5ynSu5WPpXQ0CMvuuLXOR4CbZ -kzE5kQXAeQzfwwPcWMMhwt5F4hXAXY/+jM0TfuSc7Z9PYYRWdoDg3awRNlQJS5rk -wDwbMHKUcdi4XSwIhEdbZ4zBo0o7xXKn2XSBNy+Ym39pKWMwOrffsPtWDV+qTHpD -P5rhXEW8aD01kMkxjO2pRZ7nEDQ7VGpi3MuW9FAjXZjcc1grnZSxbKz437K9obI7 -zHXGcodAXjU4Oxhs/m8JHVaHsDG5LNxLSmArsmbwcu65mlBCXcefcwOUNoVQDmZG -ENSAMlVAUX7RJ9nxXI9AqKKuVAYSMznseY3V22BCDrRR0nbVkAWjuBTKU6ajfjQ6 -W2uhAzl8kH+9Xi10jt5Fhn93b8z4VjrcpZmW/gJuEHm8FY2UFF0P1y3YMCDqCPaz -gXsFZVKPPIswg/xBHB9GhiCSHnvivu/K8Ni4FcBW7smBHuqkk0VoN7CdZvNF0CuX -EANCwDD53hSmfyxR/SY6y1PQCEsX5AiRS8rMGV2bqOjP6/oM9+2JmkRZ3I6Jht2f -sxJsRUTyjecCcwnGDgmLDCZDJqLJoOE6zlRuAOvY1U7m2cFrf9EouOlYUXSeBaEh diff --git a/tests/selenium.js b/tests/selenium.js index d6f154f..baeb30d 100644 --- a/tests/selenium.js +++ b/tests/selenium.js @@ -1,15 +1,19 @@ (function(){ "use strict;"; - var Bacon = require('baconjs').Bacon, - wd = require('wd'), + var wd = require('wd'), http = require("http"), https = require("https"), url = require("url"), path = require("path"), base64_arraybuffer = require('base64-arraybuffer'), PNG = require('png-js'), + Promise = require('bluebird'), + _ = require('lodash'), + humanizeDuration = require("humanize-duration"), fs = require("fs"); + Promise.promisifyAll(fs); + var port = 8080, colors = { red: "\x1b[1;31m", @@ -19,41 +23,19 @@ clear: "\x1b[0m" }; - function mapStat(item) { - return Bacon.combineTemplate({ - stat: Bacon.fromNodeCallback(fs.stat, item), - item: item + function getTests(path) { + return fs.readdirAsync(path).map(function(name) { + var filename = path + "/" + name; + return fs.statAsync(filename).then(function(stat) { + return stat.isDirectory() ? getTests(filename) : filename; + }); }); } - function isDirectory(item) { - return item.stat.isDirectory(); - } - - function getItem(item) { - return item.item; - } - - function isFile(item) { - return !isDirectory(item); - } - - function arrayStream(arr) { - return Bacon.fromArray(arr); - } - - function getTests(path) { - var items = Bacon.fromNodeCallback(fs.readdir, path).flatMap(arrayStream).map(function(name) { - return path + "/" + name; - }).flatMap(mapStat); - return items.filter(isFile).map(getItem).merge(items.filter(isDirectory).map(getItem).flatMap(getTests)); - } - - function getPixelArray(base64) { - return Bacon.fromCallback(function(callback) { + return new Promise(function(resolve) { var arraybuffer = base64_arraybuffer.decode(base64); - (new PNG(arraybuffer)).decodePixels(callback); + (new PNG(arraybuffer)).decodePixels(resolve); }); } @@ -67,154 +49,104 @@ return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); } - function findResult(testName, tests) { - var item = null; - return tests.some(function(testCase) { - item = testCase; - return testCase.test === testName; - }) ? item : null; - } + function initBrowser(settings) { + var browser = wd.remote({ + hostname: 'localhost', + port: 4445, + user: process.env.SAUCE_USERNAME, + pwd: process.env.SAUCE_ACCESS_KEY + }, 'promiseChain'); - function compareResults(oldResults, newResults, browser) { - var improved = [], - regressed = [], - newItems = []; - - newResults.forEach(function(testCase){ - var testResult = testCase.result, - oldResult = findResult(testCase.test, oldResults), - oldResultValue = oldResult ? oldResult.result : null, - dataObject = { - amount: (Math.abs(testResult - oldResultValue) < 0.01) ? 0 : testResult - oldResultValue, - test: testCase.test - }; - if (oldResultValue === null) { - newItems.push(dataObject); - } else if (dataObject.amount > 0) { - improved.push(dataObject); - } else if (dataObject.amount < 0) { - regressed.push(dataObject); - } - }); - - reportChanges(browser, improved, regressed, newItems); - } - - function reportChanges(browser, improved, regressed, newItems) { - if (newItems.length > 0 || improved.length > 0 || regressed.length > 0) { - console.log((regressed.length > 0) ? colors.red : colors.green, browser); - - regressed.forEach(function(item) { - console.log(colors.red, item.amount + "%", item.test); - }); - - improved.forEach(function(item) { - console.log(colors.green, item.amount + "%", item.test); - }); - - newItems.forEach(function(item) { - console.log(colors.blue, "NEW", item.test); - }); - } - } - - function formatResultName(navigator) { - return (navigator.browserName + "-" + ((navigator.version) ? navigator.version : "release") + "-" + navigator.platform).replace(/ /g, "").toLowerCase(); - } - - function webdriverStream(test) { - var browser = wd.remote("localhost", 4445, process.env.SAUCE_USERNAME, process.env.SAUCE_ACCESS_KEY); - var browserStream = new Bacon.Bus(); if (process.env.TRAVIS_JOB_NUMBER) { - test.capabilities["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER; - test.capabilities["name"] = process.env.TRAVIS_COMMIT.substring(0, 10); - test.capabilities["build"] = process.env.TRAVIS_BUILD_NUMBER; + settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER; + settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10); + settings["build"] = process.env.TRAVIS_BUILD_NUMBER; } else { - test.capabilities["name"] = "Manual run"; + settings["name"] = "Manual run"; } - var resultStream = Bacon.fromNodeCallback(browser, "init", test.capabilities) - .flatMap(Bacon.fromNodeCallback(browser, "setImplicitWaitTimeout", 15000) - .flatMap(function() { - Bacon.later(0, formatResultName(test.capabilities)).onValue(browserStream.push); - return Bacon.fromArray(test.cases).zip(browserStream.take(test.cases.length)).flatMap(function(options) { - var testCase = options[0]; - var name = options[1]; - console.log(colors.green, "STARTING", name, testCase, colors.clear); - return Bacon.fromNodeCallback(browser, "get", "http://localhost:" + port + "/" + testCase + "?selenium") - .flatMap(Bacon.combineTemplate({ - dataUrl: Bacon.fromNodeCallback(browser, "elementByCssSelector", ".html2canvas").flatMap(function(canvas) { - return Bacon.fromNodeCallback(browser, "execute", "return arguments[0].toDataURL('image/png').substring(22)", [canvas]); - }), - screenshot: Bacon.fromNodeCallback(browser, "takeScreenshot") - })) - .flatMap(function(result) { - return Bacon.combineTemplate({ - browser: name, - testCase: testCase, - accuracy: Bacon.constant(result.dataUrl).flatMap(getPixelArray).combine(Bacon.constant(result.screenshot).flatMap(getPixelArray), calculateDifference), - dataUrl: result.dataUrl, - screenshot: result.screenshot - }); - }); - }); - })); - - resultStream.onError(function(error) { - var name = formatResultName(test.capabilities); - console.log(colors.red, "ERROR", name, error.message); - browserStream.push(name); - browser.quit(); + return browser.resolve(Promise).init(settings).setImplicitWaitTimeout(15000).then(function() { + return settings; }); + } - resultStream.onValue(function(result) { - console.log(colors.green, "COMPLETE", result.browser, result.testCase, result.accuracy, "%", colors.clear); - browserStream.push(result.browser); - }); - - return resultStream.fold([], pushToArray).flatMap(function(value) { - return Bacon.fromCallback(function(callback) { - browser.quit(function() { - callback(value); - }); + function loadTestPage(browser, test) { + return function(settings) { + return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() { + return settings; }); - }); + }; } - function createImages(data) { - var dataurlFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-html2canvas.png"; - var screenshotFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-screencapture.png"; - return Bacon.combineTemplate({ - name: data.testCase, - dataurl: Bacon.fromNodeCallback(fs.writeFile, dataurlFileName, data.dataUrl, "base64").map(function() { - return dataurlFileName; - }), - screenshot: Bacon.fromNodeCallback(fs.writeFile, screenshotFileName, data.screenshot, "base64").map(function() { - return screenshotFileName; - }) - }); + function captureScreenshots(browser) { + return function() { + return Promise.props({ + dataUrl: browser.elementByCss(".html2canvas").then(function(node) { + return browser.execute("return arguments[0].toDataURL('image/png').substring(22)", [node]); + }), + screenshot: browser.takeScreenshot() + }); + }; } - function pushToArray(array, item) { - array.push(item); - return array; + function analyzeResults(test) { + return function(result) { + return Promise.all([getPixelArray(result.dataUrl), getPixelArray(result.screenshot)]).spread(calculateDifference).then(function(accuracy) { + return { + testCase: test, + accuracy: accuracy, + dataUrl: result.dataUrl, + screenshot: result.screenshot + }; + }); + }; } - function runWebDriver(browsers, cases) { - var availableBrowsers = new Bacon.Bus(); - var result = Bacon.combineTemplate({ - capabilities: Bacon.fromArray(browsers).zip(availableBrowsers.take(browsers.length), function(first) { return first; }), - cases: cases - }).flatMap(webdriverStream).doAction(function() { - availableBrowsers.push("ready"); - }); + function runTestWithRetries(browser, test, retries) { + retries = retries || 0; + return runTest(browser, test) + .timeout(60000) + .catch(Promise.TimeoutError, function() { + if (retries < 3) { + console.log("Retry", (retries + 1), test); + return runTestWithRetries(browser, test, retries + 1); + } else { + throw new Error("Couldn't run test after 3 retries"); + } + }); + } - Bacon.fromArray([1, 2, 3, 4]).onValue(availableBrowsers.push); - - return result.fold([], pushToArray); + function runTest(browser, test) { + return browser + .then(loadTestPage(browser, test)) + .then(captureScreenshots(browser)) + .then(analyzeResults(test)); } exports.tests = function(browsers, singleTest) { - return (singleTest ? Bacon.constant([singleTest]) : getTests("tests/cases").fold([], pushToArray)).flatMap(runWebDriver.bind(null, browsers)).mapError(false); + var path = "tests/cases"; + return (singleTest ? Promise.resolve([singleTest]) : getTests(path)).then(function(t) { + var tests = _.flatten(t); + return Promise.map(browsers, function(settings) { + var browser = initBrowser(settings); + var name = [settings.browserName, settings.version, settings.platform].join("-"); + var count = 0; + return Promise.map(tests, function(test, index, total) { + console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear); + var start = Date.now(); + return runTestWithRetries(browser, test).then(function(result) { + console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, result.testCase.substring(path.length), result.accuracy.toFixed(2) + "%", colors.clear); + }); + }, {concurrency: 1}) + .settle() + .catch(function(error) { + console.log(colors.red, "ERROR", name, error.message); + throw error; + }) + .finally(function() { + return browser.quit(); + }); + }, {concurrency: 3}); + }); }; })();