Simplify webdriver tests

Gruntfile.js

@@ -157,7 +157,10 @@ module.exports = function(grunt) {
         var selenium = require("./tests/selenium.js");
         var done = this.async();
         var browsers = (browser) ? [grunt.config.get(this.name + "." + browser)] : _.values(grunt.config.get(this.name));
-        selenium.tests(browsers, test).onValue(done);
+        selenium.tests(browsers, test).finally(function() {
+            console.log("Done");
+            done();
+        });
     });
 
     grunt.loadNpmTasks('grunt-contrib-watch');

package.json

@@ -20,8 +20,8 @@
     "url": "https://github.com/niklasvh/html2canvas/issues"
   },
   "devDependencies": {
-    "baconjs": "^0.7.11",
     "base64-arraybuffer": ">= 0.1.0",
+    "bluebird": "^2.3.2",
     "grunt": "^0.4.5",
     "grunt-contrib-concat": "*",
     "grunt-contrib-connect": "0.7.1",
@@ -30,6 +30,7 @@
     "grunt-contrib-uglify": "*",
     "grunt-contrib-watch": "~0.5.1",
     "grunt-execute": "^0.2.2",
+    "humanize-duration": "^2.0.1",
     "lodash": "^2.4.1",
     "png-js": ">= 0.1.1",
     "wd": "^0.2.21"

tests/certificate.pem.enc

@@ -1,43 +0,0 @@
-U2FsdGVkX1/1OhTi++Zu8+GzAWDR9xD8SFu4sB/1mvbeKt7eF3z/t217AJii/H52
-FYMoPSnwFzEvXuiS/EIpVzRYugcfc4qSkOA2PE726V1b6UQoxbcJnHK4UW2464Yb
-a8dw+saN7KFpzaY0+L09QzIGqVMvtYQG2dTQk3dfAZ4Y/hduoGuCDReyRP7DbIJw
-WD933bja/p1HL91AnNo7lIGD529nXugBdlCAEeYGkeAfa7/Or+68sw7fhki3MQJw
-xWbo/oND/Fr4WzlGDeTQMTVfEVGYmT8K+aNtcKrgAp3SDQf37zuMnxVaKREmlXOF
-IRKaoR4jzDEgiwlsMZ7CvASwKDZDckFKjg3v/yKLjg+NKg2eET+VD+3CvTai5PHQ
-xRzPqkmyh+gh3Op2ZpTCtqnlgx7CfobAXjL4BOtu8aT6LZsoS/mMmqfbX49SDBw3
-wVXFuhPPGsXDJD2F39kG5vX6LeygOGPMOPbzqimUftWlxcUJhb4mGKV5qcLz+WzH
-17E2fv6kEwMRTfjGjUR9zMR8JjBBu0GcKGyiW6MqQwjF+BXvWoIRuwv99x2cpOjN
-fAHk9Y9IgJlzWmrTpg1i5sqTNKgmQaKqarJqXwvuq86kJuCMkYr0SxleUlAmbzpC
-w25JAMcNqabQLDaTJfV6V5EWAh3sHh+lAcL+QKs2Ddx5K9RwwCnNXtIhatECsu0n
-oNDeWcFtaLmGVF/zXpkCkKBg0EsEcFIXZM662TaGCqukIktq61/UsuR7hVQdPXyH
-/4rJTITQAXbi+FaQXzrrxKrikU7Rdovn6az5YNXBXh0pMQ9FrahxhYhDrpzpCsNc
-STL2JN50u3X28ZgfRs7rB5fY9Xr5pERIAFSrNzFPCHzlStoYNSphJP8lO/XjNkcn
-CTOzZrzxVpzqqzXlmArCfx3QtVo+WUUUW9hdOBgKLl19Wt0x5ZvYDPm6ODht9BjD
-wfbmkRvdP9CTICBR17l2tPiGcu+4paTKBF6W6aSxQU7Z48uZP19/4cuOiITbcYnt
-4wHoNk+M5G9SjLSsOlfmYNMcl0D57AOcbyUUTMZ2Wu0zc3/StJ9aotzEcFSxA1Gs
-wPZr/2iqeEynJmUQeFjs01JSwxwYlLF+HOStdn6MEbsW4ftxsXO9Mr2NGAOTaO1/
-Y8pyMCRJKOH6kRvHPpzza/stIZQQZk+PMRzS3PgMXsnJoF7cuanrWXJTgiNT6B37
-y/ARNr3KtZTGOjc5gE+k32hPuqPi5tVKPrFVUrG2l/87Gqjs4JqNDTjTHBlUIViT
-y87uB2f8/8/V50lXf+RYGPKaJ+jzbyaUxp6QRZdqJk8CEZQnrGwpUEzBVMREd+Ow
-jn99f/Isn5Vadsdmt/aMy7+TnLVyjPqucIo8dUdkWdfLKF1Q7M06rYgu9MCrsbgi
-hGp1/YjqSmYxWZuSUZHvd4ps2x9YI6wuwdWXAH/7+L0IVUghpSNoARrsmi/D7Rg5
-4zF6eCOqH13z1luMRg1ZMpRyVQcI/Yu3IbyepnuIeYIqPklDJ6jl3kIdubEtXZTV
-1XMlTvZjFuRKLeUXsF/ecTmlob+rc06YvOu91vwpxSMkpYUIc/Yt4tTWey1AbmTG
-t4pf8q0cVXrVRuXf8HFEEqQlXumJGWi/WCxAawj35znEIquQDu9ERLgSemdTFH2z
-ncoJseApHD9Gf2UpM4Vva+XdI9wrlYURr83/gOxenCgMq7wCWH87yuCG1IsOryPx
-1DbES6axy6P+XBQJTDHK560zKSNF+3VVBGEeTQRMAnQs/IHlwWu7Af0DTmUAEO2Z
-S1R+Ex2HWMM3i8/Sr8P1JDkm0u7gn2wkfI0xEIf5B10md1iniKb0GovMlfMGJTTo
-+mytGTp3XUZl8lXDpbLTR9nqfpB+hL5Qi30Dv08CAQdvX9OzOEQz5T//t13wAKlo
-bsbzwIuLhyYbua+wz/3uMS6QAbNNVCgIm/J1rqlwLRmzvSTJyhraamjd2UuwwIau
-fgdIA5bYqdagzbUuIk8msItfUrAbhxkyBElVb5RgLVJsUbBQTHW3anrbsTxxMrcR
-9jnZZV2Is2ahbnmcqqIjr7fxSqTCtwpyBxRkW8Bh+Sfu2R8x+DLcJ+ID25fbxWto
-QrDSGEifpW/5bjGx4lM0j8IxoWqOl0600hlzGsD5ynSu5WPpXQ0CMvuuLXOR4CbZ
-kzE5kQXAeQzfwwPcWMMhwt5F4hXAXY/+jM0TfuSc7Z9PYYRWdoDg3awRNlQJS5rk
-wDwbMHKUcdi4XSwIhEdbZ4zBo0o7xXKn2XSBNy+Ym39pKWMwOrffsPtWDV+qTHpD
-P5rhXEW8aD01kMkxjO2pRZ7nEDQ7VGpi3MuW9FAjXZjcc1grnZSxbKz437K9obI7
-zHXGcodAXjU4Oxhs/m8JHVaHsDG5LNxLSmArsmbwcu65mlBCXcefcwOUNoVQDmZG
-ENSAMlVAUX7RJ9nxXI9AqKKuVAYSMznseY3V22BCDrRR0nbVkAWjuBTKU6ajfjQ6
-W2uhAzl8kH+9Xi10jt5Fhn93b8z4VjrcpZmW/gJuEHm8FY2UFF0P1y3YMCDqCPaz
-gXsFZVKPPIswg/xBHB9GhiCSHnvivu/K8Ni4FcBW7smBHuqkk0VoN7CdZvNF0CuX
-EANCwDD53hSmfyxR/SY6y1PQCEsX5AiRS8rMGV2bqOjP6/oM9+2JmkRZ3I6Jht2f
-sxJsRUTyjecCcwnGDgmLDCZDJqLJoOE6zlRuAOvY1U7m2cFrf9EouOlYUXSeBaEh

tests/selenium.js

@@ -1,15 +1,19 @@
 (function(){
     "use strict;";
-    var Bacon = require('baconjs').Bacon,
+    var wd = require('wd'),
-        wd = require('wd'),
         http = require("http"),
         https = require("https"),
         url = require("url"),
         path = require("path"),
         base64_arraybuffer = require('base64-arraybuffer'),
         PNG = require('png-js'),
+        Promise = require('bluebird'),
+        _ = require('lodash'),
+        humanizeDuration = require("humanize-duration"),
         fs = require("fs");
 
+    Promise.promisifyAll(fs);
+
     var port = 8080,
         colors = {
             red: "\x1b[1;31m",
@@ -19,41 +23,19 @@
             clear: "\x1b[0m"
         };
 
-    function mapStat(item) {
+    function getTests(path) {
-        return Bacon.combineTemplate({
+        return fs.readdirAsync(path).map(function(name) {
-            stat: Bacon.fromNodeCallback(fs.stat, item),
+            var filename = path + "/" + name;
-            item: item
+            return fs.statAsync(filename).then(function(stat) {
+                return stat.isDirectory() ? getTests(filename) : filename;
+            });
         });
     }
 
-    function isDirectory(item) {
-        return item.stat.isDirectory();
-    }
-
-    function getItem(item) {
-        return item.item;
-    }
-
-    function isFile(item) {
-        return !isDirectory(item);
-    }
-
-    function arrayStream(arr) {
-        return Bacon.fromArray(arr);
-    }
-
-    function getTests(path) {
-        var items = Bacon.fromNodeCallback(fs.readdir, path).flatMap(arrayStream).map(function(name) {
-            return path + "/" + name;
-        }).flatMap(mapStat);
-        return items.filter(isFile).map(getItem).merge(items.filter(isDirectory).map(getItem).flatMap(getTests));
-    }
-
-
     function getPixelArray(base64) {
-        return Bacon.fromCallback(function(callback) {
+        return new Promise(function(resolve) {
             var arraybuffer = base64_arraybuffer.decode(base64);
-            (new PNG(arraybuffer)).decodePixels(callback);
+            (new PNG(arraybuffer)).decodePixels(resolve);
         });
     }
 
@@ -67,154 +49,104 @@
         return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100));
     }
 
-    function findResult(testName, tests) {
+    function initBrowser(settings) {
-        var item = null;
+        var browser = wd.remote({
-        return tests.some(function(testCase) {
+            hostname: 'localhost',
-            item = testCase;
+            port: 4445,
-            return testCase.test === testName;
+            user: process.env.SAUCE_USERNAME,
-        }) ? item : null;
+            pwd: process.env.SAUCE_ACCESS_KEY
-    }
+        }, 'promiseChain');
 
-    function compareResults(oldResults, newResults, browser) {
-        var improved = [],
-            regressed = [],
-            newItems = [];
-
-        newResults.forEach(function(testCase){
-            var testResult = testCase.result,
-                oldResult = findResult(testCase.test, oldResults),
-                oldResultValue = oldResult ? oldResult.result : null,
-                dataObject = {
-                    amount: (Math.abs(testResult - oldResultValue) < 0.01) ? 0 : testResult - oldResultValue,
-                    test: testCase.test
-                };
-            if (oldResultValue === null) {
-                newItems.push(dataObject);
-            } else if (dataObject.amount > 0) {
-                improved.push(dataObject);
-            } else if (dataObject.amount < 0) {
-                regressed.push(dataObject);
-            }
-        });
-
-        reportChanges(browser, improved, regressed, newItems);
-    }
-
-    function reportChanges(browser, improved, regressed, newItems) {
-        if (newItems.length > 0 || improved.length > 0 || regressed.length > 0) {
-            console.log((regressed.length > 0) ? colors.red : colors.green, browser);
-
-            regressed.forEach(function(item) {
-                console.log(colors.red, item.amount + "%", item.test);
-            });
-
-            improved.forEach(function(item) {
-                console.log(colors.green, item.amount + "%", item.test);
-            });
-
-            newItems.forEach(function(item) {
-                console.log(colors.blue, "NEW", item.test);
-            });
-        }
-    }
-
-    function formatResultName(navigator) {
-        return (navigator.browserName + "-" + ((navigator.version) ? navigator.version : "release") + "-" + navigator.platform).replace(/ /g, "").toLowerCase();
-    }
-
-    function webdriverStream(test) {
-        var browser = wd.remote("localhost", 4445, process.env.SAUCE_USERNAME, process.env.SAUCE_ACCESS_KEY);
-        var browserStream = new Bacon.Bus();
         if (process.env.TRAVIS_JOB_NUMBER) {
-            test.capabilities["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER;
+            settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER;
-            test.capabilities["name"] = process.env.TRAVIS_COMMIT.substring(0, 10);
+            settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10);
-            test.capabilities["build"] = process.env.TRAVIS_BUILD_NUMBER;
+            settings["build"] = process.env.TRAVIS_BUILD_NUMBER;
         } else {
-            test.capabilities["name"] = "Manual run";
+            settings["name"] = "Manual run";
         }
 
-        var resultStream = Bacon.fromNodeCallback(browser, "init", test.capabilities)
+        return browser.resolve(Promise).init(settings).setImplicitWaitTimeout(15000).then(function() {
-            .flatMap(Bacon.fromNodeCallback(browser, "setImplicitWaitTimeout", 15000)
+            return settings;
-            .flatMap(function() {
-                Bacon.later(0, formatResultName(test.capabilities)).onValue(browserStream.push);
-                return Bacon.fromArray(test.cases).zip(browserStream.take(test.cases.length)).flatMap(function(options) {
-                    var testCase = options[0];
-                    var name = options[1];
-                    console.log(colors.green, "STARTING", name, testCase, colors.clear);
-                    return Bacon.fromNodeCallback(browser, "get", "http://localhost:" + port + "/" + testCase + "?selenium")
-                        .flatMap(Bacon.combineTemplate({
-                            dataUrl: Bacon.fromNodeCallback(browser, "elementByCssSelector", ".html2canvas").flatMap(function(canvas) {
-                                return Bacon.fromNodeCallback(browser, "execute", "return arguments[0].toDataURL('image/png').substring(22)", [canvas]);
-                            }),
-                            screenshot: Bacon.fromNodeCallback(browser, "takeScreenshot")
-                        }))
-                        .flatMap(function(result) {
-                            return Bacon.combineTemplate({
-                                browser: name,
-                                testCase: testCase,
-                                accuracy: Bacon.constant(result.dataUrl).flatMap(getPixelArray).combine(Bacon.constant(result.screenshot).flatMap(getPixelArray), calculateDifference),
-                                dataUrl: result.dataUrl,
-                                screenshot: result.screenshot
-                            });
-                        });
-                });
-            }));
-
-        resultStream.onError(function(error) {
-            var name = formatResultName(test.capabilities);
-            console.log(colors.red, "ERROR", name, error.message);
-            browserStream.push(name);
-            browser.quit();
         });
+    }
 
-        resultStream.onValue(function(result) {
+    function loadTestPage(browser, test) {
-            console.log(colors.green, "COMPLETE", result.browser, result.testCase, result.accuracy, "%", colors.clear);
+        return function(settings) {
-            browserStream.push(result.browser);
+            return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() {
-        });
+                return settings;
-
-        return resultStream.fold([], pushToArray).flatMap(function(value) {
-            return Bacon.fromCallback(function(callback) {
-                browser.quit(function() {
-                    callback(value);
-                });
             });
-        });
+        };
     }
 
-    function createImages(data) {
+    function captureScreenshots(browser) {
-        var dataurlFileName = "tests/results/" + data.browser + "-" +  data.testCase.replace(/\//g, "-") + "-html2canvas.png";
+        return function() {
-        var screenshotFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-screencapture.png";
+            return Promise.props({
-        return Bacon.combineTemplate({
+                dataUrl: browser.elementByCss(".html2canvas").then(function(node) {
-            name: data.testCase,
+                    return browser.execute("return arguments[0].toDataURL('image/png').substring(22)", [node]);
-            dataurl: Bacon.fromNodeCallback(fs.writeFile, dataurlFileName, data.dataUrl, "base64").map(function() {
+                }),
-                return dataurlFileName;
+                screenshot: browser.takeScreenshot()
-            }),
+            });
-            screenshot: Bacon.fromNodeCallback(fs.writeFile, screenshotFileName, data.screenshot, "base64").map(function() {
+        };
-                return screenshotFileName;
-            })
-        });
     }
 
-    function pushToArray(array, item) {
+    function analyzeResults(test) {
-        array.push(item);
+        return function(result) {
-        return array;
+            return Promise.all([getPixelArray(result.dataUrl), getPixelArray(result.screenshot)]).spread(calculateDifference).then(function(accuracy) {
+                return {
+                    testCase: test,
+                    accuracy: accuracy,
+                    dataUrl: result.dataUrl,
+                    screenshot: result.screenshot
+                };
+            });
+        };
     }
 
-    function runWebDriver(browsers, cases) {
+    function runTestWithRetries(browser, test, retries) {
-        var availableBrowsers = new Bacon.Bus();
+        retries = retries || 0;
-        var result = Bacon.combineTemplate({
+        return runTest(browser, test)
-            capabilities: Bacon.fromArray(browsers).zip(availableBrowsers.take(browsers.length), function(first) { return first; }),
+            .timeout(60000)
-            cases: cases
+            .catch(Promise.TimeoutError, function() {
-        }).flatMap(webdriverStream).doAction(function() {
+                if (retries < 3) {
-            availableBrowsers.push("ready");
+                    console.log("Retry", (retries + 1), test);
-        });
+                    return runTestWithRetries(browser, test, retries + 1);
+                } else {
+                    throw new Error("Couldn't run test after 3 retries");
+                }
+            });
+    }
 
-        Bacon.fromArray([1, 2, 3, 4]).onValue(availableBrowsers.push);
+    function runTest(browser, test) {
-
+        return browser
-        return result.fold([], pushToArray);
+            .then(loadTestPage(browser, test))
+            .then(captureScreenshots(browser))
+            .then(analyzeResults(test));
     }
 
     exports.tests = function(browsers, singleTest) {
-        return (singleTest ? Bacon.constant([singleTest]) : getTests("tests/cases").fold([], pushToArray)).flatMap(runWebDriver.bind(null, browsers)).mapError(false);
+        var path = "tests/cases";
+        return (singleTest ? Promise.resolve([singleTest]) : getTests(path)).then(function(t) {
+            var tests = _.flatten(t);
+            return Promise.map(browsers, function(settings) {
+                var browser = initBrowser(settings);
+                var name = [settings.browserName, settings.version, settings.platform].join("-");
+                var count = 0;
+                return Promise.map(tests, function(test, index, total) {
+                    console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear);
+                    var start = Date.now();
+                    return runTestWithRetries(browser, test).then(function(result) {
+                        console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, result.testCase.substring(path.length), result.accuracy.toFixed(2) + "%", colors.clear);
+                    });
+                }, {concurrency: 1})
+                .settle()
+                .catch(function(error) {
+                    console.log(colors.red, "ERROR", name, error.message);
+                    throw error;
+                })
+                .finally(function() {
+                    return browser.quit();
+                });
+            }, {concurrency: 3});
+        });
     };
 })();