Add task for running mocha tests with webdriver for testing proxies

Gruntfile.js

@@ -13,7 +13,48 @@ module.exports = function(grunt) {
         post: '\n}).call({}, window, document);'
     };
 
-    // Project configuration.
+    var browsers = {
+        chrome: {
+            browserName: "chrome",
+            platform: "Windows 7",
+            version: "39"
+        },
+        firefox: {
+            browserName: "firefox",
+            version: "15",
+            platform: "Windows 7"
+        },
+        ie9: {
+            browserName: "internet explorer",
+            version: "9",
+            platform: "Windows 7"
+        },
+        ie10: {
+            browserName: "internet explorer",
+            version: "10",
+            platform: "Windows 8"
+        },
+        ie11: {
+            browserName: "internet explorer",
+            version: "11",
+            platform: "Windows 8.1"
+        },
+        safari6: {
+            browserName: "safari",
+            version: "6",
+            platform: "OS X 10.8"
+        },
+        safari7:{
+            browserName: "safari",
+            platform: "OS X 10.9",
+            version: "7"
+        },
+        chromeOSX:{
+            browserName: "chrome",
+            platform: "OS X 10.8",
+            version: "39"
+        }
+    };
     grunt.initConfig({
 
         pkg: grunt.file.readJSON('package.json'),
@@ -124,48 +165,8 @@ module.exports = function(grunt) {
         mocha_phantomjs: {
             all: ['tests/mocha/**/*.html']
         },
-        webdriver: {
+        mocha_webdriver: browsers,
-            chrome: {
+        webdriver: browsers
-                browserName: "chrome",
-                platform: "Windows 7",
-                version: "34"
-            },
-            firefox: {
-                browserName: "firefox",
-                version: "15",
-                platform: "Windows 7"
-            },
-            ie9: {
-                browserName: "internet explorer",
-                version: "9",
-                platform: "Windows 7"
-            },
-            ie10: {
-                browserName: "internet explorer",
-                version: "10",
-                platform: "Windows 8"
-            },
-            ie11: {
-                browserName: "internet explorer",
-                version: "11",
-                platform: "Windows 8.1"
-            },
-            safari6: {
-                browserName: "safari",
-                version: "6",
-                platform: "OS X 10.8"
-            },
-            safari7:{
-                browserName: "safari",
-                platform: "OS X 10.9",
-                version: "7"
-            },
-            chromeOSX:{
-                browserName: "chrome",
-                platform: "OS X 10.8",
-                version: "34"
-            }
-        }
     });
 
     grunt.registerTask('webdriver', 'Browser render tests', function(browser, test) {
@@ -180,6 +181,17 @@ module.exports = function(grunt) {
         });
     });
 
+    grunt.registerTask('mocha_webdriver', 'Browser mocha tests', function(browser, test) {
+        var selenium = require("./tests/mocha/selenium.js");
+        var done = this.async();
+        var browsers = (browser) ? [grunt.config.get(this.name + "." + browser)] : _.values(grunt.config.get(this.name));
+        selenium.tests(browsers, test).catch(function() {
+            done(false);
+        }).finally(function() {
+            done();
+        });
+    });
+
     grunt.loadNpmTasks('grunt-mocha-phantomjs');
     grunt.loadNpmTasks('grunt-contrib-watch');
     grunt.loadNpmTasks('grunt-contrib-concat');

tests/mocha/background.html

@@ -26,6 +26,7 @@
             width: 200px;
             height: 200px;
             background-image: -webkit-linear-gradient(top, #008000, #008000);
+            background-image:    -moz-linear-gradient(to bottom, #008000, #008000);
             background-image:         linear-gradient(to bottom, #008000, #008000);
         }
     </style>
@@ -126,6 +127,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/colors.html

@@ -136,6 +136,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/cropping.html

@@ -136,6 +136,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/form-rendering.html

@@ -158,6 +158,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/gradients.js

@@ -102,6 +102,9 @@ describe("Gradients", function() {
         var value = container.css("backgroundImage");
         it(value, function() {
             var parsedBackground = parseBackgrounds(value);
+            if (parsedBackground[0].args[0] === "0% 50%") {
+                parsedBackground[0].args[0] = 'left';
+            }
             expect(parsedBackground[0].args).to.eql(expected[i].args);
             expect(parsedBackground[0].method).to.eql(expected[i].method);
         });

tests/mocha/multiple-renders.html

@@ -27,7 +27,7 @@
 <script>
     describe("Multiple renders", function() {
         it("render correctly", function(done) {
-            this.timeout(5000);
+            this.timeout(10000);
             var d = 0;
             var count = 3;
             for (var i = 0; i < count; i++) {
@@ -45,7 +45,7 @@
         });
 
         it("render correctly when non sequential", function(done) {
-            this.timeout(5000);
+            this.timeout(10000);
             var d = 0;
             var count = 3;
             for (var i = 0; i < count; i++) {
@@ -73,13 +73,18 @@
     function validCanvasPixels(canvas) {
         var ctx = canvas.getContext("2d");
         var data = ctx.getImageData(0, 0, canvas.width, canvas.height).data;
-        for (var i = 0, len = data.length; i < len; i+=4) {
+        for (var i = 0, len = 200*199*4; i < len; i+=4) {
             if (data[i] !== 0 || data[i+1] !== 128 || data[i+2] !== 0 || data[i+3] !== 255) {
+                console.log(i, data[i], data[i+1], data[i+2], data[i+3]);
                 expect().fail("Invalid canvas data");
             }
         }
     }
 
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
+
     mocha.checkLeaks();
     mocha.globals(['jQuery']);
     if (window.mochaPhantomJS) {

tests/mocha/options.onclone.html

@@ -66,6 +66,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/parsing.html

@@ -191,6 +191,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/scrolling.html

@@ -61,6 +61,9 @@
     else {
         mocha.run();
     }
+    mocha.suite.afterAll(function() {
+        document.body.setAttribute('data-complete', 'true');
+    });
 </script>
 </body>
 </html>

tests/mocha/selenium.js

@@ -0,0 +1,73 @@
+var wd = require('wd');
+var http = require("http");
+var https = require("https");
+var url = require("url");
+var path = require("path");
+var Promise = require('bluebird');
+var _ = require('lodash');
+var humanizeDuration = require("humanize-duration");
+var utils = require('../utils');
+var colors = utils.colors;
+var port = 8080;
+
+function runTestWithRetries(browser, test, retries) {
+    retries = retries || 0;
+    return runTest(browser, test)
+        .timeout(30000)
+        .catch(Promise.TimeoutError, function() {
+            if (retries < 3) {
+                console.log(colors.violet, "Retry", (retries + 1), test);
+                return runTestWithRetries(browser, test, retries + 1);
+            } else {
+                throw new Error("Couldn't run test after 3 retries");
+            }
+        });
+}
+
+function getResults(browser) {
+    return function() {
+        return Promise.props({
+            dataUrl: browser.waitForElementByCss("body[data-complete='true']", 90000).then(function() {
+                return browser.elementsByCssSelector('.test.fail');
+            }).then(function(nodes) {
+                return Array.isArray(nodes) ? Promise.map(nodes, function(node) {
+                    return browser.text(node).then(function(error) {
+                        return Promise.reject(error);
+                    });
+                }) : Promise.resolve([]);
+            })
+        });
+    };
+}
+
+function runTest(browser, test) {
+    return Promise.resolve(browser
+        .then(utils.loadTestPage(browser, test, port))
+        .then(getResults(browser))
+    ).cancellable();
+}
+
+exports.tests = function(browsers, singleTest) {
+    var path = "tests/mocha";
+    return (singleTest ? Promise.resolve([singleTest]) : utils.getTests(path)).then(function(tests) {
+        return Promise.map(browsers, function(settings) {
+            var name = [settings.browserName, settings.version, settings.platform].join("-");
+            var count = 0;
+            var browser = utils.initBrowser(settings);
+            return Promise.using(browser, function() {
+                return Promise.map(tests, function(test, index, total) {
+                    console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear);
+                    var start = Date.now();
+                    return runTestWithRetries(browser, test).then(function() {
+                        console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, colors.clear);
+                    });
+                }, {concurrency: 1})
+                    .settle()
+                    .catch(function(error) {
+                        console.error(colors.red, "ERROR", name, error);
+                        throw error;
+                    });
+            });
+        }, {concurrency: 3});
+    });
+};

tests/selenium.js

@@ -12,25 +12,12 @@
         humanizeDuration = require("humanize-duration"),
         fs = require("fs");
 
+    var utils = require('./utils');
+    var colors = utils.colors;
+
     Promise.promisifyAll(fs);
 
-    var port = 8080,
+    var port = 8080;
-        colors = {
-            red: "\x1b[1;31m",
-            blue: "\x1b[1;36m",
-            violet: "\x1b[0;35m",
-            green: "\x1b[0;32m",
-            clear: "\x1b[0m"
-        };
-
-    function getTests(path) {
-        return fs.readdirAsync(path).map(function(name) {
-            var filename = path + "/" + name;
-            return fs.statAsync(filename).then(function(stat) {
-                return stat.isDirectory() ? getTests(filename) : filename;
-            });
-        });
-    }
 
     function getPixelArray(base64) {
         return new Promise(function(resolve) {
@@ -49,39 +36,10 @@
         return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100));
     }
 
-    function initBrowser(settings) {
-        var browser = wd.remote({
-            hostname: 'localhost',
-            port: 4445,
-            user: process.env.SAUCE_USERNAME,
-            pwd: process.env.SAUCE_ACCESS_KEY
-        }, 'promiseChain');
-
-        if (process.env.TRAVIS_JOB_NUMBER) {
-            settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER;
-            settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10);
-            settings["build"] = process.env.TRAVIS_BUILD_NUMBER;
-        } else {
-            settings["name"] = "Manual run";
-        }
-
-        return browser.resolve(Promise).init(settings).setImplicitWaitTimeout(15000).then(function() {
-            return settings;
-        });
-    }
-
-    function loadTestPage(browser, test) {
-        return function(settings) {
-            return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() {
-                return settings;
-            });
-        };
-    }
-
     function captureScreenshots(browser) {
         return function() {
             return Promise.props({
-                dataUrl: browser.elementByCss(".html2canvas").then(function(node) {
+                dataUrl: browser.waitForElementByCss(".html2canvas", 15000).then(function(node) {
                     return browser.execute("return arguments[0].toDataURL('image/png').substring(22)", [node]);
                 }),
                 screenshot: browser.takeScreenshot()
@@ -118,19 +76,19 @@
 
     function runTest(browser, test) {
         return Promise.resolve(browser
-            .then(loadTestPage(browser, test))
+            .then(utils.loadTestPage(browser, test, port))
             .then(captureScreenshots(browser))
             .then(analyzeResults(test))).cancellable();
     }
 
     exports.tests = function(browsers, singleTest) {
         var path = "tests/cases";
-        return (singleTest ? Promise.resolve([singleTest]) : getTests(path)).then(function(t) {
+        return (singleTest ? Promise.resolve([singleTest]) : utils.getTests(path)).then(function(tests) {
-            var tests = _.flatten(t);
             return Promise.map(browsers, function(settings) {
-                var browser = initBrowser(settings);
+                var browser = utils.initBrowser(settings);
                 var name = [settings.browserName, settings.version, settings.platform].join("-");
                 var count = 0;
+                return Promise.using(browser, function() {
                     return Promise.map(tests, function(test, index, total) {
                         console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear);
                         var start = Date.now();
@@ -142,9 +100,7 @@
                         .catch(function(error) {
                             console.log(colors.red, "ERROR", name, error.message);
                             throw error;
-                })
+                        });
-                .finally(function() {
-                    return browser.quit();
                 });
             }, {concurrency: 3});
         });

tests/utils.js

@@ -0,0 +1,66 @@
+var fs = require('fs');
+var wd = require('wd');
+var path = require('path');
+var Promise = require('bluebird');
+var _ = require('lodash');
+
+Promise.promisifyAll(fs);
+
+var colors = {
+    red: "\x1b[1;31m",
+    blue: "\x1b[1;36m",
+    violet: "\x1b[0;35m",
+    green: "\x1b[0;32m",
+    clear: "\x1b[0m"
+};
+
+function isHtmlFile(filename) {
+    return path.extname(filename) === '.html';
+}
+
+function getTests(path) {
+    return fs.readdirAsync(path).map(function(name) {
+        var filename = path + "/" + name;
+        return fs.statAsync(filename).then(function(stat) {
+            return stat.isDirectory() ? getTests(filename) : filename;
+        });
+    }).then(function(t) {
+        return _.flatten(t).filter(isHtmlFile);
+    });
+}
+
+function initBrowser(settings) {
+    var browser = wd.remote({
+        hostname: 'localhost',
+        port: 4445,
+        user: process.env.SAUCE_USERNAME,
+        pwd: process.env.SAUCE_ACCESS_KEY
+    }, 'promiseChain');
+
+    if (process.env.TRAVIS_JOB_NUMBER) {
+        settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER;
+        settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10);
+        settings["build"] = process.env.TRAVIS_BUILD_NUMBER;
+    } else {
+        settings["name"] = "Manual run";
+    }
+
+    return browser.resolve(Promise).init(settings).then(function(b) {
+        return Promise.resolve(b).disposer(function() {
+            return browser.quit();
+        });
+    });
+}
+
+function loadTestPage(browser, test, port) {
+    return function(settings) {
+        return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() {
+            return settings;
+        });
+    };
+}
+
+module.exports.colors = colors;
+module.exports.getTests = getTests;
+module.exports.initBrowser = initBrowser;
+module.exports.loadTestPage = loadTestPage;