diff --git a/Gruntfile.js b/Gruntfile.js index 716cf6d..b0e54cb 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -13,7 +13,48 @@ module.exports = function(grunt) { post: '\n}).call({}, window, document);' }; - // Project configuration. + var browsers = { + chrome: { + browserName: "chrome", + platform: "Windows 7", + version: "39" + }, + firefox: { + browserName: "firefox", + version: "15", + platform: "Windows 7" + }, + ie9: { + browserName: "internet explorer", + version: "9", + platform: "Windows 7" + }, + ie10: { + browserName: "internet explorer", + version: "10", + platform: "Windows 8" + }, + ie11: { + browserName: "internet explorer", + version: "11", + platform: "Windows 8.1" + }, + safari6: { + browserName: "safari", + version: "6", + platform: "OS X 10.8" + }, + safari7:{ + browserName: "safari", + platform: "OS X 10.9", + version: "7" + }, + chromeOSX:{ + browserName: "chrome", + platform: "OS X 10.8", + version: "39" + } + }; grunt.initConfig({ pkg: grunt.file.readJSON('package.json'), @@ -124,48 +165,8 @@ module.exports = function(grunt) { mocha_phantomjs: { all: ['tests/mocha/**/*.html'] }, - webdriver: { - chrome: { - browserName: "chrome", - platform: "Windows 7", - version: "34" - }, - firefox: { - browserName: "firefox", - version: "15", - platform: "Windows 7" - }, - ie9: { - browserName: "internet explorer", - version: "9", - platform: "Windows 7" - }, - ie10: { - browserName: "internet explorer", - version: "10", - platform: "Windows 8" - }, - ie11: { - browserName: "internet explorer", - version: "11", - platform: "Windows 8.1" - }, - safari6: { - browserName: "safari", - version: "6", - platform: "OS X 10.8" - }, - safari7:{ - browserName: "safari", - platform: "OS X 10.9", - version: "7" - }, - chromeOSX:{ - browserName: "chrome", - platform: "OS X 10.8", - version: "34" - } - } + mocha_webdriver: browsers, + webdriver: browsers }); grunt.registerTask('webdriver', 'Browser render tests', function(browser, test) { @@ -180,6 +181,17 @@ module.exports = function(grunt) { }); }); + grunt.registerTask('mocha_webdriver', 'Browser mocha tests', function(browser, test) { + var selenium = require("./tests/mocha/selenium.js"); + var done = this.async(); + var browsers = (browser) ? [grunt.config.get(this.name + "." + browser)] : _.values(grunt.config.get(this.name)); + selenium.tests(browsers, test).catch(function() { + done(false); + }).finally(function() { + done(); + }); + }); + grunt.loadNpmTasks('grunt-mocha-phantomjs'); grunt.loadNpmTasks('grunt-contrib-watch'); grunt.loadNpmTasks('grunt-contrib-concat'); diff --git a/tests/mocha/background.html b/tests/mocha/background.html index 2a06789..7ecb5d9 100644 --- a/tests/mocha/background.html +++ b/tests/mocha/background.html @@ -26,6 +26,7 @@ width: 200px; height: 200px; background-image: -webkit-linear-gradient(top, #008000, #008000); + background-image: -moz-linear-gradient(to bottom, #008000, #008000); background-image: linear-gradient(to bottom, #008000, #008000); } @@ -126,6 +127,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/colors.html b/tests/mocha/colors.html index 1a2f3aa..4e2a541 100644 --- a/tests/mocha/colors.html +++ b/tests/mocha/colors.html @@ -136,6 +136,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/cropping.html b/tests/mocha/cropping.html index 06cebfb..fbb1444 100644 --- a/tests/mocha/cropping.html +++ b/tests/mocha/cropping.html @@ -136,6 +136,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/form-rendering.html b/tests/mocha/form-rendering.html index bdddac1..dbd4e3a 100644 --- a/tests/mocha/form-rendering.html +++ b/tests/mocha/form-rendering.html @@ -158,6 +158,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/gradients.js b/tests/mocha/gradients.js index c74b808..98638c0 100644 --- a/tests/mocha/gradients.js +++ b/tests/mocha/gradients.js @@ -102,6 +102,9 @@ describe("Gradients", function() { var value = container.css("backgroundImage"); it(value, function() { var parsedBackground = parseBackgrounds(value); + if (parsedBackground[0].args[0] === "0% 50%") { + parsedBackground[0].args[0] = 'left'; + } expect(parsedBackground[0].args).to.eql(expected[i].args); expect(parsedBackground[0].method).to.eql(expected[i].method); }); diff --git a/tests/mocha/multiple-renders.html b/tests/mocha/multiple-renders.html index 75b3095..5b36062 100644 --- a/tests/mocha/multiple-renders.html +++ b/tests/mocha/multiple-renders.html @@ -27,7 +27,7 @@ diff --git a/tests/mocha/parsing.html b/tests/mocha/parsing.html index 81a38c1..9d8156a 100644 --- a/tests/mocha/parsing.html +++ b/tests/mocha/parsing.html @@ -191,6 +191,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/scrolling.html b/tests/mocha/scrolling.html index 83c7f86..b27550d 100644 --- a/tests/mocha/scrolling.html +++ b/tests/mocha/scrolling.html @@ -61,6 +61,9 @@ else { mocha.run(); } + mocha.suite.afterAll(function() { + document.body.setAttribute('data-complete', 'true'); + }); diff --git a/tests/mocha/selenium.js b/tests/mocha/selenium.js new file mode 100644 index 0000000..79e152f --- /dev/null +++ b/tests/mocha/selenium.js @@ -0,0 +1,73 @@ +var wd = require('wd'); +var http = require("http"); +var https = require("https"); +var url = require("url"); +var path = require("path"); +var Promise = require('bluebird'); +var _ = require('lodash'); +var humanizeDuration = require("humanize-duration"); +var utils = require('../utils'); +var colors = utils.colors; +var port = 8080; + +function runTestWithRetries(browser, test, retries) { + retries = retries || 0; + return runTest(browser, test) + .timeout(30000) + .catch(Promise.TimeoutError, function() { + if (retries < 3) { + console.log(colors.violet, "Retry", (retries + 1), test); + return runTestWithRetries(browser, test, retries + 1); + } else { + throw new Error("Couldn't run test after 3 retries"); + } + }); +} + +function getResults(browser) { + return function() { + return Promise.props({ + dataUrl: browser.waitForElementByCss("body[data-complete='true']", 90000).then(function() { + return browser.elementsByCssSelector('.test.fail'); + }).then(function(nodes) { + return Array.isArray(nodes) ? Promise.map(nodes, function(node) { + return browser.text(node).then(function(error) { + return Promise.reject(error); + }); + }) : Promise.resolve([]); + }) + }); + }; +} + +function runTest(browser, test) { + return Promise.resolve(browser + .then(utils.loadTestPage(browser, test, port)) + .then(getResults(browser)) + ).cancellable(); +} + +exports.tests = function(browsers, singleTest) { + var path = "tests/mocha"; + return (singleTest ? Promise.resolve([singleTest]) : utils.getTests(path)).then(function(tests) { + return Promise.map(browsers, function(settings) { + var name = [settings.browserName, settings.version, settings.platform].join("-"); + var count = 0; + var browser = utils.initBrowser(settings); + return Promise.using(browser, function() { + return Promise.map(tests, function(test, index, total) { + console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear); + var start = Date.now(); + return runTestWithRetries(browser, test).then(function() { + console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, colors.clear); + }); + }, {concurrency: 1}) + .settle() + .catch(function(error) { + console.error(colors.red, "ERROR", name, error); + throw error; + }); + }); + }, {concurrency: 3}); + }); +}; diff --git a/tests/selenium.js b/tests/selenium.js index 5417ba0..678b98f 100644 --- a/tests/selenium.js +++ b/tests/selenium.js @@ -12,25 +12,12 @@ humanizeDuration = require("humanize-duration"), fs = require("fs"); + var utils = require('./utils'); + var colors = utils.colors; + Promise.promisifyAll(fs); - var port = 8080, - colors = { - red: "\x1b[1;31m", - blue: "\x1b[1;36m", - violet: "\x1b[0;35m", - green: "\x1b[0;32m", - clear: "\x1b[0m" - }; - - function getTests(path) { - return fs.readdirAsync(path).map(function(name) { - var filename = path + "/" + name; - return fs.statAsync(filename).then(function(stat) { - return stat.isDirectory() ? getTests(filename) : filename; - }); - }); - } + var port = 8080; function getPixelArray(base64) { return new Promise(function(resolve) { @@ -49,39 +36,10 @@ return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); } - function initBrowser(settings) { - var browser = wd.remote({ - hostname: 'localhost', - port: 4445, - user: process.env.SAUCE_USERNAME, - pwd: process.env.SAUCE_ACCESS_KEY - }, 'promiseChain'); - - if (process.env.TRAVIS_JOB_NUMBER) { - settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER; - settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10); - settings["build"] = process.env.TRAVIS_BUILD_NUMBER; - } else { - settings["name"] = "Manual run"; - } - - return browser.resolve(Promise).init(settings).setImplicitWaitTimeout(15000).then(function() { - return settings; - }); - } - - function loadTestPage(browser, test) { - return function(settings) { - return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() { - return settings; - }); - }; - } - function captureScreenshots(browser) { return function() { return Promise.props({ - dataUrl: browser.elementByCss(".html2canvas").then(function(node) { + dataUrl: browser.waitForElementByCss(".html2canvas", 15000).then(function(node) { return browser.execute("return arguments[0].toDataURL('image/png').substring(22)", [node]); }), screenshot: browser.takeScreenshot() @@ -118,33 +76,31 @@ function runTest(browser, test) { return Promise.resolve(browser - .then(loadTestPage(browser, test)) + .then(utils.loadTestPage(browser, test, port)) .then(captureScreenshots(browser)) .then(analyzeResults(test))).cancellable(); } exports.tests = function(browsers, singleTest) { var path = "tests/cases"; - return (singleTest ? Promise.resolve([singleTest]) : getTests(path)).then(function(t) { - var tests = _.flatten(t); + return (singleTest ? Promise.resolve([singleTest]) : utils.getTests(path)).then(function(tests) { return Promise.map(browsers, function(settings) { - var browser = initBrowser(settings); + var browser = utils.initBrowser(settings); var name = [settings.browserName, settings.version, settings.platform].join("-"); var count = 0; - return Promise.map(tests, function(test, index, total) { - console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear); - var start = Date.now(); - return runTestWithRetries(browser, test).then(function(result) { - console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, result.testCase.substring(path.length), result.accuracy.toFixed(2) + "%", colors.clear); - }); - }, {concurrency: 1}) - .settle() - .catch(function(error) { - console.log(colors.red, "ERROR", name, error.message); - throw error; - }) - .finally(function() { - return browser.quit(); + return Promise.using(browser, function() { + return Promise.map(tests, function(test, index, total) { + console.log(colors.green, "STARTING", "(" + (++count) + "/" + total + ")", name, test, colors.clear); + var start = Date.now(); + return runTestWithRetries(browser, test).then(function(result) { + console.log(colors.green, "COMPLETE", humanizeDuration(Date.now() - start), "(" + count + "/" + total + ")", name, result.testCase.substring(path.length), result.accuracy.toFixed(2) + "%", colors.clear); + }); + }, {concurrency: 1}) + .settle() + .catch(function(error) { + console.log(colors.red, "ERROR", name, error.message); + throw error; + }); }); }, {concurrency: 3}); }); diff --git a/tests/utils.js b/tests/utils.js new file mode 100644 index 0000000..db1e3e5 --- /dev/null +++ b/tests/utils.js @@ -0,0 +1,66 @@ +var fs = require('fs'); +var wd = require('wd'); +var path = require('path'); +var Promise = require('bluebird'); +var _ = require('lodash'); + +Promise.promisifyAll(fs); + +var colors = { + red: "\x1b[1;31m", + blue: "\x1b[1;36m", + violet: "\x1b[0;35m", + green: "\x1b[0;32m", + clear: "\x1b[0m" +}; + +function isHtmlFile(filename) { + return path.extname(filename) === '.html'; +} + +function getTests(path) { + return fs.readdirAsync(path).map(function(name) { + var filename = path + "/" + name; + return fs.statAsync(filename).then(function(stat) { + return stat.isDirectory() ? getTests(filename) : filename; + }); + }).then(function(t) { + return _.flatten(t).filter(isHtmlFile); + }); +} + +function initBrowser(settings) { + var browser = wd.remote({ + hostname: 'localhost', + port: 4445, + user: process.env.SAUCE_USERNAME, + pwd: process.env.SAUCE_ACCESS_KEY + }, 'promiseChain'); + + if (process.env.TRAVIS_JOB_NUMBER) { + settings["tunnel-identifier"] = process.env.TRAVIS_JOB_NUMBER; + settings["name"] = process.env.TRAVIS_COMMIT.substring(0, 10); + settings["build"] = process.env.TRAVIS_BUILD_NUMBER; + } else { + settings["name"] = "Manual run"; + } + + return browser.resolve(Promise).init(settings).then(function(b) { + return Promise.resolve(b).disposer(function() { + return browser.quit(); + }); + }); +} + +function loadTestPage(browser, test, port) { + return function(settings) { + return browser.get("http://localhost:" + port + "/" + test + "?selenium").then(function() { + return settings; + }); + }; +} + +module.exports.colors = colors; +module.exports.getTests = getTests; +module.exports.initBrowser = initBrowser; +module.exports.loadTestPage = loadTestPage;