diff --git a/.gitignore b/.gitignore index 2ad031c..4a2b018 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ node_modules/ .envrc server.js *.sublime-workspace -chromedriver.log \ No newline at end of file +chromedriver.log +.baseline \ No newline at end of file diff --git a/tests/results/chrome-23.0.1271.97-windows-vista.json b/tests/results/chrome-23.0.1271.97-windows-vista.json new file mode 100644 index 0000000..8c8cf53 --- /dev/null +++ b/tests/results/chrome-23.0.1271.97-windows-vista.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":98,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T16:48:04.248Z","version":"23.0.1271.97"} \ No newline at end of file diff --git a/tests/results/chrome-23.0.1271.97-windows-vista.json.baseline b/tests/results/chrome-23.0.1271.97-windows-vista.json.baseline new file mode 100644 index 0000000..478cb85 --- /dev/null +++ b/tests/results/chrome-23.0.1271.97-windows-vista.json.baseline @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T17:20:56.098Z","version":"23.0.1271.97"} \ No newline at end of file diff --git a/tests/results/chrome.json b/tests/results/chrome.json new file mode 100644 index 0000000..01acad7 --- /dev/null +++ b/tests/results/chrome.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T16:48:04.248Z","version":"23.0.1271.97"} \ No newline at end of file diff --git a/tests/results/firefox-12.0-windows-vista.json b/tests/results/firefox-12.0-windows-vista.json new file mode 100644 index 0000000..ffe0de2 --- /dev/null +++ b/tests/results/firefox-12.0-windows-vista.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":92.64},"date":"2012-12-28T16:48:10.772Z","version":"12.0"} \ No newline at end of file diff --git a/tests/results/firefox.json b/tests/results/firefox.json new file mode 100644 index 0000000..5b0f7bb --- /dev/null +++ b/tests/results/firefox.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":85.64},"date":"2012-12-28T16:48:10.772Z","version":"12.0"} \ No newline at end of file diff --git a/tests/results/iexplorer-9-windows-vista.json b/tests/results/iexplorer-9-windows-vista.json new file mode 100644 index 0000000..526cd67 --- /dev/null +++ b/tests/results/iexplorer-9-windows-vista.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":100},"date":"2012-12-28T16:48:04.855Z","version":"9"} \ No newline at end of file diff --git a/tests/results/iexplorer.json b/tests/results/iexplorer.json new file mode 100644 index 0000000..526cd67 --- /dev/null +++ b/tests/results/iexplorer.json @@ -0,0 +1 @@ +{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":100},"date":"2012-12-28T16:48:04.855Z","version":"9"} \ No newline at end of file diff --git a/tests/selenium.js b/tests/selenium.js index 5e9b5fe..b8e4074 100644 --- a/tests/selenium.js +++ b/tests/selenium.js @@ -1,154 +1,223 @@ -var webdriver = require("webdriver.js").webdriver, -http = require("http"), -url = require("url"), -path = require("path"), -base64_arraybuffer = require('base64-arraybuffer'), -PNG = require('png-js'), -fs = require("fs"); +(function(){ + "use strict;" + var webdriver = require("webdriver.js").webdriver, + http = require("http"), + url = require("url"), + path = require("path"), + base64_arraybuffer = require('base64-arraybuffer'), + PNG = require('png-js'), + fs = require("fs"); + function createServer(port) { + return http.createServer(function(request, response) { + var uri = url.parse(request.url).pathname, + filename = path.join(process.cwd(), "../" + uri); -function createServer(port) { - return http.createServer(function(request, response) { - var uri = url.parse(request.url).pathname, - filename = path.join(process.cwd(), "../" + uri); - - fs.exists(filename, function(exists) { - if(!exists) { - response.writeHead(404, { - "Content-Type": "text/plain" - }); - response.write("404 Not Found\n"); - response.end(); - return; - } - - if (fs.statSync(filename).isDirectory()) filename += '/index.html'; - - fs.readFile(filename, "binary", function(err, file) { - if(err) { - response.writeHead(500, { + fs.exists(filename, function(exists) { + if(!exists) { + response.writeHead(404, { "Content-Type": "text/plain" }); - response.write(err + "\n"); + response.write("404 Not Found\n"); response.end(); return; } - response.writeHead(200); - response.write(file, "binary"); - response.end(); + if (fs.statSync(filename).isDirectory()) filename += '/index.html'; + + fs.readFile(filename, "binary", function(err, file) { + if(err) { + response.writeHead(500, { + "Content-Type": "text/plain" + }); + response.write(err + "\n"); + response.end(); + return; + } + + response.writeHead(200); + response.write(file, "binary"); + response.end(); + }); }); - }); - }).listen(port); -} + }).listen(port); + } -function walkDir(dir, done) { - var results = []; - fs.readdir(dir, function(err, list) { - if (err) return done(err); - var i = 0; - (function next() { - var file = list[i++]; - if (!file) return done(null, results); - file = dir + '/' + file; - fs.stat(file, function(err, stat) { - if (stat && stat.isDirectory()) { - walkDir(file, function(err, res) { - results = results.concat(res); + function walkDir(dir, done) { + var results = []; + fs.readdir(dir, function(err, list) { + if (err) return done(err); + var i = 0; + (function next() { + var file = list[i++]; + if (!file) return done(null, results); + file = dir + '/' + file; + fs.stat(file, function(err, stat) { + if (stat && stat.isDirectory()) { + walkDir(file, function(err, res) { + results = results.concat(res); + next(); + }); + } else { + results.push(file); next(); - }); - } else { - results.push(file); - next(); - } - }); - })(); - }); -}; - -function getPixelArray(base64, func) { - var arraybuffer = base64_arraybuffer.decode(base64); - (new PNG(arraybuffer)).decode(func); -} - - -function testPage(browser, url, done) { - - browser.url(url) - .$(".html2canvas", 5000, function(){ - this.execute(function(){ - var canvas = $('.html2canvas')[0]; - return canvas.toDataURL("image/png").substring(22); - },[], function(dataurl) { - getPixelArray(dataurl, function(h2cPixels) { - browser.screenshot(function(base64){ - getPixelArray(base64, function(screenPixels) { - var len = h2cPixels.length, index = 0, diff = 0; - for (; index < len; index++) { - if (screenPixels[index] - h2cPixels[index] !== 0) { - diff++; - } - } - done(100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); - }); - }) - }); + } + }); + })(); }); - }); -} - -function runBrowsers(pages){ - - var port = 5555, - stats = {}, - browsers = ["chrome", "firefox", "internet explorer"], - browsersDone = 0, - server = createServer(port), - numPages = pages.length; - - var browserDone = function() { - if (++browsersDone >= browsers.length) { - server.close(); - console.log(stats); - } }; - browsers.forEach(function(browserName){ - var browser = new webdriver({ - browser: browserName - }), - browserType; + function getPixelArray(base64, func) { + var arraybuffer = base64_arraybuffer.decode(base64); + (new PNG(arraybuffer)).decode(func); + } - browser.status(function(browserInfo){ - browserType = [browserName, browser.version, browserInfo.os.name.replace(/\s+/g, "-").toLowerCase()].join("-"); - var date = new Date(), - obj = { - tests: {}, - date: date.toISOString() + + function testPage(browser, url, done) { + + browser.url(url) + .$(".html2canvas", 5000, function(){ + this.execute(function(){ + var canvas = $('.html2canvas')[0]; + return canvas.toDataURL("image/png").substring(22); + },[], function(dataurl) { + getPixelArray(dataurl, function(h2cPixels) { + browser.screenshot(function(base64){ + getPixelArray(base64, function(screenPixels) { + var len = h2cPixels.length, index = 0, diff = 0; + for (; index < len; index++) { + if (screenPixels[index] - h2cPixels[index] !== 0) { + diff++; + } + } + done(100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); + }); + }) + }); + }); + }); + } + + var writeResultFile = function(filename, json, append) { + fs.writeFile(filename + (append || ""), json); + }; + + var openResultFile = function(stats, browser) { + var tests = stats[browser].tests, + filename = "results/" + browser + ".json", + write = writeResultFile.bind(null, filename, JSON.stringify(stats[browser])); + + fs.exists(filename, function(exists) { + if(exists) { + fs.readFile(filename, "binary", parseResultFile.bind(null, tests, browser, write)); + } else { + write(); + } + }); + }; + + var setColor = function(color, text) { + return color + text.amount + "% " + text.test; + }; + + var parseResultFile = function(tests, browser, createResultFile, err, file) { + if (err) throw err; + var data = JSON.parse(file), + improved = [], + colors = { + red: "\x1b[1;31m", + green: "\x1b[0;32m" + }, + regressed = []; + + Object.keys(tests).forEach(function(test){ + var testResult = tests[test], + dataResult = data.tests[test], + dataObject = { + amount: testResult - dataResult, + test: test }; - stats[browserType] = obj; - stats[browserName] = obj; - processPage(0); + + if (testResult > dataResult) { + improved.push(dataObject); + } else if (testResult < dataResult) { + regressed.push(dataObject); + } + }); - function processPage(index) { - var page = pages[index++]; - testPage(browser, "http://localhost:" + port + "/tests/" + page + "?selenium", function(result) { - if (numPages > index) { - processPage(index); - } else { - browser.close(browserDone); - } - stats[browserType].tests[page] = result; + if (improved.length > 0 || regressed.length > 0) { + if (regressed.length === 0) { + createResultFile(".baseline"); + } + + console.log((regressed.length > 0) ? colors.red : colors.green, browser); + + improved.map(setColor.bind(null, colors.green)).concat(regressed.map(setColor.bind(null, colors.red))).forEach(function(item) { + console.log(" *", item); }); } + }; + + function handleResults(stats) { + Object.keys(stats).forEach(openResultFile.bind(null, stats)); + } + + function runBrowsers(pages){ + + var port = 5555, + stats = {}, + browsers = ["chrome", "firefox", "internet explorer"], + browsersDone = 0, + server = createServer(port), + numPages = pages.length; + + var browserDone = function() { + if (++browsersDone >= browsers.length) { + server.close(); + handleResults(stats); + } + }; + + browsers.forEach(function(browserName){ + var browser = new webdriver({ + browser: browserName + }), + browserType; + browserName = browserName.replace("internet explorer", "iexplorer"); + browser.status(function(browserInfo){ + browserType = [browserName, browser.version, browserInfo.os.name.replace(/\s+/g, "-").toLowerCase()].join("-"); + var date = new Date(), + obj = { + tests: {}, + date: date.toISOString(), + version: browser.version + }; + stats[browserType] = obj; + stats[browserName] = obj; + processPage(0); + }); + + function processPage(index) { + var page = pages[index++]; + testPage(browser, "http://localhost:" + port + "/tests/" + page + "?selenium", function(result) { + if (numPages > index) { + processPage(index); + } else { + browser.close(browserDone); + } + stats[browserType].tests[page] = result; + }); + } + + }); + } + + walkDir("cases", function(err, results) { + if (err) throw err; + runBrowsers(results.slice(0, 2)); }); -} - -walkDir("cases", function(err, results) { - if (err) throw err; - runBrowsers(results); -}); +})();