From b91fd9bc87231516f954262b6a6d3fb075f6b9ea Mon Sep 17 00:00:00 2001 From: Niklas von Hertzen Date: Wed, 29 May 2013 17:53:47 +0300 Subject: [PATCH] use sync-webdriver for selenium tests --- package.json | 19 +- tests/selenium.js | 538 +++++++++++++++++++++------------------------- 2 files changed, 261 insertions(+), 296 deletions(-) diff --git a/package.json b/package.json index 28cb563..2752a68 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "Screenshots with JavaScript", "version": "0.4.0", "author": { - "name":"Niklas von Hertzen (@niklasvh)" + "name": "Niklas von Hertzen (@niklasvh)" }, "engines": { "node": ">=0.8.0" @@ -12,9 +12,11 @@ "dependencies": { "base64-arraybuffer": ">= 0.1.0", "png-js": ">= 0.1.1", - "webdriver.js": ">= 0.1.0" + "sync-webdriver": ">=0.1.1", + "express": "~3.2.3", + "baconjs": "~0.3.15" }, - "devDependencies":{ + "devDependencies": { "grunt": ">=0.4.0", "grunt-contrib-concat": "*", "grunt-contrib-uglify": "*", @@ -22,8 +24,9 @@ "grunt-contrib-qunit": "*" }, "homepage": "http://html2canvas.hertzen.com", - "licenses": [{ - "type": "MIT" - }] - -} \ No newline at end of file + "licenses": [ + { + "type": "MIT" + } + ] +} diff --git a/tests/selenium.js b/tests/selenium.js index e8ad78e..c88be68 100644 --- a/tests/selenium.js +++ b/tests/selenium.js @@ -1,303 +1,265 @@ (function(){ - "use strict;" - var webdriver = require("webdriver.js").webdriver, - http = require("http"), - url = require("url"), - path = require("path"), - base64_arraybuffer = require('base64-arraybuffer'), - PNG = require('png-js'), - fs = require("fs"); + "use strict;"; + var WebDriver = require('sync-webdriver'), + Bacon = require('baconjs').Bacon, + express = require('express'), + http = require("http"), + url = require("url"), + path = require("path"), + base64_arraybuffer = require('base64-arraybuffer'), + PNG = require('png-js'), + fs = require("fs"); - function createServer(port) { - return http.createServer(function(request, response) { - var uri = url.parse(request.url).pathname, - filename = path.join(process.cwd(), uri); + var port = 8080, + app = express(), + colors = { + red: "\x1b[1;31m", + blue: "\x1b[1;36m", + violet: "\x1b[0;35m", + green: "\x1b[0;32m", + clear: "\x1b[0m" + }; - fs.exists(filename, function(exists) { - if(!exists) { - response.writeHead(404, { - "Content-Type": "text/plain" - }); - response.write("404 Not Found\n"); - response.end(); - return; + var server = app.listen(port); + app.use('/', express.static(__dirname + "/../")); + + function mapStat(item) { + return Bacon.combineTemplate({ + stat: Bacon.fromNodeCallback(fs.stat, item), + item: item + }); + } + + function isDirectory(item) { + return item.stat.isDirectory(); + } + + function getItem(item) { + return item.item; + } + + function isFile(item) { + return !isDirectory(item); + } + + function arrayStream(arr) { + return Bacon.fromArray(arr); + } + + function getTests(path) { + var items = Bacon.fromNodeCallback(fs.readdir, path).flatMap(arrayStream).map(function(name) { + return path + "/" + name; + }).flatMap(mapStat); + return items.filter(isFile).map(getItem).merge(items.filter(isDirectory).map(getItem).flatMap(getTests)); + } + + + function getPixelArray(base64) { + return Bacon.fromCallback(function(callback) { + var arraybuffer = base64_arraybuffer.decode(base64); + (new PNG(arraybuffer)).decode(callback); + }); + } + + function calculateDifference(h2cPixels, screenPixels) { + var len = h2cPixels.length, index = 0, diff = 0; + for (; index < len; index++) { + if (screenPixels[index] - h2cPixels[index] !== 0) { + diff++; + } + } + return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); + } + + function canvasToDataUrl(canvas) { + return canvas.toDataURL("image/png").substring(22); + } + + function createImages(data) { + return Bacon.combineTemplate({ + dataurl: Bacon.fromNodeCallback(fs.writeFile, "tests/results/captures/" + data.testCase.replace(/\//g, "-") + "-html2canvas.png", data.dataUrl, "base64"), + screenshot: Bacon.fromNodeCallback(fs.writeFile, "tests/results/captures/" + data.testCase.replace(/\//g, "-") + "-screencapture.png", data.screenshot, "base64"), + data: Bacon.constant(data) + }); + } + + function closeServer() { + server.close(); + } + + function findResult(testName, tests) { + var item = null; + return tests.some(function(testCase) { + item = testCase; + return testCase.test === testName; + }) ? item : null; + } + + function compareResults(oldResults, newResults, browser) { + var improved = [], + regressed = [], + newItems = []; + + newResults.forEach(function(testCase){ + var testResult = testCase.result, + oldResult = findResult(testCase.test, oldResults), + oldResultValue = oldResult ? oldResult.result : null, + dataObject = { + amount: (Math.abs(testResult - oldResultValue) < 0.01) ? 0 : testResult - oldResultValue, + test: testCase.test + }; + if (oldResultValue === null) { + newItems.push(dataObject); + } else if (dataObject.amount > 0) { + improved.push(dataObject); + } else if (dataObject.amount < 0) { + regressed.push(dataObject); + } + }); + + reportChanges(browser, improved, regressed, newItems); + } + + function reportChanges(browser, improved, regressed, newItems) { + if (newItems.length > 0 || improved.length > 0 || regressed.length > 0) { + console.log((regressed.length > 0) ? colors.red : colors.green, browser); + + regressed.forEach(function(item) { + console.log(colors.red, item.amount + "%", item.test); + }); + + improved.forEach(function(item) { + console.log(colors.green, item.amount + "%", item.test); + }); + + newItems.forEach(function(item) { + console.log(colors.blue, "NEW", item.test); + }); + } + } + + function writeResults() { + Object.keys(results).forEach(function(browser) { + var filename = "tests/results/" + browser + ".json"; + try { + var oldResults = JSON.parse(fs.readFileSync(filename)); + compareResults(oldResults, results[browser], browser); + } catch(e) {} + + console.log(colors.violet, "Writing", browser + ".json"); + fs.writeFile(filename, JSON.stringify(results[browser])); + }); + } + + function webdriverOptions(browserName, version, platform) { + var options = {}; + if (process.env.SAUCE_USERNAME && process.env.SAUCE_ACCESS_KEY) { + options = { + port: 4445, + hostname: "localhost", + username: process.env.SAUCE_USERNAME, + password: process.env.SAUCE_ACCESS_KEY, + desiredCapabilities: { + browserName: browserName, + version: version, + platform: platform, + "tunnel-identifier": process.env.TRAVIS_JOB_NUMBER + } + }; + } + return options; + } + + function mapResults(result) { + if (!results[result.browser]) { + results[result.browser] = []; } - if (fs.statSync(filename).isDirectory()) filename += '/index.html'; - - fs.readFile(filename, "binary", function(err, file) { - if(err) { - response.writeHead(500, { - "Content-Type": "text/plain" - }); - response.write(err + "\n"); - response.end(); - return; - } - - response.writeHead(200); - response.write(file, "binary"); - response.end(); + results[result.browser].push({ + test: result.testCase, + result: result.accuracy }); - }); - - }).listen(port); - } - - function walkDir(dir, done) { - var results = []; - fs.readdir(dir, function(err, list) { - if (err) return done(err); - var i = 0; - (function next() { - var file = list[i++]; - if (!file) return done(null, results); - file = dir + '/' + file; - fs.stat(file, function(err, stat) { - if (stat && stat.isDirectory()) { - walkDir(file, function(err, res) { - results = results.concat(res); - next(); - }); - } else { - results.push(file); - next(); - } - }); - })(); - }); - }; - - function getPixelArray(base64, func) { - var arraybuffer = base64_arraybuffer.decode(base64); - (new PNG(arraybuffer)).decode(func); - } - - function getBaselineFiles() { - return fs.readdirSync("tests/results/").filter(function(name) { - return /\.baseline$/.test(name); - }).map(function(item) { - return "tests/results/" + item; - }); - } - - function testPage(browser, url, done) { - browser.url(url) - .$(".html2canvas", 5000, function(){ - this.execute(function(){ - var canvas = $('.html2canvas')[0]; - return canvas.toDataURL("image/png").substring(22); - },[], function(dataurl) { - getPixelArray(dataurl, function(h2cPixels) { - browser.screenshot(function(base64){ - getPixelArray(base64, function(screenPixels) { - var len = h2cPixels.length, index = 0, diff = 0; - for (; index < len; index++) { - if (screenPixels[index] - h2cPixels[index] !== 0) { - diff++; - } - } - done(100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); - }); - }); - }); - }); - }); - } - - var writeResultFile = function(filename, json, append) { - fs.writeFile(filename + (append || ""), json); - }; - - var openResultFile = function(stats, browser) { - var tests = stats[browser].tests, - filename = "tests/results/" + browser + ".json", - write = writeResultFile.bind(null, filename, JSON.stringify(stats[browser])); - - fs.exists(filename, function(exists) { - if(exists) { - fs.readFile(filename, "binary", parseResultFile.bind(null, tests, browser, write)); - } else { - write(); - } - }); - }; - - var setColor = function(color, text) { - return [color, " * ", ((isNaN(text.amount)) ? "NEW" : text.amount + "%"), " ", text.test].join(""); - }; - - var parseResultFile = function(tests, browser, createResultFile, err, file) { - if (err) throw err; - var data = JSON.parse(file), - improved = [], - regressed = [], - newItems = [], - colors = { - red: "\x1b[1;31m", - blue: "\x1b[1;36m", - violet: "\x1b[0;35m", - green: "\x1b[0;32m" - }; - - Object.keys(tests).forEach(function(test){ - var testResult = tests[test], - dataResult = data.tests[test], - dataObject = { - amount: (Math.abs(testResult - dataResult) < 0.02) ? 0 : testResult - dataResult, - test: test - }; - - if (dataObject.amount > 0) { - improved.push(dataObject); - } else if (dataObject.amount < 0) { - regressed.push(dataObject); - } else if (dataResult === undefined) { - newItems.push(dataObject); - } - - }); - - if (newItems.length > 0 || improved.length > 0 || regressed.length > 0) { - if (regressed.length === 0) { - createResultFile(".baseline"); - } - - console.log(colors.violet, "********************"); - console.log((regressed.length > 0) ? colors.red : colors.green, browser); - - improved.map(setColor.bind(null, colors.green)) - .concat(regressed.map(setColor.bind(null, colors.red))) - .concat(newItems.map(setColor.bind(null, colors.blue))) - .forEach(function(item) { - console.log(item); - }); } - }; - - function handleResults(stats) { - Object.keys(stats).forEach(openResultFile.bind(null, stats)); - } - - function runBrowsers(pages){ - - var port = 5555, - stats = {}, - browsers = ["chrome", "firefox", "internet explorer"], - browsersDone = 0, - server = createServer(port), - numPages = pages.length; - - var browserDone = function() { - if (++browsersDone >= browsers.length) { - server.close(); - handleResults(stats); - } - }; - - browsers.forEach(function(browserName){ - var browser = new webdriver({ - browser: browserName - }), - browserType; - browserName = browserName.replace("internet explorer", "iexplorer"); - browser.status(function(browserInfo){ - browserType = [browserName, browser.version, browserInfo.os.name.replace(/\s+/g, "-").toLowerCase()].join("-"); - var date = new Date(), - obj = { - tests: {}, - date: date.toISOString(), - version: browser.version - }; - stats[browserType] = obj; - stats[browserName] = obj; - processPage(0); - }); - - function processPage(index) { - var page = pages[index++]; - testPage(browser, "http://localhost:" + port + "/" + page + "?selenium", function(result) { - if (numPages > index) { - processPage(index); - } else { - browser.close(browserDone); - } - stats[browserType].tests[page] = result; - }); - } - - }); - } - - exports.tests = function() { - getBaselineFiles().forEach(fs.unlinkSync.bind(fs)); - walkDir("tests/cases", function(err, results) { - if (err) throw err; - runBrowsers(results); - }); - }; - - exports.baseline = function() { - getBaselineFiles().forEach(function(file) { - var newName = file.substring(0, file.length - 9); - fs.renameSync(file, newName); - console.log(newName, "created"); - }); - }; - - exports.markdown = function() { - var data = {}, - html = "", - md = " | ", - browsers = ["chrome", "firefox", "iexplorer", "safari"], - activeBrowsers = []; - - // Create row for browsers - browsers.forEach(function(browser) { - - if (fs.existsSync("tests/results/" + browser + ".json")) { - - var fileContents = fs.readFileSync("tests/results/" + browser + ".json"); - data[browser] = JSON.parse(fileContents); - - activeBrowsers.push(browser); - - html += ""; - md += browser + data[browser].version + " | "; - } else { - console.log("Browser report not found. ", browser + ".json"); - } - - }); - - html += "\n"; - md += "\n ----"; - for (var i = activeBrowsers.length - 1; i >= 0; i--) { - md += "|---- "; + function formatResultName(navigator) { + return (navigator.browser + "-" + ((navigator.version) ? navigator.version : "release") + "-" + navigator.platform).replace(/ /g, "").toLowerCase(); } - md += "\n"; - Object.keys(data[activeBrowsers[0]].tests).forEach(function(testFile) { + function webdriverStream(navigator) { + return Bacon.fromCallback(function(callback) { + new WebDriver.Session(webdriverOptions(navigator.browser, navigator.version, navigator.platform), function() { + var browser = this; - html += ""; - md += testFile.substring(12); - activeBrowsers.forEach(function(activeBrowsers) { - html += ""; - md += " | " + Math.round(data[activeBrowsers].tests[testFile] * 100) / 100 + "%"; - }); - html += "\n"; - md += "\n"; + var resultStream = Bacon.fromArray(tests).flatMap(function(testCase) { + console.log(colors.green, "STARTING",formatResultName(navigator), testCase, colors.clear); + browser.url = "http://localhost:" + port + "/" + testCase + "?selenium"; + var canvas = browser.element(".html2canvas", 15000); + var dataUrl = Bacon.constant(browser.execute(canvasToDataUrl, canvas)); + var screenshot = Bacon.constant(browser.screenshot()); + var result = dataUrl.flatMap(getPixelArray).combine(screenshot.flatMap(getPixelArray), calculateDifference); + console.log(colors.green, "COMPLETE", formatResultName(navigator), testCase, colors.clear); + return Bacon.combineTemplate({ + browser: formatResultName(navigator), + testCase: testCase, + accuracy: result, + dataUrl: dataUrl, + screenshot: screenshot + }); + }); + resultStream.onValue(mapResults); + resultStream.onEnd(callback); + }); + }); + } + + function runWebDriver() { + var browsers = [ + { + browser: "chrome", + platform: "Windows 7" + },{ + browser: "firefox", + version: "15", + platform: "Windows 7" + },{ + browser: "internet explorer", + version: "9", + platform: "Windows 7" + },{ + browser: "internet explorer", + version: "10", + platform: "Windows 8" + },{ + browser: "safari", + version: "6", + platform: "OS X 10.8" + },{ + browser: "chrome", + platform: "OS X 10.8" + } + ]; + + var testRunnerStream = Bacon.sequentially(1000, browsers).flatMap(webdriverStream); + testRunnerStream.onEnd(writeResults); + testRunnerStream.onEnd(closeServer); + } + + var tests = [], + outputImages = false, + results = {}, + testStream = getTests("tests/cases"); + + testStream.onValue(function(test) { + tests.push(test); }); - html += "
" + browser + "
" + data[browser].version + "
" + testFile.substring(12) + "" + Math.round(data[activeBrowsers].tests[testFile] * 100) / 100 + "%
"; - - // if (isMarkdown){ - // fs.writeFileSync("tests/readme.md", md); - // } else { - fs.writeFileSync("tests/readme.md", html); - // } - - }; + /* + if (outputImages) { + resultStream.flatMap(createImages).onValue(function(test){ + console.log(test.data.testCase, "screenshots created"); + }); + } + */ + testStream.onEnd(runWebDriver); })(); \ No newline at end of file