Test result outputs

This commit is contained in:
Niklas von Hertzen 2012-12-28 19:41:14 +02:00
parent cdc7a744e3
commit 3bbcfe36e0
9 changed files with 208 additions and 131 deletions

3
.gitignore vendored
View File

@ -12,4 +12,5 @@ node_modules/
.envrc
server.js
*.sublime-workspace
chromedriver.log
chromedriver.log
.baseline

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":98,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T16:48:04.248Z","version":"23.0.1271.97"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T17:20:56.098Z","version":"23.0.1271.97"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":82.27},"date":"2012-12-28T16:48:04.248Z","version":"23.0.1271.97"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":92.64},"date":"2012-12-28T16:48:10.772Z","version":"12.0"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":85.64},"date":"2012-12-28T16:48:10.772Z","version":"12.0"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":100},"date":"2012-12-28T16:48:04.855Z","version":"9"}

View File

@ -0,0 +1 @@
{"tests":{"cases/background/encoded.html":100,"cases/background/linear-gradient.html":100},"date":"2012-12-28T16:48:04.855Z","version":"9"}

View File

@ -1,154 +1,223 @@
var webdriver = require("webdriver.js").webdriver,
http = require("http"),
url = require("url"),
path = require("path"),
base64_arraybuffer = require('base64-arraybuffer'),
PNG = require('png-js'),
fs = require("fs");
(function(){
"use strict;"
var webdriver = require("webdriver.js").webdriver,
http = require("http"),
url = require("url"),
path = require("path"),
base64_arraybuffer = require('base64-arraybuffer'),
PNG = require('png-js'),
fs = require("fs");
function createServer(port) {
return http.createServer(function(request, response) {
var uri = url.parse(request.url).pathname,
filename = path.join(process.cwd(), "../" + uri);
function createServer(port) {
return http.createServer(function(request, response) {
var uri = url.parse(request.url).pathname,
filename = path.join(process.cwd(), "../" + uri);
fs.exists(filename, function(exists) {
if(!exists) {
response.writeHead(404, {
"Content-Type": "text/plain"
});
response.write("404 Not Found\n");
response.end();
return;
}
if (fs.statSync(filename).isDirectory()) filename += '/index.html';
fs.readFile(filename, "binary", function(err, file) {
if(err) {
response.writeHead(500, {
fs.exists(filename, function(exists) {
if(!exists) {
response.writeHead(404, {
"Content-Type": "text/plain"
});
response.write(err + "\n");
response.write("404 Not Found\n");
response.end();
return;
}
response.writeHead(200);
response.write(file, "binary");
response.end();
if (fs.statSync(filename).isDirectory()) filename += '/index.html';
fs.readFile(filename, "binary", function(err, file) {
if(err) {
response.writeHead(500, {
"Content-Type": "text/plain"
});
response.write(err + "\n");
response.end();
return;
}
response.writeHead(200);
response.write(file, "binary");
response.end();
});
});
});
}).listen(port);
}
}).listen(port);
}
function walkDir(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walkDir(file, function(err, res) {
results = results.concat(res);
function walkDir(dir, done) {
var results = [];
fs.readdir(dir, function(err, list) {
if (err) return done(err);
var i = 0;
(function next() {
var file = list[i++];
if (!file) return done(null, results);
file = dir + '/' + file;
fs.stat(file, function(err, stat) {
if (stat && stat.isDirectory()) {
walkDir(file, function(err, res) {
results = results.concat(res);
next();
});
} else {
results.push(file);
next();
});
} else {
results.push(file);
next();
}
});
})();
});
};
function getPixelArray(base64, func) {
var arraybuffer = base64_arraybuffer.decode(base64);
(new PNG(arraybuffer)).decode(func);
}
function testPage(browser, url, done) {
browser.url(url)
.$(".html2canvas", 5000, function(){
this.execute(function(){
var canvas = $('.html2canvas')[0];
return canvas.toDataURL("image/png").substring(22);
},[], function(dataurl) {
getPixelArray(dataurl, function(h2cPixels) {
browser.screenshot(function(base64){
getPixelArray(base64, function(screenPixels) {
var len = h2cPixels.length, index = 0, diff = 0;
for (; index < len; index++) {
if (screenPixels[index] - h2cPixels[index] !== 0) {
diff++;
}
}
done(100 - (Math.round((diff/h2cPixels.length) * 10000) / 100));
});
})
});
}
});
})();
});
});
}
function runBrowsers(pages){
var port = 5555,
stats = {},
browsers = ["chrome", "firefox", "internet explorer"],
browsersDone = 0,
server = createServer(port),
numPages = pages.length;
var browserDone = function() {
if (++browsersDone >= browsers.length) {
server.close();
console.log(stats);
}
};
browsers.forEach(function(browserName){
var browser = new webdriver({
browser: browserName
}),
browserType;
function getPixelArray(base64, func) {
var arraybuffer = base64_arraybuffer.decode(base64);
(new PNG(arraybuffer)).decode(func);
}
browser.status(function(browserInfo){
browserType = [browserName, browser.version, browserInfo.os.name.replace(/\s+/g, "-").toLowerCase()].join("-");
var date = new Date(),
obj = {
tests: {},
date: date.toISOString()
function testPage(browser, url, done) {
browser.url(url)
.$(".html2canvas", 5000, function(){
this.execute(function(){
var canvas = $('.html2canvas')[0];
return canvas.toDataURL("image/png").substring(22);
},[], function(dataurl) {
getPixelArray(dataurl, function(h2cPixels) {
browser.screenshot(function(base64){
getPixelArray(base64, function(screenPixels) {
var len = h2cPixels.length, index = 0, diff = 0;
for (; index < len; index++) {
if (screenPixels[index] - h2cPixels[index] !== 0) {
diff++;
}
}
done(100 - (Math.round((diff/h2cPixels.length) * 10000) / 100));
});
})
});
});
});
}
var writeResultFile = function(filename, json, append) {
fs.writeFile(filename + (append || ""), json);
};
var openResultFile = function(stats, browser) {
var tests = stats[browser].tests,
filename = "results/" + browser + ".json",
write = writeResultFile.bind(null, filename, JSON.stringify(stats[browser]));
fs.exists(filename, function(exists) {
if(exists) {
fs.readFile(filename, "binary", parseResultFile.bind(null, tests, browser, write));
} else {
write();
}
});
};
var setColor = function(color, text) {
return color + text.amount + "% " + text.test;
};
var parseResultFile = function(tests, browser, createResultFile, err, file) {
if (err) throw err;
var data = JSON.parse(file),
improved = [],
colors = {
red: "\x1b[1;31m",
green: "\x1b[0;32m"
},
regressed = [];
Object.keys(tests).forEach(function(test){
var testResult = tests[test],
dataResult = data.tests[test],
dataObject = {
amount: testResult - dataResult,
test: test
};
stats[browserType] = obj;
stats[browserName] = obj;
processPage(0);
if (testResult > dataResult) {
improved.push(dataObject);
} else if (testResult < dataResult) {
regressed.push(dataObject);
}
});
function processPage(index) {
var page = pages[index++];
testPage(browser, "http://localhost:" + port + "/tests/" + page + "?selenium", function(result) {
if (numPages > index) {
processPage(index);
} else {
browser.close(browserDone);
}
stats[browserType].tests[page] = result;
if (improved.length > 0 || regressed.length > 0) {
if (regressed.length === 0) {
createResultFile(".baseline");
}
console.log((regressed.length > 0) ? colors.red : colors.green, browser);
improved.map(setColor.bind(null, colors.green)).concat(regressed.map(setColor.bind(null, colors.red))).forEach(function(item) {
console.log(" *", item);
});
}
};
function handleResults(stats) {
Object.keys(stats).forEach(openResultFile.bind(null, stats));
}
function runBrowsers(pages){
var port = 5555,
stats = {},
browsers = ["chrome", "firefox", "internet explorer"],
browsersDone = 0,
server = createServer(port),
numPages = pages.length;
var browserDone = function() {
if (++browsersDone >= browsers.length) {
server.close();
handleResults(stats);
}
};
browsers.forEach(function(browserName){
var browser = new webdriver({
browser: browserName
}),
browserType;
browserName = browserName.replace("internet explorer", "iexplorer");
browser.status(function(browserInfo){
browserType = [browserName, browser.version, browserInfo.os.name.replace(/\s+/g, "-").toLowerCase()].join("-");
var date = new Date(),
obj = {
tests: {},
date: date.toISOString(),
version: browser.version
};
stats[browserType] = obj;
stats[browserName] = obj;
processPage(0);
});
function processPage(index) {
var page = pages[index++];
testPage(browser, "http://localhost:" + port + "/tests/" + page + "?selenium", function(result) {
if (numPages > index) {
processPage(index);
} else {
browser.close(browserDone);
}
stats[browserType].tests[page] = result;
});
}
});
}
walkDir("cases", function(err, results) {
if (err) throw err;
runBrowsers(results.slice(0, 2));
});
}
walkDir("cases", function(err, results) {
if (err) throw err;
runBrowsers(results);
});
})();