Fix webdriver tests

This commit is contained in:
Niklas von Hertzen 2014-05-18 17:39:24 +03:00
parent 281e6bbedf
commit b7595e19e9
3 changed files with 79 additions and 278 deletions

View File

@ -37,6 +37,13 @@ module.exports = function(grunt) {
base: './', base: './',
keepalive: true keepalive: true
} }
},
ci: {
options: {
port: 8080,
base: './',
keepalive: false
}
} }
}, },
uglify: { uglify: {
@ -65,7 +72,7 @@ module.exports = function(grunt) {
if (arguments.length) { if (arguments.length) {
selenium[arg1].apply(null, arguments); selenium[arg1].apply(null, arguments);
} else { } else {
selenium.tests(); selenium.tests().onValue(done);
} }
}); });
@ -81,6 +88,6 @@ module.exports = function(grunt) {
grunt.registerTask('server', ['connect']); grunt.registerTask('server', ['connect']);
grunt.registerTask('build', ['concat', 'uglify']); grunt.registerTask('build', ['concat', 'uglify']);
grunt.registerTask('default', ['jshint', 'concat', 'qunit', 'uglify']); grunt.registerTask('default', ['jshint', 'concat', 'qunit', 'uglify']);
grunt.registerTask('travis', ['jshint', 'concat','qunit', 'uglify', 'webdriver']); grunt.registerTask('travis', ['jshint', 'concat','qunit', 'uglify', 'connect:ci', 'webdriver']);
}; };

View File

@ -26,13 +26,9 @@
"grunt-contrib-jshint": "*", "grunt-contrib-jshint": "*",
"grunt-contrib-qunit": "*", "grunt-contrib-qunit": "*",
"grunt-contrib-watch": "~0.5.1", "grunt-contrib-watch": "~0.5.1",
"googleapis": "~0.4.3",
"jwt-sign": "~0.1.0",
"base64-arraybuffer": ">= 0.1.0", "base64-arraybuffer": ">= 0.1.0",
"png-js": ">= 0.1.1", "png-js": ">= 0.1.1",
"sync-webdriver": ">=0.1.1", "baconjs": "0.7.11",
"express": "~3.2.3",
"baconjs": "~0.3.15",
"wd": "~0.2.7", "wd": "~0.2.7",
"grunt-contrib-connect": "~0.6.0" "grunt-contrib-connect": "~0.6.0"
}, },

View File

@ -1,20 +1,16 @@
(function(){ (function(){
"use strict;"; "use strict;";
var WebDriver = require('sync-webdriver'), var Bacon = require('baconjs').Bacon,
Bacon = require('baconjs').Bacon, wd = require('wd'),
express = require('express'),
http = require("http"), http = require("http"),
https = require("https"), https = require("https"),
url = require("url"), url = require("url"),
path = require("path"), path = require("path"),
base64_arraybuffer = require('base64-arraybuffer'), base64_arraybuffer = require('base64-arraybuffer'),
PNG = require('png-js'), PNG = require('png-js'),
fs = require("fs"), fs = require("fs");
googleapis = require('googleapis'),
jwt = require('jwt-sign');
var port = 8080, var port = 8080,
app = express(),
colors = { colors = {
red: "\x1b[1;31m", red: "\x1b[1;31m",
blue: "\x1b[1;36m", blue: "\x1b[1;36m",
@ -23,16 +19,6 @@
clear: "\x1b[0m" clear: "\x1b[0m"
}; };
var server = app.listen(port);
app.use('/index.html', function(req, res){
res.send("<ul>" + tests.map(function(test) {
return "<li><a href='" + test + "'>" + test + "</a></li>";
}).join("") + "</ul>");
});
app.use('/', express.static(__dirname + "/../"));
function mapStat(item) { function mapStat(item) {
return Bacon.combineTemplate({ return Bacon.combineTemplate({
stat: Bacon.fromNodeCallback(fs.stat, item), stat: Bacon.fromNodeCallback(fs.stat, item),
@ -81,14 +67,6 @@
return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100)); return (100 - (Math.round((diff/h2cPixels.length) * 10000) / 100));
} }
function canvasToDataUrl(canvas) {
return canvas.toDataURL("image/png").substring(22);
}
function closeServer() {
server.close();
}
function findResult(testName, tests) { function findResult(testName, tests) {
var item = null; var item = null;
return tests.some(function(testCase) { return tests.some(function(testCase) {
@ -140,288 +118,108 @@
} }
} }
function httpget(options) {
return Bacon.fromCallback(function(callback) {
https.get(options, function(res){
var data = '';
res.on('data', function (chunk){
data += chunk;
});
res.on('end',function(){
callback(data);
});
});
});
}
function parseJSON(str) {
return JSON.parse(str);
}
function writeResults() {
Object.keys(results).forEach(function(browser) {
var filename = "tests/results/" + browser + ".json";
try {
var oldResults = JSON.parse(fs.readFileSync(filename));
compareResults(oldResults, results[browser], browser);
} catch(e) {}
var date = new Date();
var result = JSON.stringify({
browser: browser,
results: results[browser],
timestamp: date.toISOString()
});
if (process.env.MONGOLAB_APIKEY) {
var options = {
host: "api.mongolab.com",
port: 443,
path: "/api/1/databases/html2canvas/collections/webdriver-results?apiKey=" + process.env.MONGOLAB_APIKEY + '&q={"browser":"' + browser + '"}&fo=true&s={"timestamp":-1}'
};
httpget(options).map(parseJSON).onValue(function(data) {
compareResults(data.results, results[browser], browser);
options.method = 'POST';
options.path = "/api/1/databases/html2canvas/collections/webdriver-results?apiKey=" + process.env.MONGOLAB_APIKEY;
options.headers = {
'Content-Type': 'application/json',
'Content-Length': result.length
};
console.log("Sending results for", browser);
var request = https.request(options, function(res) {
console.log(colors.green, "Results sent for", browser);
});
request.write(result);
request.end();
});
}
console.log(colors.violet, "Writing", browser + ".json");
fs.writeFile(filename, result);
});
}
function webdriverOptions(browserName, version, platform) {
var options = {};
if (process.env.SAUCE_USERNAME && process.env.SAUCE_ACCESS_KEY) {
options = {
port: 4445,
hostname: "localhost",
name: process.env.TRAVIS_JOB_ID || "Manual run",
username: process.env.SAUCE_USERNAME,
password: process.env.SAUCE_ACCESS_KEY,
desiredCapabilities: {
browserName: browserName,
version: version,
platform: platform,
"tunnel-identifier": process.env.TRAVIS_JOB_NUMBER
}
};
}
return options;
}
function mapResults(result) {
if (!results[result.browser]) {
results[result.browser] = [];
}
results[result.browser].push({
test: result.testCase,
result: result.accuracy
});
}
function formatResultName(navigator) { function formatResultName(navigator) {
return (navigator.browser + "-" + ((navigator.version) ? navigator.version : "release") + "-" + navigator.platform).replace(/ /g, "").toLowerCase(); return (navigator.browserName + "-" + ((navigator.version) ? navigator.version : "release") + "-" + navigator.platform).replace(/ /g, "").toLowerCase();
} }
function webdriverStream(navigator) { function webdriverStream(test) {
var drive = Bacon.fromCallback(discover, "drive", "v2").toProperty(); var browser = wd.remote("ondemand.saucelabs.com", 80, process.env.SAUCE_USERNAME, process.env.SAUCE_ACCESS_KEY);
var auth = Bacon.fromCallback(createToken, "95492219822.apps.googleusercontent.com").toProperty(); var browserStream = new Bacon.Bus();
var resultStream = Bacon.fromNodeCallback(browser, "init", test.capabilities)
return Bacon.fromCallback(function(callback) { .flatMap(Bacon.fromNodeCallback(browser, "setImplicitWaitTimeout", 15000)
new WebDriver.Session(webdriverOptions(navigator.browser, navigator.version, navigator.platform), function() { .flatMap(function() {
var browser = this; Bacon.later(0, formatResultName(test.capabilities)).onValue(browserStream.push);
return Bacon.fromArray(test.cases).zip(browserStream.take(test.cases.length)).flatMap(function(options) {
var resultStream = Bacon.fromArray(tests).flatMap(function(testCase) { var testCase = options[0];
console.log(colors.green, "STARTING",formatResultName(navigator), testCase, colors.clear); var name = options[1];
browser.url = "http://localhost:" + port + "/" + testCase + "?selenium"; console.log(colors.green, "STARTING", name, testCase, colors.clear);
var canvas = browser.element(".html2canvas", 15000); return Bacon.fromNodeCallback(browser, "get", "http://localhost:" + port + "/" + testCase + "?selenium")
var dataUrl = Bacon.constant(browser.execute(canvasToDataUrl, canvas)); .flatMap(Bacon.combineTemplate({
var screenshot = Bacon.constant(browser.screenshot()); dataUrl: Bacon.fromNodeCallback(browser, "elementByCssSelector", ".html2canvas").flatMap(function(canvas) {
var result = dataUrl.flatMap(getPixelArray).combine(screenshot.flatMap(getPixelArray), calculateDifference); return Bacon.fromNodeCallback(browser, "execute", "return arguments[0].toDataURL('image/png').substring(22)", [canvas]);
console.log(colors.green, "COMPLETE", formatResultName(navigator), testCase, colors.clear); }),
return Bacon.combineTemplate({ screenshot: Bacon.fromNodeCallback(browser, "takeScreenshot")
browser: formatResultName(navigator), })).flatMap(function(result) {
testCase: testCase, return Bacon.combineTemplate({
accuracy: result, browser: name,
dataUrl: dataUrl, testCase: testCase,
screenshot: screenshot accuracy: Bacon.constant(result.dataUrl).flatMap(getPixelArray).combine(Bacon.constant(result.screenshot).flatMap(getPixelArray), calculateDifference),
dataUrl: result.dataUrl,
screenshot: result.screenshot
});
});
}); });
}); }));
if (fs.existsSync('tests/certificate.pem')) { resultStream.onError(function(error) {
Bacon.combineWith(permissionRequest, drive, auth, Bacon.combineWith(uploadRequest, drive, auth, resultStream.doAction(mapResults).flatMap(createImages)).flatMap(executeRequest)).flatMap(executeRequestOriginal).onValue(uploadImages); console.log(colors.red, "ERROR", test.capabilities.browserName, error);
} browserStream.push(formatResultName(test.capabilities));
resultStream.onEnd(callback);
});
}); });
}
function permissionRequest(client, authClient, images) { resultStream.onValue(function(result) {
var body = { console.log(colors.green, "COMPLETE", result.browser, result.testCase, result.accuracy, "%", colors.clear);
value: 'me', browserStream.push(result.browser);
type: 'anyone',
role: 'reader'
};
return images.map(function(data) {
var request = client.drive.permissions.insert({fileId: data.id}).withAuthClient(authClient);
request.body = body;
request.fileData = data;
return request;
}); });
}
function executeRequest(requests) { resultStream.onEnd(function() {
return Bacon.combineAsArray(requests.map(function(request) { browser.quit();
return Bacon.fromCallback(function(callback) { });
request.execute(function(err, result) {
if (!err) {
callback(result);
} else {
console.log("Google drive error", err);
}
});
});
}));
}
function executeRequestOriginal(requests) { return resultStream.fold([], pushToArray);
return Bacon.combineAsArray(requests.map(function(request) {
return Bacon.fromCallback(function(callback) {
request.execute(function(err, result) {
if (!err) {
callback(request.fileData);
} else {
console.log("Google drive error", err);
}
});
});
}));
} }
function createImages(data) { function createImages(data) {
var dataurlFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-html2canvas.png"; var dataurlFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-html2canvas.png";
var screenshotFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-screencapture.png"; var screenshotFileName = "tests/results/" + data.browser + "-" + data.testCase.replace(/\//g, "-") + "-screencapture.png";
return Bacon.combineTemplate({ return Bacon.combineTemplate({
name: data.testCase, name: data.testCase,
dataurl: Bacon.fromNodeCallback(fs.writeFile, dataurlFileName, data.dataUrl, "base64").map(function() { dataurl: Bacon.fromNodeCallback(fs.writeFile, dataurlFileName, data.dataUrl, "base64").map(function() {
return dataurlFileName; return dataurlFileName;
}), }),
screenshot: Bacon.fromNodeCallback(fs.writeFile, screenshotFileName, data.screenshot, "base64").map(function() { screenshot: Bacon.fromNodeCallback(fs.writeFile, screenshotFileName, data.screenshot, "base64").map(function() {
return screenshotFileName; return screenshotFileName;
}) })
});
}
function uploadImages(results) {
results.forEach(function(result) {
console.log(result.webContentLink);
}); });
} }
function discover(api, version, callback) { function pushToArray(array, item) {
googleapis.discover(api, version).execute(function(err, client) { array.push(item);
if (!err) { return array;
callback(client);
}
});
} }
function createToken(account, callback) { function runWebDriver(cases) {
var payload = {
"iss": '95492219822@developer.gserviceaccount.com',
"scope": 'https://www.googleapis.com/auth/drive',
"aud":"https://accounts.google.com/o/oauth2/token",
"exp": ~~(new Date().getTime() / 1000) + (30 * 60),
"iat": ~~(new Date().getTime() / 1000 - 60)
},
key = fs.readFileSync('tests/certificate.pem', 'utf8'),
transporterTokenRequest = {
method: 'POST',
uri: 'https://accounts.google.com/o/oauth2/token',
form: {
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion: jwt.sign(payload, key)
},
json: true
},
oauth2Client = new googleapis.OAuth2Client(account, "", "");
oauth2Client.transporter.request(transporterTokenRequest, function(err, result) {
if (!err) {
oauth2Client.credentials = result;
callback(oauth2Client);
}
});
}
function uploadRequest(client, authClient, data) {
return [
client.drive.files.insert({title: data.dataurl, mimeType: 'image/png', description: process.env.TRAVIS_JOB_ID}).withMedia('image/png', fs.readFileSync(data.dataurl)).withAuthClient(authClient),
client.drive.files.insert({title: data.screenshot, mimeType: 'image/png', description: process.env.TRAVIS_JOB_ID}).withMedia('image/png', fs.readFileSync(data.screenshot)).withAuthClient(authClient)
];
}
function runWebDriver() {
var browsers = [ var browsers = [
{ {
browser: "chrome", browserName: "chrome",
platform: "Windows 7" platform: "Windows 7"
},{ },{
browser: "firefox", browserName: "firefox",
version: "15", version: "15",
platform: "Windows 7" platform: "Windows 7"
},{ },{
browser: "internet explorer", browserName: "internet explorer",
version: "9", version: "9",
platform: "Windows 7" platform: "Windows 7"
},{ },{
browser: "internet explorer", browserName: "internet explorer",
version: "10", version: "10",
platform: "Windows 8" platform: "Windows 8"
},{ },{
browser: "safari", browserName: "safari",
version: "6", version: "6",
platform: "OS X 10.8" platform: "OS X 10.8"
},{ },{
browser: "chrome", browserName: "chrome",
platform: "OS X 10.8" platform: "OS X 10.8"
} }
]; ];
var testRunnerStream = Bacon.sequentially(1000, browsers).flatMap(webdriverStream); return Bacon.combineTemplate({
testRunnerStream.onEnd(writeResults); capabilities: Bacon.sequentially(1000, browsers),
testRunnerStream.onEnd(closeServer); cases: cases
}).flatMap(webdriverStream);
} }
var tests = [], exports.tests = function() {
results = {}, return getTests("tests/cases").fold([], pushToArray).flatMap(runWebDriver).mapError(false);
testStream = getTests("tests/cases"); };
})();
testStream.onValue(function(test) {
tests.push(test);
});
exports.tests = function() {
testStream.onEnd(runWebDriver);
};
})();