Skip to content

Commit

Permalink
refactor test runner
Browse files Browse the repository at this point in the history
  • Loading branch information
@krausest
krausest committed Sep 16, 2019
1 parent bb12708 commit 366ab12
Show file tree
Hide file tree
Showing 6 changed files with 76 additions and 59 deletions.
11 changes: 5 additions & 6 deletions build.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@ Or it can be restrict matching to keyed or non-keyed frameworks:
--restartWith keyed/angu
(Matches only keyed angular, angularjs, angular-ivy, ...)
The distinction is made by checking whether a slash is contained in the parameter
Changed behaviour: Builds only the frameworks. To build webdriver-ts or the results use the npm run install-local task
*/

let args = yargs(process.argv)
.usage("npm run build [-- [--check] [--skipIrrelevant] [--restartWith] [--benchmarks_only]]")
.usage("npm run build [-- [--check] [--skipIrrelevant] [--restartWith]]")
.help('help')
.boolean('check')
.boolean('benchmarks_only')
.boolean('skipIrrelevant')
.string('restartWith')
.argv;
Expand All @@ -31,8 +32,6 @@ var referenceBranch = "origin/master";

var restartWithFramework = args.restartWith || '';

var core = args.benchmarks_only ? [] : ["webdriver-ts", "webdriver-ts-results"].map(f => ["", f]);

var frameworks = [].concat(
fs.readdirSync('./frameworks/keyed').map(f => ['keyed', f]),
fs.readdirSync('./frameworks/non-keyed').map(f => ['non-keyed', f]));
Expand All @@ -49,13 +48,13 @@ var notRestarter = ([dir, name]) => {
let skippable = _.takeWhile(frameworks, notRestarter);
let buildable = _.slice(frameworks, skippable.length);

var relevant = args.skipIrrelevant && !_.some(core, isDifferent)
var relevant = args.skipIrrelevant
? _.filter(buildable, isDifferent)
: buildable;

_.each(skippable, ([dir,name]) => console.log("*** Skipping " + dir + "/" + name));

_.each([].concat(relevant, core), function([dir,name]) {
_.each(relevant, function([dir,name]) {
let fullname = path.join("frameworks", dir, name);
if(fs.statSync(fullname).isDirectory() && fs.existsSync(path.join(fullname, "package.json"))) {
console.log("*** Executing npm install in "+fullname);
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"docker-start": "cross-env-shell docker run --rm -d -p 8080:8080 --name js-framework-benchmark --volume ${INIT_CWD}:/src --volume js-framework-benchmark:/build js-framework-benchmark-centos",
"docker-stop": "docker stop js-framework-benchmark",
"docker-sync": "docker exec -it js-framework-benchmark rsync -avC --exclude /index.html --exclude /framework/**/dist --exclude /webdriver-ts*/dist --exclude package-lock.json --exclude node_modules --exclude /webdriver-ts/results*/ /src/ /build/",
"docker-build-frameworks": "npm run docker-sync && docker exec -it js-framework-benchmark npm install && docker exec -it js-framework-benchmark node build.js -- --benchmarks_only",
"docker-build-frameworks": "npm run docker-sync && docker exec -it js-framework-benchmark npm install && docker exec -it js-framework-benchmark node build.js",
"docker-install-webdriver": "npm run docker-sync && docker exec -it js-framework-benchmark npm run install-local",
"docker-rebuild": "npm run docker-sync && docker exec -it -w /build/webdriver-ts js-framework-benchmark npm run rebuild --headless",
"docker-bench": "docker exec -it -w /build/webdriver-ts js-framework-benchmark node dist/benchmarkRunner.js --headless",
Expand Down
85 changes: 47 additions & 38 deletions webdriver-ts/src/benchmarkRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,50 @@ function forkedRun(frameworks: FrameworkData[], frameworkName: string, keyed: bo
}
}

async function performRetryableRun(runFrameworks: FrameworkData[], framework: FrameworkData, benchmark: Benchmark) {
let errors: String[] = [];
let warnings: String[] = [];
let retry = 1;
let attemptRetry = true;

for (; retry<=5 && attemptRetry; retry++) {
errors = [];
warnings = [];
console.log(`Executing benchmark ${framework.name} and benchmark ${benchmark.id} retry # ${retry}`);

attemptRetry = false;
let benchmarkOptions: BenchmarkOptions = {
port: config.PORT.toFixed(),
remoteDebuggingPort: config.REMOTE_DEBUGGING_PORT,
chromePort: config.CHROME_PORT,
headless: args.headless,
chromeBinaryPath: args.chromeBinary,
numIterationsForCPUBenchmarks: config.REPEAT_RUN,
numIterationsForMemBenchmarks: config.REPEAT_RUN_MEM,
numIterationsForStartupBenchmark: config.REPEAT_RUN_STARTUP
}
// Assumption: For all errors we can handle it won't throw but return a result
let benchMsg: any = await forkedRun(runFrameworks, framework.name, framework.keyed, benchmark.id, benchmarkOptions);
if (benchMsg.failure) {
console.log(`Executing ${framework.uri} and benchmark ${benchmark.id} failed with a technical error: ${benchMsg.failure}`);
errors.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed with a technical error: ${benchMsg.failure}`);
if (config.EXIT_ON_ERROR) throw "Exiting because of an technical error and config.EXIT_ON_ERROR = true";
} else {
let errorsAndWarnings = benchMsg as ErrorAndWarning;
if (errorsAndWarnings.error) errors.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed: ` + errorsAndWarnings.error);
for (let warning of errorsAndWarnings.warnings) {
if (errorsAndWarnings.error) warnings.push(`Executing ${framework.uri} and benchmark ${benchmark.id} failed: ` + errorsAndWarnings.error);
}
if (errorsAndWarnings.error && errorsAndWarnings.error.indexOf("Server terminated early with status 1")>-1) {
console.log("******* STRANGE selenium error found - retry");
attemptRetry = true;
}
if (errorsAndWarnings.error && config.EXIT_ON_ERROR) throw "Exiting because of an error and config.EXIT_ON_ERROR = true";
}
}
return {errors: errors, warnings: warnings};
}

async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]) {
let errors: String[] = [];
let warnings: String[] = [];
Expand All @@ -57,43 +101,9 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
for (let i = 0; i < data.length; i++) {
let framework = data[i][0];
let benchmark = data[i][1];

// let retry = 1;
// for (; retry<=5; retry++) {
console.log(`Executing benchmark ${framework.name} and benchmark ${benchmark.id} `); //retry # ${retry}`);

let benchmarkOptions: BenchmarkOptions = {
port: config.PORT.toFixed(),
remoteDebuggingPort: config.REMOTE_DEBUGGING_PORT,
chromePort: config.CHROME_PORT,
headless: args.headless,
chromeBinaryPath: args.chromeBinary,
numIterationsForCPUBenchmarks: config.REPEAT_RUN,
numIterationsForMemBenchmarks: config.REPEAT_RUN_MEM,
numIterationsForStartupBenchmark: config.REPEAT_RUN_STARTUP
}

try {
let benchMsg: any = await forkedRun(runFrameworks, framework.name, framework.keyed, benchmark.id, benchmarkOptions);
// Note: The following code has not yet been tested
// The "Server terminated early" issue stopped as soon as the code was added ;-(
if (benchMsg.failure) {
console.log(`Executing benchmark ${framework.name} and benchmark ${benchmark.id} failed with a technical error: `, benchMsg.failure);
throw `Executing benchmark failed`;
} else {
let errorsAndWarnings = benchMsg as ErrorAndWarning;
if (errorsAndWarnings.error) errors.push(`Executing benchmark ${framework.name} and benchmark ${benchmark.id} failed: ` + errorsAndWarnings.error);
for (let warning of errorsAndWarnings.warnings) {
if (errorsAndWarnings.error) warnings.push(`Executing benchmark ${framework.name} and benchmark ${benchmark.id} failed: ` + errorsAndWarnings.error);
}
if (errorsAndWarnings.error) throw `Executing benchmark failed`;
}
} catch (err) {
console.log(`Error executing benchmark ${framework.name} and benchmark ${benchmark.id}: `, err);
if (config.EXIT_ON_ERROR) throw "Exiting because of an error and config.EXIT_ON_ERROR = true";
}
// if (retry>1) throw "RETRY > 1 CHECK CONSOLE OUTPUT";
// }
let result = await performRetryableRun(runFrameworks, framework, benchmark);
errors = errors.concat(result.errors);
warnings = warnings.concat(result.warnings);
}

if (warnings.length > 0) {
Expand All @@ -113,7 +123,6 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]

errors.forEach(e => {
console.log(e);
console.log();
});
throw "Benchmarking failed with errors";
}
Expand Down
2 changes: 1 addition & 1 deletion webdriver-ts/src/common.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export let config = {
LOG_DETAILS: false,
LOG_DEBUG: false,
LOG_TIMELINE: false,
EXIT_ON_ERROR: true,
EXIT_ON_ERROR: null as boolean, // set from command line
STARTUP_DURATION_FROM_EVENTLOG: true,
STARTUP_SLEEP_DURATION: 1000,
FORK_CHROMEDRIVER: true,
Expand Down
7 changes: 7 additions & 0 deletions webdriver-ts/src/createResultJS.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ async function main() {
let resultNice = {framework:data.framework, benchmark:data.benchmark, values:data.values.filter(v => v!=null)};
resultJS += '\n' + JSON.stringify(result) + ',';
jsonResult.push(resultNice)
if (benchmarkInfo.type === BenchmarkType.CPU && resultNice.values.length != config.REPEAT_RUN) {
console.log(`WARNING: for ${framework.uri} and benchmark ${benchmarkInfo.id} count was ${resultNice.values.length }. We expected ${config.REPEAT_RUN}`);
} else if (benchmarkInfo.type === BenchmarkType.MEM && resultNice.values.length != config.REPEAT_RUN_MEM) {
console.log(`WARNING: for ${framework.uri} and benchmark ${benchmarkInfo.id} count was ${resultNice.values.length }. We expected ${config.REPEAT_RUN_MEM}`);
} else if (benchmarkInfo.type === BenchmarkType.STARTUP && resultNice.values.length != config.REPEAT_RUN_STARTUP) {
console.log(`WARNING: for ${framework.uri} and benchmark ${benchmarkInfo.id} count was ${resultNice.values.length }. We expected ${config.REPEAT_RUN_STARTUP}`);
}
} else {
console.log("MISSING FILE",file);
}
Expand Down
28 changes: 15 additions & 13 deletions webdriver-ts/src/forkedBenchmarkRunner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,11 @@ async function forceGC(framework: FrameworkData, driver: WebDriver): Promise<any
async function snapMemorySize(driver: WebDriver): Promise<number> {
// currently needed due to https://github.com/krausest/js-framework-benchmark/issues/538
let heapSnapshot: any = await driver.executeScript(":takeHeapSnapshot");
if (typeof(heapSnapshot) === 'string') {
console.log("INFO: heapSnapshot was a JSON string");
heapSnapshot = JSON.parse(heapSnapshot);
}
console.log("****** heapSnapshot.snapshot.meta", typeof(heapSnapshot));
let node_fields: any = heapSnapshot.snapshot.meta.node_fields;
let nodes: any = heapSnapshot.nodes;

Expand Down Expand Up @@ -448,7 +453,6 @@ async function runCPUBenchmark(framework: FrameworkData, benchmark: Benchmark, b
}
} catch (err) {
console.log("ERROR cleaning up driver", err);
throw "Quitting driver failed";
}
}
return {error, warnings};
Expand Down Expand Up @@ -506,7 +510,6 @@ async function runMemBenchmark(framework: FrameworkData, benchmark: Benchmark, b
}
} catch (err) {
console.log("ERROR cleaning up driver", err);
throw "Quitting driver failed";
}
}
return {error, warnings};
Expand Down Expand Up @@ -538,22 +541,22 @@ export async function executeBenchmark(frameworks: FrameworkData[], keyed: boole
let framework = runFrameworks[0];
let benchmark = runBenchmarks[0];

let errorsAndWarnings : ErrorAndWarning;
let errorAndWarnings : ErrorAndWarning;
if (benchmark.type == BenchmarkType.STARTUP) {
errorsAndWarnings = await runStartupBenchmark(framework, benchmark, benchmarkOptions);
errorAndWarnings = await runStartupBenchmark(framework, benchmark, benchmarkOptions);
} else if (benchmark.type == BenchmarkType.CPU) {
errorsAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
errorAndWarnings = await runCPUBenchmark(framework, benchmark, benchmarkOptions);
} else {
errorsAndWarnings = await runMemBenchmark(framework, benchmark, benchmarkOptions);
errorAndWarnings = await runMemBenchmark(framework, benchmark, benchmarkOptions);
}

return errorsAndWarnings;
return errorAndWarnings;
}

export async function performBenchmark(frameworks: FrameworkData[], keyed: boolean, frameworkName: string, benchmarkName: string, benchmarkOptions: BenchmarkOptions): Promise<ErrorAndWarning> {
let errorsAndWarnings = await executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
if (config.LOG_DEBUG) console.log("benchmark finished - got errors promise", errorsAndWarnings);
return errorsAndWarnings;
let errorAndWarnings = await executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
if (config.LOG_DEBUG) console.log("benchmark finished - got errors promise", errorAndWarnings);
return errorAndWarnings;
}

process.on('message', (msg) => {
Expand All @@ -564,12 +567,11 @@ process.on('message', (msg) => {
let {frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions} : {frameworks: FrameworkData[], keyed: boolean, frameworkName: string, benchmarkName: string, benchmarkOptions: BenchmarkOptions} = msg;
if (!benchmarkOptions.port) benchmarkOptions.port = config.PORT.toFixed();
performBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions).then(result => {
console.log("THEN: Returning result", result);
process.send(result);
process.exit(0);
}).catch((err) => {
console.log("CATCH: Error in forkedBenchmarkRunner", JSON.stringify(err), typeof err, err);
process.send({failure: "Unhandled error"});
console.log("CATCH: Error in forkedBenchmarkRunner");
process.send({failure: convertError(err)});
process.exit(0);
});
});

0 comments on commit 366ab12

Please sign in to comment.