feat(test): Miscellaneous improvements to test_tasks.js (#6615)

* feat(tests): Make runTestBlock able to run any gulp task

  Modify runTestBlock so that it can run any async task, not just
  ones that return a Promise, by using the async-done package
  (part of Gulp, and already an indirect dependency) to detect
  task completion.

  Celebrate by renaming it to runTestTask.

* refactor(tests): Create Tester class to encapsulate test infrastructure

  - Create Tester class to encapsulate the runTestTask,
    and reportTestResult and runAll functions.

  - Remove the unnecessary id parameter from runTestTask (code was
    already using the .name of the task function object).

  - Remove --silent flag from npm scripts so as not to suppress
    syntax error in gulpfiles.

* refactor(tests): Invoke buildAdvancedCompilationTest task directly

  Have the test task invoke the buildAdvancedCompilationTest
  (via onlyBuildAdvancedCompilationTest, to skip already-run
  prerequisites) directly, rather than by running npm.
This commit is contained in:
Christopher Allen
2022-11-16 18:17:54 +00:00
committed by GitHub
parent d37223d8ab
commit 54670d534e
4 changed files with 227 additions and 139 deletions

View File

@@ -52,9 +52,6 @@ module.exports = {
buildAdvancedCompilationTest: buildTasks.buildAdvancedCompilationTest,
gitCreateRC: gitTasks.createRC,
docs: docsTasks.docs,
// Targets intended only for invocation by scripts; may omit prerequisites.
onlyBuildAdvancedCompilationTest: buildTasks.onlyBuildAdvancedCompilationTest,
// Legacy targets, to be deleted.
recompile: releaseTasks.recompile,

114
package-lock.json generated
View File

@@ -21,6 +21,7 @@
"@microsoft/api-extractor": "^7.29.5",
"@typescript-eslint/eslint-plugin": "^5.33.1",
"@wdio/selenium-standalone-service": "^7.10.1",
"async-done": "^2.0.0",
"chai": "^4.2.0",
"clang-format": "^1.6.0",
"closure-calculate-chunks": "^3.0.2",
@@ -2481,18 +2482,17 @@
"dev": true
},
"node_modules/async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-2.0.0.tgz",
"integrity": "sha512-j0s3bzYq9yKIVLKGE/tWlCpa3PfFLcrDZLTSVdnnCTGagXuXBJO4SsY9Xdk/fQBirCkH4evW5xOeJXqlAQFdsw==",
"dev": true,
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
"end-of-stream": "^1.4.4",
"once": "^1.4.0",
"stream-exhaust": "^1.0.2"
},
"engines": {
"node": ">= 0.10"
"node": ">= 10.13.0"
}
},
"node_modules/async-each": {
@@ -2523,6 +2523,21 @@
"node": ">= 0.10"
}
},
"node_modules/async-settle/node_modules/async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -2577,6 +2592,21 @@
"node": ">= 0.10"
}
},
"node_modules/bach/node_modules/async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/balanced-match": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
@@ -6277,6 +6307,21 @@
"node": ">=0.10.0"
}
},
"node_modules/glob-watcher/node_modules/async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/glob-watcher/node_modules/binary-extensions": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",
@@ -16487,15 +16532,14 @@
"dev": true
},
"async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-2.0.0.tgz",
"integrity": "sha512-j0s3bzYq9yKIVLKGE/tWlCpa3PfFLcrDZLTSVdnnCTGagXuXBJO4SsY9Xdk/fQBirCkH4evW5xOeJXqlAQFdsw==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
"end-of-stream": "^1.4.4",
"once": "^1.4.0",
"stream-exhaust": "^1.0.2"
}
},
"async-each": {
@@ -16518,6 +16562,20 @@
"dev": true,
"requires": {
"async-done": "^1.2.2"
},
"dependencies": {
"async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
}
}
}
},
"asynckit": {
@@ -16560,6 +16618,20 @@
"async-done": "^1.2.2",
"async-settle": "^1.0.0",
"now-and-later": "^2.0.0"
},
"dependencies": {
"async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
}
}
}
},
"balanced-match": {
@@ -19501,6 +19573,18 @@
}
}
},
"async-done": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/async-done/-/async-done-1.3.2.tgz",
"integrity": "sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.2",
"process-nextick-args": "^2.0.0",
"stream-exhaust": "^1.0.1"
}
},
"binary-extensions": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz",

View File

@@ -42,7 +42,6 @@
"lint:fix": "eslint . --fix",
"langfiles": "gulp langfiles",
"minify": "gulp minify",
"only:compile:advanced": "gulp onlyBuildAdvancedCompilationTest --debug",
"package": "gulp package",
"postinstall": "patch-package",
"prepare": "gulp prepare",
@@ -53,8 +52,8 @@
"release": "gulp gitCreateRC",
"start": "concurrently -n tsc,server \"tsc --watch --preserveWatchOutput --outDir 'build/src' --declarationDir 'build/declarations'\" \"http-server ./ -s -o /tests/playground.html -c-1\"",
"tsc": "gulp tsc",
"test": "gulp --silent test",
"test:generators": "gulp --silent testGenerators",
"test": "gulp test",
"test:generators": "gulp testGenerators",
"test:mocha:interactive": "http-server ./ -o /tests/mocha/index.html -c-1",
"test:compile:advanced": "gulp buildAdvancedCompilationTest --debug",
"updateGithubPages": "npm ci && gulp gitUpdateGithubPages"
@@ -78,6 +77,7 @@
"@microsoft/api-extractor": "^7.29.5",
"@typescript-eslint/eslint-plugin": "^5.33.1",
"@wdio/selenium-standalone-service": "^7.10.1",
"async-done": "^2.0.0",
"chai": "^4.2.0",
"clang-format": "^1.6.0",
"closure-calculate-chunks": "^3.0.2",

View File

@@ -9,6 +9,7 @@
*/
/* eslint-env node */
const asyncDone = require('async-done');
const gulp = require('gulp');
const gzip = require('gulp-gzip');
const fs = require('fs');
@@ -16,6 +17,7 @@ const path = require('path');
const {execSync} = require('child_process');
const rimraf = require('rimraf');
const buildTasks = require('./build_tasks');
const {BUILD_DIR, RELEASE_DIR} = require('./config');
const runMochaTestsInBrowser =
@@ -31,46 +33,84 @@ const BOLD_GREEN = '\x1b[1;32m';
const BOLD_RED = '\x1b[1;31m';
const ANSI_RESET = '\x1b[0m';
let failerCount = 0;
class Tester {
constructor(tasks = []) {
this.successCount = 0;
this.failCount = 0;
this.tasks = tasks;
}
/**
* Run all tests in sequence.
*/
async runAll() {
for (const task of this.tasks) {
await this.runTestTask(task)
}
this.reportTestResult();
}
/**
* Helper method for running test code block.
* @param {string} id test id
* @param {function} block test code block
* @return {Promise} asynchronous result
*/
function runTestBlock(id, block) {
return new Promise((resolve) => {
/**
* Create a Gulp task to run all tests.
*/
asTask() {
return this.runAll.bind(this);
}
/**
* Run an arbitrary Gulp task as a test.
* @param {function} task Any gulp task
* @return {Promise} asynchronous result
*/
async runTestTask(task) {
const id = task.name;
console.log('=======================================');
console.log(`== ${id}`);
if (process.env.CI) console.log('::group::');
block()
.then((result) => {
try {
try {
await new Promise((resolve, reject) => {
asyncDone(task, (error, result) => {
if (error) reject(error);
resolve(result);
});
});
} finally {
if (process.env.CI) console.log('::endgroup::');
console.log(`${BOLD_GREEN}SUCCESS:${ANSI_RESET} ${id}`);
resolve(result);
})
.catch((err) => {
failerCount++;
console.error(err.message);
if (process.env.CI) console.log('::endgroup::');
console.log(`${BOLD_RED}FAILED:${ANSI_RESET} ${id}`);
// Always continue.
resolve(err);
});
});
}
}
this.successCount++;
console.log(`${BOLD_GREEN}SUCCESS:${ANSI_RESET} ${id}`);
} catch (error) {
this.failCount++;
console.error(error.message);
console.log(`${BOLD_RED}FAILED:${ANSI_RESET} ${id}`);
}
}
/**
* Print test results.
*/
reportTestResult() {
console.log('=======================================');
// Check result.
if (this.failCount === 0) {
console.log(
`${BOLD_GREEN}All ${this.successCount} tests passed.${ANSI_RESET}`);
} else {
console.log(
`${BOLD_RED}Failures in ${this.failCount} test groups.${ANSI_RESET}`);
}
}
};
/**
* Helper method for running test command.
* @param {string} id test id
* @param {string} command command line to run
* @return {Promise} asynchronous result
*/
function runTestCommand(id, command) {
return runTestBlock(id, async() => {
return execSync(command, {stdio: 'inherit'});
}, false);
async function runTestCommand(command) {
execSync(command, {stdio: 'inherit'});
}
/**
@@ -83,7 +123,7 @@ function eslint() {
console.log('Skip linting.');
return Promise.resolve();
}
return runTestCommand('eslint', 'eslint .');
return runTestCommand('eslint .');
}
/**
@@ -92,8 +132,7 @@ function eslint() {
* @return {Promise} asynchronous result
*/
function build() {
return runTestCommand('build + package',
'npm run package -- --verbose --debug');
return runTestCommand('npm run package -- --verbose --debug');
}
/**
@@ -101,7 +140,7 @@ function build() {
* @return {Promise} asynchronous result
*/
function renamings() {
return runTestCommand('renamings', 'node tests/migration/validate-renamings.js');
return runTestCommand('node tests/migration/validate-renamings.js');
}
/**
@@ -165,50 +204,46 @@ function zippingFiles() {
* Check the sizes of built files for unexpected growth.
* @return {Promise} asynchronous result
*/
function metadata() {
return runTestBlock('metadata', async() => {
// Zipping the compressed files.
await zippingFiles();
// Read expected size from script.
const contents = fs.readFileSync('tests/scripts/check_metadata.sh')
async function metadata() {
// Zipping the compressed files.
await zippingFiles();
// Read expected size from script.
const contents = fs.readFileSync('tests/scripts/check_metadata.sh')
.toString();
const pattern = /^readonly (?<key>[A-Z_]+)=(?<value>\d+)$/gm;
const matches = contents.matchAll(pattern);
const expected = {};
for (const match of matches) {
expected[match.groups.key] = match.groups.value;
}
const pattern = /^readonly (?<key>[A-Z_]+)=(?<value>\d+)$/gm;
const matches = contents.matchAll(pattern);
const expected = {};
for (const match of matches) {
expected[match.groups.key] = match.groups.value;
}
// Check the sizes of the files.
let failed = 0;
failed += compareSize('blockly_compressed.js',
expected.BLOCKLY_SIZE_EXPECTED);
failed += compareSize('blocks_compressed.js',
expected.BLOCKS_SIZE_EXPECTED);
failed += compareSize('blockly_compressed.js.gz',
expected.BLOCKLY_GZ_SIZE_EXPECTED);
failed += compareSize('blocks_compressed.js.gz',
expected.BLOCKS_GZ_SIZE_EXPECTED);
if (failed > 0) {
throw new Error('Unexpected growth was detected.');
}
});
// Check the sizes of the files.
let failed = 0;
failed += compareSize('blockly_compressed.js',
expected.BLOCKLY_SIZE_EXPECTED);
failed += compareSize('blocks_compressed.js',
expected.BLOCKS_SIZE_EXPECTED);
failed += compareSize('blockly_compressed.js.gz',
expected.BLOCKLY_GZ_SIZE_EXPECTED);
failed += compareSize('blocks_compressed.js.gz',
expected.BLOCKS_GZ_SIZE_EXPECTED);
if (failed > 0) {
throw new Error('Unexpected growth was detected.');
}
}
/**
* Run Mocha tests inside a browser.
* @return {Promise} asynchronous result
*/
function mocha() {
return runTestBlock('mocha', async() => {
const result = await runMochaTestsInBrowser().catch(e => {
throw e;
});
if (result) {
throw new Error('Mocha tests failed');
}
console.log('Mocha tests passed');
async function mocha() {
const result = await runMochaTestsInBrowser().catch(e => {
throw e;
});
if (result) {
throw new Error('Mocha tests failed');
}
console.log('Mocha tests passed');
}
/**
@@ -264,28 +299,26 @@ function checkResult(suffix) {
* Run generator tests inside a browser and check the results.
* @return {Promise} asynchronous result
*/
function generators() {
return runTestBlock('generators', async() => {
// Clean up.
rimraf.sync(OUTPUT_DIR);
fs.mkdirSync(OUTPUT_DIR);
async function generators() {
// Clean up.
rimraf.sync(OUTPUT_DIR);
fs.mkdirSync(OUTPUT_DIR);
await runGeneratorsInBrowser(OUTPUT_DIR).catch(() => {});
await runGeneratorsInBrowser(OUTPUT_DIR).catch(() => {});
const generatorSuffixes = ['js', 'py', 'dart', 'lua', 'php'];
let failed = 0;
generatorSuffixes.forEach((suffix) => {
failed += checkResult(suffix);
});
if (failed === 0) {
console.log(`${BOLD_GREEN}All generator tests passed.${ANSI_RESET}`);
} else {
console.log(
`${BOLD_RED}Failures in ${failed} generator tests.${ANSI_RESET}`);
throw new Error('Generator tests failed.');
}
const generatorSuffixes = ['js', 'py', 'dart', 'lua', 'php'];
let failed = 0;
generatorSuffixes.forEach((suffix) => {
failed += checkResult(suffix);
});
if (failed === 0) {
console.log(`${BOLD_GREEN}All generator tests passed.${ANSI_RESET}`);
} else {
console.log(
`${BOLD_RED}Failures in ${failed} generator tests.${ANSI_RESET}`);
throw new Error('Generator tests failed.');
}
}
/**
@@ -293,35 +326,12 @@ function generators() {
* @return {Promise} asynchronous result
*/
function node() {
return runTestCommand('node', 'mocha tests/node --config tests/node/.mocharc.js');
return runTestCommand('mocha tests/node --config tests/node/.mocharc.js');
}
/**
* Attempt advanced compilation of a Blockly app.
* @return {Promise} asynchronous result
*/
function advancedCompile() {
return runTestCommand('advanced_compile', 'npm run only:compile:advanced');
}
/**
* Report test result.
* @return {Promise} asynchronous result
*/
function reportTestResult() {
console.log('=======================================');
// Check result.
if (failerCount === 0) {
console.log(`${BOLD_GREEN}All tests passed.${ANSI_RESET}`);
return Promise.resolve();
} else {
console.log(`${BOLD_RED}Failures in ${failerCount} test groups.${ANSI_RESET}`);
return Promise.reject();
}
}
// Indivisual tasks.
const testTasks = [
// Run all tests in sequence.
const test = new Tester([
eslint,
build,
renamings,
@@ -329,12 +339,9 @@ const testTasks = [
mocha,
generators,
node,
advancedCompile,
reportTestResult,
];
buildTasks.onlyBuildAdvancedCompilationTest,
]).asTask();
// Run all tests in sequence.
const test = gulp.series(...testTasks);
module.exports = {
test,