From 9b8704f4bcb5c6f78443887271ffb19264a5dcce Mon Sep 17 00:00:00 2001 From: Andrew n marshall Date: Tue, 22 May 2018 17:23:01 -0700 Subject: [PATCH] Adding travis_fold logging around each test (#1879) Replace individual .travis.yml commands and test_runner.js with tests/run_all_tests.sh, which manages the travis_fold statements and accumulating the output of all the test phases. This script is now in charge of test_setup.sh (previously handled by npm pretest script). It also adds green/red color coding to each test section, based on success and failure. --- .travis.yml | 5 +- package.json | 3 +- tests/jsunit/run_jsunit_tests_in_browser.js | 84 ++++++++++++++------- tests/run_all_tests.sh | 75 ++++++++++++++++++ tests/test_runner.js | 44 ----------- 5 files changed, 135 insertions(+), 76 deletions(-) create mode 100755 tests/run_all_tests.sh delete mode 100644 tests/test_runner.js diff --git a/.travis.yml b/.travis.yml index d2f147c39..de6192206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,4 @@ before_script: - sleep 2 script: - - set -x - - npm run lint - - npm test - - tests/compile/compile.sh + - tests/run_all_tests.sh diff --git a/package.json b/package.json index 0637984c5..4c5f9069d 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,7 @@ "scripts": { "prepare": "gulp blockly_javascript_en", "lint": "eslint .", - "pretest": "tests/scripts/test_setup.sh", - "test": "node tests/test_runner.js" + "test": "tests/run_all_tests.sh" }, "license": "Apache-2.0", "private": true, diff --git a/tests/jsunit/run_jsunit_tests_in_browser.js b/tests/jsunit/run_jsunit_tests_in_browser.js index ae14cc394..2764dc237 100644 --- a/tests/jsunit/run_jsunit_tests_in_browser.js +++ b/tests/jsunit/run_jsunit_tests_in_browser.js @@ -27,7 +27,7 @@ var webdriverio = require('webdriverio'); * Runs the JsUnit tests in this directory in Chrome. It uses webdriverio to * launch Chrome and load index.html. Outputs a summary of the test results * to the console. - * @throws If any error occurs when attempting to run the tests. + * @return the Thenable managing the processing of the browser tests. */ function runJsUnitTestsInBrowser() { var options = { @@ -36,41 +36,73 @@ function runJsUnitTestsInBrowser() { } }; - //TODO: change pause to waitunitl - var browser = webdriverio + var url = 'file://' + __dirname + '/index.html'; + console.log('Starting webdriverio...'); + return webdriverio .remote(options) .init() - .url('file://' + __dirname + '/index.html').pause(5000); + .then(function() { + console.log('Initialized.\nLoading url: ' + url); + }) + .url(url) + .then(function() { + console.log('Loaded.\nPausing to allow processing.'); + }) + .pause(5000) //TODO: change pause to waitunitl + .then(function() { + console.log('Retrieving results...'); + }) + .getHTML('#closureTestRunnerLog') + .then(function(result) { + // call js to parse html + var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i; + var numOfFailure = regex.exec(result)[1]; + var regex2 = /Unit Tests for Blockly .*]/; + var testStatus = regex2.exec(result)[0]; + console.log('============Blockly Unit Test Summary================='); + console.log(testStatus); + var regex3 = /\d+ passed,\s\d+ failed/; + var detail = regex3.exec(result)[0]; + console.log(detail); + console.log('============Blockly Unit Test Summary================='); + if (parseInt(numOfFailure) !== 0) { + console.log(result); + process.exit(1); + } + }) + .catch(function(e) { + console.error('Error: ', e); + if (require.main === module) { + // .catch() doesn't seem to work in the calling code, + // even if the error is rethrown. To ensure the script + // exit code is non-zero, shutdown the process here. + process.exit(1); + } - browser - .getHTML('#closureTestRunnerLog') - .then(function(result) { - // call js to parse html - var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i; - var numOfFailure = regex.exec(result)[1]; - var regex2 = /Unit Tests for Blockly .*]/; - var testStatus = regex2.exec(result)[0]; - console.log('============Blockly Unit Test Summary================='); - console.log(testStatus); - var regex3 = /\d+ passed,\s\d+ failed/; - var detail = regex3.exec(result)[0]; - console.log(detail); - console.log('============Blockly Unit Test Summary================='); - if ( parseInt(numOfFailure) !== 0) { - console.log(result); - process.exit(1); - } - }) + // WARNING: Catching this outside of runJsUnitTestsInBrowser() is not + // working. However, killing the process doesn't seem good, either. + throw e; + }); } module.exports = runJsUnitTestsInBrowser; if (require.main === module) { try { - runJsUnitTestsInBrowser(); - } catch(errorStr) { - console.error(errorStr); + runJsUnitTestsInBrowser() + .catch(function(e) { + // TODO: Never called during errors. Fix. + console.error('Error: ' + e); + process.exit(1); + }) + .endAll() + .then(function() { + console.log('JSUnit tests completed'); + process.exit(0); + }); + } catch(e) { + console.error('Uncaught error: ', e); process.exit(1); } } diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh new file mode 100755 index 000000000..ae6b8ca3b --- /dev/null +++ b/tests/run_all_tests.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +if [ ! -z $TRAVIS ]; then echo "Executing run_all_tests.sh from $(pwd)"; fi + +# ANSI colors +BOLD_GREEN='\033[1;32m' +BOLD_RED='\033[1;31m' +ANSI_RESET='\033[0m' + +travis_fold () { + local startOrEnd=$1 # Either "start" or "end" + local id=$2 # The fold id. No spaces. + + if [ ! -z $TRAVIS ]; then + echo "travis_fold:$startOrEnd:$id" + fi +} + +# Find the Blockly project root if pwd is the root +# or if pwd is the directory containing this script. +if [ -f ./run_all_tests.js ]; then + BLOCKLY_ROOT=".." +elif [ -f tests/run_all_tests.sh ]; then + BLOCKLY_ROOT="." +else + echo -e "${BOLD_RED}ERROR: Cannot determine BLOCKLY_ROOT${ANSI_RESET}" 1>&2; + exit 1 +fi +pushd $BLOCKLY_ROOT +echo "pwd: $(pwd)" + +FAILURE_COUNT=0 + +run_test_command () { + local test_id=$1 # The id to use for folds and similar. No spaces. + local command=$2 # The command to run. + + echo "=======================================" + echo "== $test_id" + travis_fold start $test_id + $command + local test_result=$? + travis_fold end $test_id + if [ $test_result -eq 0 ]; then + echo -e "${BOLD_GREEN}SUCCESS:${ANSI_RESET} ${test_id}" + else + echo -e "${BOLD_RED}FAILED:${ANSI_RESET} ${test_id}" + FAILURE_COUNT=$((FAILURE_COUNT+1)) + fi +} + +# Setup the environment (Chrome, Selenium, etc.) +run_test_command "test_setup" "tests/scripts/test_setup.sh" + +# Lint the codebase. +run_test_command "eslint" "eslint ." + +# Run JSUnit tests inside a browser. +run_test_command "jsunit" "node tests/jsunit/run_jsunit_tests_in_browser.js" +# TODO: Make sure jsunit output is captured. Child process? + +# Attempt advanced compilation of a Blockly app. +run_test_command "compile" "tests/compile/compile.sh" + + +# End of tests. +popd +echo "=======================================" +if [ "$FAILURE_COUNT" -eq "0" ]; then + echo -e "${BOLD_GREEN}All tests passed.${ANSI_RESET}" + exit 0 +else + echo -e "${BOLD_RED}Failures in ${FAILURE_COUNT} test groups.${ANSI_RESET}" + exit 1 +fi diff --git a/tests/test_runner.js b/tests/test_runner.js deleted file mode 100644 index e21f3ec0a..000000000 --- a/tests/test_runner.js +++ /dev/null @@ -1,44 +0,0 @@ -/** - * @license - * Visual Blocks Editor - * - * Copyright 2018 Google Inc. - * https://developers.google.com/blockly/ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @fileoverview Node.js script to run all tests. - */ -if (require.main !== module) { - throw __filename + ' must be called directly.'; -} - -var testFns = [ - require('./jsunit/run_jsunit_tests_in_browser') - ]; - -var errored = false; -testFns.forEach((testFn) => { - try { - testFn(); - } catch (errorStr) { - errored = true; - console.error(errorStr + '\n\n'); - } -}); - -if (errored) { - process.exit(1); -}