Adding travis_fold logging around each test (#1879)

Replace individual .travis.yml commands and test_runner.js with tests/run_all_tests.sh, which manages the travis_fold statements and accumulating the output of all the test phases.

This script is now in charge of test_setup.sh (previously handled by npm pretest script). It also adds green/red color coding to each test section, based on success and failure.

.travis.yml

@@ -22,7 +22,4 @@ before_script:
   - sleep 2
 
 script:
-  - set -x
+  - tests/run_all_tests.sh
-  - npm run lint
-  - npm test
-  - tests/compile/compile.sh

package.json

@@ -20,8 +20,7 @@
   "scripts": {
     "prepare": "gulp blockly_javascript_en",
     "lint": "eslint .",
-    "pretest": "tests/scripts/test_setup.sh",
+    "test": "tests/run_all_tests.sh"
-    "test": "node tests/test_runner.js"
   },
   "license": "Apache-2.0",
   "private": true,

tests/jsunit/run_jsunit_tests_in_browser.js

@@ -27,7 +27,7 @@ var webdriverio = require('webdriverio');
  * Runs the JsUnit tests in this directory in Chrome. It uses webdriverio to
  * launch Chrome and load index.html. Outputs a summary of the test results
  * to the console.
- * @throws If any error occurs when attempting to run the tests.
+ * @return the Thenable managing the processing of the browser tests.
  */
 function runJsUnitTestsInBrowser() {
   var options = {
@@ -36,41 +36,73 @@ function runJsUnitTestsInBrowser() {
       }
   };
 
-  //TODO: change pause to waitunitl
+  var url = 'file://' + __dirname + '/index.html';
-  var browser = webdriverio
+  console.log('Starting webdriverio...');
+  return webdriverio
       .remote(options)
       .init()
-      .url('file://' + __dirname + '/index.html').pause(5000);
+      .then(function() {
+        console.log('Initialized.\nLoading url: ' + url);
+      })
+      .url(url)
+      .then(function() {
+        console.log('Loaded.\nPausing to allow processing.');
+      })
+      .pause(5000) //TODO: change pause to waitunitl
+      .then(function() {
+        console.log('Retrieving results...');
+      })
+      .getHTML('#closureTestRunnerLog')
+      .then(function(result) {
+        // call js to parse html
+        var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i;
+        var numOfFailure = regex.exec(result)[1];
+        var regex2 = /Unit Tests for Blockly .*]/;
+        var testStatus = regex2.exec(result)[0];
+        console.log('============Blockly Unit Test Summary=================');
+        console.log(testStatus);
+        var regex3 = /\d+ passed,\s\d+ failed/;
+        var detail = regex3.exec(result)[0];
+        console.log(detail);
+        console.log('============Blockly Unit Test Summary=================');
+        if (parseInt(numOfFailure) !== 0) {
+          console.log(result);
+          process.exit(1);
+        }
+      })
+      .catch(function(e) {
+        console.error('Error: ', e);
 
+        if (require.main === module) {
+          // .catch() doesn't seem to work in the calling code,
+          // even if the error is rethrown. To ensure the script
+          // exit code is non-zero, shutdown the process here.
+          process.exit(1);
+        }
 
-  browser
+        // WARNING: Catching this outside of runJsUnitTestsInBrowser() is not
-  .getHTML('#closureTestRunnerLog')
+        // working. However, killing the process doesn't seem good, either.
-  .then(function(result) {
+        throw e;
-    // call js to parse html
+      });
-    var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i;
-    var numOfFailure = regex.exec(result)[1];
-    var regex2 = /Unit Tests for Blockly .*]/;
-    var testStatus = regex2.exec(result)[0];
-    console.log('============Blockly Unit Test Summary=================');
-    console.log(testStatus);
-    var regex3 = /\d+ passed,\s\d+ failed/;
-    var detail = regex3.exec(result)[0];
-    console.log(detail);
-    console.log('============Blockly Unit Test Summary=================');
-    if ( parseInt(numOfFailure) !== 0) {
-      console.log(result);
-      process.exit(1);
-    }
-  })
 }
 
 module.exports = runJsUnitTestsInBrowser;
 
 if (require.main === module) {
   try {
-    runJsUnitTestsInBrowser();
+    runJsUnitTestsInBrowser()
-  } catch(errorStr) {
+    .catch(function(e) {
-    console.error(errorStr);
+      // TODO: Never called during errors. Fix.
+      console.error('Error: ' + e);
+      process.exit(1);
+    })
+    .endAll()
+    .then(function() {
+      console.log('JSUnit tests completed');
+      process.exit(0);
+    });
+  } catch(e) {
+    console.error('Uncaught error: ', e);
     process.exit(1);
   }
 }

tests/run_all_tests.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ ! -z $TRAVIS ]; then echo "Executing run_all_tests.sh from $(pwd)"; fi
+
+# ANSI colors
+BOLD_GREEN='\033[1;32m'
+BOLD_RED='\033[1;31m'
+ANSI_RESET='\033[0m'
+
+travis_fold () {
+  local startOrEnd=$1 # Either "start" or "end"
+  local id=$2         # The fold id. No spaces.
+
+  if [ ! -z $TRAVIS ]; then
+    echo "travis_fold:$startOrEnd:$id"
+  fi
+}
+
+# Find the Blockly project root if pwd is the root
+# or if pwd is the directory containing this script.
+if [ -f ./run_all_tests.js ]; then
+  BLOCKLY_ROOT=".."
+elif [ -f tests/run_all_tests.sh ]; then
+  BLOCKLY_ROOT="."
+else
+  echo -e "${BOLD_RED}ERROR: Cannot determine BLOCKLY_ROOT${ANSI_RESET}" 1>&2;
+  exit 1
+fi
+pushd $BLOCKLY_ROOT
+echo "pwd: $(pwd)"
+
+FAILURE_COUNT=0
+
+run_test_command () {
+  local test_id=$1  # The id to use for folds and similar. No spaces.
+  local command=$2  # The command to run.
+
+  echo "======================================="
+  echo "== $test_id"
+  travis_fold start $test_id
+  $command
+  local test_result=$?
+  travis_fold end $test_id
+  if [ $test_result -eq 0 ]; then
+    echo -e "${BOLD_GREEN}SUCCESS:${ANSI_RESET} ${test_id}"
+  else
+    echo -e "${BOLD_RED}FAILED:${ANSI_RESET} ${test_id}"
+    FAILURE_COUNT=$((FAILURE_COUNT+1))
+  fi
+}
+
+# Setup the environment (Chrome, Selenium, etc.)
+run_test_command "test_setup" "tests/scripts/test_setup.sh"
+
+# Lint the codebase.
+run_test_command "eslint" "eslint ."
+
+# Run JSUnit tests inside a browser.
+run_test_command "jsunit" "node tests/jsunit/run_jsunit_tests_in_browser.js"
+# TODO: Make sure jsunit output is captured.  Child process?
+
+# Attempt advanced compilation of a Blockly app.
+run_test_command "compile" "tests/compile/compile.sh"
+
+
+# End of tests.
+popd
+echo "======================================="
+if [ "$FAILURE_COUNT" -eq "0" ]; then
+  echo -e "${BOLD_GREEN}All tests passed.${ANSI_RESET}"
+  exit 0
+else
+  echo -e "${BOLD_RED}Failures in ${FAILURE_COUNT} test groups.${ANSI_RESET}"
+  exit 1
+fi

tests/test_runner.js

@@ -1,44 +0,0 @@
-/**
- * @license
- * Visual Blocks Editor
- *
- * Copyright 2018 Google Inc.
- * https://developers.google.com/blockly/
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * @fileoverview Node.js script to run all tests.
- */
-if (require.main !== module) {
-  throw __filename + ' must be called directly.';
-}
-
-var testFns = [
-      require('./jsunit/run_jsunit_tests_in_browser')
-    ];
-
-var errored = false;
-testFns.forEach((testFn) => {
-  try {
-    testFn();
-  } catch (errorStr) {
-    errored = true;
-    console.error(errorStr + '\n\n');
-  }
-});
-
-if (errored) {
-  process.exit(1);
-}