Merge pull request #2424 from rachel-fenichel/testing/webdriverio_upgrade

Update to webdriverio v5; fix resulting test breakages

.travis.yml

@@ -6,6 +6,7 @@ node_js:
   - 10
 sudo: required
 addons:
+  firefox: latest
   apt:
     packages:
     - google-chrome-stable

package.json

@@ -34,7 +34,7 @@
     "gulp-series": "^1.0.2",
     "gulp-shell": "^0.6.5",
     "jshint": "^2.10.1",
-    "webdriverio": "^4.14.1"
+    "webdriverio": "^5.8.0"
   },
   "jshintConfig": {
     "globalstrict": true,

tests/generators/run_generators_in_browser.js

@@ -24,6 +24,8 @@
 var webdriverio = require('webdriverio');
 var fs = require('fs');
 
+module.exports = runGeneratorsInBrowser;
+
 /**
  * Run the generator for a given language and save the results to a file.
  * @param {Thenable} browser A Thenable managing the processing of the browser
@@ -31,23 +33,16 @@ var fs = require('fs');
  * @param {string} filename Where to write the output file.
  * @param {Function} codegenFn The function to run for code generation for this
  *     language.
- * @return the Thenable managing the processing of the browser tests.
  */
-function runLangGeneratorInBrowser(browser, filename, codegenFn) {
-  return browser
-      .pause(5000)
-      .then(function() {
-        this.execute(codegenFn)
-      })
-      .pause(10000)
-      .getValue("#importExport")
-      .then(function(result) {
-        fs.writeFile(filename, result, function(err) {
-          if (err) {
-            return console.log(err);
-          }
-        });
-      });
+async function runLangGeneratorInBrowser(browser, filename, codegenFn) {
+  await browser.execute(codegenFn);
+  var elem = await browser.$("#importExport");
+  var result = await elem.getValue();
+  fs.writeFile(filename, result, function(err) {
+    if (err) {
+      return console.log(err);
+    }
+  });
 }
 
 /**
@@ -56,90 +51,61 @@ function runLangGeneratorInBrowser(browser, filename, codegenFn) {
  * to the console and outputs files for later validation.
  * @return the Thenable managing the processing of the browser tests.
  */
-function runGeneratorsInBrowser() {
+async function runGeneratorsInBrowser() {
   var options = {
-      desiredCapabilities: {
+      capabilities: {
           browserName: 'firefox'
       }
   };
 
   var url = 'file://' + __dirname + '/index.html';
-  var prefix = 'tests/generators/tmp/generated'
+  var prefix = 'tests/generators/tmp/generated';
+
   console.log('Starting webdriverio...');
-  return webdriverio
-      .remote(options)
-      .init()
-      .then(function() {
-        console.log('Initialized.\nLoading url: ' + url);
-      })
-      .url(url)
-      .then(function() {
-        console.log('about to load');
-        this.execute(function() {
-          checkAll();
-          loadSelected();
-        })
-      })
-      .pause(10000)
-      .then(function() {
-        return runLangGeneratorInBrowser(this, prefix + '.js', function() {
-          toJavaScript();
-        });
-      })
-      .then(function() {
-        return runLangGeneratorInBrowser(this, prefix + '.py', function() {
-          toPython();
-        });
-      })
-      .then(function() {
-        return runLangGeneratorInBrowser(this, prefix + '.dart', function() {
-          toDart();
-        });
-      })
-      .then(function() {
-        return runLangGeneratorInBrowser(this, prefix + '.lua', function() {
-          toLua();
-        });
-      })
-      .then(function() {
-        return runLangGeneratorInBrowser(this, prefix + '.php', function() {
-          toPhp();
-        });
-      })
-      .pause(10000)
-      .catch(function(e) {
-        console.error('Error: ', e);
+  const browser = await webdriverio.remote(options);
+  console.log('Initialized.\nLoading url: ' + url);
+  await browser.url(url);
 
-        if (require.main === module) {
-          // .catch() doesn't seem to work in the calling code,
-          // even if the error is rethrown. To ensure the script
-          // exit code is non-zero, shutdown the process here.
-          process.exit(1);
-        }
+  await browser.execute(function() {
+    checkAll();
+    loadSelected();
+  });
 
-        // WARNING: Catching this outside of runJsUnitTestsInBrowser() is not
-        // working. However, killing the process doesn't seem good, either.
-        throw e;
+  await runLangGeneratorInBrowser(browser, prefix + '.js',
+      function() {
+        toJavaScript();
+      });
+  await runLangGeneratorInBrowser(browser, prefix + '.py',
+      function() {
+        toPython();
+      });
+  await runLangGeneratorInBrowser(browser, prefix + '.dart',
+      function() {
+        toDart();
+      });
+  await runLangGeneratorInBrowser(browser, prefix + '.lua',
+      function() {
+        toLua();
+      });
+  await runLangGeneratorInBrowser(browser, prefix + '.php',
+      function() {
+        toPhp();
       });
-}
 
-module.exports = runGeneratorsInBrowser;
+  await browser.deleteSession();
+}
 
 if (require.main === module) {
-  try {
-    runGeneratorsInBrowser()
-    .catch(function(e) {
-      // TODO: Never called during errors. Fix.
-      console.error('Error: ' + e);
-      process.exit(1);
-    })
-    .endAll()
-    .then(function() {
-      console.log('JSUnit tests completed');
-      process.exit(0);
-    });
-  } catch(e) {
-    console.error('Uncaught error: ', e);
+  runGeneratorsInBrowser().catch(e => {
+    console.error(e);
     process.exit(1);
-  }
+  }).then(function(result) {
+    if (result) {
+      console.log('Generator tests failed');
+      process.exit(1);
+    } else {
+      console.log('Generator tests passed');
+      process.exit(0);
+    }
+  });
 }

tests/jsunit/run_jsunit_tests_in_browser.js

@@ -23,86 +23,60 @@
  */
 var webdriverio = require('webdriverio');
 
+module.exports = runJsUnitTestsInBrowser;
+
 /**
  * Runs the JsUnit tests in this directory in Chrome. It uses webdriverio to
  * launch Chrome and load index.html. Outputs a summary of the test results
  * to the console.
- * @return the Thenable managing the processing of the browser tests.
+ * @return 0 on success, 1 on failure.
  */
-function runJsUnitTestsInBrowser() {
+async function runJsUnitTestsInBrowser() {
   var options = {
-      desiredCapabilities: {
+      capabilities: {
           browserName: 'chrome'
       }
   };
 
   var url = 'file://' + __dirname + '/index.html';
   console.log('Starting webdriverio...');
-  return webdriverio
-      .remote(options)
-      .init()
-      .then(function() {
-        console.log('Initialized.\nLoading url: ' + url);
-      })
-      .url(url)
-      .then(function() {
-        console.log('Loaded.\nPausing to allow processing.');
-      })
-      .pause(5000) //TODO: change pause to waitunitl
-      .then(function() {
-        console.log('Retrieving results...');
-      })
-      .getHTML('#closureTestRunnerLog')
-      .then(function(result) {
-        // call js to parse html
-        var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i;
-        var numOfFailure = regex.exec(result)[1];
-        var regex2 = /Unit Tests for Blockly .*]/;
-        var testStatus = regex2.exec(result)[0];
-        console.log('============Blockly Unit Test Summary=================');
-        console.log(testStatus);
-        var regex3 = /\d+ passed,\s\d+ failed/;
-        var detail = regex3.exec(result)[0];
-        console.log(detail);
-        console.log('============Blockly Unit Test Summary=================');
-        if (parseInt(numOfFailure) !== 0) {
-          console.log(result);
-          process.exit(1);
-        }
-      })
-      .catch(function(e) {
-        console.error('Error: ', e);
+  const browser = await webdriverio.remote(options);
+  console.log('Initialized.\nLoading url: ' + url);
+  await browser.url(url);
 
-        if (require.main === module) {
-          // .catch() doesn't seem to work in the calling code,
-          // even if the error is rethrown. To ensure the script
-          // exit code is non-zero, shutdown the process here.
-          process.exit(1);
-        }
+  const elem = await browser.$('#closureTestRunnerLog')
+  const result = await elem.getHTML();
 
-        // WARNING: Catching this outside of runJsUnitTestsInBrowser() is not
-        // working. However, killing the process doesn't seem good, either.
-        throw e;
-      });
+  // call js to parse html
+  var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i;
+  var numOfFailure = regex.exec(result)[1];
+  var regex2 = /Unit Tests for Blockly .*]/;
+  var testStatus = regex2.exec(result)[0];
+  console.log('============Blockly Unit Test Summary=================');
+  console.log(testStatus);
+  var regex3 = /\d+ passed,\s\d+ failed/;
+  var detail = regex3.exec(result)[0];
+  console.log(detail);
+  console.log('============Blockly Unit Test Summary=================');
+  if (parseInt(numOfFailure) !== 0) {
+    await browser.deleteSession();
+    return 1;
+  }
+  await browser.deleteSession();
+  return 0;
 }
 
-module.exports = runJsUnitTestsInBrowser;
-
 if (require.main === module) {
-  try {
-    runJsUnitTestsInBrowser()
-    .catch(function(e) {
-      // TODO: Never called during errors. Fix.
-      console.error('Error: ' + e);
-      process.exit(1);
-    })
-    .endAll()
-    .then(function() {
-      console.log('JSUnit tests completed');
-      process.exit(0);
-    });
-  } catch(e) {
-    console.error('Uncaught error: ', e);
+  runJsUnitTestsInBrowser().catch(e => {
+    console.error(e);
     process.exit(1);
-  }
+  }).then(function(result) {
+    if (result) {
+      console.log('JSUnit tests failed');
+      process.exit(1);
+    } else {
+      console.log('JSUnit tests passed');
+      process.exit(0);
+    }
+  });
 }

tests/scripts/get_geckdriver.sh

@@ -7,9 +7,9 @@ fi
 echo "downloading gechdriver"
 
 if [[ $os_name == 'Linux' ]]; then
-  cd ../ && curl -L https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz | tar xz
+  cd ../ && curl -L https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-linux64.tar.gz | tar xz
   sleep 5
 elif [[ $os_name == 'Darwin' ]]; then
-  cd ../ &&  curl -L https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-macos.tar.gz | tar xz
+  cd ../ &&  curl -L https://github.com/mozilla/geckodriver/releases/download/v0.21.0/geckodriver-v0.21.0-macos.tar.gz | tar xz
   sleep 5
 fi