refactor(tests): Migrate generator tests to import shims; delete bootstrap.js (#7414)

* refactor(tests): Use shims instead of bootstrap to load Blockly

  - Modify tests/generators/index.html to import the test shims
    instead of using bootstrap.js to load Blockly.

  - Modify test/generators/webdriver.js to have it wait for the
    workspace to exist before calling loadSelected().  There was
    previously a race which index.html had been winning, but
    now webdriver.js is winning (and the tests failing because
    there is no workspace yet when start() is called.

* chore(tests): Delete bootstrap.js etc.

  - Delete bootstrap.js, bootstrap_helper.js, and bootstrap_done.mjs.
  - Remove remaining references to bootstrap.js

* refactor(build): Remove deps npm script

  buildDeps is now only needed by buildCompiled, not ever for
  runnning in uncompressed mode, so:

  - Remove the deps gulp task (and the deps npm script.
  - Have the minify task run buildJavaScript and buildDeps directly.

  Additionally, the buildAdvanceCompilationTest target hasn't
  needed deps.js for some time (if ever), so skip having it run
  buildDeps entirely.

* refactor(build): Repatriate DEPS_FILE to build_tasks.js

  Since this is no longer used anywhere else it doesn't need to
  live in common.js.

* fix(scripts): Remove vestigial references to deps.mocha.js

* docs(tests): Add additional explanatory note

core/main.js

@@ -7,7 +7,7 @@
 /**
  * @fileoverview The entrypoint for blockly_compressed.js.  Provides
  *     various backwards-compatibility hacks.  Not used when loading
- *     in uncompiled (uncompressed) mode via bootstrap.js.
+ *     in uncompiled (uncompressed) mode via blockly.loader.mjs.
  */
 'use strict';
 

package.json

@@ -24,14 +24,12 @@
     "build-strict-log": "npm run build:strict > build-debug.log 2>&1 && tail -3 build-debug.log",
     "build:compiled": "exit 1 # Deprecated; use \"npm run minify\" instead.",
     "build:compressed": "exit 1 # Deprecated; use \"npm run minify\" instead.",
-    "build:deps": "exit 1 # Deprecated; use \"npm run deps\" instead.",
     "build:js": "exit 1 # Deprecated; use \"npm run tsc\" instead.",
     "build:langfiles": "exit 1 # Deprecated; use \"npm run langfiles\" instead.",
     "bump": "npm --no-git-tag-version version 4.$(date +'%Y%m%d').0",
     "clean": "gulp clean",
     "deployDemos": "npm ci && gulp deployDemos",
     "deployDemos:beta": "npm ci && gulp deployDemosBeta",
-    "deps": "gulp deps",
     "docs": "gulp docs",
     "format": "prettier --write .",
     "format:check": "prettier --check .",

scripts/gulpfiles/build_tasks.js

@@ -22,7 +22,7 @@ const closureCompiler = require('google-closure-compiler').gulp();
 const argv = require('yargs').argv;
 const {rimraf} = require('rimraf');
 
-const {BUILD_DIR, DEPS_FILE, RELEASE_DIR, TSC_OUTPUT_DIR, TYPINGS_BUILD_DIR} = require('./config');
+const {BUILD_DIR, RELEASE_DIR, TSC_OUTPUT_DIR, TYPINGS_BUILD_DIR} = require('./config');
 const {getPackageJson} = require('./helper_tasks');
 
 const {posixPath, quote} = require('../helpers');
@@ -43,6 +43,11 @@ const PYTHON = process.platform === 'win32' ? 'python' : 'python3';
  */
 const COMPILED_SUFFIX = '_compressed';
 
+/**
+ * Dependencies file (used by buildCompiled for chunking.
+ */
+const DEPS_FILE = path.join(BUILD_DIR, 'deps.js');
+
 /**
  * Name of an object to be used as a shared "global" namespace by
  * chunks generated by the Closure Compiler with the
@@ -302,8 +307,9 @@ function buildJavaScript(done) {
 }
 
 /**
- * This task updates DEPS_FILE (deps.js), used by the debug module
+ * This task updates DEPS_FILE (deps.js), used by
- * loader (via bootstrap.js) when loading Blockly in uncompiled mode.
+ * closure-calculate-chunks when determining how to organise .js
+ * source files into chunks.
  *
  * Prerequisite: buildJavaScript.
  */
@@ -354,7 +360,6 @@ error message above, try running:
             reject(error);
           } else {
             log(stderr);
-            // Anything not about mocha goes in DEPS_FILE.
             fs.writeFileSync(DEPS_FILE, stdout);
             resolve();
           }
@@ -710,7 +715,7 @@ ${Object.keys(exports).map((name) => `  ${name},`).join('\n')}
  * This task builds Blockly core, blocks and generators together and uses
  * Closure Compiler's ADVANCED_COMPILATION mode.
  *
- * Prerequisite: buildDeps.
+ * Prerequisite: buildJavaScript.
  */
 function buildAdvancedCompilationTest() {
   // If main_compressed.js exists (from a previous run) delete it so that
@@ -761,14 +766,13 @@ function cleanBuildDir() {
 exports.cleanBuildDir = cleanBuildDir;
 exports.langfiles = buildLangfiles;  // Build build/msg/*.js from msg/json/*.
 exports.tsc = buildJavaScript;
-exports.deps = gulp.series(exports.tsc, buildDeps);
+exports.minify = gulp.series(exports.tsc, buildDeps, buildCompiled, buildShims);
-exports.minify = gulp.series(exports.deps, buildCompiled, buildShims);
 exports.build = gulp.parallel(exports.minify, exports.langfiles);
 
 // Manually-invokable targets, with prerequisites where required.
 exports.messages = generateMessages;  // Generate msg/json/en.json et al.
 exports.buildAdvancedCompilationTest =
-    gulp.series(exports.deps, buildAdvancedCompilationTest);
+    gulp.series(exports.tsc, buildAdvancedCompilationTest);
 
 // Targets intended only for invocation by scripts; may omit prerequisites.
 exports.onlyBuildAdvancedCompilationTest = buildAdvancedCompilationTest;

scripts/gulpfiles/config.js

@@ -19,15 +19,10 @@ const path = require('path');
 // - tests/scripts/compile_typings.sh
 // - tests/scripts/check_metadata.sh
 // - tests/scripts/update_metadata.sh
-// - tests/bootstrap.js (for location of deps.js)
-// - tests/mocha/index.html (for location of deps.mocha.js)
 
 // Directory to write compiled output to.
 exports.BUILD_DIR = 'build';
 
-// Dependencies file (used by bootstrap.js in uncompiled mode):
-exports.DEPS_FILE = path.join(exports.BUILD_DIR, 'deps.js');
-
 // Directory to write typings output to.
 exports.TYPINGS_BUILD_DIR = path.join(exports.BUILD_DIR, 'declarations');
 

scripts/migration/js2ts

@@ -54,7 +54,6 @@ goog.addDependency = function(relPath, provides, _requires, opt_loadFlags) {
 
 // Load deps files relative to this script's location.
 require(path.resolve(__dirname, '../../build/deps.js'));
-require(path.resolve(__dirname, '../../build/deps.mocha.js'));
 
 //////////////////////////////////////////////////////////////////////
 // Process files mentioned on the command line.

tests/bootstrap.js

@@ -1,276 +0,0 @@
-/**
- * @license
- * Copyright 2021 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-
-/**
- * @fileoverview Bootstrap code to load Blockly, typically in
- *     uncompressed mode.
- *
- *     Load this file in a <script> tag in a web page to use the
- *     Closure Library debug module loader to load Blockly in
- *     uncompressed mode.
- *
- *     You must use a <script src=> tag to load this script first
- *     (after setting BLOCKLY_BOOTSTRAP_OPTIONS in a preceeding
- *     <script> tag, if desired - see below), then import
- *     bootstrap_done.mjs in a <script type=module> to wait for
- *     bootstrapping to finish.
- *
- *     See tests/playground.html for example usage.
- *
- *     Exception: for speed, if this is script is from a host other
- *     than localhost, blockly_compressed.js (et al.) will be loaded
- *     instead.  Because of the sequential, non-parallel module loading
- *     carried out by the debug module loader, loading can be painfully
- *     tedious over a slow network connection. (This can be overridden
- *     by the page if desired.)
- *
- *     The bootstrap code will consult a BLOCKLY_BOOTSTRAP_OPTIONS
- *     global variable to determine what to load.  This must be set
- *     before loading this script.  See documentation inline below.
- */
-'use strict';
-
-(function () {
-  // Values used to compute default bootstrap options.
-  const localhosts = ['localhost', '127.0.0.1', '[::1]'];
-  const isLocalhost = localhosts.includes(location.hostname);
-  const isFileUrl = location.origin === 'file://';
-
-  // Default bootstrap options.  These can be overridden by setting
-  // the same property on BLOCKLY_BOOTSTRAP_OPTIONS.
-  const options = {
-    // Decide whether to use compressed mode or not.  Please see issue
-    // #5557 for more information.
-    loadCompressed: !(isLocalhost || isFileUrl),
-
-    // URL of the blockly repository.  This is needed for a few reasons:
-    //
-    // - We need an absolute path instead of relative path because the
-    //   advanced_playground and the regular playground are in
-    //   different folders.
-    // - We need to get the root directory for blockly because it is
-    //   different for github.io, appspot and local.
-    //
-    // Default value will work so long as top-level page is loaded
-    // from somewhere in tests/.
-    root: window.location.href.replace(/\/tests\/.*$/, '/'),
-
-    // List of deps files to load.  Paths relative to root.
-    depsFiles: ['build/deps.js'],
-
-    // List of modules to load.
-    // - id: goog.module ID to require in uncompressed mode.
-    // - script: path, relative to root, to .js file to load via
-    //   <script> in compressed mode.
-    // - scriptExport: path at which script will save exports object
-    //   (see chunks in build_tasks.js); defaults to id.
-    // - importAt: gloal variable to set to export object.
-    // - destructure: map of globalVariable: exportName; globals will
-    //   be set to the corresponding exports.
-    modules: [
-      {
-        id: 'Blockly',
-        script: 'dist/blockly_compressed.js',
-        scriptExport: 'Blockly',
-        importAt: 'Blockly',
-      },
-      {
-        id: 'Blockly.libraryBlocks',
-        script: 'dist/blocks_compressed.js',
-        scriptExport: 'Blockly.libraryBlocks',
-        importAt: 'libraryBlocks',
-      },
-      {
-        id: 'Blockly.Dart.all',
-        script: 'dist/dart_compressed.js',
-        scriptExport: 'dart',
-        destructure: {dartGenerator: 'dartGenerator'},
-      },
-      {
-        id: 'Blockly.JavaScript.all',
-        script: 'dist/javascript_compressed.js',
-        scriptExport: 'javascript',
-        destructure: {javascriptGenerator: 'javascriptGenerator'},
-      },
-      {
-        id: 'Blockly.Lua.all',
-        script: 'dist/lua_compressed.js',
-        scriptExport: 'lua',
-        destructure: {luaGenerator: 'luaGenerator'},
-      },
-      {
-        id: 'Blockly.PHP.all',
-        script: 'dist/php_compressed.js',
-        scriptExport: 'php',
-        destructure: {phpGenerator: 'phpGenerator'},
-      },
-      {
-        id: 'Blockly.Python.all',
-        script: 'dist/python_compressed.js',
-        scriptExport: 'python',
-        destructure: {pythonGenerator: 'pythonGenerator'},
-      },
-    ],
-
-    // Additional goog.modules to goog.require (for side-effects only,
-    // in uncompressed mode only).
-    requires: [],
-
-    // Additional scripts to be loaded after Blockly is loaded,
-    // whether Blockly is loaded from compressed or uncompressed.
-    // Paths relative to root.
-    scripts: ['build/msg/en.js'],
-  };
-  if (typeof window.BLOCKLY_BOOTSTRAP_OPTIONS === 'object') {
-    Object.assign(options, window.BLOCKLY_BOOTSTRAP_OPTIONS);
-  }
-
-  /* eslint-disable-next-line no-undef */
-  if (typeof module === 'object' && typeof module.exports === 'object') {
-    // Running in node.js.  Maybe we wish to support this.
-    // blockly_uncompressed formerly did, though it appears that the
-    // code had not been working for some time (at least since PR
-    // #5718 back in December 2021.  For now just throw an error.
-    throw new Error('Bootstrapping in node.js not implemented.');
-  }
-
-  // Create a global variable to remember some state that will be
-  // needed by later scripts.
-  window.bootstrapInfo = {
-    /** boolean */ compressed: options.loadCompressed,
-    /** Object<{id: string, script: string, scriptExport: string,
-     *          destructure: Object<string>}> */ modules: options.modules,
-    /** ?Array<string> */ requires: null,
-    /** ?Promise */ done: null,
-  };
-
-  if (!options.loadCompressed) {
-    // We can load Blockly in uncompressed mode.
-
-    // Disable loading of closure/goog/deps.js (which doesn't exist).
-    window.CLOSURE_NO_DEPS = true;
-    // Load the Closure Library's base.js (the only part of the
-    // library we use, mainly for goog.require / goog.provide /
-    // goog.module).
-    document.write(
-      `<script src="${options.root}build/src/closure/goog/base.js"></script>`,
-    );
-
-    // Prevent spurious transpilation warnings.
-    document.write('<script>goog.TRANSPILE = "never";</script>');
-
-    // Load dependency graph info from the specified deps files -
-    // typically just build/deps.js.  To update deps after changing
-    // any module's goog.requires / imports, run `npm run build:deps`.
-    for (const depsFile of options.depsFiles) {
-      document.write(`<script src="${options.root + depsFile}"></script>`);
-    }
-
-    // Assemble a list of module targets to bootstrap.
-    //
-    // The first group of targets are those listed in options.modules
-    // and options.requires.  These are recorded on bootstrapInfo so
-    // so bootstrap_helper.js can goog.require() them to force loading
-    // to complete.
-    //
-    // The next target is a fake one that will load
-    // bootstrap_helper.js.  We generate a call to goog.addDependency
-    // to tell the debug module loader that it can be loaded via a
-    // fake module name, and that it depends on all the targets in the
-    // first group (and indeed bootstrap_helper.js will make a call to
-    // goog.require for each one).
-    //
-    // We then create another target for each of options.scripts,
-    // again generating calls to goog.addDependency for each one
-    // making it dependent on the previous one.
-    let requires = (window.bootstrapInfo.requires = [
-      ...options.modules.map((module) => module.id),
-      ...options.requires,
-    ]);
-
-    const scripts = ['tests/bootstrap_helper.js', ...options.scripts];
-    const scriptDeps = [];
-    for (const script of scripts) {
-      const fakeModuleName = `script.${script.replace(/[./]/g, '-')}`;
-      scriptDeps.push(
-        `goog.addDependency(${quote('../../../../' + script)}, ` +
-          `[${quote(fakeModuleName)}], [${requires.map(quote).join()}], ` +
-          `{'lang': 'es6'});`,
-      );
-      requires = [fakeModuleName];
-    }
-
-    // Finally, write out a script containing the generated
-    // goog.addDependency calls and a call to goog.bootstrap
-    // requesting the loading of the final target, which will cause
-    // all the previous ones to be loaded recursively.  Wrap this in a
-    // promise and save it so it can be awaited in bootstrap_done.mjs.
-    document.write(`<script>
-  ${scriptDeps.join('\n  ')}
-  window.bootstrapInfo.done = new Promise((resolve, reject) => {
-    goog.bootstrap([${requires.map(quote).join()}], resolve);
-  });
-</script>`);
-  } else {
-    // We need to load Blockly in compressed mode.  Load
-    // blockly_compressed.js et al. using <script> tags.
-    const scripts = [
-      ...options.modules.map((module) => module.script),
-      'tests/bootstrap_helper.js',
-      ...options.scripts,
-    ];
-    for (const script of scripts) {
-      document.write(`<script src="${options.root + script}"></script>`);
-    }
-  }
-
-  return; // All done.  Only helper functions after this point.
-
-  /**
-   * Convert a string into a string literal.  Strictly speaking we
-   * only need to escape backslash, \r, \n, \u2028 (line separator),
-   * \u2029 (paragraph separator) and whichever quote character we're
-   * using, but for simplicity we escape all the control characters.
-   *
-   * Based on https://github.com/google/CodeCity/blob/master/server/code.js
-   *
-   * @param {string} str The string to convert.
-   * @return {string} The value s as a eval-able string literal.
-   */
-  function quote(str) {
-    /* eslint-disable no-control-regex, no-multi-spaces */
-    /** Regexp for characters to be escaped in a single-quoted string. */
-    const singleRE = /[\x00-\x1f\\\u2028\u2029']/g;
-
-    /** Map of control character replacements. */
-    const replacements = {
-      '\x00': '\\0',
-      '\x01': '\\x01',
-      '\x02': '\\x02',
-      '\x03': '\\x03',
-      '\x04': '\\x04',
-      '\x05': '\\x05',
-      '\x06': '\\x06',
-      '\x07': '\\x07',
-      '\x08': '\\b',
-      '\x09': '\\t',
-      '\x0a': '\\n',
-      '\x0b': '\\v',
-      '\x0c': '\\f',
-      '\x0d': '\\r',
-      '\x0e': '\\x0e',
-      '\x0f': '\\x0f',
-      '"': '\\"',
-      "'": "\\'",
-      '\\': '\\\\',
-      '\u2028': '\\u2028',
-      '\u2029': '\\u2029',
-    };
-    /* eslint-enable no-control-regex, no-multi-spaces */
-
-    return "'" + str.replace(singleRE, (c) => replacements[c]) + "'";
-  }
-})();

tests/bootstrap_done.mjs

@@ -1,41 +0,0 @@
-/**
- * @license
- * Copyright 2022 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-
-/**
- * @fileoverview Finishes loading Blockly and exports it as this
- *     module's default export.
- *
- *     It is exported as the default export to avoid having to
- *     re-export each property on Blockly individually, because you
- *     can't do:
- *
- *         export * from <dynamically computed source>;  // SYNTAX ERROR
- *
- *     You must use a <script> tag to load prepare.js first, before
- *     importing this module in a <script type=module> to obtain the
- *     loaded value.
- *
- *     See tests/playground.html for example usage.
- */
-if (!window.bootstrapInfo) {
-  throw new Error(
-    'window.bootstrapInfo not found.  ' +
-      'Make sure to load bootstrap.js before importing bootstrap_done.mjs.',
-  );
-}
-
-if (window.bootstrapInfo.compressed) {
-  // Compiled mode.  Nothing more to do.
-} else {
-  // Uncompressed mode.  Use top-level await
-  // (https://v8.dev/features/top-level-await) to block loading of
-  // this module until goog.bootstrap()ping of Blockly is finished.
-  await window.bootstrapInfo.done;
-  // Note that this module previously did an export default of the
-  // value returned by the bootstrapInfo.done promise.  This was
-  // changed in PR #5995 because library blocks and generators cannot
-  // be accessed via that the core/blockly.js exports object.
-}

tests/bootstrap_helper.js

@@ -1,56 +0,0 @@
-/**
- * @license
- * Copyright 2022 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-
-/**
- * @fileoverview Helper script for bootstrap.js
- *
- * This is loaded, via goog.bootstrap(), after the other top-level
- * Blockly modules.  It simply calls goog.require() for each of them,
- * to force the debug module loader to finish loading them before any
- * non-module scripts (like msg/en.js) that might have
- * undeclared dependencies on them.
- */
-
-(function () {
-  const info = window.bootstrapInfo;
-
-  if (!info.compressed) {
-    // Force debug module loader to finish loading all modules.
-    for (const require of info.requires) {
-      goog.require(require);
-    }
-  }
-
-  // Create global names for named and destructured imports.
-  for (const module of info.modules) {
-    const exports = info.compressed
-      ? get(module.scriptExport)
-      : goog.module.get(module.id);
-    if (module.importAt) {
-      window[module.importAt] = exports;
-    }
-    for (const location in module.destructure) {
-      window[location] = exports[module.destructure[location]];
-    }
-  }
-
-  return; // All done.  Only helper functions after this point.
-
-  /**
-   * Get the object referred to by a doted-itentifier path
-   * (e.g. foo.bar.baz).
-   * @param {string} path The path referring to the object.
-   * @return {string|null} The object, or null if not found.
-   */
-  function get(path) {
-    let obj = window;
-    for (const part of path.split('.')) {
-      obj = obj[part];
-      if (!obj) return null;
-    }
-    return obj;
-  }
-})();

tests/generators/index.html

@@ -4,20 +4,12 @@
 <meta charset="utf-8">
 <title>Blockly Generator Tests</title>
 <script>
-  var BLOCKLY_BOOTSTRAP_OPTIONS = {
+// N.B.: This script depends on the following (module) script to load
-    scripts: [
+// Blockly, the code generators, and the unittest*.js scripts.  The
-      'build/msg/en.js',
+// module script actually gets executed after this one, so although
-      'tests/generators/unittest_javascript.js',
+// Blockly will be available when the defined global functions are
-      'tests/generators/unittest_python.js',
+// called it isn't yet available when this script runs.
-      'tests/generators/unittest_php.js',
+
-      'tests/generators/unittest_lua.js',
-      'tests/generators/unittest_dart.js',
-      'tests/generators/unittest.js',
-    ],
-  }
-</script>
-<script src="../bootstrap.js"></script>
-<script>
 var demoWorkspace = null;
 
 function start() {
@@ -199,9 +191,31 @@ function changeIndex() {
   demoWorkspace.getToolbox().flyout_.workspace_.options.oneBasedIndex = oneBasedIndex;
 }
 </script>
+
 <script type="module">
-// Wait for Blockly to finish loading before running tests.
+import {loadScript} from '../scripts/load.mjs';
-import '../bootstrap_done.mjs';
+
+import * as Blockly from '../../build/blockly.loader.mjs';
+import '../../build/blocks.loader.mjs';
+import {dartGenerator} from '../../build/dart.loader.mjs';
+import {luaGenerator} from '../../build/lua.loader.mjs';
+import {javascriptGenerator} from '../../build/javascript.loader.mjs';
+import {phpGenerator} from '../../build/php.loader.mjs';
+import {pythonGenerator} from '../../build/python.loader.mjs';
+
+globalThis.dartGenerator = dartGenerator;
+globalThis.luaGenerator = luaGenerator;
+globalThis.javascriptGenerator = javascriptGenerator;
+globalThis.phpGenerator = phpGenerator;
+globalThis.pythonGenerator = pythonGenerator;
+
+await loadScript('../../build/msg/en.js');
+await loadScript('./unittest_javascript.js');
+await loadScript('./unittest_python.js');
+await loadScript('./unittest_php.js');
+await loadScript('./unittest_lua.js');
+await loadScript('./unittest_dart.js');
+await loadScript('./unittest.js');
 
 start();
 </script>

tests/generators/webdriver.js

@@ -73,6 +73,10 @@ async function runGeneratorsInBrowser(outputDir) {
   console.log('Loading url: ' + url);
   await browser.url(url);
 
+  await browser
+    .$('.blocklySvg .blocklyWorkspace > .blocklyBlockCanvas')
+    .waitForExist({timeout: 2000});
+
   await browser.execute(function() {
     checkAll();
     loadSelected();

tests/mocha/index.html

@@ -18,9 +18,9 @@
     <div id="mocha"></div>
     <div id="failureCount" style="display: none" tests_failed="unset"></div>
     <div id="failureMessages" style="display: none"></div>
-    <!-- Load mocha et al. before bootstrapping so that we can safely
+    <!-- Load mocha et al. before Blockly and the test modules so that
-         goog.require() the test modules that make calls to (e.g.)
+         we can safely import the test modules that make calls
-         suite() at the top level. -->
+         to (e.g.)  suite() at the top level. -->
     <script src="../../node_modules/chai/chai.js"></script>
     <script src="../../node_modules/mocha/mocha.js"></script>
     <script src="../../node_modules/sinon/pkg/sinon.js"></script>