From 64b531b87d1b88be55a96750a9fee5dd2c91cf0f Mon Sep 17 00:00:00 2001
From: Christopher Allen <cpcallen+git@google.com>
Date: Thu, 10 Aug 2023 17:01:10 +0100
Subject: [PATCH] feat(tests): Introduce chunk loading shims

- Add a buildShims task to build_tasks.js that, for each chunk,
  creates a correspondingly-named build/<chunk>.mjs that will
  either (in uncompressed mode) import and reexport that chunk's
  entry point module (e.g. core/blockly.js) or (in compressed
  mode) load dist/<chunk>_compressed.js using a <script> tag
  and then export the corresponding properties on the chunk's
  exports object.

- Provide helper methods used by these shims in
  tests/scripts/loading.mjs, including code to detect whether
  to load in compressed or uncompressed mode.

- Add a quote() function to scripts/helpers.js, used by
  buildShims.  This is copied from tests/bootstrap_helper.js,
  which will be removed in a later commit.
---
 scripts/gulpfiles/build_tasks.js |  56 ++++++++++++-
 scripts/helpers.js               |  46 ++++++++++
 tests/scripts/load.mjs           | 140 +++++++++++++++++++++++++++++++
 3 files changed, 240 insertions(+), 2 deletions(-)
 create mode 100644 tests/scripts/load.mjs

diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js
index 4479c0be7..cd80b132e 100644
--- a/scripts/gulpfiles/build_tasks.js
+++ b/scripts/gulpfiles/build_tasks.js
@@ -15,6 +15,7 @@ gulp.sourcemaps = require('gulp-sourcemaps');
 
 const path = require('path');
 const fs = require('fs');
+const fsPromises = require('fs/promises');
 const {exec, execSync} = require('child_process');
 
 const closureCompiler = require('google-closure-compiler').gulp();
@@ -24,7 +25,7 @@ const {rimraf} = require('rimraf');
 const {BUILD_DIR, DEPS_FILE, RELEASE_DIR, TEST_DEPS_FILE, TSC_OUTPUT_DIR, TYPINGS_BUILD_DIR} = require('./config');
 const {getPackageJson} = require('./helper_tasks');
 
-const {posixPath} = require('../helpers');
+const {posixPath, quote} = require('../helpers');
 
 ////////////////////////////////////////////////////////////
 //                        Build                           //
@@ -106,6 +107,7 @@ const chunks = [
   {
     name: 'blockly',
     entry: path.join(TSC_OUTPUT_DIR, 'core', 'main.js'),
+    moduleEntry: path.join(TSC_OUTPUT_DIR, 'core', 'blockly.js'),
     exports: 'module$build$src$core$blockly',
     scriptExport: 'Blockly',
   },
@@ -672,6 +674,56 @@ function buildCompiled() {
       .pipe(gulp.dest(RELEASE_DIR));
 }
 
+/**
+ * This task builds the shims used by the playgrounds and tests to
+ * load Blockly in either compressed or uncompressed mode, creating
+ * build/blockly.mjs, blocks.mjs, javascript.mjs, etc.
+ *
+ * Prerequisite: getChunkOptions (via buildCompiled, for chunks[].parent).
+ */
+async function buildShims() {
+  // Install a package.json file in BUILD_DIR to tell node.js that the
+  // .js files therein are ESM not CJS, so we can import the
+  // entrypoints to enumerate their exported names.
+  //
+  // N.B.: There is an exception: core/main.js is a goog.module not
+  // ESM, but fortunately we don't attempt to import or require this
+  // file from node.js - we only feed it to Closure Compiler, which
+  // uses the type information in deps.js rather than package.json.
+  await fsPromises.writeFile(
+    path.join(BUILD_DIR, 'package.json'),
+    '{"type": "module"}'
+  );
+
+  // Import each entrypoint module, enumerate its exports, and write
+  // a shim to load the chunk either by importing the entrypoint
+  // module or by loading the compiled script.
+  await Promise.all(chunks.map(async (chunk) => {
+    const modulePath = posixPath(chunk.moduleEntry ?? chunk.entry);
+    const scriptPath =
+        path.posix.join(RELEASE_DIR, `${chunk.name}${COMPILED_SUFFIX}.js`);
+    const shimPath = path.join(BUILD_DIR, `${chunk.name}.mjs`);
+    const parentImport =
+        chunk.parent ? `import ${quote(`./${chunk.parent.name}.mjs`)};` : '';
+    const exports = await import(`../../${modulePath}`);
+
+    await fsPromises.writeFile(shimPath,
+        `import {loadChunk} from '../tests/scripts/load.mjs';
+${parentImport}
+
+export const {
+${Object.keys(exports).map((name) => `  ${name},`).join('\n')}
+} = await loadChunk(
+  ${quote(modulePath)},
+  ${quote(scriptPath)},
+  ${quote(chunk.scriptExport)},
+);
+`);
+  }));
+}
+
+
+
 /**
  * This task builds Blockly core, blocks and generators together and uses
  * Closure Compiler's ADVANCED_COMPILATION mode.
@@ -728,7 +780,7 @@ exports.cleanBuildDir = cleanBuildDir;
 exports.langfiles = buildLangfiles;  // Build build/msg/*.js from msg/json/*.
 exports.tsc = buildJavaScript;
 exports.deps = gulp.series(exports.tsc, buildDeps);
-exports.minify = gulp.series(exports.deps, buildCompiled);
+exports.minify = gulp.series(exports.deps, buildCompiled, buildShims);
 exports.build = gulp.parallel(exports.minify, exports.langfiles);
 
 // Manually-invokable targets, with prerequisites where required.
diff --git a/scripts/helpers.js b/scripts/helpers.js
index 6346014da..8150593dd 100644
--- a/scripts/helpers.js
+++ b/scripts/helpers.js
@@ -23,6 +23,52 @@ function posixPath(target) {
   return target.split(path.sep).join(path.posix.sep);
 }
 
+/**
+ * Convert a string into a string literal.  Strictly speaking we
+ * only need to escape backslash, \r, \n, \u2028 (line separator),
+ * \u2029 (paragraph separator) and whichever quote character we're
+ * using, but for simplicity we escape all the control characters.
+ *
+ * Based on https://github.com/google/CodeCity/blob/master/server/code.js
+ *
+ * @param {string} str The string to convert.
+ * @return {string} The value s as a eval-able string literal.
+ */
+function quote(str) {
+  /* eslint-disable no-control-regex, no-multi-spaces */
+  /** Regexp for characters to be escaped in a single-quoted string. */
+  const singleRE = /[\x00-\x1f\\\u2028\u2029']/g;
+
+  /** Map of control character replacements. */
+  const replacements = {
+    '\x00': '\\0',
+    '\x01': '\\x01',
+    '\x02': '\\x02',
+    '\x03': '\\x03',
+    '\x04': '\\x04',
+    '\x05': '\\x05',
+    '\x06': '\\x06',
+    '\x07': '\\x07',
+    '\x08': '\\b',
+    '\x09': '\\t',
+    '\x0a': '\\n',
+    '\x0b': '\\v',
+    '\x0c': '\\f',
+    '\x0d': '\\r',
+    '\x0e': '\\x0e',
+    '\x0f': '\\x0f',
+    '"': '\\"',
+    "'": "\\'",
+    '\\': '\\\\',
+    '\u2028': '\\u2028',
+    '\u2029': '\\u2029',
+  };
+  /* eslint-enable no-control-regex, no-multi-spaces */
+
+  return "'" + str.replace(singleRE, (c) => replacements[c]) + "'";
+}
+
 module.exports = {
   posixPath,
+  quote,
 };
diff --git a/tests/scripts/load.mjs b/tests/scripts/load.mjs
new file mode 100644
index 000000000..9c604e58c
--- /dev/null
+++ b/tests/scripts/load.mjs
@@ -0,0 +1,140 @@
+/**
+ * @license
+ * Copyright 2023 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @fileoverview Helper functions for loading Blockly in tests.
+ *
+ * Used by playgrounds and test harnesses, both directly and via the
+ * shims generated by the buildShims function in
+ * scripts/gulpfiles/build_tasks.js.
+ *
+ */
+
+if (typeof window !== 'object') {
+  // Not running in a browser.  Maybe we wish to support this?
+  // blockly_uncompressed formerly supported node.js, though it
+  // appears that the code had not been working for some time (at
+  // least since PR #5718 back in December 2021.  For now just throw
+  // an error.
+  throw new Error('Bootstrapping without window is not supported.');
+}
+
+/**
+ * URL of the blockly repository.  This is needed for a few reasons:
+ *
+ * - We need an absolute path instead of relative path because the
+ *   advanced_playground and the regular playground are in
+ *   different folders.
+ * - We need to get the root directory for blockly because it is
+ *   different for github.io, appspot and local.
+ *
+ * The formulation given here will work so long as top-level page is loaded from
+ * somewhere in tests/.
+ */
+export const ROOT = window.location.href.replace(/\/tests\/.*$/, '/');
+
+/**
+ * Decide whether to use compressed mode or not.
+ *
+ * Honours a "?compressed=" query parameter if present; otherwise uses
+ * uncompressed for when loading from local machine and compressed
+ * otherwise.  See issue #5557 for additional background.
+ *
+ * @return {boolean} True if shoudl load in compressed mode.
+ */
+function compressed() {
+  const param = location.search.match(/compressed=([^&]+)/)?.[1];
+  if (param) {
+    if (['y', 'yes', 'true', '1'].includes(param)) return true;
+    if (['n', 'no', 'false', '0'].includes(param)) return false;
+    console.error(`Unrecognised compressed parameter "${param}"`);
+  }
+
+  const LOCALHOSTS = ['localhost', '127.0.0.1', '[::1]'];
+  const isLocalhost = LOCALHOSTS.includes(location.hostname);
+  const isFileUrl = location.origin === 'file://';
+  return !(isLocalhost || isFileUrl);
+}
+
+/** @type {boolean} Load in compressed mode. */
+export const COMPRESSED = compressed();
+
+/**
+ * Load a chunk, either by importing its ESM entrypoint or loading the
+ * compressed chunk script.
+ *
+ * When loading in uncompressed mode, if scriptExports is a simple
+ * variable name (e.g. 'Blockly') then globalThis[scriptExports] will
+ * be set to the the chunk's Module object.  This attempts provide
+ * backward compatibility with loading the compressed chunk as a
+ * script, where this is done by the compressed chunk's UMD wrapper.
+ * The compatibility is not complete, however: since Module objects
+ * are immutable, it is not possible to set (e.g.)
+ * Blockly.libraryBlocks.
+ *
+ * The intention is to allow the chunk to be accessed accessed either
+ * the returned Module / exports object (preferred) or via the global
+ * scope (needed by dev-tools) regardless of whether it was loaded
+ * compressed or uncompressed.
+ *
+ * Paths should be relative to the repository root.
+ *
+ * @param {string} modulePath Path to the chunk's ESM entry point.
+ * @param {string} scriptPath Path to the chunk's _compressed.js file.
+ * @param {string} scriptExports The global variable name (or
+ *     dotted-identifier path relative from a global variable) in
+ *     which the compressed chunk's UMD wrapper will save the module's
+ *     exports object.  Should be the same as the correspodning
+ *     chunks[].scriptExport property in
+ *     scripts/gulpfiles/build_tasks.js.
+ * @return {Module} The module's Module or exports object.
+ */
+export async function loadChunk(modulePath, scriptPath, scriptExports) {
+  if (COMPRESSED) {
+    await loadScript(`${ROOT}${scriptPath}`);
+    return get(scriptExports);
+  } else {
+    const exports = await import(`../../${modulePath}`);
+    if (!scriptExports.includes('.')) globalThis[scriptExports] = exports;
+    return exports;
+  }
+}
+
+/**
+ * Load a given URL using a <script> tag.
+ *
+ * Unlike document.write('<script src=...>'), this is safe to use even
+ * after page loading is complete.
+ *
+ * @param {string} src The src attribute for the <script> tag.
+ * @return {Promise} A promise that resolves (or rejects) once the
+ *     script is loaded.
+ */
+export function loadScript(src) {
+  return new Promise((resolve, reject) => {
+    const script = document.createElement('script');
+    script.src = src;
+    script.onload = resolve;
+    script.onerror = reject;
+    document.getElementsByTagName('head')[0].appendChild(script);
+  });
+}
+
+/**
+ * Get the value referred to by a doted-itentifier path
+ * (e.g. foo.bar.baz).  Throws TypeError if path is is not valid,
+ * i.e., if any component but the last does not exist).
+ *
+ * @param {string} path The path referring to the desired value.
+ * @return {string|null} The value referred to.
+ */
+function get(path) {
+  let obj = globalThis;
+  for (const part of path.split('.')) {
+    obj = obj[part];
+  }
+  return obj;
+}