From b348e18dfb646dcfbf5524547f597a2aee823442 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Mon, 21 Jun 2021 19:13:59 +0100 Subject: [PATCH 01/20] Don't use path.join to create globs for gulp.src Per the gulp documentation[1], globs passed to gulp.src should use '/' as the path separator regardless of the path separator used on whatever OS we are running on. [1] https://gulpjs.com/docs/en/getting-started/explaining-globs#segments-and-separators --- scripts/gulpfiles/appengine_tasks.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gulpfiles/appengine_tasks.js b/scripts/gulpfiles/appengine_tasks.js index e07b9f60f..e8114587a 100644 --- a/scripts/gulpfiles/appengine_tasks.js +++ b/scripts/gulpfiles/appengine_tasks.js @@ -47,8 +47,8 @@ function copyStaticSrc(done) { */ function copyAppengineSrc() { const appengineSrc = [ - path.join(demoStaticTmpDir, 'appengine/**/*'), - path.join(demoStaticTmpDir, 'appengine/.gcloudignore'), + `${demoStaticTmpDir}/appengine/**/*`, + `${demoStaticTmpDir}/appengine/.gcloudignore`, ]; return gulp.src(appengineSrc).pipe(gulp.dest(demoTmpDir)); } From 082fd1fc6fb44187fed45cb6a0677261ba963b85 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 18 Jun 2021 13:38:58 +0100 Subject: [PATCH 02/20] Move build and package directory config into new config.js Make the destination directories for certain build/package/release steps more easily (and centrally) configurable. This only deals with building *_compressed* files; blockly_uncompressed.js and the various msg/js/*.js files are not affected by this commit. --- scripts/gulpfiles/build_tasks.js | 10 ++-- scripts/gulpfiles/config.js | 22 ++++++++ scripts/gulpfiles/package_tasks.js | 91 +++++++++++++++--------------- scripts/gulpfiles/release_tasks.js | 27 +++++---- 4 files changed, 89 insertions(+), 61 deletions(-) create mode 100644 scripts/gulpfiles/config.js diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index 6553e8872..5b196d7ac 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -21,8 +21,8 @@ var through2 = require('through2'); var closureCompiler = require('google-closure-compiler').gulp(); var closureDeps = require('google-closure-deps'); var argv = require('yargs').argv; -var { getPackageJson } = require('./helper_tasks'); - +var {BUILD_DIR} = require('./config'); +var {getPackageJson} = require('./helper_tasks'); //////////////////////////////////////////////////////////// // Build // @@ -217,7 +217,7 @@ function buildCompressed() { })) .pipe( gulp.sourcemaps.write('.', {includeContent: false, sourceRoot: './'})) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** @@ -242,7 +242,7 @@ function buildBlocks() { includeContent: false, sourceRoot: './' })) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** @@ -268,7 +268,7 @@ function buildGenerator(language, namespace) { includeContent: false, sourceRoot: './' })) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** diff --git a/scripts/gulpfiles/config.js b/scripts/gulpfiles/config.js new file mode 100644 index 000000000..370b5a79b --- /dev/null +++ b/scripts/gulpfiles/config.js @@ -0,0 +1,22 @@ +/** + * @license + * Copyright 2021 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @fileoverview Common configuration for Gulp scripts. + */ + +var path = require('path'); + +// Paths are all relative to the repository root. Do not include +// trailing slash. +module.exports = { + // Directory to write compiled output to. + BUILD_DIR: '.', + + // Directory in which to assemble (and from which to publish) the + // blockly npm package. + RELEASE_DIR: 'dist', +}; diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index bf79b7a6e..f1dfd121a 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -17,13 +17,11 @@ gulp.umd = require('gulp-umd'); var path = require('path'); var fs = require('fs'); -var { getPackageJson } = require('./helper_tasks'); - -const blocklyRoot = '../../'; - -// The destination path where all the NPM distribution files will go. -const packageDistribution = 'dist'; +var {getPackageJson} = require('./helper_tasks'); +var {BUILD_DIR, RELEASE_DIR} = require('./config'); +// Path to template files for gulp-umd. +const TEMPLATE_DIR = 'scripts/package/templates'; /** * A helper method for wrapping a file into a Universal Module Definition. @@ -35,7 +33,7 @@ function packageUMD(namespace, dependencies) { dependencies: function () { return dependencies; }, namespace: function () { return namespace; }, exports: function () { return namespace; }, - template: path.join(__dirname, `${blocklyRoot}/scripts/package/templates/umd.template`) + template: path.join(TEMPLATE_DIR, 'umd.template') }); }; @@ -49,26 +47,26 @@ function packageCommonJS(namespace, dependencies) { dependencies: function () { return dependencies; }, namespace: function () { return namespace; }, exports: function () { return namespace; }, - template: path.join(__dirname, `${blocklyRoot}/scripts/package/templates/node.template`) + template: path.join(TEMPLATE_DIR, 'node.template') }); }; /** - * This task copies source files into the distribution directory. + * This task copies source files into the release directory. */ function packageSources() { return gulp.src(['core/**/**.js', 'blocks/**.js', 'generators/**/**.js'], {base: '.'}) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** - * This task copies the compressed files and their source maps into the - * distribution directory. + * This task copies the compressed files and their source maps into + * the release directory. */ function packageCompressed() { - return gulp.src('*_compressed.js?(.map)') - .pipe(gulp.dest(packageDistribution)); + return gulp.src('*_compressed.js?(.map)', {cwd: BUILD_DIR}) + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -83,7 +81,7 @@ function packageBlockly() { cjs: './blockly_compressed', }])) .pipe(gulp.rename('blockly.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -98,7 +96,7 @@ function packageBlocks() { cjs: './blocks_compressed', }])) .pipe(gulp.rename('blocks.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -115,7 +113,7 @@ function packageIndex() { cjs: './node', }])) .pipe(gulp.rename('index.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -147,7 +145,7 @@ function packageBrowser() { cjs: './javascript', }])) .pipe(gulp.rename('browser.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -166,7 +164,7 @@ function packageCore() { cjs: './blockly', }])) .pipe(gulp.rename('core-browser.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -205,7 +203,7 @@ function packageNode() { cjs: './dart', }])) .pipe(gulp.rename('node.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -224,7 +222,7 @@ function packageNodeCore() { cjs: './blockly', }])) .pipe(gulp.rename('core.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -245,7 +243,7 @@ function packageGenerator(file, rename, namespace) { cjs: `./${file}`, }])) .pipe(gulp.rename(rename)) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -303,7 +301,7 @@ function packageLocales() { amd: '../core', cjs: '../core', }])) - .pipe(gulp.dest(`${packageDistribution}/msg`)); + .pipe(gulp.dest(`${RELEASE_DIR}/msg`)); }; /** @@ -314,60 +312,63 @@ function packageLocales() { */ function packageUMDBundle() { var srcs = [ - 'blockly_compressed.js', + `${BUILD_DIR}/blockly_compressed.js`, 'msg/js/en.js', - 'blocks_compressed.js', - 'javascript_compressed.js' + `${BUILD_DIR}/blocks_compressed.js`, + `${BUILD_DIR}/javascript_compressed.js`, ]; return gulp.src(srcs) - .pipe(gulp.concat('blockly.min.js')) - .pipe(gulp.dest(`${packageDistribution}`)) + .pipe(gulp.concat('blockly.min.js')) + .pipe(gulp.dest(`${RELEASE_DIR}`)); }; /** - * This task copies all the media/* files into the distribution directory. + * This task copies all the media/* files into the release directory. */ function packageMedia() { - return gulp.src('./media/*') - .pipe(gulp.dest(`${packageDistribution}/media`)); + return gulp.src('media/*') + .pipe(gulp.dest(`${RELEASE_DIR}/media`)); }; /** - * This task copies the package.json file into the distribution directory. + * This task copies the package.json file into the release directory. */ function packageJSON(cb) { const packageJson = getPackageJson(); const json = Object.assign({}, packageJson); delete json['scripts']; - if (!fs.existsSync(packageDistribution)) { - fs.mkdirSync(packageDistribution); + if (!fs.existsSync(RELEASE_DIR)) { + fs.mkdirSync(RELEASE_DIR); } - fs.writeFileSync(`${packageDistribution}/package.json`, + fs.writeFileSync(`${RELEASE_DIR}/package.json`, JSON.stringify(json, null, 2)); cb(); }; /** - * This task copies the scripts/package/README.md file into the distribution directory. - * This file is what developers will see at https://www.npmjs.com/package/blockly. + * This task copies the scripts/package/README.md file into the + * release directory. This file is what developers will see at + * https://www.npmjs.com/package/blockly . */ function packageReadme() { - return gulp.src('./scripts/package/README.md') - .pipe(gulp.dest(`${packageDistribution}`)); + return gulp.src('scripts/package/README.md') + .pipe(gulp.dest(`${RELEASE_DIR}`)); }; /** - * This task copies the typings/blockly.d.ts TypeScript definition file into the - * distribution directory. - * The bundled declaration file is referenced in package.json in the types property. + * This task copies the typings/blockly.d.ts TypeScript definition + * file into the release directory. The bundled declaration file is + * referenced in package.json in the types property. */ function packageDTS() { - return gulp.src(['./typings/*.d.ts', './typings/msg/*.d.ts'], {base: './typings'}) - .pipe(gulp.dest(`${packageDistribution}`)); + return gulp.src(['typings/*.d.ts', 'typings/msg/*.d.ts'], + {base: './typings'}) + .pipe(gulp.dest(`${RELEASE_DIR}`)); }; /** - * This task prepares the NPM distribution files under the /dist directory. + * This task prepares the files to be included in the NPM by copying + * them into the release directory. */ const package = gulp.parallel( packageIndex, diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index 5e14fc08c..e98639b48 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -17,9 +17,9 @@ var typings = require('./typings'); var buildTasks = require('./build_tasks'); var gitTasks = require('./git_tasks'); var packageTasks = require('./package_tasks'); -var { getPackageJson } = require('./helper_tasks'); +var {getPackageJson} = require('./helper_tasks'); +var {RELEASE_DIR} = require('./config'); -const RELEASE_DIR = 'dist'; // Gets the current major version. function getMajorVersion() { @@ -80,20 +80,25 @@ function checkBranch(done) { } -// Sanity check that the dist folder exists, and that certain files are in the dist folder. -function checkDist(done) { - const sanityFiles = ['blockly_compressed.js', 'blocks_compressed.js', 'core', 'blocks', 'generators']; - // Check that dist exists. +// Sanity check that the RELASE_DIR directory exists, and that certain +// files are in it. +function checkReleaseDir(done) { + const sanityFiles = ['blockly_compressed.js', 'blocks_compressed.js', + 'core', 'blocks', 'generators']; + // Check that directory exists. if (fs.existsSync(RELEASE_DIR)) { - // Sanity check that certain files exist in dist. + // Sanity check that certain files exist in RELASE_DIR. sanityFiles.forEach((fileName) => { if (!fs.existsSync(`${RELEASE_DIR}/${fileName}`)) { - done(new Error(`Your dist folder does not contain:${fileName}`)); + done(new Error( + `Your ${RELEASE_DIR} directory does not contain ${fileName}`)); + return; } }); done(); } else { - done(new Error('The dist directory does not exist. Is packageTasks.package being run?')); + done(new Error(`The ${RELEASE_DIR} directory does not exist. ` + + 'Has packageTasks.package been run?')); } } @@ -148,7 +153,7 @@ function updateBetaVersion(done) { const publish = gulp.series( packageTasks.package, checkBranch, - checkDist, + checkReleaseDir, loginAndPublish ); @@ -158,7 +163,7 @@ const publishBeta = gulp.series( buildTasks.build, packageTasks.package, checkBranch, - checkDist, + checkReleaseDir, loginAndPublishBeta ); From 4111073292848ce60921619c4b5f8f3166ac9a6b Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 18 Jun 2021 21:01:42 +0100 Subject: [PATCH 03/20] Write build output to built/ instead of repository root I have verified that npm run build && npm run package produces an identical dist/ directory compared to the one produced prior to this and the previous commit. --- .gitignore | 1 + scripts/gulpfiles/config.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1e7a1e198..ea2fce879 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ local_build/*compiler*.jar local_build/local_*_compressed.js chromedriver typings/tmp/* +built/ dist/ diff --git a/scripts/gulpfiles/config.js b/scripts/gulpfiles/config.js index 370b5a79b..4008b6304 100644 --- a/scripts/gulpfiles/config.js +++ b/scripts/gulpfiles/config.js @@ -14,7 +14,7 @@ var path = require('path'); // trailing slash. module.exports = { // Directory to write compiled output to. - BUILD_DIR: '.', + BUILD_DIR: 'built', // Directory in which to assemble (and from which to publish) the // blockly npm package. From a03cd29b6868887c9b524ff23e46a896f0d9ee5d Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 18 Jun 2021 22:08:33 +0100 Subject: [PATCH 04/20] Separate script to rebuild msg/json/en.json, qqq.json etc. There are some files in msg/json/ (currently en.json, qqq.json, constants.json and synonyms.json) that are generated by scripts/i18n/js_to_json.py as part of the language file build process - but this only needs to be done when messages.js is updated and and usually requires some manual cleanup, so remove this step from the existing buildLangfiles gulp script and create a separate command ('npm run generate:langfiles') to do this when required. --- gulpfile.js | 1 + msg/messages.js | 12 +++++---- package.json | 1 + scripts/gulpfiles/build_tasks.js | 42 +++++++++++++++++++++++++------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 496211472..887e18855 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -23,6 +23,7 @@ var cleanupTasks = require('./scripts/gulpfiles/cleanup_tasks'); module.exports = { deployDemos: appengineTasks.deployDemos, default: buildTasks.build, + generateLangfiles: buildTasks.generateLangfiles, build: buildTasks.build, buildCore: buildTasks.core, buildBlocks: buildTasks.blocks, diff --git a/msg/messages.js b/msg/messages.js index 884c97245..636de4bd5 100644 --- a/msg/messages.js +++ b/msg/messages.js @@ -8,13 +8,15 @@ * @fileoverview English strings. * @author fraser@google.com (Neil Fraser) * - * After modifying this file, either run "build.py" from the parent directory, - * or run (from this directory): - * ../scripts/i18n/js_to_json.py - * to regenerate json/{en,qqq,synonyms}.json. + * After modifying this file, run: + * + * npm run generate:langfiles + * + * to regenerate json/{en,qqq,constants,synonyms}.json. * * To convert all of the json files to .js files, run: - * ../scripts/i18n/create_messages.py json/*.json + * + * npm run build:langfiles */ 'use strict'; diff --git a/package.json b/package.json index 6adb5f0a1..341b118ce 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "deployDemos": "gulp deployDemos", "format": "git-clang-format", "format:sortrequires": "gulp sortRequires", + "generate:langfiles": "gulp generateLangfiles", "license": "gulp checkLicenses", "lint": "eslint .", "package": "gulp package", diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index 5b196d7ac..f57e0d178 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -408,22 +408,45 @@ goog.require('Blockly.requires'); }); }; +/** + * This task regenrates msg/json/en.js and msg/json/qqq.js from + * msg/messages.js. + */ +function generateLangfiles(done) { + // Run js_to_json.py + const jsToJsonCmd = `python scripts/i18n/js_to_json.py \ + --input_file ${path.join('msg', 'messages.js')} \ + --output_dir ${path.join('msg', 'json')} \ + --quiet`; + execSync(jsToJsonCmd, { stdio: 'inherit' }); + + console.log(` +Regenerated several flies in msg/json/. Now run + + git diff msg/json/*.json + +and check that operation has not overwritten any modifications made to +hints, etc. by the TranslateWiki volunteers. If it has, backport +their changes to msg/messages.js and re-run 'npm run generate:langfiles'. + +Once you are satisfied that any new hints have been backported you may +go ahead and commit the changes, but note that the generate script +will have removed the translator credits - be careful not to commit +this removal! +`); + + done(); +}; + /** * This task builds Blockly's lang files. * msg/*.js */ function buildLangfiles(done) { - // Run js_to_json.py - const jsToJsonCmd = `python ./scripts/i18n/js_to_json.py \ ---input_file ${path.join('msg', 'messages.js')} \ ---output_dir ${path.join('msg', 'json')} \ ---quiet`; - execSync(jsToJsonCmd, { stdio: 'inherit' }); - // Run create_messages.py let json_files = fs.readdirSync(path.join('msg', 'json')); json_files = json_files.filter(file => file.endsWith('json') && - !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file))); + !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file))); json_files = json_files.map(file => path.join('msg', 'json', file)); const createMessagesCmd = `python ./scripts/i18n/create_messages.py \ --source_lang_file ${path.join('msg', 'json', 'en.json')} \ @@ -432,7 +455,7 @@ function buildLangfiles(done) { --key_file ${path.join('msg', 'json', 'keys.json')} \ --output_dir ${path.join('msg', 'js')} \ --quiet ${json_files.join(' ')}`; - execSync(createMessagesCmd, { stdio: 'inherit' }); + execSync(createMessagesCmd, { stdio: 'inherit' }); done(); }; @@ -514,6 +537,7 @@ module.exports = { build: build, core: buildCore, blocks: buildBlocks, + generateLangfiles: generateLangfiles, langfiles: buildLangfiles, uncompressed: buildUncompressed, compressed: buildCompressed, From 25f073d0e37451cc009905d6ef2f9f52098924c3 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Tue, 22 Jun 2021 12:04:30 +0100 Subject: [PATCH 05/20] Build msg/js/*.js langfiles in BUILD_DIR --- scripts/gulpfiles/build_tasks.js | 11 ++++++++--- scripts/gulpfiles/package_tasks.js | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index f57e0d178..69fda0a23 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -443,7 +443,12 @@ this removal! * msg/*.js */ function buildLangfiles(done) { - // Run create_messages.py + // Create output directory. + // BUG(cpcallen): this probably doesn't work on Windows. + const outputDir = path.join(BUILD_DIR, 'msg', 'js'); + execSync(`mkdir -p ${outputDir}`, {stdio: 'inherit'}); + + // Run create_messages.py. let json_files = fs.readdirSync(path.join('msg', 'json')); json_files = json_files.filter(file => file.endsWith('json') && !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file))); @@ -453,9 +458,9 @@ function buildLangfiles(done) { --source_synonym_file ${path.join('msg', 'json', 'synonyms.json')} \ --source_constants_file ${path.join('msg', 'json', 'constants.json')} \ --key_file ${path.join('msg', 'json', 'keys.json')} \ - --output_dir ${path.join('msg', 'js')} \ + --output_dir ${outputDir} \ --quiet ${json_files.join(' ')}`; - execSync(createMessagesCmd, { stdio: 'inherit' }); + execSync(createMessagesCmd, {stdio: 'inherit'}); done(); }; diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index f1dfd121a..84f79668c 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -287,12 +287,12 @@ function packagePHP() { }; /** - * This task wraps each of the msg/js/* files into a UMD module. + * This task wraps each of the ${BUILD_DIR}/msg/js/* files into a UMD module. * @example import * as En from 'blockly/msg/en'; */ function packageLocales() { // Remove references to goog.provide and goog.require. - return gulp.src('msg/js/*.js') + return gulp.src(`${BUILD_DIR}/msg/js/*.js`) .pipe(gulp.replace(/goog\.[^\n]+/g, '')) .pipe(gulp.insert.prepend(` var Blockly = {};Blockly.Msg={};`)) From e2c4a9b7f99e95e3d20894c3389d9596da585802 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Tue, 22 Jun 2021 13:39:31 +0100 Subject: [PATCH 06/20] Script to copy built files from BUILD_DIR to repository This is to allow built files to be checked in. --- gulpfile.js | 1 + package.json | 1 + scripts/gulpfiles/build_tasks.js | 12 ++++++++++++ 3 files changed, 14 insertions(+) diff --git a/gulpfile.js b/gulpfile.js index 887e18855..e1fd6f369 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -32,6 +32,7 @@ module.exports = { buildCompressed: buildTasks.compressed, buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, + checkinBuilt: buildTasks.checkinBuilt, gitSyncDevelop: gitTasks.syncDevelop, gitSyncMaster: gitTasks.syncMaster, gitCreateRC: gitTasks.createRC, diff --git a/package.json b/package.json index 341b118ce..ad0401410 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "build:langfiles": "gulp buildLangfiles", "build:uncompressed": "gulp buildUncompressed", "bump": "npm --no-git-tag-version version 4.$(date +'%Y%m%d').0", + "checkin": "gulp checkinBuilt", "deployDemos": "gulp deployDemos", "format": "git-clang-format", "format:sortrequires": "gulp sortRequires", diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index 69fda0a23..41b41a962 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -538,6 +538,17 @@ const build = gulp.parallel( buildLangfiles ); +/** + * This task copies built files from BUILD_DIR back to the repository + * so they can be committed to git. + */ +function checkinBuilt() { + return gulp.src([ + `${BUILD_DIR}/**.js`, + `${BUILD_DIR}/**/**.js`, + ]).pipe(gulp.dest('.')); +}; + module.exports = { build: build, core: buildCore, @@ -547,5 +558,6 @@ module.exports = { uncompressed: buildUncompressed, compressed: buildCompressed, generators: buildGenerators, + checkinBuilt: checkinBuilt, advancedCompilationTest: buildAdvancedCompilationTest, } From 961319b3eb24f4bbba8ffd710b26139df5c9a265 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Tue, 22 Jun 2021 18:10:02 +0100 Subject: [PATCH 07/20] Script to clean build directory --- gulpfile.js | 1 + package.json | 1 + scripts/gulpfiles/build_tasks.js | 15 +++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/gulpfile.js b/gulpfile.js index e1fd6f369..ccb9ed072 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -33,6 +33,7 @@ module.exports = { buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, checkinBuilt: buildTasks.checkinBuilt, + clean: buildTasks.cleanBuildDir, gitSyncDevelop: gitTasks.syncDevelop, gitSyncMaster: gitTasks.syncMaster, gitCreateRC: gitTasks.createRC, diff --git a/package.json b/package.json index ad0401410..81925ec18 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "build:langfiles": "gulp buildLangfiles", "build:uncompressed": "gulp buildUncompressed", "bump": "npm --no-git-tag-version version 4.$(date +'%Y%m%d').0", + "clean": "gulp clean", "checkin": "gulp checkinBuilt", "deployDemos": "gulp deployDemos", "format": "git-clang-format", diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index 41b41a962..e87b1cf69 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -21,6 +21,8 @@ var through2 = require('through2'); var closureCompiler = require('google-closure-compiler').gulp(); var closureDeps = require('google-closure-deps'); var argv = require('yargs').argv; +var rimraf = require('rimraf'); + var {BUILD_DIR} = require('./config'); var {getPackageJson} = require('./helper_tasks'); @@ -549,6 +551,18 @@ function checkinBuilt() { ]).pipe(gulp.dest('.')); }; +/** + * This task cleans the build directory (by deleting it). + */ +function cleanBuildDir(done) { + // Sanity check. + if (BUILD_DIR === '.' || BUILD_DIR === '/') { + throw new Error(`Refusing to rm -rf ${BUILD_DIR}`); + } + rimraf.sync(BUILD_DIR); + done(); +} + module.exports = { build: build, core: buildCore, @@ -559,5 +573,6 @@ module.exports = { compressed: buildCompressed, generators: buildGenerators, checkinBuilt: checkinBuilt, + cleanBuildDir: cleanBuildDir, advancedCompilationTest: buildAdvancedCompilationTest, } From e8eb30fa8b3c2e4575de6c9832cd91c7e14d7428 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Wed, 23 Jun 2021 13:07:20 +0100 Subject: [PATCH 08/20] Script to clean release directory You can now do npm run clean:buildDir, ... clean:releaseDir, or just ... clean, which does both. The release directory is automatically cleaned before packaging commences. --- gulpfile.js | 6 +++- package.json | 2 ++ scripts/gulpfiles/build_tasks.js | 3 +- scripts/gulpfiles/package_tasks.js | 58 ++++++++++++++++++------------ 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index ccb9ed072..65cecb977 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -33,7 +33,11 @@ module.exports = { buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, checkinBuilt: buildTasks.checkinBuilt, - clean: buildTasks.cleanBuildDir, + clean: gulp.series( + buildTasks.cleanBuildDir, + packageTasks.cleanReleaseDir), + cleanBuildDir: buildTasks.cleanBuildDir, + cleanReleaseDir: packageTasks.cleanReleaseDir, gitSyncDevelop: gitTasks.syncDevelop, gitSyncMaster: gitTasks.syncMaster, gitCreateRC: gitTasks.createRC, diff --git a/package.json b/package.json index 81925ec18..181613046 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,8 @@ "build:uncompressed": "gulp buildUncompressed", "bump": "npm --no-git-tag-version version 4.$(date +'%Y%m%d').0", "clean": "gulp clean", + "clean:build": "gulp cleanBuildDir", + "clean:release": "gulp cleanReleaseDir", "checkin": "gulp checkinBuilt", "deployDemos": "gulp deployDemos", "format": "git-clang-format", diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index e87b1cf69..ef1ca6ffc 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -559,8 +559,7 @@ function cleanBuildDir(done) { if (BUILD_DIR === '.' || BUILD_DIR === '/') { throw new Error(`Refusing to rm -rf ${BUILD_DIR}`); } - rimraf.sync(BUILD_DIR); - done(); + rimraf(BUILD_DIR, done); } module.exports = { diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index 84f79668c..de4c5dad0 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -366,33 +366,47 @@ function packageDTS() { .pipe(gulp.dest(`${RELEASE_DIR}`)); }; +/** + * This task cleans the release directory (by deleting it). + */ +function cleanReleaseDir(done) { + // Sanity check. + if (RELEASE_DIR === '.' || RELEASE_DIR === '/') { + throw new Error(`Refusing to rm -rf ${RELEASE_DIR}`); + } + rimraf(RELEASE_DIR, done); +} + /** * This task prepares the files to be included in the NPM by copying * them into the release directory. */ -const package = gulp.parallel( - packageIndex, - packageSources, - packageCompressed, - packageBrowser, - packageNode, - packageCore, - packageNodeCore, - packageBlockly, - packageBlocks, - packageJavascript, - packagePython, - packageLua, - packageDart, - packagePHP, - packageLocales, - packageMedia, - packageUMDBundle, - packageJSON, - packageReadme, - packageDTS -); +const package = gulp.series( + cleanReleaseDir, + gulp.parallel( + packageIndex, + packageSources, + packageCompressed, + packageBrowser, + packageNode, + packageCore, + packageNodeCore, + packageBlockly, + packageBlocks, + packageJavascript, + packagePython, + packageLua, + packageDart, + packagePHP, + packageLocales, + packageMedia, + packageUMDBundle, + packageJSON, + packageReadme, + packageDTS) + ); module.exports = { + cleanReleaseDir: cleanReleaseDir, package: package, }; From c521fa9bfab9fd3fbd5d56e2a3ac2c961bea2f4d Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Wed, 23 Jun 2021 13:11:29 +0100 Subject: [PATCH 09/20] Fix release process The documented release process is to do npm run recompile, merge the resulting branch to develop, and then do npm run relase, which does not do another build. This process should probably be changed, but for the moment ensure that npm run recompile (as well as npm run package:beta) runs buildTasks.checkinBuilt after each .build to preserve the old procedure. --- scripts/gulpfiles/git_tasks.js | 2 ++ scripts/gulpfiles/package_tasks.js | 10 +++++++++- scripts/gulpfiles/release_tasks.js | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/gulpfiles/git_tasks.js b/scripts/gulpfiles/git_tasks.js index f08ab9e00..bfc28df4c 100644 --- a/scripts/gulpfiles/git_tasks.js +++ b/scripts/gulpfiles/git_tasks.js @@ -103,7 +103,9 @@ const updateGithubPages = gulp.series( execSync('git reset --hard upstream/develop', { stdio: 'inherit' }); done(); }, + buildTasks.cleanBuildDir, buildTasks.build, + buildTasks.checkinBuilt, function(done) { execSync('git commit -am "Rebuild"', { stdio: 'inherit' }); execSync('git push ' + upstream_url + ' gh-pages --force', { stdio: 'inherit' }); diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index de4c5dad0..6fef12152 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -17,12 +17,20 @@ gulp.umd = require('gulp-umd'); var path = require('path'); var fs = require('fs'); +var rimraf = require('rimraf'); var {getPackageJson} = require('./helper_tasks'); -var {BUILD_DIR, RELEASE_DIR} = require('./config'); +var {RELEASE_DIR} = require('./config'); // Path to template files for gulp-umd. const TEMPLATE_DIR = 'scripts/package/templates'; +// Path from which to pull files to package. +// +// TODO(cpcallen): Use BUILD_DIR from config.js once release_tasks are +// updated to do build-package-release all in one go, instead of doing +// build-checkin and then package-release as separate steps. +var BUILD_DIR = '.'; + /** * A helper method for wrapping a file into a Universal Module Definition. * @param {string} namespace The export namespace. diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index e98639b48..b0faa759f 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -160,7 +160,9 @@ const publish = gulp.series( // Publish a beta version of Blockly. const publishBeta = gulp.series( updateBetaVersion, + buildTasks.cleanBuildDir, buildTasks.build, + buildTasks.checkinBuilt, packageTasks.package, checkBranch, checkReleaseDir, @@ -172,7 +174,9 @@ const recompile = gulp.series( gitTasks.syncDevelop(), gitTasks.createRebuildBranch, updateVersionPrompt, + buildTasks.cleanBuildDir, buildTasks.build, + buildTasks.checkinBuilt, typings.typings, gitTasks.pushRebuildBranch ); From daab45f67814e0151d35362543c056a1f02d4de8 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Thu, 24 Jun 2021 12:00:37 +0100 Subject: [PATCH 10/20] Fix eslint tests Don't try to lint built/. --- .eslintignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintignore b/.eslintignore index 373bf7a87..f5fc2f625 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,6 +2,7 @@ *_uncompressed*.js gulpfile.js /msg/* +/built/* /dist/* /core/utils/global.js /tests/blocks/* From 96da5ed01423bd4609ba9f30c3669e3f01a94260 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Thu, 24 Jun 2021 12:09:06 +0100 Subject: [PATCH 11/20] Fix metadata tests - Do not run npm install. - Do not re-run build that has already been run by a previous test. - Check files in built/ instead repo root. - Fix formatting, styleguide issues. --- tests/scripts/check_metadata.sh | 83 +++++++++++++++++---------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/tests/scripts/check_metadata.sh b/tests/scripts/check_metadata.sh index 8dbae6ccc..d3e674b5d 100755 --- a/tests/scripts/check_metadata.sh +++ b/tests/scripts/check_metadata.sh @@ -1,9 +1,13 @@ #!/bin/bash # Checks the size of generated files and verifies they aren't growing -# unreasonably. +# unreasonably. Assumes the compressed files have already been built. -# These values should be updated with each release +# Location of the pre-built compressed files. +readonly BUILD_DIR='built' + +# These values should be updated with each release. (Note that the +# historic values are tab-delimited.) # Size of blockly_compressed.js # Q2 2019 2.20190722.0 812688 @@ -12,9 +16,9 @@ # Q1 2020 3.20200402.0 619341 # Q2 2020 3.20200625.0 621811 # Q3 2020 3.20200924.0 641216 -# Q4 2020 4.20201217.0 653624 -# Q1 2021 5.20210325.0 653957 -blockly_size_expected=653957 +# Q4 2020 4.20201217.0 653624 +# Q1 2021 5.20210325.0 653957 +readonly BLOCKLY_SIZE_EXPECTED=653957 # Size of blocks_compressed.js # Q2 2019 2.20190722.0 75618 @@ -23,9 +27,9 @@ blockly_size_expected=653957 # Q1 2020 3.20200402.0 75805 # Q2 2020 3.20200625.0 76360 # Q3 2020 3.20200924.0 76429 -# Q4 2020 4.20201217.0 76693 -# Q1 2021 5.20210325.0 76693 -blocks_size_expected=76693 +# Q4 2020 4.20201217.0 76693 +# Q1 2021 5.20210325.0 76693 +readonly BLOCKS_SIZE_EXPECTED=76693 # Size of blockly_compressed.js.gz # Q2 2019 2.20190722.0 180925 @@ -34,9 +38,9 @@ blocks_size_expected=76693 # Q1 2020 3.20200402.0 134133 # Q2 2020 3.20200625.0 135181 # Q3 2020 3.20200924.0 138003 -# Q4 2020 4.20201217.0 138115 -# Q1 2021 5.20210325.0 136118 -blockly_gz_size_expected=136118 +# Q4 2020 4.20201217.0 138115 +# Q1 2021 5.20210325.0 136118 +readonly BLOCKLY_GZ_SIZE_EXPECTED=136118 # Size of blocks_compressed.js.gz # Q2 2019 2.20190722.0 14552 @@ -45,49 +49,46 @@ blockly_gz_size_expected=136118 # Q1 2020 3.20200402.0 14966 # Q2 2020 3.20200625.0 15195 # Q3 2020 3.20200924.0 15231 -# Q4 2020 4.20201217.0 15224 -# Q1 2021 5.20210325.0 15285 -blocks_gz_size_expected=15285 +# Q4 2020 4.20201217.0 15224 +# Q1 2021 5.20210325.0 15285 +readonly BLOCKS_GZ_SIZE_EXPECTED=15285 # ANSI colors -BOLD_GREEN='\033[1;32m' -BOLD_RED='\033[1;31m' -ANSI_RESET='\033[0m' +readonly BOLD_GREEN='\033[1;32m' +readonly BOLD_RED='\033[1;31m' +readonly ANSI_RESET='\033[0m' -# Build the compressed files for core and blocks -echo "Building files" -npm install -gulp buildCompressed -gulp buildBlocks - -# GZip them for additional size comparisons +# GZip them for additional size comparisons (keep originals, force +# overwite previously-gzipped copies). echo "Zipping the compressed files" -gzip -c blockly_compressed.js > blockly_compressed.js.gz -gzip -c blocks_compressed.js > blocks_compressed.js.gz +gzip -kf "${BUILD_DIR}/blockly_compressed.js" +gzip -kf "${BUILD_DIR}/blocks_compressed.js" # Check the sizes of the files has_failed=0 compare_size() { - local name=$1 - local expected=$2 - local compare=$(echo "$expected * 1.1 / 1" | bc) + local name="$1" + local expected="$2" + local compare=$(echo "${expected} * 1.1 / 1" | bc) - local size=$(wc -c <"$name") + local size=$(wc -c <"${name}") - if (( $size > $compare)) - then - echo -e "${BOLD_RED}Failed: Size of $name has grown more than 10%. $size vs $expected ${ANSI_RESET}" >&2 - has_failed=1 - else - echo -e "${BOLD_GREEN}Size of $name at $size compared to previous $expected.${ANSI_RESET}" - fi + if (( $size > $compare)) + then + echo -ne "${BOLD_RED}Failed: Size of ${name} has grown more than 10%. " >&2 + echo -e "${size} vs ${expected} ${ANSI_RESET}" >&2 + has_failed=1 + else + echo -ne "${BOLD_GREEN}Size of ${name} at ${size} compared to previous " >&2 + echo -e "${expected}.${ANSI_RESET}" + fi } -compare_size "blockly_compressed.js" $blockly_size_expected -compare_size "blocks_compressed.js" $blocks_size_expected -compare_size "blockly_compressed.js.gz" $blockly_gz_size_expected -compare_size "blocks_compressed.js.gz" $blocks_gz_size_expected +compare_size "${BUILD_DIR}/blockly_compressed.js" $BLOCKLY_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blocks_compressed.js" $BLOCKS_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blockly_compressed.js.gz" $BLOCKLY_GZ_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blocks_compressed.js.gz" $BLOCKS_GZ_SIZE_EXPECTED exit $has_failed From 37ecce8d8098c53d5a3da7a33ddaf8a6db11ab61 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Tue, 29 Jun 2021 16:25:36 +0100 Subject: [PATCH 12/20] Have npm run typings use built directory; fix typescript tests * Modify scripts/gulpfiles/typings.js to write typings to BUILD_DIR. * Modify tests/scripts/compile_typings.sh to check compilability of resulting output from BUILD_DIR. * Rename checkin script to checkin:built, add a checkin:typings script to do the same for .d.ts files, and a new checkin script to do both. * Have recompile run checkin:typings. --- gulpfile.js | 6 +++--- package.json | 4 +++- scripts/gulpfiles/release_tasks.js | 1 + scripts/gulpfiles/typings.js | 19 ++++++++++++++++--- tests/scripts/compile_typings.sh | 10 +++++----- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 65cecb977..9f707bfa0 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -32,10 +32,9 @@ module.exports = { buildCompressed: buildTasks.compressed, buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, + checkin: gulp.series(buildTasks.checkinBuilt, typings.checkinTypings), checkinBuilt: buildTasks.checkinBuilt, - clean: gulp.series( - buildTasks.cleanBuildDir, - packageTasks.cleanReleaseDir), + clean: gulp.series(buildTasks.cleanBuildDir, packageTasks.cleanReleaseDir), cleanBuildDir: buildTasks.cleanBuildDir, cleanReleaseDir: packageTasks.cleanReleaseDir, gitSyncDevelop: gitTasks.syncDevelop, @@ -43,6 +42,7 @@ module.exports = { gitCreateRC: gitTasks.createRC, gitUpdateGithubPages: gitTasks.updateGithubPages, typings: gulp.series(typings.typings, typings.msgTypings), + checkinTypings: typings.checkinTypings, package: packageTasks.package, checkLicenses: licenseTasks.checkLicenses, recompile: releaseTasks.recompile, diff --git a/package.json b/package.json index 7f345709d..41308cf5e 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,9 @@ "clean": "gulp clean", "clean:build": "gulp cleanBuildDir", "clean:release": "gulp cleanReleaseDir", - "checkin": "gulp checkinBuilt", + "checkin": "gulp checkin", + "checkin:built": "gulp checkinBuilt", + "checkin:typings": "gulp checkinTypings", "deployDemos": "gulp deployDemos", "format": "git-clang-format", "format:sortrequires": "gulp sortRequires", diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index b0faa759f..c4480f06c 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -178,6 +178,7 @@ const recompile = gulp.series( buildTasks.build, buildTasks.checkinBuilt, typings.typings, + typings.checkinTypings, gitTasks.pushRebuildBranch ); diff --git a/scripts/gulpfiles/typings.js b/scripts/gulpfiles/typings.js index 67b94f59c..4f4a82845 100644 --- a/scripts/gulpfiles/typings.js +++ b/scripts/gulpfiles/typings.js @@ -16,6 +16,7 @@ var path = require('path'); var fs = require('fs'); var rimraf = require('rimraf'); var execSync = require('child_process').execSync; +var {BUILD_DIR} = require('./config'); /** * Recursively generates a list of file paths with the specified extension @@ -94,7 +95,7 @@ function typings() { ]; return gulp.src(srcs) .pipe(gulp.concat('blockly.d.ts')) - .pipe(gulp.dest('typings')) + .pipe(gulp.dest(BUILD_DIR)) .on('end', function () { // Clean up tmp directory. if (fs.existsSync(tmpDir)) { @@ -110,12 +111,24 @@ function msgTypings(cb) { msgFiles.forEach(msg => { const localeName = msg.substring(0, msg.indexOf('.json')); const msgTypings = template.slice().replace(/<%= locale %>/gi, localeName); - fs.writeFileSync(path.join('typings', 'msg', localeName + '.d.ts'), msgTypings, 'utf-8'); + fs.writeFileSync(path.join(BUILD_DIR, 'msg', localeName + '.d.ts'), msgTypings, 'utf-8'); }) cb(); } +/** + * This task copies built files from BUILD_DIR back to the repository + * so they can be committed to git. + */ +function checkinTypings() { + return gulp.src([ + `${BUILD_DIR}/**.d.ts`, + `${BUILD_DIR}/**/**.d.ts`, + ]).pipe(gulp.dest('typings')); +}; + module.exports = { typings: typings, - msgTypings: msgTypings + msgTypings: msgTypings, + checkinTypings: checkinTypings, }; diff --git a/tests/scripts/compile_typings.sh b/tests/scripts/compile_typings.sh index ee16beb56..893407732 100755 --- a/tests/scripts/compile_typings.sh +++ b/tests/scripts/compile_typings.sh @@ -1,21 +1,21 @@ #!/bin/bash +# Location that npm run typings will write .d.ts files to. +readonly BUILD_DIR='built' + # ANSI colors BOLD_GREEN='\033[1;32m' BOLD_RED='\033[1;31m' ANSI_RESET='\033[0m' -# Download TypeScript to obtain the compiler. -echo "Downloading TypeScript" -npm install typescript - # Generate Blockly typings. echo "Generating Blockly typings" npm run typings # Use the TypeScript compiler to compile the generated typings. echo "Compiling typings" -cd typings + +cd "${BUILD_DIR}" ../node_modules/.bin/tsc blockly.d.ts From 5410ba6ab37563b03dacf43323970850bbe63db4 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Wed, 30 Jun 2021 16:12:10 +0100 Subject: [PATCH 13/20] Do trivial gulp tasks in parallel where possible N.B. can't run typings.typings and typings.msgTypings in parallel yet because the latter depends on the existence of an output directory created by the former. --- gulpfile.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gulpfile.js b/gulpfile.js index 9f707bfa0..8c198cd86 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -32,9 +32,9 @@ module.exports = { buildCompressed: buildTasks.compressed, buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, - checkin: gulp.series(buildTasks.checkinBuilt, typings.checkinTypings), + checkin: gulp.parallel(buildTasks.checkinBuilt, typings.checkinTypings), checkinBuilt: buildTasks.checkinBuilt, - clean: gulp.series(buildTasks.cleanBuildDir, packageTasks.cleanReleaseDir), + clean: gulp.parallel(buildTasks.cleanBuildDir, packageTasks.cleanReleaseDir), cleanBuildDir: buildTasks.cleanBuildDir, cleanReleaseDir: packageTasks.cleanReleaseDir, gitSyncDevelop: gitTasks.syncDevelop, From c265eb87b5fb97c26b09ab0b06b33f4a0a3baa2e Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 2 Jul 2021 10:23:10 +0100 Subject: [PATCH 14/20] Update metadata sizes for Q2 2021 release Transcribed (with tweaks) from PR #4986. --- tests/scripts/check_metadata.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/scripts/check_metadata.sh b/tests/scripts/check_metadata.sh index d3e674b5d..4b02649b7 100755 --- a/tests/scripts/check_metadata.sh +++ b/tests/scripts/check_metadata.sh @@ -3,6 +3,9 @@ # Checks the size of generated files and verifies they aren't growing # unreasonably. Assumes the compressed files have already been built. +# The ..._EXPECTED values should be updated with each release. +# Run this script to get the new values. + # Location of the pre-built compressed files. readonly BUILD_DIR='built' @@ -18,7 +21,8 @@ readonly BUILD_DIR='built' # Q3 2020 3.20200924.0 641216 # Q4 2020 4.20201217.0 653624 # Q1 2021 5.20210325.0 653957 -readonly BLOCKLY_SIZE_EXPECTED=653957 +# Q2 2021 6.20210701.0 664497 +readonly BLOCKLY_SIZE_EXPECTED=664497 # Size of blocks_compressed.js # Q2 2019 2.20190722.0 75618 @@ -29,7 +33,8 @@ readonly BLOCKLY_SIZE_EXPECTED=653957 # Q3 2020 3.20200924.0 76429 # Q4 2020 4.20201217.0 76693 # Q1 2021 5.20210325.0 76693 -readonly BLOCKS_SIZE_EXPECTED=76693 +# Q2 2021 6.20210701.0 76669 +readonly BLOCKS_SIZE_EXPECTED=76669 # Size of blockly_compressed.js.gz # Q2 2019 2.20190722.0 180925 @@ -40,7 +45,8 @@ readonly BLOCKS_SIZE_EXPECTED=76693 # Q3 2020 3.20200924.0 138003 # Q4 2020 4.20201217.0 138115 # Q1 2021 5.20210325.0 136118 -readonly BLOCKLY_GZ_SIZE_EXPECTED=136118 +# Q2 2021 6.20210701.0 142112 +readonly BLOCKLY_GZ_SIZE_EXPECTED=142112 # Size of blocks_compressed.js.gz # Q2 2019 2.20190722.0 14552 @@ -50,8 +56,9 @@ readonly BLOCKLY_GZ_SIZE_EXPECTED=136118 # Q2 2020 3.20200625.0 15195 # Q3 2020 3.20200924.0 15231 # Q4 2020 4.20201217.0 15224 -# Q1 2021 5.20210325.0 15285 -readonly BLOCKS_GZ_SIZE_EXPECTED=15285 +# Q1 2021 5.20210325.0 15285 +# Q2 2021 6.20210701.0 15275 +readonly BLOCKS_GZ_SIZE_EXPECTED=15275 # ANSI colors readonly BOLD_GREEN='\033[1;32m' From e6929decdeabb135cdfc527118f6c5c56a31a2c3 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Wed, 7 Jul 2021 15:36:39 +0100 Subject: [PATCH 15/20] Also check in sourcemaps The .js.map files generated by buildCompressed, buildBlocks etc. were not being copied back by checkinBuilt. --- scripts/gulpfiles/build_tasks.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index ef1ca6ffc..ad5e1fb4a 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -547,7 +547,9 @@ const build = gulp.parallel( function checkinBuilt() { return gulp.src([ `${BUILD_DIR}/**.js`, + `${BUILD_DIR}/**.js.map`, `${BUILD_DIR}/**/**.js`, + `${BUILD_DIR}/**/**.js.map`, ]).pipe(gulp.dest('.')); }; From ec14cc3ed8872419c55cd63bfb86c93fa5aa1b36 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Wed, 7 Jul 2021 16:22:42 +0100 Subject: [PATCH 16/20] Package build output; fix node tests. - Reconfigure package_tasks.js to use BUILD_DIR from config.js (i.e., build/) rather than repository root as source of files to package. - Add check to packageTasks.package to ensure that certain required files exist in BUILD_DIR, to verify that buildTasks.build and typings.typings have been run. - Fix packageUMDBundle to use generated, rather than checked-in version of en.js. - Fix packageDTS to use the generated (rather than checked-in) versions of blockly.d.ts and msg/*.d.ts. - Modify run_all_tests.sh to run packageTasks.package before running node tests, since they depend on it. Previously this was only working because 'npm install' runs the 'prepare' script, which would run the 'package' script - so the code being tested by the node tests was not the current source but whatever precomipled code had previously been checked in. - Remove the 'prepare' script from package.json, since it is no longer needed (and is now broken, since it requires that build and typings have been done first.) Note that no scripts at all are included in the version of package.json that is created in dist/ and subsequently included in the published npms, so this deletion does not affect what happens when the Blockly npm in installed - it only affects what happens when 'npm install' is run after the blockly repo is cloned. - Factor out the actual recompilation steps from releaseTasks.recompile. - Rename releaseTasks.recomple to recompileDevelop, since it deals specifically with the develop branch. (The npm script name is unchanged.) - Ensure that a full recompile and repackage is done before publishing (beta and non-beta) npms. --- package.json | 1 - scripts/gulpfiles/package_tasks.js | 64 ++++++++++++++++++++++++------ scripts/gulpfiles/release_tasks.js | 27 ++++++++----- tests/run_all_tests.sh | 14 ++++--- 4 files changed, 76 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index 41308cf5e..f43578ac3 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,6 @@ "license": "gulp checkLicenses", "lint": "eslint .", "package": "gulp package", - "prepare": "npm run package", "publish": "gulp publish", "publish:beta": "gulp publishBeta", "recompile": "gulp recompile", diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index 6fef12152..6b791d2e6 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -19,18 +19,11 @@ var path = require('path'); var fs = require('fs'); var rimraf = require('rimraf'); var {getPackageJson} = require('./helper_tasks'); -var {RELEASE_DIR} = require('./config'); +var {BUILD_DIR, RELEASE_DIR} = require('./config'); // Path to template files for gulp-umd. const TEMPLATE_DIR = 'scripts/package/templates'; -// Path from which to pull files to package. -// -// TODO(cpcallen): Use BUILD_DIR from config.js once release_tasks are -// updated to do build-package-release all in one go, instead of doing -// build-checkin and then package-release as separate steps. -var BUILD_DIR = '.'; - /** * A helper method for wrapping a file into a Universal Module Definition. * @param {string} namespace The export namespace. @@ -59,6 +52,41 @@ function packageCommonJS(namespace, dependencies) { }); }; +// Sanity check that the BUILD_DIR directory exists, and that certain +// files are in it. +function checkBuildDir(done) { + // Check that directory exists. + if (!fs.existsSync(BUILD_DIR)) { + done(new Error(`The ${BUILD_DIR} directory does not exist. ` + + 'Have both packageTasks.build and typingsTasks.typings been run?')); + return; + } + // Check files built by buildTasks.build exist in BUILD_DIR. + for (const fileName of [ + 'blockly_compressed.js', // buildTasks.buildCompressed + 'blocks_compressed.js', // buildTasks.buildBlocks + 'javascript_compressed.js', // buildTasks.buildGenerators + 'msg/js/en.js', // buildTaks.buildLangfiles + ]) { + if (!fs.existsSync(`${BUILD_DIR}/${fileName}`)) { + done(new Error( + `Your ${BUILD_DIR} directory does not contain ${fileName}. ` + + 'Has packageTasks.build been run? Try "npm run build".')); + return; + } + } + // Check files built by typings.typings exist in BUILD_DIR. + for (const fileName of ['blockly.d.ts', 'msg/en.d.ts']) { + if (!fs.existsSync(`${BUILD_DIR}/${fileName}`)) { + done(new Error( + `Your ${BUILD_DIR} directory does not contain ${fileName}. ` + + 'Has typings.typings been run? Try "npm run typings".')); + return; + } + } + done(); +} + /** * This task copies source files into the release directory. */ @@ -321,7 +349,7 @@ function packageLocales() { function packageUMDBundle() { var srcs = [ `${BUILD_DIR}/blockly_compressed.js`, - 'msg/js/en.js', + `${BUILD_DIR}/msg/js/en.js`, `${BUILD_DIR}/blocks_compressed.js`, `${BUILD_DIR}/javascript_compressed.js`, ]; @@ -360,7 +388,7 @@ function packageJSON(cb) { */ function packageReadme() { return gulp.src('scripts/package/README.md') - .pipe(gulp.dest(`${RELEASE_DIR}`)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -369,9 +397,18 @@ function packageReadme() { * referenced in package.json in the types property. */ function packageDTS() { - return gulp.src(['typings/*.d.ts', 'typings/msg/*.d.ts'], - {base: './typings'}) - .pipe(gulp.dest(`${RELEASE_DIR}`)); + const handwrittenSrcs = [ + 'typings/*.d.ts', + '!typings/blockly.d.ts', // Exclude checked-in copy of blockly.d.ts. + 'typings/msg/msg.d.ts', + ]; + const builtSrcs = [ + `${BUILD_DIR}/blockly.d.ts`, // Use freshly-built one instead. + `${BUILD_DIR}/msg/*.d.ts`, + ]; + return gulp.src(handwrittenSrcs, {base: 'typings'}) + .pipe(gulp.src(builtSrcs, {base: BUILD_DIR})) + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -390,6 +427,7 @@ function cleanReleaseDir(done) { * them into the release directory. */ const package = gulp.series( + checkBuildDir, cleanReleaseDir, gulp.parallel( packageIndex, diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index c4480f06c..a55edd380 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -149,8 +149,18 @@ function updateBetaVersion(done) { done(); } +// Build Blockly and prepare to check in the resulting built files. +const recompile = gulp.series( + buildTasks.cleanBuildDir, + buildTasks.build, + buildTasks.checkinBuilt, + typings.typings, + typings.checkinTypings, + ); + // Package and publish to npm. const publish = gulp.series( + recompile, packageTasks.package, checkBranch, checkReleaseDir, @@ -160,30 +170,25 @@ const publish = gulp.series( // Publish a beta version of Blockly. const publishBeta = gulp.series( updateBetaVersion, - buildTasks.cleanBuildDir, - buildTasks.build, - buildTasks.checkinBuilt, + recompile, packageTasks.package, checkBranch, checkReleaseDir, loginAndPublishBeta ); -// Switch to a new branch, update the version number, and build Blockly. -const recompile = gulp.series( +// Switch to a new branch, update the version number, build Blockly +// and check in the resulting built files. +const recompileDevelop = gulp.series( gitTasks.syncDevelop(), gitTasks.createRebuildBranch, updateVersionPrompt, - buildTasks.cleanBuildDir, - buildTasks.build, - buildTasks.checkinBuilt, - typings.typings, - typings.checkinTypings, + recompile, gitTasks.pushRebuildBranch ); module.exports = { - recompile: recompile, + recompile: recompileDevelop, publishBeta: publishBeta, publish: publish } diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 5107a6ebf..f6c7c0f8c 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -51,14 +51,15 @@ run_test_command () { # Lint the codebase. run_test_command "eslint" "eslint ." -# Run the closure compiler. -run_test_command "compile" "npm run build" +# Run the full usual build process. +run_test_command "build" "npm run build" -# Run the closure compiler ensuring there are no compiler warnings / errors. -run_test_command "compile:warnings" "npm run build:debug" +# Run the debug build, to ensure there are no closure compiler +# warnings / errors. +run_test_command "build:debug" "npm run build:debug" # Generate TypeScript typings and ensure there are no errors. -run_test_command "typings" "tests/scripts/compile_typings.sh" +run_test_command "typings" "npm run typings" # Check the sizes of built files for unexpected growth. run_test_command "metadata" "tests/scripts/check_metadata.sh" @@ -69,6 +70,9 @@ run_test_command "mocha" "node tests/mocha/run_mocha_tests_in_browser.js" # Run generator tests inside a browser and check the results. run_test_command "generators" "tests/scripts/run_generators.sh" +# Run the package build process, as Node tests depend on it. +run_test_command "package" "npm run package" + # Run Node tests. run_test_command "node" "./node_modules/.bin/mocha tests/node --config tests/node/.mocharc.js" From 311b2c16d8377ace7799c6fd8e98ddc101aea3b9 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Thu, 8 Jul 2021 16:55:15 +0100 Subject: [PATCH 17/20] Rename BUILD_DIR to "build" (was "built") Per discussion in PR #4968, the dirname "build" is widely used for this purpose. --- scripts/gulpfiles/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gulpfiles/config.js b/scripts/gulpfiles/config.js index 4008b6304..30434f8c0 100644 --- a/scripts/gulpfiles/config.js +++ b/scripts/gulpfiles/config.js @@ -14,7 +14,7 @@ var path = require('path'); // trailing slash. module.exports = { // Directory to write compiled output to. - BUILD_DIR: 'built', + BUILD_DIR: 'build', // Directory in which to assemble (and from which to publish) the // blockly npm package. From 1e4fa8005a4d3f87d681dce3f8eb0a2395a9b2b4 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Thu, 8 Jul 2021 17:29:26 +0100 Subject: [PATCH 18/20] Fix comment TODO formatting. --- scripts/gulpfiles/build_tasks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index ad5e1fb4a..b27d21a5b 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -446,7 +446,7 @@ this removal! */ function buildLangfiles(done) { // Create output directory. - // BUG(cpcallen): this probably doesn't work on Windows. + // TODO(#5000): does mkidr -p work on Windows? const outputDir = path.join(BUILD_DIR, 'msg', 'js'); execSync(`mkdir -p ${outputDir}`, {stdio: 'inherit'}); From 9e72378b6959e2e4d43f129649ba0bec73ea67a7 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 9 Jul 2021 14:06:59 +0100 Subject: [PATCH 19/20] Finish renaming BUILD_DIR; make tests more error-prone I had forgotten that I needed to change the value of BUILD_DIR in several different places. Added comments warning future editors about this as well as filing issue #5007 to track fixing this properly. Despite being misconfigured and therefore failing, the typescript and metadata test scripts were exiting with status 0, indicating successful completion. These have been fixed so they should fail on any error, including misconfiguration. --- .eslintignore | 1 - .gitignore | 2 +- scripts/gulpfiles/config.js | 6 ++++++ tests/scripts/check_metadata.sh | 13 ++++++++++++- tests/scripts/compile_typings.sh | 23 +++++++++++++---------- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.eslintignore b/.eslintignore index f5fc2f625..373bf7a87 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,7 +2,6 @@ *_uncompressed*.js gulpfile.js /msg/* -/built/* /dist/* /core/utils/global.js /tests/blocks/* diff --git a/.gitignore b/.gitignore index ea2fce879..02387abda 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,5 @@ local_build/*compiler*.jar local_build/local_*_compressed.js chromedriver typings/tmp/* -built/ +build/ dist/ diff --git a/scripts/gulpfiles/config.js b/scripts/gulpfiles/config.js index 30434f8c0..8f8424c6c 100644 --- a/scripts/gulpfiles/config.js +++ b/scripts/gulpfiles/config.js @@ -12,6 +12,12 @@ var path = require('path'); // Paths are all relative to the repository root. Do not include // trailing slash. +// +// TODO(#5007): If you modify these values, you must also modify the +// corresponding values in the following files: +// +// - tests/scripts/compile_typings.sh +// - tests/scripts/check_metadata.sh module.exports = { // Directory to write compiled output to. BUILD_DIR: 'build', diff --git a/tests/scripts/check_metadata.sh b/tests/scripts/check_metadata.sh index 4b02649b7..a35257c8e 100755 --- a/tests/scripts/check_metadata.sh +++ b/tests/scripts/check_metadata.sh @@ -7,7 +7,10 @@ # Run this script to get the new values. # Location of the pre-built compressed files. -readonly BUILD_DIR='built' +# +# (TODO(#5007): Should fetch this from scripts/gulpfiles/config.js +# instead of hardcoding it here. +readonly BUILD_DIR='build' # These values should be updated with each release. (Note that the # historic values are tab-delimited.) @@ -65,6 +68,14 @@ readonly BOLD_GREEN='\033[1;32m' readonly BOLD_RED='\033[1;31m' readonly ANSI_RESET='\033[0m' +# Terminate immediately with non-zero status if any command exits +# with non-zero status, printing a nice message. +set -e +function fail { + echo -e "${BOLD_RED}Error while checking metadata.${ANSI_RESET}" >&2 +} +trap fail ERR + # GZip them for additional size comparisons (keep originals, force # overwite previously-gzipped copies). echo "Zipping the compressed files" diff --git a/tests/scripts/compile_typings.sh b/tests/scripts/compile_typings.sh index 893407732..35e9d56dc 100755 --- a/tests/scripts/compile_typings.sh +++ b/tests/scripts/compile_typings.sh @@ -1,13 +1,24 @@ #!/bin/bash # Location that npm run typings will write .d.ts files to. -readonly BUILD_DIR='built' +# +# (TODO(#5007): Should fetch this from scripts/gulpfiles/config.js +# instead of hardcoding it here. +readonly BUILD_DIR='build' # ANSI colors BOLD_GREEN='\033[1;32m' BOLD_RED='\033[1;31m' ANSI_RESET='\033[0m' +# Terminate immediately with non-zero status if any command exits +# with non-zero status, printing a nice message. +set -e +function fail { + echo -e "${BOLD_RED}Failed to compile TypeScript typings.${ANSI_RESET}" >&2 +} +trap fail ERR + # Generate Blockly typings. echo "Generating Blockly typings" npm run typings @@ -18,12 +29,4 @@ echo "Compiling typings" cd "${BUILD_DIR}" ../node_modules/.bin/tsc blockly.d.ts - -if [ $? -eq 0 ] -then - echo -e "${BOLD_GREEN}TypeScript typings compiled successfully.${ANSI_RESET}" - exit 0 -else - echo -e "${BOLD_RED}Failed to compile TypeScript typings.${ANSI_RESET}" >&2 - exit 1 -fi +echo -e "${BOLD_GREEN}TypeScript typings compiled successfully.${ANSI_RESET}" From c45929db22535dcaac3e58aee50d9f575e444231 Mon Sep 17 00:00:00 2001 From: Christopher Allen Date: Fri, 9 Jul 2021 23:08:59 +0100 Subject: [PATCH 20/20] Rename internal recompile function to rebuildAll --- scripts/gulpfiles/release_tasks.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index a55edd380..793ce50d3 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -150,7 +150,7 @@ function updateBetaVersion(done) { } // Build Blockly and prepare to check in the resulting built files. -const recompile = gulp.series( +const rebuildAll = gulp.series( buildTasks.cleanBuildDir, buildTasks.build, buildTasks.checkinBuilt, @@ -160,7 +160,7 @@ const recompile = gulp.series( // Package and publish to npm. const publish = gulp.series( - recompile, + rebuildAll, packageTasks.package, checkBranch, checkReleaseDir, @@ -170,7 +170,7 @@ const publish = gulp.series( // Publish a beta version of Blockly. const publishBeta = gulp.series( updateBetaVersion, - recompile, + rebuildAll, packageTasks.package, checkBranch, checkReleaseDir, @@ -183,7 +183,7 @@ const recompileDevelop = gulp.series( gitTasks.syncDevelop(), gitTasks.createRebuildBranch, updateVersionPrompt, - recompile, + rebuildAll, gitTasks.pushRebuildBranch );