diff --git a/.gitignore b/.gitignore index 1e7a1e198..02387abda 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ local_build/*compiler*.jar local_build/local_*_compressed.js chromedriver typings/tmp/* +build/ dist/ diff --git a/gulpfile.js b/gulpfile.js index 3f0dc46a4..53a756874 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -23,6 +23,7 @@ var cleanupTasks = require('./scripts/gulpfiles/cleanup_tasks'); module.exports = { deployDemos: appengineTasks.deployDemos, default: buildTasks.build, + generateLangfiles: buildTasks.generateLangfiles, build: buildTasks.build, buildCore: buildTasks.core, buildBlocks: buildTasks.blocks, @@ -31,11 +32,17 @@ module.exports = { buildCompressed: buildTasks.compressed, buildGenerators: buildTasks.generators, buildAdvancedCompilationTest: buildTasks.advancedCompilationTest, + checkin: gulp.parallel(buildTasks.checkinBuilt, typings.checkinTypings), + checkinBuilt: buildTasks.checkinBuilt, + clean: gulp.parallel(buildTasks.cleanBuildDir, packageTasks.cleanReleaseDir), + cleanBuildDir: buildTasks.cleanBuildDir, + cleanReleaseDir: packageTasks.cleanReleaseDir, gitSyncDevelop: gitTasks.syncDevelop, gitSyncMaster: gitTasks.syncMaster, gitCreateRC: gitTasks.createRC, gitUpdateGithubPages: gitTasks.updateGithubPages, typings: gulp.series(typings.typings, typings.msgTypings), + checkinTypings: typings.checkinTypings, package: packageTasks.package, checkLicenses: licenseTasks.checkLicenses, recompile: releaseTasks.recompile, diff --git a/msg/messages.js b/msg/messages.js index a142f6fe9..636de4bd5 100644 --- a/msg/messages.js +++ b/msg/messages.js @@ -8,13 +8,15 @@ * @fileoverview English strings. * @author fraser@google.com (Neil Fraser) * - * After modifying this file, either run "npm run build:langfiles" from the - * parent directory, or run (from this directory): - * ../scripts/i18n/js_to_json.py - * to regenerate json/{en,qqq,synonyms}.json. + * After modifying this file, run: + * + * npm run generate:langfiles + * + * to regenerate json/{en,qqq,constants,synonyms}.json. * * To convert all of the json files to .js files, run: - * ../scripts/i18n/create_messages.py json/*.json + * + * npm run build:langfiles */ 'use strict'; diff --git a/package.json b/package.json index e83f97aa9..75d294dc0 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,19 @@ "build:langfiles": "gulp buildLangfiles", "build:uncompressed": "gulp buildUncompressed", "bump": "npm --no-git-tag-version version 4.$(date +'%Y%m%d').0", + "clean": "gulp clean", + "clean:build": "gulp cleanBuildDir", + "clean:release": "gulp cleanReleaseDir", + "checkin": "gulp checkin", + "checkin:built": "gulp checkinBuilt", + "checkin:typings": "gulp checkinTypings", "deployDemos": "gulp deployDemos", "format": "git-clang-format", "format:sortrequires": "gulp sortRequires", + "generate:langfiles": "gulp generateLangfiles", "license": "gulp checkLicenses", "lint": "eslint .", "package": "gulp package", - "prepare": "npm run package", "prepareDemos": "gulp prepareDemos", "publish": "gulp publish", "publish:beta": "gulp publishBeta", diff --git a/scripts/gulpfiles/appengine_tasks.js b/scripts/gulpfiles/appengine_tasks.js index 66573d3f5..5e3f7d681 100644 --- a/scripts/gulpfiles/appengine_tasks.js +++ b/scripts/gulpfiles/appengine_tasks.js @@ -47,8 +47,8 @@ function copyStaticSrc(done) { */ function copyAppengineSrc() { const appengineSrc = [ - path.join(demoStaticTmpDir, 'appengine/**/*'), - path.join(demoStaticTmpDir, 'appengine/.gcloudignore'), + `${demoStaticTmpDir}/appengine/**/*`, + `${demoStaticTmpDir}/appengine/.gcloudignore`, ]; return gulp.src(appengineSrc).pipe(gulp.dest(demoTmpDir)); } diff --git a/scripts/gulpfiles/build_tasks.js b/scripts/gulpfiles/build_tasks.js index 6553e8872..b27d21a5b 100644 --- a/scripts/gulpfiles/build_tasks.js +++ b/scripts/gulpfiles/build_tasks.js @@ -21,8 +21,10 @@ var through2 = require('through2'); var closureCompiler = require('google-closure-compiler').gulp(); var closureDeps = require('google-closure-deps'); var argv = require('yargs').argv; -var { getPackageJson } = require('./helper_tasks'); +var rimraf = require('rimraf'); +var {BUILD_DIR} = require('./config'); +var {getPackageJson} = require('./helper_tasks'); //////////////////////////////////////////////////////////// // Build // @@ -217,7 +219,7 @@ function buildCompressed() { })) .pipe( gulp.sourcemaps.write('.', {includeContent: false, sourceRoot: './'})) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** @@ -242,7 +244,7 @@ function buildBlocks() { includeContent: false, sourceRoot: './' })) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** @@ -268,7 +270,7 @@ function buildGenerator(language, namespace) { includeContent: false, sourceRoot: './' })) - .pipe(gulp.dest('./')); + .pipe(gulp.dest(BUILD_DIR)); }; /** @@ -408,31 +410,59 @@ goog.require('Blockly.requires'); }); }; +/** + * This task regenrates msg/json/en.js and msg/json/qqq.js from + * msg/messages.js. + */ +function generateLangfiles(done) { + // Run js_to_json.py + const jsToJsonCmd = `python scripts/i18n/js_to_json.py \ + --input_file ${path.join('msg', 'messages.js')} \ + --output_dir ${path.join('msg', 'json')} \ + --quiet`; + execSync(jsToJsonCmd, { stdio: 'inherit' }); + + console.log(` +Regenerated several flies in msg/json/. Now run + + git diff msg/json/*.json + +and check that operation has not overwritten any modifications made to +hints, etc. by the TranslateWiki volunteers. If it has, backport +their changes to msg/messages.js and re-run 'npm run generate:langfiles'. + +Once you are satisfied that any new hints have been backported you may +go ahead and commit the changes, but note that the generate script +will have removed the translator credits - be careful not to commit +this removal! +`); + + done(); +}; + /** * This task builds Blockly's lang files. * msg/*.js */ function buildLangfiles(done) { - // Run js_to_json.py - const jsToJsonCmd = `python ./scripts/i18n/js_to_json.py \ ---input_file ${path.join('msg', 'messages.js')} \ ---output_dir ${path.join('msg', 'json')} \ ---quiet`; - execSync(jsToJsonCmd, { stdio: 'inherit' }); - - // Run create_messages.py + // Create output directory. + // TODO(#5000): does mkidr -p work on Windows? + const outputDir = path.join(BUILD_DIR, 'msg', 'js'); + execSync(`mkdir -p ${outputDir}`, {stdio: 'inherit'}); + + // Run create_messages.py. let json_files = fs.readdirSync(path.join('msg', 'json')); json_files = json_files.filter(file => file.endsWith('json') && - !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file))); + !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file))); json_files = json_files.map(file => path.join('msg', 'json', file)); const createMessagesCmd = `python ./scripts/i18n/create_messages.py \ --source_lang_file ${path.join('msg', 'json', 'en.json')} \ --source_synonym_file ${path.join('msg', 'json', 'synonyms.json')} \ --source_constants_file ${path.join('msg', 'json', 'constants.json')} \ --key_file ${path.join('msg', 'json', 'keys.json')} \ - --output_dir ${path.join('msg', 'js')} \ + --output_dir ${outputDir} \ --quiet ${json_files.join(' ')}`; - execSync(createMessagesCmd, { stdio: 'inherit' }); + execSync(createMessagesCmd, {stdio: 'inherit'}); done(); }; @@ -510,13 +540,40 @@ const build = gulp.parallel( buildLangfiles ); +/** + * This task copies built files from BUILD_DIR back to the repository + * so they can be committed to git. + */ +function checkinBuilt() { + return gulp.src([ + `${BUILD_DIR}/**.js`, + `${BUILD_DIR}/**.js.map`, + `${BUILD_DIR}/**/**.js`, + `${BUILD_DIR}/**/**.js.map`, + ]).pipe(gulp.dest('.')); +}; + +/** + * This task cleans the build directory (by deleting it). + */ +function cleanBuildDir(done) { + // Sanity check. + if (BUILD_DIR === '.' || BUILD_DIR === '/') { + throw new Error(`Refusing to rm -rf ${BUILD_DIR}`); + } + rimraf(BUILD_DIR, done); +} + module.exports = { build: build, core: buildCore, blocks: buildBlocks, + generateLangfiles: generateLangfiles, langfiles: buildLangfiles, uncompressed: buildUncompressed, compressed: buildCompressed, generators: buildGenerators, + checkinBuilt: checkinBuilt, + cleanBuildDir: cleanBuildDir, advancedCompilationTest: buildAdvancedCompilationTest, } diff --git a/scripts/gulpfiles/config.js b/scripts/gulpfiles/config.js new file mode 100644 index 000000000..8f8424c6c --- /dev/null +++ b/scripts/gulpfiles/config.js @@ -0,0 +1,28 @@ +/** + * @license + * Copyright 2021 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @fileoverview Common configuration for Gulp scripts. + */ + +var path = require('path'); + +// Paths are all relative to the repository root. Do not include +// trailing slash. +// +// TODO(#5007): If you modify these values, you must also modify the +// corresponding values in the following files: +// +// - tests/scripts/compile_typings.sh +// - tests/scripts/check_metadata.sh +module.exports = { + // Directory to write compiled output to. + BUILD_DIR: 'build', + + // Directory in which to assemble (and from which to publish) the + // blockly npm package. + RELEASE_DIR: 'dist', +}; diff --git a/scripts/gulpfiles/git_tasks.js b/scripts/gulpfiles/git_tasks.js index f08ab9e00..bfc28df4c 100644 --- a/scripts/gulpfiles/git_tasks.js +++ b/scripts/gulpfiles/git_tasks.js @@ -103,7 +103,9 @@ const updateGithubPages = gulp.series( execSync('git reset --hard upstream/develop', { stdio: 'inherit' }); done(); }, + buildTasks.cleanBuildDir, buildTasks.build, + buildTasks.checkinBuilt, function(done) { execSync('git commit -am "Rebuild"', { stdio: 'inherit' }); execSync('git push ' + upstream_url + ' gh-pages --force', { stdio: 'inherit' }); diff --git a/scripts/gulpfiles/package_tasks.js b/scripts/gulpfiles/package_tasks.js index bf79b7a6e..6b791d2e6 100644 --- a/scripts/gulpfiles/package_tasks.js +++ b/scripts/gulpfiles/package_tasks.js @@ -17,13 +17,12 @@ gulp.umd = require('gulp-umd'); var path = require('path'); var fs = require('fs'); -var { getPackageJson } = require('./helper_tasks'); - -const blocklyRoot = '../../'; - -// The destination path where all the NPM distribution files will go. -const packageDistribution = 'dist'; +var rimraf = require('rimraf'); +var {getPackageJson} = require('./helper_tasks'); +var {BUILD_DIR, RELEASE_DIR} = require('./config'); +// Path to template files for gulp-umd. +const TEMPLATE_DIR = 'scripts/package/templates'; /** * A helper method for wrapping a file into a Universal Module Definition. @@ -35,7 +34,7 @@ function packageUMD(namespace, dependencies) { dependencies: function () { return dependencies; }, namespace: function () { return namespace; }, exports: function () { return namespace; }, - template: path.join(__dirname, `${blocklyRoot}/scripts/package/templates/umd.template`) + template: path.join(TEMPLATE_DIR, 'umd.template') }); }; @@ -49,26 +48,61 @@ function packageCommonJS(namespace, dependencies) { dependencies: function () { return dependencies; }, namespace: function () { return namespace; }, exports: function () { return namespace; }, - template: path.join(__dirname, `${blocklyRoot}/scripts/package/templates/node.template`) + template: path.join(TEMPLATE_DIR, 'node.template') }); }; +// Sanity check that the BUILD_DIR directory exists, and that certain +// files are in it. +function checkBuildDir(done) { + // Check that directory exists. + if (!fs.existsSync(BUILD_DIR)) { + done(new Error(`The ${BUILD_DIR} directory does not exist. ` + + 'Have both packageTasks.build and typingsTasks.typings been run?')); + return; + } + // Check files built by buildTasks.build exist in BUILD_DIR. + for (const fileName of [ + 'blockly_compressed.js', // buildTasks.buildCompressed + 'blocks_compressed.js', // buildTasks.buildBlocks + 'javascript_compressed.js', // buildTasks.buildGenerators + 'msg/js/en.js', // buildTaks.buildLangfiles + ]) { + if (!fs.existsSync(`${BUILD_DIR}/${fileName}`)) { + done(new Error( + `Your ${BUILD_DIR} directory does not contain ${fileName}. ` + + 'Has packageTasks.build been run? Try "npm run build".')); + return; + } + } + // Check files built by typings.typings exist in BUILD_DIR. + for (const fileName of ['blockly.d.ts', 'msg/en.d.ts']) { + if (!fs.existsSync(`${BUILD_DIR}/${fileName}`)) { + done(new Error( + `Your ${BUILD_DIR} directory does not contain ${fileName}. ` + + 'Has typings.typings been run? Try "npm run typings".')); + return; + } + } + done(); +} + /** - * This task copies source files into the distribution directory. + * This task copies source files into the release directory. */ function packageSources() { return gulp.src(['core/**/**.js', 'blocks/**.js', 'generators/**/**.js'], {base: '.'}) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** - * This task copies the compressed files and their source maps into the - * distribution directory. + * This task copies the compressed files and their source maps into + * the release directory. */ function packageCompressed() { - return gulp.src('*_compressed.js?(.map)') - .pipe(gulp.dest(packageDistribution)); + return gulp.src('*_compressed.js?(.map)', {cwd: BUILD_DIR}) + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -83,7 +117,7 @@ function packageBlockly() { cjs: './blockly_compressed', }])) .pipe(gulp.rename('blockly.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -98,7 +132,7 @@ function packageBlocks() { cjs: './blocks_compressed', }])) .pipe(gulp.rename('blocks.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -115,7 +149,7 @@ function packageIndex() { cjs: './node', }])) .pipe(gulp.rename('index.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -147,7 +181,7 @@ function packageBrowser() { cjs: './javascript', }])) .pipe(gulp.rename('browser.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -166,7 +200,7 @@ function packageCore() { cjs: './blockly', }])) .pipe(gulp.rename('core-browser.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -205,7 +239,7 @@ function packageNode() { cjs: './dart', }])) .pipe(gulp.rename('node.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -224,7 +258,7 @@ function packageNodeCore() { cjs: './blockly', }])) .pipe(gulp.rename('core.js')) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -245,7 +279,7 @@ function packageGenerator(file, rename, namespace) { cjs: `./${file}`, }])) .pipe(gulp.rename(rename)) - .pipe(gulp.dest(packageDistribution)); + .pipe(gulp.dest(RELEASE_DIR)); }; /** @@ -289,12 +323,12 @@ function packagePHP() { }; /** - * This task wraps each of the msg/js/* files into a UMD module. + * This task wraps each of the ${BUILD_DIR}/msg/js/* files into a UMD module. * @example import * as En from 'blockly/msg/en'; */ function packageLocales() { // Remove references to goog.provide and goog.require. - return gulp.src('msg/js/*.js') + return gulp.src(`${BUILD_DIR}/msg/js/*.js`) .pipe(gulp.replace(/goog\.[^\n]+/g, '')) .pipe(gulp.insert.prepend(` var Blockly = {};Blockly.Msg={};`)) @@ -303,7 +337,7 @@ function packageLocales() { amd: '../core', cjs: '../core', }])) - .pipe(gulp.dest(`${packageDistribution}/msg`)); + .pipe(gulp.dest(`${RELEASE_DIR}/msg`)); }; /** @@ -314,84 +348,111 @@ function packageLocales() { */ function packageUMDBundle() { var srcs = [ - 'blockly_compressed.js', - 'msg/js/en.js', - 'blocks_compressed.js', - 'javascript_compressed.js' + `${BUILD_DIR}/blockly_compressed.js`, + `${BUILD_DIR}/msg/js/en.js`, + `${BUILD_DIR}/blocks_compressed.js`, + `${BUILD_DIR}/javascript_compressed.js`, ]; return gulp.src(srcs) - .pipe(gulp.concat('blockly.min.js')) - .pipe(gulp.dest(`${packageDistribution}`)) + .pipe(gulp.concat('blockly.min.js')) + .pipe(gulp.dest(`${RELEASE_DIR}`)); }; /** - * This task copies all the media/* files into the distribution directory. + * This task copies all the media/* files into the release directory. */ function packageMedia() { - return gulp.src('./media/*') - .pipe(gulp.dest(`${packageDistribution}/media`)); + return gulp.src('media/*') + .pipe(gulp.dest(`${RELEASE_DIR}/media`)); }; /** - * This task copies the package.json file into the distribution directory. + * This task copies the package.json file into the release directory. */ function packageJSON(cb) { const packageJson = getPackageJson(); const json = Object.assign({}, packageJson); delete json['scripts']; - if (!fs.existsSync(packageDistribution)) { - fs.mkdirSync(packageDistribution); + if (!fs.existsSync(RELEASE_DIR)) { + fs.mkdirSync(RELEASE_DIR); } - fs.writeFileSync(`${packageDistribution}/package.json`, + fs.writeFileSync(`${RELEASE_DIR}/package.json`, JSON.stringify(json, null, 2)); cb(); }; /** - * This task copies the scripts/package/README.md file into the distribution directory. - * This file is what developers will see at https://www.npmjs.com/package/blockly. + * This task copies the scripts/package/README.md file into the + * release directory. This file is what developers will see at + * https://www.npmjs.com/package/blockly . */ function packageReadme() { - return gulp.src('./scripts/package/README.md') - .pipe(gulp.dest(`${packageDistribution}`)); + return gulp.src('scripts/package/README.md') + .pipe(gulp.dest(RELEASE_DIR)); }; /** - * This task copies the typings/blockly.d.ts TypeScript definition file into the - * distribution directory. - * The bundled declaration file is referenced in package.json in the types property. + * This task copies the typings/blockly.d.ts TypeScript definition + * file into the release directory. The bundled declaration file is + * referenced in package.json in the types property. */ function packageDTS() { - return gulp.src(['./typings/*.d.ts', './typings/msg/*.d.ts'], {base: './typings'}) - .pipe(gulp.dest(`${packageDistribution}`)); + const handwrittenSrcs = [ + 'typings/*.d.ts', + '!typings/blockly.d.ts', // Exclude checked-in copy of blockly.d.ts. + 'typings/msg/msg.d.ts', + ]; + const builtSrcs = [ + `${BUILD_DIR}/blockly.d.ts`, // Use freshly-built one instead. + `${BUILD_DIR}/msg/*.d.ts`, + ]; + return gulp.src(handwrittenSrcs, {base: 'typings'}) + .pipe(gulp.src(builtSrcs, {base: BUILD_DIR})) + .pipe(gulp.dest(RELEASE_DIR)); }; /** - * This task prepares the NPM distribution files under the /dist directory. + * This task cleans the release directory (by deleting it). */ -const package = gulp.parallel( - packageIndex, - packageSources, - packageCompressed, - packageBrowser, - packageNode, - packageCore, - packageNodeCore, - packageBlockly, - packageBlocks, - packageJavascript, - packagePython, - packageLua, - packageDart, - packagePHP, - packageLocales, - packageMedia, - packageUMDBundle, - packageJSON, - packageReadme, - packageDTS -); +function cleanReleaseDir(done) { + // Sanity check. + if (RELEASE_DIR === '.' || RELEASE_DIR === '/') { + throw new Error(`Refusing to rm -rf ${RELEASE_DIR}`); + } + rimraf(RELEASE_DIR, done); +} + +/** + * This task prepares the files to be included in the NPM by copying + * them into the release directory. + */ +const package = gulp.series( + checkBuildDir, + cleanReleaseDir, + gulp.parallel( + packageIndex, + packageSources, + packageCompressed, + packageBrowser, + packageNode, + packageCore, + packageNodeCore, + packageBlockly, + packageBlocks, + packageJavascript, + packagePython, + packageLua, + packageDart, + packagePHP, + packageLocales, + packageMedia, + packageUMDBundle, + packageJSON, + packageReadme, + packageDTS) + ); module.exports = { + cleanReleaseDir: cleanReleaseDir, package: package, }; diff --git a/scripts/gulpfiles/release_tasks.js b/scripts/gulpfiles/release_tasks.js index 5e14fc08c..793ce50d3 100644 --- a/scripts/gulpfiles/release_tasks.js +++ b/scripts/gulpfiles/release_tasks.js @@ -17,9 +17,9 @@ var typings = require('./typings'); var buildTasks = require('./build_tasks'); var gitTasks = require('./git_tasks'); var packageTasks = require('./package_tasks'); -var { getPackageJson } = require('./helper_tasks'); +var {getPackageJson} = require('./helper_tasks'); +var {RELEASE_DIR} = require('./config'); -const RELEASE_DIR = 'dist'; // Gets the current major version. function getMajorVersion() { @@ -80,20 +80,25 @@ function checkBranch(done) { } -// Sanity check that the dist folder exists, and that certain files are in the dist folder. -function checkDist(done) { - const sanityFiles = ['blockly_compressed.js', 'blocks_compressed.js', 'core', 'blocks', 'generators']; - // Check that dist exists. +// Sanity check that the RELASE_DIR directory exists, and that certain +// files are in it. +function checkReleaseDir(done) { + const sanityFiles = ['blockly_compressed.js', 'blocks_compressed.js', + 'core', 'blocks', 'generators']; + // Check that directory exists. if (fs.existsSync(RELEASE_DIR)) { - // Sanity check that certain files exist in dist. + // Sanity check that certain files exist in RELASE_DIR. sanityFiles.forEach((fileName) => { if (!fs.existsSync(`${RELEASE_DIR}/${fileName}`)) { - done(new Error(`Your dist folder does not contain:${fileName}`)); + done(new Error( + `Your ${RELEASE_DIR} directory does not contain ${fileName}`)); + return; } }); done(); } else { - done(new Error('The dist directory does not exist. Is packageTasks.package being run?')); + done(new Error(`The ${RELEASE_DIR} directory does not exist. ` + + 'Has packageTasks.package been run?')); } } @@ -144,36 +149,46 @@ function updateBetaVersion(done) { done(); } +// Build Blockly and prepare to check in the resulting built files. +const rebuildAll = gulp.series( + buildTasks.cleanBuildDir, + buildTasks.build, + buildTasks.checkinBuilt, + typings.typings, + typings.checkinTypings, + ); + // Package and publish to npm. const publish = gulp.series( + rebuildAll, packageTasks.package, checkBranch, - checkDist, + checkReleaseDir, loginAndPublish ); // Publish a beta version of Blockly. const publishBeta = gulp.series( updateBetaVersion, - buildTasks.build, + rebuildAll, packageTasks.package, checkBranch, - checkDist, + checkReleaseDir, loginAndPublishBeta ); -// Switch to a new branch, update the version number, and build Blockly. -const recompile = gulp.series( +// Switch to a new branch, update the version number, build Blockly +// and check in the resulting built files. +const recompileDevelop = gulp.series( gitTasks.syncDevelop(), gitTasks.createRebuildBranch, updateVersionPrompt, - buildTasks.build, - typings.typings, + rebuildAll, gitTasks.pushRebuildBranch ); module.exports = { - recompile: recompile, + recompile: recompileDevelop, publishBeta: publishBeta, publish: publish } diff --git a/scripts/gulpfiles/typings.js b/scripts/gulpfiles/typings.js index 67b94f59c..4f4a82845 100644 --- a/scripts/gulpfiles/typings.js +++ b/scripts/gulpfiles/typings.js @@ -16,6 +16,7 @@ var path = require('path'); var fs = require('fs'); var rimraf = require('rimraf'); var execSync = require('child_process').execSync; +var {BUILD_DIR} = require('./config'); /** * Recursively generates a list of file paths with the specified extension @@ -94,7 +95,7 @@ function typings() { ]; return gulp.src(srcs) .pipe(gulp.concat('blockly.d.ts')) - .pipe(gulp.dest('typings')) + .pipe(gulp.dest(BUILD_DIR)) .on('end', function () { // Clean up tmp directory. if (fs.existsSync(tmpDir)) { @@ -110,12 +111,24 @@ function msgTypings(cb) { msgFiles.forEach(msg => { const localeName = msg.substring(0, msg.indexOf('.json')); const msgTypings = template.slice().replace(/<%= locale %>/gi, localeName); - fs.writeFileSync(path.join('typings', 'msg', localeName + '.d.ts'), msgTypings, 'utf-8'); + fs.writeFileSync(path.join(BUILD_DIR, 'msg', localeName + '.d.ts'), msgTypings, 'utf-8'); }) cb(); } +/** + * This task copies built files from BUILD_DIR back to the repository + * so they can be committed to git. + */ +function checkinTypings() { + return gulp.src([ + `${BUILD_DIR}/**.d.ts`, + `${BUILD_DIR}/**/**.d.ts`, + ]).pipe(gulp.dest('typings')); +}; + module.exports = { typings: typings, - msgTypings: msgTypings + msgTypings: msgTypings, + checkinTypings: checkinTypings, }; diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 5107a6ebf..f6c7c0f8c 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -51,14 +51,15 @@ run_test_command () { # Lint the codebase. run_test_command "eslint" "eslint ." -# Run the closure compiler. -run_test_command "compile" "npm run build" +# Run the full usual build process. +run_test_command "build" "npm run build" -# Run the closure compiler ensuring there are no compiler warnings / errors. -run_test_command "compile:warnings" "npm run build:debug" +# Run the debug build, to ensure there are no closure compiler +# warnings / errors. +run_test_command "build:debug" "npm run build:debug" # Generate TypeScript typings and ensure there are no errors. -run_test_command "typings" "tests/scripts/compile_typings.sh" +run_test_command "typings" "npm run typings" # Check the sizes of built files for unexpected growth. run_test_command "metadata" "tests/scripts/check_metadata.sh" @@ -69,6 +70,9 @@ run_test_command "mocha" "node tests/mocha/run_mocha_tests_in_browser.js" # Run generator tests inside a browser and check the results. run_test_command "generators" "tests/scripts/run_generators.sh" +# Run the package build process, as Node tests depend on it. +run_test_command "package" "npm run package" + # Run Node tests. run_test_command "node" "./node_modules/.bin/mocha tests/node --config tests/node/.mocharc.js" diff --git a/tests/scripts/check_metadata.sh b/tests/scripts/check_metadata.sh index 8dbae6ccc..a35257c8e 100755 --- a/tests/scripts/check_metadata.sh +++ b/tests/scripts/check_metadata.sh @@ -1,9 +1,19 @@ #!/bin/bash # Checks the size of generated files and verifies they aren't growing -# unreasonably. +# unreasonably. Assumes the compressed files have already been built. -# These values should be updated with each release +# The ..._EXPECTED values should be updated with each release. +# Run this script to get the new values. + +# Location of the pre-built compressed files. +# +# (TODO(#5007): Should fetch this from scripts/gulpfiles/config.js +# instead of hardcoding it here. +readonly BUILD_DIR='build' + +# These values should be updated with each release. (Note that the +# historic values are tab-delimited.) # Size of blockly_compressed.js # Q2 2019 2.20190722.0 812688 @@ -12,9 +22,10 @@ # Q1 2020 3.20200402.0 619341 # Q2 2020 3.20200625.0 621811 # Q3 2020 3.20200924.0 641216 -# Q4 2020 4.20201217.0 653624 -# Q1 2021 5.20210325.0 653957 -blockly_size_expected=653957 +# Q4 2020 4.20201217.0 653624 +# Q1 2021 5.20210325.0 653957 +# Q2 2021 6.20210701.0 664497 +readonly BLOCKLY_SIZE_EXPECTED=664497 # Size of blocks_compressed.js # Q2 2019 2.20190722.0 75618 @@ -23,9 +34,10 @@ blockly_size_expected=653957 # Q1 2020 3.20200402.0 75805 # Q2 2020 3.20200625.0 76360 # Q3 2020 3.20200924.0 76429 -# Q4 2020 4.20201217.0 76693 -# Q1 2021 5.20210325.0 76693 -blocks_size_expected=76693 +# Q4 2020 4.20201217.0 76693 +# Q1 2021 5.20210325.0 76693 +# Q2 2021 6.20210701.0 76669 +readonly BLOCKS_SIZE_EXPECTED=76669 # Size of blockly_compressed.js.gz # Q2 2019 2.20190722.0 180925 @@ -34,9 +46,10 @@ blocks_size_expected=76693 # Q1 2020 3.20200402.0 134133 # Q2 2020 3.20200625.0 135181 # Q3 2020 3.20200924.0 138003 -# Q4 2020 4.20201217.0 138115 -# Q1 2021 5.20210325.0 136118 -blockly_gz_size_expected=136118 +# Q4 2020 4.20201217.0 138115 +# Q1 2021 5.20210325.0 136118 +# Q2 2021 6.20210701.0 142112 +readonly BLOCKLY_GZ_SIZE_EXPECTED=142112 # Size of blocks_compressed.js.gz # Q2 2019 2.20190722.0 14552 @@ -45,49 +58,55 @@ blockly_gz_size_expected=136118 # Q1 2020 3.20200402.0 14966 # Q2 2020 3.20200625.0 15195 # Q3 2020 3.20200924.0 15231 -# Q4 2020 4.20201217.0 15224 -# Q1 2021 5.20210325.0 15285 -blocks_gz_size_expected=15285 +# Q4 2020 4.20201217.0 15224 +# Q1 2021 5.20210325.0 15285 +# Q2 2021 6.20210701.0 15275 +readonly BLOCKS_GZ_SIZE_EXPECTED=15275 # ANSI colors -BOLD_GREEN='\033[1;32m' -BOLD_RED='\033[1;31m' -ANSI_RESET='\033[0m' +readonly BOLD_GREEN='\033[1;32m' +readonly BOLD_RED='\033[1;31m' +readonly ANSI_RESET='\033[0m' -# Build the compressed files for core and blocks -echo "Building files" -npm install -gulp buildCompressed -gulp buildBlocks +# Terminate immediately with non-zero status if any command exits +# with non-zero status, printing a nice message. +set -e +function fail { + echo -e "${BOLD_RED}Error while checking metadata.${ANSI_RESET}" >&2 +} +trap fail ERR -# GZip them for additional size comparisons +# GZip them for additional size comparisons (keep originals, force +# overwite previously-gzipped copies). echo "Zipping the compressed files" -gzip -c blockly_compressed.js > blockly_compressed.js.gz -gzip -c blocks_compressed.js > blocks_compressed.js.gz +gzip -kf "${BUILD_DIR}/blockly_compressed.js" +gzip -kf "${BUILD_DIR}/blocks_compressed.js" # Check the sizes of the files has_failed=0 compare_size() { - local name=$1 - local expected=$2 - local compare=$(echo "$expected * 1.1 / 1" | bc) + local name="$1" + local expected="$2" + local compare=$(echo "${expected} * 1.1 / 1" | bc) - local size=$(wc -c <"$name") + local size=$(wc -c <"${name}") - if (( $size > $compare)) - then - echo -e "${BOLD_RED}Failed: Size of $name has grown more than 10%. $size vs $expected ${ANSI_RESET}" >&2 - has_failed=1 - else - echo -e "${BOLD_GREEN}Size of $name at $size compared to previous $expected.${ANSI_RESET}" - fi + if (( $size > $compare)) + then + echo -ne "${BOLD_RED}Failed: Size of ${name} has grown more than 10%. " >&2 + echo -e "${size} vs ${expected} ${ANSI_RESET}" >&2 + has_failed=1 + else + echo -ne "${BOLD_GREEN}Size of ${name} at ${size} compared to previous " >&2 + echo -e "${expected}.${ANSI_RESET}" + fi } -compare_size "blockly_compressed.js" $blockly_size_expected -compare_size "blocks_compressed.js" $blocks_size_expected -compare_size "blockly_compressed.js.gz" $blockly_gz_size_expected -compare_size "blocks_compressed.js.gz" $blocks_gz_size_expected +compare_size "${BUILD_DIR}/blockly_compressed.js" $BLOCKLY_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blocks_compressed.js" $BLOCKS_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blockly_compressed.js.gz" $BLOCKLY_GZ_SIZE_EXPECTED +compare_size "${BUILD_DIR}/blocks_compressed.js.gz" $BLOCKS_GZ_SIZE_EXPECTED exit $has_failed diff --git a/tests/scripts/compile_typings.sh b/tests/scripts/compile_typings.sh index ee16beb56..35e9d56dc 100755 --- a/tests/scripts/compile_typings.sh +++ b/tests/scripts/compile_typings.sh @@ -1,13 +1,23 @@ #!/bin/bash +# Location that npm run typings will write .d.ts files to. +# +# (TODO(#5007): Should fetch this from scripts/gulpfiles/config.js +# instead of hardcoding it here. +readonly BUILD_DIR='build' + # ANSI colors BOLD_GREEN='\033[1;32m' BOLD_RED='\033[1;31m' ANSI_RESET='\033[0m' -# Download TypeScript to obtain the compiler. -echo "Downloading TypeScript" -npm install typescript +# Terminate immediately with non-zero status if any command exits +# with non-zero status, printing a nice message. +set -e +function fail { + echo -e "${BOLD_RED}Failed to compile TypeScript typings.${ANSI_RESET}" >&2 +} +trap fail ERR # Generate Blockly typings. echo "Generating Blockly typings" @@ -15,15 +25,8 @@ npm run typings # Use the TypeScript compiler to compile the generated typings. echo "Compiling typings" -cd typings + +cd "${BUILD_DIR}" ../node_modules/.bin/tsc blockly.d.ts - -if [ $? -eq 0 ] -then - echo -e "${BOLD_GREEN}TypeScript typings compiled successfully.${ANSI_RESET}" - exit 0 -else - echo -e "${BOLD_RED}Failed to compile TypeScript typings.${ANSI_RESET}" >&2 - exit 1 -fi +echo -e "${BOLD_GREEN}TypeScript typings compiled successfully.${ANSI_RESET}"