diff --git a/.eslintignore b/.eslintignore index 0022114ea..8dd3beb37 100644 --- a/.eslintignore +++ b/.eslintignore @@ -9,6 +9,7 @@ gulpfile.js /tests/jsunit/* /tests/generators/* /tests/mocha/run_mocha_tests_in_browser.js +/tests/screenshot/* /tests/test_runner.js /tests/workspace_svg/* /generators/* diff --git a/.gitignore b/.gitignore index bcfe7f406..5d5b37e20 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,7 @@ npm-debug.log tests/compile/main_compressed.js tests/compile/*compiler*.jar +tests/screenshot/outputs/* local_build/*compiler*.jar local_build/local_*_compressed.js +chromedriver diff --git a/tests/screenshot/diff-reporter.js b/tests/screenshot/diff-reporter.js new file mode 100644 index 000000000..726d5d8e0 --- /dev/null +++ b/tests/screenshot/diff-reporter.js @@ -0,0 +1,164 @@ +// diff-reporter.js + +/** + * @license + * Visual Blocks Editor + * + * Copyright 2019 Google Inc. + * https://developers.google.com/blockly/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Reporter that prints results to the console with the same + * format as the spec reporter, but also saves a test_output.js file with a + * variable that just wraps a json object, for use in diff_viewer.html. + */ +var mocha = require('mocha'); +var fs = require("fs"); +module.exports = DiffReporter; + + +function DiffReporter(runner) { + mocha.reporters.Base.call(this, runner); + var passes = 0; + var failures = 0; + + // Values for the JSON output. + var json_tests = []; + + // From the spec reporter. + var self = this; + var indents = 0; + var n = 0; + var colours = { + pass: 32, + fail: 31, + 'bright pass': 92, + 'bright fail': 91, + 'bright yellow': 93, + pending: 36, + suite: 0, + 'error title': 0, + 'error message': 31, + 'error stack': 90, + checkmark: 32, + fast: 90, + medium: 33, + slow: 31, + green: 32, + light: 90, + 'diff gutter': 90, + 'diff added': 32, + 'diff removed': 31 + }; + + var symbols = { + ok: '✓', + err: '✖', + dot: '․', + comma: ',', + bang: '!' + }; + + /** + * colour `str` with the given `type`, + * allowing colours to be disabled, + * as well as user-defined colour + * schemes. + * + * @private + * @param {string} type + * @param {string} str + * @return {string} + */ + var colour = function(type, str) { + if (!colours) { + return String(str); + } + return '\u001b[' + colours[type] + 'm' + str + '\u001b[0m'; + }; + + function indent() { + return Array(indents).join(' '); + } + + // Indent/unindent correctly. + runner.on('start', function() { + console.log(); + }); + runner.on('suite', function(suite) { + ++indents; + console.log(colour('suite', '%s%s'), indent(), suite.title); + }); + runner.on('suite end', function() { + --indents; + if (indents === 1) { + console.log(); + } + }); + runner.on('pass', function(test) { + passes++; + json_tests.push(test); + var logStr = + indent() + + colour('checkmark', ' ' + symbols.ok) + + colour('pass', ' ' + test.title); + console.log(logStr); + }); + + runner.on('fail', function(test, err) { + failures++; + json_tests.push(test); + // Print test information the way the spec reporter would. + console.log(indent() + colour('fail', ' %d) %s'), ++n, test.title); + }); + + runner.on('end', function() { + console.log('\n%d/%d tests passed\n', passes, passes + failures); + var jsonObj = { + passes: passes, + failures: failures, + total: passes + failures, + tests: json_tests.map(clean) + } + runner.testResults = jsonObj; + + let json = JSON.stringify(jsonObj, null, 2) + let jsonOutput = writeJSON(json, 'test_output') + }); + + + function clean(test) { + return { + title: test.title, + fullTitle: test.fullTitle(), + state: test.state + }; + } + + function writeJSON(data, filename){ + let output_dir = `${process.cwd()}/tests/screenshot/outputs` + let output= `${output_dir}/${filename}` + + if (!fs.existsSync(output_dir)){ + fs.mkdirSync(output_dir); + } + fs.writeFileSync(output + '.json', data) + + fs.writeFileSync(output + '.js', 'var results = ' + data); + return output + } +} +mocha.utils.inherits(DiffReporter, mocha.reporters.Spec); diff --git a/tests/screenshot/diff_screenshots.js b/tests/screenshot/diff_screenshots.js new file mode 100644 index 000000000..2a96a691f --- /dev/null +++ b/tests/screenshot/diff_screenshots.js @@ -0,0 +1,86 @@ +/** + * @license + * Visual Blocks Editor + * + * Copyright 2019 Google Inc. + * https://developers.google.com/blockly/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Mocha tests that diff images and save the diffs as artifacts. + */ + +var chai = require('chai'); +var fs = require('fs'), + PNG = require('pngjs').PNG, + pixelmatch = require('pixelmatch'); + +var old_dir = 'tests/screenshot/outputs/old/'; +var new_dir = 'tests/screenshot/outputs/new/'; +var diff_dir = 'tests/screenshot/outputs/diff/'; +var test_list_location ='tests/screenshot/test_cases/test_cases.json'; + +if (!fs.existsSync(diff_dir)){ + fs.mkdirSync(diff_dir); +} + +function getTestList() { + var file = fs.readFileSync(test_list_location); + var json = JSON.parse(file); + var testSpecArr = json.tests; + var testList = []; + for (var i = 0, testSpec; testSpec = testSpecArr[i]; i++) { + if (!testSpec.skip) { + testList.push(testSpec.title); + } + } + return testList; +} + +var test_list = getTestList(); + +suite('Rendering', function() { + /** + * - Load the old and new files as PNGs + * - Diff the files + * - Assert that the files are the same + * - Save the visual diff to a file. + */ + function diffScreenshots(name) { + + var file1 = fs.readFileSync(old_dir + name + '.png'); + var img1 = PNG.sync.read(file1); + + var file2 = fs.readFileSync(new_dir + name + '.png'); + var img2 = PNG.sync.read(file2); + + var diff = new PNG({width: img1.width, height: img1.height}); + + var mismatch_num = pixelmatch( + img1.data, + img2.data, + diff.data, + img1.width, + img1.height, {threshold: 0.1}); + diff.pack().pipe(fs.createWriteStream(diff_dir + name + '.png')); + chai.assert.equal(mismatch_num, 0); + } + + test_list.forEach(function(testName) { + test(testName, function() { + diffScreenshots(testName); + }) + }); +}); diff --git a/tests/screenshot/diff_viewer.html b/tests/screenshot/diff_viewer.html new file mode 100644 index 000000000..90426a81b --- /dev/null +++ b/tests/screenshot/diff_viewer.html @@ -0,0 +1,132 @@ + + + + + +Diff Viewer + + + + + + +
+ + + + + + + + + + diff --git a/tests/screenshot/gen_screenshots.js b/tests/screenshot/gen_screenshots.js new file mode 100644 index 000000000..9866a9ad0 --- /dev/null +++ b/tests/screenshot/gen_screenshots.js @@ -0,0 +1,186 @@ +/** + * @license + * Visual Blocks Editor + * + * Copyright 2019 Google Inc. + * https://developers.google.com/blockly/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @fileoverview Node.js script to generate screenshots in Chrome, via webdriver. + */ +var webdriverio = require('webdriverio'); +var fs = require('fs'); + +module.exports = genScreenshots; + +var isCollapsed = false; +var filterText = ''; +var isInsertionMarker = false; +var isRtl = false; +var inlineInputs = false; +var externalInputs = false; + +function processArgs() { + var args = process.argv; + for (var i = 0; i < args.length; i++) { + var arg = args[i]; + if (arg === '--collapsed') { + isCollapsed = true; + } else if (arg === '--name') { + filterText = args[i + 1]; + } else if (arg === '--insertionMarker') { + isInsertionMarker = true; + } else if (arg === '--rtl') { + isRtl = true; + } else if (arg === '--inlineInputs') { + inlineInputs = true + } else if (arg === '--externalInputs') { + externalInputs = true + } + } +} + +function checkAndCreateDir(dirname) { + if (!fs.existsSync(dirname)){ + fs.mkdirSync(dirname); + } +}; + +/** + * Opens two different webdriverio browsers. One uses the hosted version of + * blockly_compressed.js; the other uses the local blockly_uncompressed.js. + * + * Each playground is a minimal Blockly instance. This loads the same XML in + * both playgrounds and saves a screenshot of each. + */ +async function genScreenshots() { + var output_url = 'tests/screenshot/outputs' + processArgs(); + checkAndCreateDir(output_url) + checkAndCreateDir(output_url + '/old'); + checkAndCreateDir(output_url + '/new'); + + var url_prefix = 'file://' + __dirname + '/playground'; + var browser_new = await buildBrowser(url_prefix + '_new.html', isRtl); + var browser_old = await buildBrowser(url_prefix + '_old.html', isRtl); + var test_list = getTestList(); + for (var i = 0, testName; testName = test_list[i]; i++) { + await genSingleScreenshot(browser_new, 'new', testName, isCollapsed, isInsertionMarker, inlineInputs, externalInputs); + if (!fs.existsSync(output_url + '/old/' + testName)) { + await genSingleScreenshot(browser_old, 'old', testName, isCollapsed, isInsertionMarker, inlineInputs, externalInputs); + } + } + + await cleanUp(browser_new, browser_old); + return 0; +} + +function getTestList() { + var file = fs.readFileSync('tests/screenshot/test_cases/test_cases.json'); + var json = JSON.parse(file); + var testSpecArr = json.tests; + var testList = []; + for (var i = 0, testSpec; testSpec = testSpecArr[i]; i++) { + if (!testSpec.skip && testSpec.title.includes(filterText)) { + testList.push(testSpec.title); + } + } + return testList; +} + +async function cleanUp(browser_new, browser_old) { + await browser_new.deleteSession(); + await browser_old.deleteSession(); +} + +async function buildBrowser(url, isRtl) { + var options = { + capabilities: { + browserName: 'chrome' + }, + logLevel: 'warn' + }; + console.log('Starting webdriverio...'); + const browser = await webdriverio.remote(options); + var injectBlockly = function(isRtl) { + workspace = Blockly.inject('blocklyDiv', + { + comments: true, + collapse: true, + disable: true, + + horizontalLayout: false, + maxBlocks: Infinity, + maxInstances: {'test_basic_limit_instances': 3}, + media: '../../media/', + oneBasedIndex: true, + readOnly: false, + rtl: isRtl, + move: { + scrollbars: false, + drag: true, + wheel: false, + }, + toolboxPosition: 'start', + zoom: + { + controls: false, + wheel: true, + startScale: 2.0, + maxScale: 4, + minScale: 0.25, + scaleSpeed: 1.1 + } + }); + } + + await browser.setWindowSize(500, 500); + console.log('Initialized.\nLoading url: ' + url); + await browser.url(url); + await browser.execute(injectBlockly, isRtl); + return browser; +} + +async function genSingleScreenshot(browser, dir, test_name, isCollapsed, isInsertionMarker, inlineInputs, externalInputs) { + var prefix = './tests/screenshot/'; + var xml_url = prefix + 'test_cases/' + test_name; + var xml = fs.readFileSync(xml_url, 'utf8'); + + var loadXmlFn = function(xml_text, isCollapsed, isInsertionMarker, inlineInputs, externalInputs) { + workspace.clear(); + var xml = Blockly.Xml.textToDom(xml_text); + Blockly.Xml.domToWorkspace(xml, workspace); + if (isCollapsed || isInsertionMarker || inlineInputs || externalInputs) { + var blocks = workspace.getAllBlocks(); + for (var i = 0, block; block = blocks[i]; i++) { + block.setCollapsed(isCollapsed); + block.setInsertionMarker(isInsertionMarker); + if (inlineInputs) { + block.setInputsInline(true); + } else if (externalInputs) { + block.setInputsInline(false); + } + } + } + }; + await browser.execute(loadXmlFn, xml, isCollapsed, isInsertionMarker, inlineInputs, externalInputs); + await browser.saveScreenshot(prefix + '/outputs/' + dir + '/' + test_name + '.png'); +} + + +if (require.main === module) { + genScreenshots(); +} diff --git a/tests/screenshot/img_viewer.html b/tests/screenshot/img_viewer.html new file mode 100644 index 000000000..37831771b --- /dev/null +++ b/tests/screenshot/img_viewer.html @@ -0,0 +1,79 @@ + + + + + + Image comparison + + + + + + + + +

+
+ + diff --git a/tests/screenshot/playground_new.html b/tests/screenshot/playground_new.html new file mode 100644 index 000000000..709ee77b2 --- /dev/null +++ b/tests/screenshot/playground_new.html @@ -0,0 +1,111 @@ + + + + + +Blockly Playground: New rendering + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + diff --git a/tests/screenshot/playground_old.html b/tests/screenshot/playground_old.html new file mode 100644 index 000000000..1541eab02 --- /dev/null +++ b/tests/screenshot/playground_old.html @@ -0,0 +1,69 @@ + + + + + +Blockly Playground: Old rendering + + + + + + + + + + + + + + + + + +
+ + diff --git a/tests/screenshot/run_differ.py b/tests/screenshot/run_differ.py new file mode 100644 index 000000000..414e4f1b0 --- /dev/null +++ b/tests/screenshot/run_differ.py @@ -0,0 +1,122 @@ +#!/usr/bin/python2.7 +# +# Copyright 2019 Google Inc. +# https://developers.google.com/blockly/ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: +# run_differ.py with no parameters runs all screenshot tests with blocks in rtl +# and not collapsed. +# --name runs all tests that contain the given name. If not given, +# runs all tests specified in test_cases.json. +# --collapsed runs all tests with the blocks collapsed. If not given, blocks are +# expanded. +# --insertionMarker runs all tests with the blocks set as insertion markers. If +# not given then will default to normal blocks. +# --inlineInputs runs all tests with the blocks set to have inline inputs. If +# not given then the blocks will be in their default state. +# --externalInputs runs all tests with the with all blocks set to have external +# inputs. If not given then the blocks will be in their default state. +# + +import os, errno, platform, shutil, sys + +NAME_ARG = "--name" +COLLAPSE_ARG = "--collapsed" +RTL_ARG = "--rtl" +INSERTION_ARG = "--insertionMarker" +INLINE_INPUTS_ARG = "--inlineInputs" +EXTERNAL_INPUTS_ARG = "--externalInputs" + +ARG_VALS = [COLLAPSE_ARG, RTL_ARG, INSERTION_ARG, INLINE_INPUTS_ARG, EXTERNAL_INPUTS_ARG] + +# Generates the screenshots according to the given parameters, diffs the +# screenshots and then displays them. +def main(): + cleanup() + check_arguments() + filter_text = find_argument_value(NAME_ARG) + argument_string = create_arg_string() + gen_screenshots(filter_text, argument_string) + diff_screenshots(filter_text) + display_screenshots() + +# Cleans up any files left over from running the script previously. +def cleanup(): + remove_dir("tests/screenshot/outputs/new") + remove_dir("tests/screenshot/outputs/diff") + remove_file("tests/screenshot/outputs/test_output.js") + remove_file("tests/screenshot/outputs/test_output.json") + +# If the --name is given find the name of the test case. +def find_argument_value(argument_name): + args = sys.argv + for i in range(len(args)): + if args[i] == argument_name: + if i + 1 < len(args): + return args[i+1] + else: + print ("Must supply a name after name arg") + sys.exit() + return "" + +# Prints an error and exits if the arguments given aren't allowed. +def check_arguments(): + if (INLINE_INPUTS_ARG in sys.argv) and (EXTERNAL_INPUTS_ARG in sys.argv): + print ("Can not have both --inlineInputs and --externalInputs") + sys.exit() + +# Create a string with all arguments. +def create_arg_string(): + arg_string = "" + for arg in sys.argv: + arg_string = arg_string + " " + arg + return arg_string + +# Generates a set of old and new screenshots according to the given parameters. +def gen_screenshots(filter_text, argument_string): + os.system("node tests/screenshot/gen_screenshots.js " + argument_string) + +# Diffs the old and new screenshots that were created in gen_screenshots. +def diff_screenshots(filter_text): + if filter_text != "": + os.system("./node_modules/.bin/mocha tests/screenshot/diff_screenshots.js --ui tdd --reporter ./tests/screenshot/diff-reporter.js" + " --fgrep " + filter_text) + else: + os.system("./node_modules/.bin/mocha tests/screenshot/diff_screenshots.js --ui tdd --reporter ./tests/screenshot/diff-reporter.js") + +# Displays the old screenshots, new screenshots, and the diff of them. +def display_screenshots(): + if (platform.system() == "Linux"): + os.system("xdg-open tests/screenshot/diff_viewer.html") + elif (platform.system() == 'Darwin'): + os.system("open tests/screenshot/diff_viewer.html") + +# Removes a file and catches the error if the file does not exist. +def remove_file(filename): + try: + os.remove(filename) + except (OSError) as e: + if e.errno != errno.ENOENT: + raise + +# Removes a directory and catches the error if the directory does not exist. +def remove_dir(dir_name): + try: + shutil.rmtree(dir_name) + except (OSError) as e: + if e.errno != errno.ENOENT: + raise + +if __name__ == "__main__": + main()
Test nameOldNewDiff