CodingPirates/blockly
1
0
Fork 0
mirror of https://github.com/google/blockly.git synced 2026-01-25 17:50:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

Make generator tests compare results against golden files; add golden files.

Browse Source
This commit is contained in:
Rachel Fenichel
2018-11-09 13:22:34 -08:00
parent a248c012f4
commit 6202919272
12 changed files with 8689 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1670
tests/generators/golden/generated.dart Normal file
View File

File diff suppressed because it is too large Load Diff

1769
tests/generators/golden/generated.js Normal file
View File

File diff suppressed because it is too large Load Diff

1872
tests/generators/golden/generated.lua Normal file
View File

File diff suppressed because it is too large Load Diff

1696
tests/generators/golden/generated.php Normal file
View File

File diff suppressed because it is too large Load Diff

1524
tests/generators/golden/generated.py Normal file
View File

File diff suppressed because it is too large Load Diff

21
tests/generators/index.html
View File

@@ -259,27 +259,6 @@ function changeIndex() {
demoWorkspace.options.oneBasedIndex = oneBasedIndex;
demoWorkspace.toolbox_.flyout_.workspace_.options.oneBasedIndex = oneBasedIndex;
}
//https://stackoverflow.com/a/18197341
function download(filename, text) {
var element = document.createElement('a');
element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(text));
element.setAttribute('download', filename);
element.style.display = 'none';
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
}
function downloadOutputCode() {
console.log('clicked!');
download('outputcode.js', outputCode);
console.log('and done');
}
</script>
<style>

32
tests/generators/lists.xml
View File

@@ -3,25 +3,25 @@
<field name="SUITE_NAME">Lists</field>
<statement name="DO">
<block type="procedures_callnoreturn">
<mutation name="test create"></mutation>
<mutation name="test create lists"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test empty"></mutation>
<mutation name="test lists empty"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test length"></mutation>
<mutation name="test lists length"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test find simple"></mutation>
<mutation name="test find lists simple"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test find complex"></mutation>
<mutation name="test find lists complex"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test get simple"></mutation>
<mutation name="test get lists simple"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test get complex"></mutation>
<mutation name="test get lists complex"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test getRemove"></mutation>
@@ -57,7 +57,7 @@
<mutation name="test sort numeric"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test reverse"></mutation>
<mutation name="test lists reverse"></mutation>
</block>
</next>
</block>
@@ -134,7 +134,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="713">
<field name="NAME">test create</field>
<field name="NAME">test create lists</field>
<comment pinned="false" h="80" w="160">Tests the "create list with" and "create empty list" blocks.</comment>
<statement name="STACK">
<block type="unittest_assertequals" inline="false">
@@ -298,7 +298,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="1238">
<field name="NAME">test empty</field>
<field name="NAME">test lists empty</field>
<comment pinned="false" h="80" w="160">Tests the "is empty" block.</comment>
<statement name="STACK">
<block type="unittest_assertvalue" inline="false">
@@ -391,7 +391,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="1563">
<field name="NAME">test length</field>
<field name="NAME">test lists length</field>
<comment pinned="false" h="80" w="160">Tests the "length" block.</comment>
<statement name="STACK">
<block type="unittest_assertequals" inline="false">
@@ -525,7 +525,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="1988">
<field name="NAME">test find simple</field>
<field name="NAME">test find lists simple</field>
<comment pinned="false" h="80" w="160">Tests the "find" block with a variable.</comment>
<statement name="STACK">
<block type="variables_set">
@@ -697,7 +697,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="2563">
<field name="NAME">test find complex</field>
<field name="NAME">test find lists complex</field>
<comment pinned="false" h="80" w="160">Tests the "find" block with a function call.</comment>
<statement name="STACK">
<block type="variables_set">
@@ -1067,7 +1067,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="3738">
<field name="NAME">test get simple</field>
<field name="NAME">test get lists simple</field>
<comment pinned="false" h="80" w="160">Tests the "get" block with a variable.</comment>
<statement name="STACK">
<block type="variables_set" inline="false">
@@ -1391,7 +1391,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="4663">
<field name="NAME">test get complex</field>
<field name="NAME">test get lists complex</field>
<comment pinned="false" h="80" w="160">Tests the "get" block with a function call.</comment>
<statement name="STACK">
<block type="variables_set" inline="false">
@@ -8138,7 +8138,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="20463">
<field name="NAME">test reverse</field>
<field name="NAME">test lists reverse</field>
<comment pinned="false" h="80" w="160">Tests the "list reverse" block.</comment>
<statement name="STACK">
<block type="variables_set">

4
tests/generators/loops2.xml
View File

@@ -3,7 +3,7 @@
<field name="SUITE_NAME">Loops 2</field>
<statement name="DO">
<block type="procedures_callnoreturn">
<mutation name="test count"></mutation>
<mutation name="test count loops"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test count by"></mutation>
@@ -580,7 +580,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="29" y="184">
<field name="NAME">test count</field>
<field name="NAME">test count loops</field>
<statement name="STACK">
<block type="variables_set" inline="false">
<field name="VAR">log</field>

125
tests/generators/run_js_generator_tests_in_browser.js → tests/generators/run_generators_in_browser.js
View File

@@ -19,27 +19,44 @@
*/
/**
* @fileoverview Node.js script to run JsUnit tests in Chrome, via webdriver.
* @fileoverview Node.js script to run generator tests in Firefox, via webdriver.
*/
var webdriverio = require('webdriverio');
var fs = require('fs');
/*
* Notes
* this.execute runs the code in the browser's context (which is what I want).
* this is browser in most of the webdriverio examples.
* tests currently fail because of CORS restrictions. Oh, that's because it's
* running in chrome. let's try making it run in firefox before giving up and
* figuring out how to start a server.
* in firefox it pops up the alerts about it but also still works. so uh...
*/
/**
* Runs the JsUnit tests in this directory in Chrome. It uses webdriverio to
* launch Chrome and load index.html. Outputs a summary of the test results
* to the console.
* Run the generator for a given language and save the results to a file.
* @param {Thenable} browser A Thenable managing the processing of the browser
* tests.
* @param {string} filename Where to write the output file.
* @param {Function} codegenFn The function to run for code generation for this
* language.
* @return the Thenable managing the processing of the browser tests.
*/
function runJsGeneratorTestsInBrowser() {
function runLangGeneratorInBrowser(browser, filename, codegenFn) {
return browser
.pause(5000)
.then(function() {
this.execute(codegenFn)
})
.pause(10000)
.getValue("#importExport")
.then(function(result) {
fs.writeFile(filename, result, function(err) {
if (err) {
return console.log(err);
}
});
});
}
/**
* Runs the generator tests in Firefox. It uses webdriverio to
* launch Firefox and load index.html. Outputs a summary of the test results
* to the console and outputs files for later validation.
* @return the Thenable managing the processing of the browser tests.
*/
function runGeneratorsInBrowser() {
var options = {
desiredCapabilities: {
browserName: 'firefox'
@@ -47,6 +64,7 @@ function runJsGeneratorTestsInBrowser() {
};
var url = 'file://' + __dirname + '/index.html';
var prefix = 'tests/generators/tmp/generated'
console.log('Starting webdriverio...');
return webdriverio
.remote(options)
@@ -56,62 +74,37 @@ function runJsGeneratorTestsInBrowser() {
})
.url(url)
.then(function() {
console.log('Loaded.\nPausing to allow processing.');
})
.pause(1000)
.then(function() {
this.execute( function() {
console.log('about to load');
this.execute(function() {
checkAll();
loadSelected();
})
// var logs = this.log('browser')
// console.log('============Start Browser Logs=================');
// console.log(logs);
// console.log('============End Browser Logs=================');
})
.pause(15000)
.then(function() {
this.execute(function() {
toJavaScript();
console.log('Code generated');
eval(outputCode);
})
}) //TODO: change pause to waitunitl
// .then(function() {
// console.log('Retrieving results...');
// })
// .getHTML('#closureTestRunnerLog')
// .then(function(result) {
// // call js to parse html
// var regex = /[\d]+\spassed,\s([\d]+)\sfailed./i;
// var numOfFailure = regex.exec(result)[1];
// var regex2 = /Unit Tests for Blockly .*]/;
// var testStatus = regex2.exec(result)[0];
// console.log('============Blockly Unit Test Summary=================');
// console.log(testStatus);
// var regex3 = /\d+ passed,\s\d+ failed/;
// var detail = regex3.exec(result)[0];
// console.log(detail);
// console.log('============Blockly Unit Test Summary=================');
// if (parseInt(numOfFailure) !== 0) {
// console.log(result);
// process.exit(1);
// }
// })
.pause(10000)
.getValue("#importExport")
.then(function(result) {
fs.writeFile("test.js", result, function(err) {
if (err) {
return console.log(err);
}
//console.log("The file was saved!");
.then(function() {
return runLangGeneratorInBrowser(this, prefix + '.js', function() {
toJavaScript();
});
})
.then(function() {
return runLangGeneratorInBrowser(this, prefix + '.py', function() {
toPython();
});
})
.then(function() {
return runLangGeneratorInBrowser(this, prefix + '.dart', function() {
toDart();
});
})
.then(function() {
return runLangGeneratorInBrowser(this, prefix + '.lua', function() {
toLua();
});
})
.then(function() {
return runLangGeneratorInBrowser(this, prefix + '.php', function() {
toPhp();
});
// console.log('start result')
// console.log('======')
// console.log(result)
// console.log('======')
// console.log('end result');
})
.pause(10000)
.catch(function(e) {
@@ -130,11 +123,11 @@ function runJsGeneratorTestsInBrowser() {
});
}
module.exports = runJsGeneratorTestsInBrowser;
module.exports = runGeneratorsInBrowser;
if (require.main === module) {
try {
runJsGeneratorTestsInBrowser()
runGeneratorsInBrowser()
.catch(function(e) {
// TODO: Never called during errors. Fix.
console.error('Error: ' + e);

36
tests/generators/text.xml
View File

@@ -3,28 +3,28 @@
<field name="SUITE_NAME">Text</field>
<statement name="DO">
<block type="procedures_callnoreturn">
<mutation name="test length"></mutation>
<mutation name="test text length"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test empty"></mutation>
<mutation name="test empty text"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test create"></mutation>
<mutation name="test create text"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test append"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test find simple"></mutation>
<mutation name="test find text simple"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test find complex"></mutation>
<mutation name="test find text complex"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test get simple"></mutation>
<mutation name="test get text simple"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test get complex"></mutation>
<mutation name="test get text complex"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test substring simple"></mutation>
@@ -39,10 +39,10 @@
<mutation name="test trim"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test count"></mutation>
<mutation name="test count text"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test reverse"></mutation>
<mutation name="test text reverse"></mutation>
<next>
<block type="procedures_callnoreturn">
<mutation name="test replace"></mutation>
@@ -114,7 +114,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="563">
<field name="NAME">test create</field>
<field name="NAME">test create text</field>
<comment pinned="false" h="80" w="160">Tests the "create text with" block with varying number of inputs.</comment>
<statement name="STACK">
<block type="unittest_assertequals" inline="false">
@@ -333,7 +333,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="1338">
<field name="NAME">test empty</field>
<field name="NAME">test empty text</field>
<comment pinned="false" h="80" w="160">Tests the "is empty" block".</comment>
<statement name="STACK">
<block type="unittest_assertvalue" inline="false">
@@ -425,7 +425,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="1663">
<field name="NAME">test length</field>
<field name="NAME">test text length</field>
<comment pinned="false" h="80" w="160">Tests the "length" block.</comment>
<statement name="STACK">
<block type="unittest_assertequals" inline="false">
@@ -643,7 +643,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="2438">
<field name="NAME">test find simple</field>
<field name="NAME">test find text simple</field>
<comment pinned="false" h="80" w="160">Tests the "find" block with a variable.</comment>
<statement name="STACK">
<block type="variables_set" inline="false">
@@ -775,7 +775,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="2938">
<field name="NAME">test find complex</field>
<field name="NAME">test find text complex</field>
<comment pinned="false" h="80" w="160">Tests the "find" block with a function call.</comment>
<statement name="STACK">
<block type="variables_set">
@@ -1145,7 +1145,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="4013">
<field name="NAME">test get simple</field>
<field name="NAME">test get text simple</field>
<comment pinned="false" h="80" w="160">Tests the "get letter" block with a variable.</comment>
<statement name="STACK">
<block type="variables_set" inline="false">
@@ -1432,7 +1432,7 @@
</value>
</block>
<block type="procedures_defnoreturn" x="13" y="4938">
<field name="NAME">test get complex</field>
<field name="NAME">test get text complex</field>
<comment pinned="false" h="80" w="160">Tests the "get letter" block with a function call.</comment>
<statement name="STACK">
<block type="variables_set" inline="false">
@@ -4108,7 +4108,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="13" y="12088">
<field name="NAME">test count</field>
<field name="NAME">test count text</field>
<comment pinned="false" h="80" w="160">Tests the "trim" block.</comment>
<statement name="STACK">
<block type="variables_set">
@@ -4318,7 +4318,7 @@
</statement>
</block>
<block type="procedures_defnoreturn" x="12" y="12812">
<field name="NAME">test reverse</field>
<field name="NAME">test text reverse</field>
<comment pinned="false" h="80" w="160">Tests the "trim" block.</comment>
<statement name="STACK">
<block type="unittest_assertequals" inline="false">

14
tests/run_all_tests.sh
View File

@@ -49,19 +49,19 @@ run_test_command () {
fi
}
# Setup the environment (Chrome, Selenium, etc.)
# Set up the environment (Chrome, Selenium, etc.)
run_test_command "test_setup" "tests/scripts/test_setup.sh"
# # Lint the codebase.
# run_test_command "eslint" "eslint ."
# Lint the codebase.
run_test_command "eslint" "eslint ."
# Run JSUnit tests inside a browser.
#run_test_command "jsunit" "node tests/jsunit/run_jsunit_tests_in_browser.js"
run_test_command "generators" "node tests/generators/run_js_generator_tests_in_browser.js"
#run_test_command "fs_test" "node tests/generators/fs_test.js"
run_test_command "jsunit" "node tests/jsunit/run_jsunit_tests_in_browser.js"
# TODO: Make sure jsunit output is captured. Child process?
# Run generator tests inside a browser and check the results.
run_test_command "generators" "tests/scripts/run_generators.sh"
# # Attempt advanced compilation of a Blockly app.
# run_test_command "compile" "tests/compile/compile.sh"

56
tests/scripts/run_generators.sh Executable file
View File

@@ -0,0 +1,56 @@
#!/bin/bash
# ANSI colors
BOLD_GREEN='\033[1;32m'
BOLD_RED='\033[1;31m'
ANSI_RESET='\033[0m'
SUCCESS_PREFIX="${BOLD_GREEN}SUCCESS:${ANSI_RESET}"
FAILURE_PREFIX="${BOLD_RED}FAILED:${ANSI_RESET}"
TMP_DIR="tests/generators/tmp/"
GOLDEN_DIR="tests/generators/golden/"
FAILURE_COUNT=0
check_result() {
local suffix=$1 # One of: js, py, dart, lua, php
local tmp_filename="${TMP_DIR}generated.$suffix"
if [ ! -f $tmp_filename ]; then
echo "File $tmp_filename not found!"
FAILURE_COUNT=$((FAILURE_COUNT+1))
fi
local golden_filename="${GOLDEN_DIR}generated.$suffix"
if [ ! -f $golden_filename ]; then
echo "File $golden_filename not found!"
FAILURE_COUNT=$((FAILURE_COUNT+1))
fi
if cmp --silent $tmp_filename $golden_filename; then
echo -e "$SUCCESS_PREFIX $suffix: $tmp_filename matches $golden_filename"
else
echo -e "$FAILURE_PREFIX $suffix: $tmp_filename does not match $golden_filename"
FAILURE_COUNT=$((FAILURE_COUNT+1))
fi
}
mkdir $TMP_DIR
node tests/generators/run_generators_in_browser.js
generator_suffixes=( "js" "py" "dart" "lua" "php" )
for i in "${generator_suffixes[@]}"
do
check_result "$i"
done
# Clean up.
rm -r $TMP_DIR
if [ "$FAILURE_COUNT" -eq "0" ]; then
echo -e "${BOLD_GREEN}All generator tests passed.${ANSI_RESET}"
exit 0
else
echo -e "${BOLD_RED}Failures in ${FAILURE_COUNT} generator tests.${ANSI_RESET}"
exit 1
fi