From d3443d6f2a862b7b3c068cb9433ff4d8bf24fac6 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Fri, 10 Jul 2020 16:08:14 +0100 Subject: [PATCH 1/5] ci: Use the timeout multiplier from the environment When running our test suite on our CI infrastructure. --- .gitlab-ci/run-tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh index 5bdbaf9637..9a6896979c 100755 --- a/.gitlab-ci/run-tests.sh +++ b/.gitlab-ci/run-tests.sh @@ -16,6 +16,7 @@ case "${backend}" in x11) xvfb-run -a -s "-screen 0 1024x768x24" \ meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ --print-errorlogs \ --setup=${backend} \ --suite=gtk \ @@ -35,6 +36,7 @@ case "${backend}" in export WAYLAND_DISPLAY=wayland-5 meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ --print-errorlogs \ --setup=${backend} \ --suite=gtk \ @@ -53,6 +55,7 @@ case "${backend}" in export BROADWAY_DISPLAY=:5 meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ --print-errorlogs \ --setup=${backend} \ --suite=gtk \ From 70e4dabe89c138ee09c6550f0020adeb191e8226 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Fri, 10 Jul 2020 16:08:59 +0100 Subject: [PATCH 2/5] ci: Increase the timeout multiplier We do have some slow tests. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3558e9f6ef..399124fa2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ variables: COMMON_MESON_FLAGS: "--fatal-meson-warnings --werror" BACKEND_FLAGS: "-Dx11-backend=true -Dwayland-backend=true -Dbroadway-backend=true -Dvulkan=yes" FEATURE_FLAGS: "-Dcloudproviders=true" - MESON_TEST_TIMEOUT_MULTIPLIER: 2 + MESON_TEST_TIMEOUT_MULTIPLIER: 3 FEDORA_IMAGE: "registry.gitlab.gnome.org/gnome/gtk/fedora:v20" FLATPAK_IMAGE: "registry.gitlab.gnome.org/gnome/gnome-runtime-images/gnome:master" DOCS_IMAGE: "registry.gitlab.gnome.org/gnome/gtk/fedora-docs:v19" From 333b013f7fd3a3adf8036c20bcb52cf69bb67511 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Fri, 10 Jul 2020 16:12:25 +0100 Subject: [PATCH 3/5] Increase the default timeout for slow tests If a test is part of the 'slow' test suite, we should increment the default timeout to compensate. --- testsuite/gtk/meson.build | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build index a93540dd80..d725d992f6 100644 --- a/testsuite/gtk/meson.build +++ b/testsuite/gtk/meson.build @@ -107,6 +107,8 @@ foreach t : tests test_srcs = ['@0@.c'.format(test_name)] + t.get(1, []) test_extra_cargs = t.get(2, []) test_extra_ldflags = t.get(3, []) + test_extra_suites = t.get(4, []) + test_timeout = 60 test_exe = executable(test_name, test_srcs, c_args : test_cargs + test_extra_cargs, @@ -117,17 +119,21 @@ foreach t : tests expect_fail = xfail.contains(test_name) + if test_extra_suites.contains('slow') + test_timeout = 90 + endif + test(test_name, test_exe, args: [ '--tap', '-k' ], protocol: 'tap', - timeout: 60, + timeout: test_timeout, env: [ 'NO_AT_BRIDGE=1', 'GSK_RENDERER=cairo', 'G_TEST_SRCDIR=@0@'.format(meson.current_source_dir()), 'G_TEST_BUILDDIR=@0@'.format(meson.current_build_dir()), ], - suite: 'gtk', + suite: ['gtk'] + test_extra_suites, should_fail: expect_fail, ) endforeach From ad04da84c965f37f7d6b3a1fbb07d2bdafa484d2 Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Fri, 10 Jul 2020 16:16:15 +0100 Subject: [PATCH 4/5] Add the filtermodel-exhaustive test to the slow suite This increases the default timeout. --- testsuite/gtk/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build index d725d992f6..fb6c387fbf 100644 --- a/testsuite/gtk/meson.build +++ b/testsuite/gtk/meson.build @@ -30,7 +30,7 @@ tests = [ ['expression'], ['filter'], ['filterlistmodel'], - ['filterlistmodel-exhaustive'], + ['filterlistmodel-exhaustive', [], [], [], ['slow']], ['flattenlistmodel'], ['floating'], ['flowbox'], From 057842261237d27549b5f80f757d8873b0b0b09e Mon Sep 17 00:00:00 2001 From: Emmanuele Bassi Date: Fri, 10 Jul 2020 16:32:15 +0100 Subject: [PATCH 5/5] build: Restructure the tests to use dictionaries Instead of an array of arrays, let's use an array of dictionaries; it's easier to add optional keys without requiring to remember where to put empty arrays. --- testsuite/gtk/meson.build | 187 +++++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 72 deletions(-) diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build index fb6c387fbf..8f1828aa0c 100644 --- a/testsuite/gtk/meson.build +++ b/testsuite/gtk/meson.build @@ -7,76 +7,119 @@ if cc.get_id() != 'msvc' gtk_tests_export_dynamic_ldflag = ['-Wl,--export-dynamic'] endif +# Available keys for each test: +# +# - 'name': the test name; used for the test and to determine the base +# source file for the test (MANDATORY) +# - 'sources': (array): additional sources needed by the test +# - 'c_args': (array): additional compiler arguments +# - 'link_args': (array): additional linker arguments +# - 'suites': (array): additional test suites tests = [ - ['accel'], - ['accessible'], - ['action'], - ['adjustment'], - ['bitset'], - ['bitmask', ['../../gtk/gtkallocatedbitmask.c'], ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG']], - ['builder', [], [], gtk_tests_export_dynamic_ldflag], - ['builderparser'], - ['cellarea'], - ['check-icon-names'], - ['constraint-solver', [ + { 'name': 'accel' }, + { 'name': 'accessible' }, + { 'name': 'action' }, + { 'name': 'adjustment' }, + { 'name': 'bitset' }, + { + 'name': 'bitmask', + 'sources': ['../../gtk/gtkallocatedbitmask.c'], + 'c_args': ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'], + }, + { + 'name': 'builder', + 'link_args': gtk_tests_export_dynamic_ldflag, + }, + { 'name': 'builderparser' }, + { 'name': 'cellarea' }, + { 'name': 'check-icon-names' }, + { + 'name': 'constraint-solver', + 'sources': [ '../../gtk/gtkconstraintsolver.c', '../../gtk/gtkconstraintexpression.c', - ], ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'] - ], - ['cssprovider'], - ['rbtree-crash', ['../../gtk/gtkrbtree.c'], ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG']], - ['defaultvalue'], - ['entry'], - ['expression'], - ['filter'], - ['filterlistmodel'], - ['filterlistmodel-exhaustive', [], [], [], ['slow']], - ['flattenlistmodel'], - ['floating'], - ['flowbox'], - #['gestures'], - ['grid'], - ['grid-layout'], - ['icontheme'], - ['listbox'], - ['main'], - ['maplistmodel'], - ['multiselection'], - ['notify'], - ['no-gtk-init'], - ['object'], - ['objects-finalize'], - ['papersize'], - #['popover'], - ['propertylookuplistmodel', ['../../gtk/gtkpropertylookuplistmodel.c'], ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG']], - ['rbtree', ['../../gtk/gtktreerbtree.c'], ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG']], - ['recentmanager'], - ['regression-tests'], - ['scrolledwindow'], - ['searchbar'], - ['shortcuts'], - ['singleselection'], - ['slicelistmodel'], - ['sorter'], - ['sortlistmodel'], - ['spinbutton'], - ['stringlist'], - ['templates'], - ['textbuffer'], - ['textiter'], - ['theme-validate'], - ['tooltips'], - ['treelistmodel'], - ['treemodel', ['treemodel.c', 'liststore.c', 'treestore.c', 'filtermodel.c', - 'modelrefcount.c', 'sortmodel.c', 'gtktreemodelrefcount.c']], - ['treepath'], - ['treesorter'], - ['treeview'], - ['typename'], - ['displayclose'], - ['revealer-size'], - ['widgetorder'], - ['widget-refcount'], + ], + 'c_args': ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'], + }, + { 'name': 'cssprovider' }, + { + 'name': 'rbtree-crash', + 'sources': ['../../gtk/gtkrbtree.c'], + 'c_args': ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'], + }, + { 'name': 'defaultvalue' }, + { 'name': 'entry' }, + { 'name': 'expression' }, + { 'name': 'filter' }, + { 'name': 'filterlistmodel' }, + { + 'name': 'filterlistmodel-exhaustive', + 'suites': ['slow'], + }, + { 'name': 'flattenlistmodel' }, + { 'name': 'floating' }, + { 'name': 'flowbox' }, + #{ 'name': 'gestures' }, + { 'name': 'grid' }, + { 'name': 'grid-layout' }, + { 'name': 'icontheme' }, + { 'name': 'listbox' }, + { 'name': 'main' }, + { 'name': 'maplistmodel' }, + { 'name': 'multiselection' }, + { 'name': 'notify' }, + { 'name': 'no-gtk-init' }, + { 'name': 'object' }, + { 'name': 'objects-finalize' }, + { 'name': 'papersize' }, + #{ 'name': 'popover' }, + { + 'name': 'propertylookuplistmodel', + 'sources': ['../../gtk/gtkpropertylookuplistmodel.c'], + 'c_args': ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'], + }, + { + 'name': 'rbtree', + 'sources': ['../../gtk/gtktreerbtree.c'], + 'c_args': ['-DGTK_COMPILATION', '-UG_ENABLE_DEBUG'], + }, + { 'name': 'recentmanager' }, + { 'name': 'regression-tests' }, + { 'name': 'scrolledwindow' }, + { 'name': 'searchbar' }, + { 'name': 'shortcuts' }, + { 'name': 'singleselection' }, + { 'name': 'slicelistmodel' }, + { 'name': 'sorter' }, + { 'name': 'sortlistmodel' }, + { 'name': 'spinbutton' }, + { 'name': 'stringlist' }, + { 'name': 'templates' }, + { 'name': 'textbuffer' }, + { 'name': 'textiter' }, + { 'name': 'theme-validate' }, + { 'name': 'tooltips' }, + { 'name': 'treelistmodel' }, + { + 'name': 'treemodel', + 'sources': [ + 'treemodel.c', + 'liststore.c', + 'treestore.c', + 'filtermodel.c', + 'modelrefcount.c', + 'sortmodel.c', + 'gtktreemodelrefcount.c', + ], + }, + { 'name': 'treepath' }, + { 'name': 'treesorter' }, + { 'name': 'treeview' }, + { 'name': 'typename' }, + { 'name': 'displayclose' }, + { 'name': 'revealer-size' }, + { 'name': 'widgetorder' }, + { 'name': 'widget-refcount' }, ] # Tests that are expected to fail @@ -103,11 +146,11 @@ foreach flag: common_cflags endforeach foreach t : tests - test_name = t.get(0) - test_srcs = ['@0@.c'.format(test_name)] + t.get(1, []) - test_extra_cargs = t.get(2, []) - test_extra_ldflags = t.get(3, []) - test_extra_suites = t.get(4, []) + test_name = t.get('name') + test_srcs = ['@0@.c'.format(test_name)] + t.get('sources', []) + test_extra_cargs = t.get('c_args', []) + test_extra_ldflags = t.get('link_args', []) + test_extra_suites = t.get('suites', []) test_timeout = 60 test_exe = executable(test_name, test_srcs,