From cdb598793a0bdbbbc5b6226635afefcd33f51753 Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@debian.org>
Date: Wed, 23 Nov 2022 21:26:50 +0000
Subject: [PATCH] testsuite: Don't create .test files for flaky or failing
 tests

These tests can be run manually, but are not suitable for use as an
acceptance test, so let's not make frameworks like Debian's autopkgtest
run these when they run ginsttest-runner in the most obvious way.

a11ytests.test doesn't seem to be reliable enough to be used as a QA
acceptance criterion, and has been disabled as a build-time test in both
Gitlab-CI and Debian since 2019. a11ystate.test is not set up to be run
at build time at all, and has been marked as flaky on ci.debian.net
since 2018.

The rest of the testsuite/a11y directory seems to have been
reliable in practice, at least on ci.debian.net, so try leaving them
enabled as installed-tests.

In principle this could be made finer-grained by having a separate .test
file and a separate Meson test() for each .ui file, but that would
require more active maintenance of GTK 3.

Signed-off-by: Simon McVittie <smcv@debian.org>
---
 testsuite/a11y/meson.build       | 7 ++++++-
 testsuite/a11y/state/meson.build | 7 +++++--
 testsuite/gtk/meson.build        | 7 ++++++-
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/testsuite/a11y/meson.build b/testsuite/a11y/meson.build
index ea6348c679..2613e3cc22 100644
--- a/testsuite/a11y/meson.build
+++ b/testsuite/a11y/meson.build
@@ -181,7 +181,12 @@ installed_test_data = [
 ]
 
 a11y_installed_tests = [
-  'a11ytests.test',
+  # This is the equivalent of a11y_state_tests above, and does not seem
+  # to be reliable enough to act as a QA gate in practice. We install the
+  # test executable and the data needed to run it, but don't hook it up
+  # to ginsttest-runner.
+  #'a11ytests.test',
+
   'a11ychildren.test',
   'a11ytree.test',
   'a11yvalue.test',
diff --git a/testsuite/a11y/state/meson.build b/testsuite/a11y/state/meson.build
index 782649ef3f..aa18a84873 100644
--- a/testsuite/a11y/state/meson.build
+++ b/testsuite/a11y/state/meson.build
@@ -16,7 +16,10 @@ test_data = [
 
 
 a11y_installed_tests = [
-  'a11ystate.test',
+  # This is not run at build time at all, and consistently fails on
+  # Debian's CI infrastructure, so don't set it up to be run by
+  # ginsttest-runner.
+  #'a11ystate.test',
 ]
 
 if get_option('installed_tests')
@@ -32,4 +35,4 @@ if get_option('installed_tests')
   endforeach
 
   install_data(test_data, install_dir: join_paths(installed_test_bindir, 'state'))
-endif
\ No newline at end of file
+endif
diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build
index 137f93a626..84a8cfc1c6 100644
--- a/testsuite/gtk/meson.build
+++ b/testsuite/gtk/meson.build
@@ -146,6 +146,11 @@ endif
 if get_option('installed_tests')
   foreach t : tests
     test_name = t.get(0)
+
+    if flaky.contains(test_name) or xfail.contains(test_name)
+      continue
+    endif
+
     conf = configuration_data()
     conf.set('testexecdir', installed_test_bindir)
     conf.set('test', test_name)
@@ -158,4 +163,4 @@ if get_option('installed_tests')
   install_subdir('icons', install_dir: installed_test_bindir)
   install_subdir('icons2', install_dir: installed_test_bindir)
   install_subdir('ui', install_dir: installed_test_bindir)
-endif
\ No newline at end of file
+endif