diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py deleted file mode 100755 index ba7d01c5ba..0000000000 --- a/.gitlab-ci/meson-html-report.py +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env python3 - -# Turns a Mason testlog.json file into an HTML report -# -# Copyright 2019 GNOME Foundation -# -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Original author: Emmanuele Bassi - -import argparse -import datetime -import json -import os -import sys -from jinja2 import Template - -REPORT_TEMPLATE = ''' - - - - {{ report.project_name }} Test Report - - - - -
-

{{ report.project_name }}/{{ report.backend }}/{{ report.branch_name }} :: Test Reports

-
- -
-
-
-

Backend: {{ report.backend }}

-

Branch: {{ report.branch_name }}

-

Date:

- {% if report.job_id %}

Job ID: {{ report.job_id }}

{% endif %} -
-
- -
-
-

Summary

-
    -
  • Total units: {{ report.total_units }}
    • -
  • Failed: {{ report.total_failures }}
    • -
  • Passed: {{ report.total_successes }}
    • -
-
-
- - {% for suite_result in report.results_list %} -
-
-

Suite: {{ suite_result.suite_name }}

- - -
-

Failures

-
    - {% for failure in suite_result.failures if failure.result in [ 'ERROR', 'FAIL', 'UNEXPECTEDPASS' ] %} -
  • {{ failure.name }} - result: {{ failure.result }}
    - {% if failure.stdout %} - Output: 
    {{ failure.stdout }}
    - {% endif %} - {% if failure.image_data is defined %} - - {% endif %} -
    • - {% else %} -
  • None
    • - {% endfor %} -
- -

Timed out

-
    - {% for failure in suite_result.failures if failure.result == 'TIMEOUT' %} -
  • {{ failure.name }} - result: {{ failure.result }}
    - {% if failure.stdout %} - Output: 
    {{ failure.stdout }}
    - {% endif %} -
    • - {% else %} -
  • None
    • - {% endfor %} -
-
- -
-

Expected failures

-
    - {% for success in suite_result.successes if success.result == 'EXPECTEDFAIL' %} -
  • {{ success.name }} - result: {{ success.result }}
    - {% if success.stdout %} - Output: 
    {{ success.stdout }}
    - {% endif %} - {% if success.image_data is defined %} -
      -
    • ref
      • -
    • out
      • -
    • diff
      • -
    - {% endif %} -
    • - {% else %} -
  • None
    • - {% endfor %} -
- -

Skipped

-
    - {% for success in suite_result.successes if success.result == 'SKIP' %} -
  • {{ success.name }} - result: {{ success.result }}
    • - {% else %} -
  • None
    • - {% endfor %} -
- -

Passed

-
    - {% for success in suite_result.successes if success.result == 'OK' %} -
  • {{ success.name }} - result: {{ success.result }}
    • - {% else %} -
  • None
    • - {% endfor %} -
-
- -
-
- {% endfor %} - -
- - -''' - -aparser = argparse.ArgumentParser(description='Turns a Meson test log into an HTML report') -aparser.add_argument('--project-name', metavar='NAME', - help='The project name', - default='Unknown') -aparser.add_argument('--backend', metavar='NAME', - help='The used backend', - default='unknown') -aparser.add_argument('--job-id', metavar='ID', - help='The job ID for the report', - default=None) -aparser.add_argument('--branch', metavar='NAME', - help='Branch of the project being tested', - default='main') -aparser.add_argument('--output', metavar='FILE', - help='The output HTML file, stdout by default', - type=argparse.FileType('w', encoding='UTF-8'), - default=sys.stdout) -aparser.add_argument('--reftest-suite', metavar='NAME', - help='The name of the reftests suite', - default='reftest') -aparser.add_argument('--reftest-output-dir', metavar='DIR', - help='The output directory for reftests data', - default=None) -aparser.add_argument('infile', metavar='FILE', - help='The input testlog.json, stdin by default', - type=argparse.FileType('r', encoding='UTF-8'), - default=sys.stdin) - -args = aparser.parse_args() - -outfile = args.output - -suites = {} -for line in args.infile: - data = json.loads(line) - (full_suite, unit_name) = data['name'].split(' / ') - (project_name, suite_name) = full_suite.split(':') - - unit = { - 'project-name': project_name, - 'suite': suite_name, - 'name': unit_name, - 'duration': data['duration'], - 'returncode': data['returncode'], - 'result': data['result'], - 'stdout': data['stdout'], - } - - if args.reftest_output_dir is not None and suite_name == args.reftest_suite: - filename = unit_name.split(' ')[1] - basename = os.path.splitext(filename)[0] - - image_data = { - 'ref': os.path.join(args.reftest_output_dir, '{}.ref.png'.format(basename)), - 'out': os.path.join(args.reftest_output_dir, '{}.out.png'.format(basename)), - 'refnode': os.path.join(args.reftest_output_dir, '{}.ref.node'.format(basename)), - 'outnode': os.path.join(args.reftest_output_dir, '{}.out.node'.format(basename)), - 'diff': os.path.join(args.reftest_output_dir, '{}.diff.png'.format(basename)), - } - - unit['image_data'] = image_data - - units = suites.setdefault(full_suite, []) - units.append(unit) - -report = {} -report['date'] = datetime.datetime.now(datetime.UTC) -report['locale_date'] = report['date'].strftime("%c") -report['project_name'] = args.project_name -report['backend'] = args.backend -report['job_id'] = args.job_id -report['branch_name'] = args.branch -report['total_successes'] = 0 -report['total_failures'] = 0 -report['total_units'] = 0 -report['results_list'] = [] - -for name, units in suites.items(): - (project_name, suite_name) = name.split(':') - print('Processing {} suite {}:'.format(project_name, suite_name)) - - def if_failed(unit): - if unit['result'] in ['FAIL', 'UNEXPECTEDPASS', 'TIMEOUT', 'ERROR',]: - return True - return False - - def if_succeded(unit): - if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']: - return True - return False - - successes = list(filter(if_succeded, units)) - failures = list(filter(if_failed, units)) - - n_units = len(units) - n_successes = len(successes) - n_failures = len(failures) - - report['total_units'] += n_units - report['total_successes'] += n_successes - report['total_failures'] += n_failures - print(' - {}: {} total, {} pass, {} fail'.format(suite_name, n_units, n_successes, n_failures)) - - suite_report = { - 'suite_name': suite_name, - 'n_units': n_units, - 'successes': successes, - 'n_successes': n_successes, - 'failures': failures, - 'n_failures': n_failures, - } - report['results_list'].append(suite_report) - -template = Template(REPORT_TEMPLATE) -outfile.write(template.render({'report': report})) diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh index dba31cd592..7e9cd96539 100755 --- a/.gitlab-ci/run-tests.sh +++ b/.gitlab-ci/run-tests.sh @@ -93,14 +93,4 @@ case "${setup}" in esac -cd ${builddir} - -$srcdir/.gitlab-ci/meson-html-report.py \ - --project-name=gtk \ - --backend="${setup}" \ - --job-id="${CI_JOB_NAME}" \ - --reftest-output-dir="testsuite/reftests/output/${setup}" \ - --output="report-${setup}.html" \ - "meson-logs/testlog-${setup}.json" - exit $exit_code diff --git a/.gitlab-ci/test-docker.sh b/.gitlab-ci/test-docker.sh index d6cafa9159..b9cb23f6fa 100755 --- a/.gitlab-ci/test-docker.sh +++ b/.gitlab-ci/test-docker.sh @@ -35,15 +35,3 @@ xvfb-run -a -s "-screen 0 1024x768x24" \ --print-errorlogs \ --suite=gtk \ --no-suite=gtk:a11y - -# Save the exit code -exit_code=$? - -$srcdir/.gitlab-ci/meson-html-report.py \ - --project-name=GTK \ - --job-id="${CI_JOB_NAME}" \ - --reftest-output-dir="testsuite/reftests/output" \ - --output=report.html \ - meson-logs/testlog.json - -exit $exit_code