diff --git a/.github/workflows/system-tests-latest-components.yml b/.github/workflows/system-tests-latest-components.yml index 07e808213..a9c74c300 100644 --- a/.github/workflows/system-tests-latest-components.yml +++ b/.github/workflows/system-tests-latest-components.yml @@ -168,11 +168,19 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" run-system-tests: - name: Trigger system tests + name: Trigger system tests (shard ${{ matrix.shard }}) needs: gather-refs + strategy: + fail-fast: false + matrix: + include: + - shard: 1 + suites: release_test_shard_1 + - shard: 2 + suites: release_test_shard_2 uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop with: - suites: ${{ inputs.suites || 'release_test' }} + suites: ${{ matrix.suites }} build_args: "PLATFORM:ubuntu_2404,\ PRECICE_REF:${{ needs.gather-refs.outputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ needs.gather-refs.outputs.ref-python-bindings }},\ diff --git a/changelog-entries/829.md b/changelog-entries/829.md new file mode 100644 index 000000000..98bc44fe8 --- /dev/null +++ b/changelog-entries/829.md @@ -0,0 +1 @@ +- Parallelized pull-request system tests via a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` [#829](https://github.com/precice/tutorials/pull/829) diff --git a/tools/tests/README.md b/tools/tests/README.md index 83a71910a..8cdfa89dd 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -54,6 +54,13 @@ To be able to fill in the right case tuple into the `tests.yaml`, you can use th ## Running the system tests on GitHub Actions +The scheduled [System tests (latest components)](https://github.com/precice/tutorials/actions/workflows/system-tests-latest-components.yml) +workflow runs as a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` +(together they cover the same cases as `release_test`). When a shard fails, re-run only the +failed matrix job. With multiple `precice-tests-vm` runners, shards may execute concurrently +and may rebuild the same component images; we have not seen Docker build conflicts in local +runs, but we can limit parallel jobs or add a shared build step if CI shows issues. + Go to Actions > [Run Testsuite (manual)](https://github.com/precice/tutorials/actions/workflows/run_testsuite_manual.yml) to see this workflow. After bringing these changes to `master`, the manual triggering option should be visible on the top right. Until that happens, we can only trigger this workflow manually from the [GitHub CLI](https://github.blog/changelog/2021-04-15-github-cli-1-9-enables-you-to-work-with-github-actions-from-your-terminal/): diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..0604324de 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -33,6 +33,15 @@ class TestSuites(list): def __init__(self, testsuites: List[TestSuite]): self.testsuites = testsuites + @staticmethod + def _iter_tutorial_cases(tutorials_section): + """Yield tutorial case dicts, flattening YAML list aliases (e.g. shard lists).""" + for item in tutorials_section: + if isinstance(item, list): + yield from TestSuites._iter_tutorial_cases(item) + else: + yield item + @classmethod def from_yaml(cls, path, parsed_tutorials: Tutorials): """ @@ -55,7 +64,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} # iterate over tutorials: - for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: + for tutorial_case in cls._iter_tutorial_cases( + test_suites_raw[test_suite_name]['tutorials']): tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) if not tutorial: raise Exception(f"No tutorial with path {tutorial_case['path']} found.") diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 1a497d904..569c0a940 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -500,8 +500,9 @@ test_suites: ##################################################################### ## Test suites referring to the test suites defined above - release_test: - tutorials: + # Shards of release_test for parallel CI. release_test is their union (YAML anchors). + release_test_shard_1: + tutorials: &release_test_shard_1_tutorials - *breaking-dam-2d_fluid-openfoam_solid-calculix - *channel-transport_fluid-openfoam_transport-nutils - *channel-transport-reaction_fluid-fenics_chemical-fenics @@ -526,6 +527,9 @@ test_suites: - *oscillator_mass-left-fmi_mass-right-fmi - *oscillator_mass-left-python_mass-right-python - *oscillator-overlap_mass-left-python_mass-right-python + + release_test_shard_2: + tutorials: &release_test_shard_2_tutorials - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics @@ -551,6 +555,11 @@ test_suites: - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + release_test: + tutorials: + - *release_test_shard_1_tutorials + - *release_test_shard_2_tutorials + # These test suites take longer to run. They are available, but not regularly executed. extra: tutorials: