Refactor batch 11 #253
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ################################################################################ | |
| # trigger CI | |
| # DataSHIELD GHA test suite - dsBaseClient | |
| # Replaces azure-pipelines.yml / opal_azure-pipelines.yml / armadillo_azure-pipelines.yml. | |
| # | |
| # This is one of three separate workflow files, each its own named check on a | |
| # PR/run: this one (all the actual test execution), check.yaml (doc-sync + | |
| # R CMD check), lint.yaml (lintr). Split so each shows as its own category | |
| # rather than one graph mixing test execution with static checks. | |
| # | |
| # Structure (all jobs below run in parallel except where "needs" says otherwise): | |
| # opal-dsbase (matrix x8) - dsBase suite against Opal, one shard per | |
| # category, plus a dsdanger entry, all grouped under | |
| # one summary box. | |
| # opal-report - needs opal-dsbase; merges results, computes its own | |
| # pass/fail, posts its own row into the shared PR | |
| # comment (see .github/scripts/post-ci-comment.js). | |
| # armadillo-dsbase (matrix x8) - dsBase suite against Armadillo, same split. | |
| # armadillo-report - needs armadillo-dsbase; same as opal-report, plus | |
| # coverage (computed here only - see comment at that | |
| # step for why one backend's figure is sufficient). | |
| # | |
| # There is no separate combining/summary job: dsBaseClient's own R/ source | |
| # never branches on backend, so Armadillo and Opal results are independently | |
| # meaningful and each report job stands alone (gate on both being required | |
| # checks in branch protection, rather than one job that waits on both). | |
| # | |
| # The dsBase suite (matching TEST_FILTER_DSBASE - same 292 files the Azure | |
| # pipelines run) is split into 7 shards - smk (116 files, 2 shards), perf (51 | |
| # fixed 30s-loop benchmarks, 3 shards - by far the slowest per-file), arg (1 | |
| # shard), and misc (1 shard) - each shard with its own testthat filter | |
| # substring. smk/perf are split by the first letter of the function name | |
| # (after any "ds." prefix) rather than an enumerated file list, so newly | |
| # added test files fall into a bucket automatically. The small dsDanger suite | |
| # is folded in as an 8th matrix entry (steps gated on matrix.category == | |
| # 'dsdanger') rather than a standalone job, so it groups under the same | |
| # summary box instead of its own. This is one job with a matrix (not split | |
| # into separate job definitions per category) so all entries share one | |
| # summary box in the run graph, and so they don't share a reusable workflow | |
| # name - GitHub's default same-name concurrency cap of 2 would otherwise | |
| # throttle them to 2-at-a-time. Setup (checkout through installing dsBase) | |
| # lives in a shared composite action (setup-armadillo-with-dsbase / | |
| # setup-opal-with-dsbase), used by every matrix entry including dsdanger, so | |
| # that part stays DRY. | |
| # | |
| # Each dsbase/dsdanger job spins up its OWN backend instance (isolated - no | |
| # shared server state / concurrency risk between categories running at once). | |
| # | |
| # Opal runs via docker-compose (docker-compose_opal.yml). | |
| # Armadillo runs as a plain `java -jar` process (not docker-compose): Armadillo | |
| # self-manages its Rock container over the host Docker socket | |
| # (docker-management-enabled: true / docker-run-in-container: false), which skips | |
| # building/pulling the old custom armadillo_citest image. See | |
| # molgenis-service-armadillo's application.template.yml for that flag pairing. | |
| # | |
| # Every job starts its backend as the very first step so it boots in the | |
| # background while R dependencies install, instead of paying for both serially. | |
| # | |
| # As of Sept. 2025 the single-backend, dsBase-only, unsharded version of this | |
| # took ~ 95 mins; the full (Opal+Armadillo, dsBase+dsDanger) unsharded run is | |
| # well over an hour. | |
| ################################################################################ | |
| name: dsBaseClient tests' suite | |
| on: | |
| push: | |
| branches: [main, master, 'v*-dev'] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| dsbase-ref: | |
| description: dsBase branch, tag or SHA to test against | |
| required: false | |
| default: v7.0-dev | |
| schedule: | |
| - cron: '0 0 * * 0' # Weekly | |
| - cron: '0 1 * * *' # Nightly | |
| # A new push to the same ref supersedes any run still in progress for it, so | |
| # we don't burn compute on stale commits. Scoped by event_name too, so a | |
| # schedule/workflow_dispatch run is never auto-cancelled by an unrelated push. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| env: | |
| _r_check_system_clock_: 0 | |
| PROJECT_NAME: dsBaseClient | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| DSBASE_REF: ${{ inputs.dsbase-ref || 'v7.0-dev' }} | |
| R_KEEP_PKG_SOURCE: yes | |
| # Selects perf_files/<driver>_<PERF_PROFILE>_perf-profile.csv as the perf | |
| # test reference rates/tolerances (see tests/testthat/perf_tests/perf_rate.R). | |
| # Reuses the existing azure-pipeline reference files rather than adding new | |
| # ones, matching what the old Azure pipelines set via perf.profile. | |
| PERF_PROFILE: github-workflows | |
| # dsBase shards get their filter from matrix.filter per entry instead. | |
| TEST_FILTER_DSDANGER: '__dgr-|datachk_dgr-|smk_dgr-|arg_dgr-|disc_dgr-|smk_expt_dgr-|expt_dgr-|math_dgr-' | |
| jobs: | |
| # Runs immediately (no `needs`) so the two test rows reset to pending as | |
| # soon as a new commit lands, rather than showing the previous commit's | |
| # result for the ~20-60 min the matrix jobs take to complete. | |
| mark-pending: | |
| name: Mark tests pending | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Checkout dsBaseClient | |
| uses: actions/checkout@v5 | |
| with: | |
| path: dsBaseClient | |
| - name: Mark pending | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js'); | |
| await postCiComment({ github, context, updates: { | |
| 'row:tests-armadillo': `<tr><td>Armadillo unit tests</td><td>⏳ pending</td></tr>`, | |
| 'row:tests-opal': `<tr><td>Opal unit tests</td><td>⏳ pending</td></tr>`, | |
| 'row:coverage': `<tr><td>Test coverage</td><td>⏳ pending</td></tr>`, | |
| 'ver:armadillo': `_pending_`, | |
| 'ver:opal': `_pending_`, | |
| 'log:tests-armadillo': `_pending_`, | |
| 'log:tests-opal': `_pending_`, | |
| 'log:coverage': `_pending_` | |
| }}); | |
| ################################################################################ | |
| # Opal - dsBase suite, sharded, plus dsDanger folded in as an extra matrix | |
| # entry. Each entry is a fully isolated job with its own Opal instance. | |
| ################################################################################ | |
| # One job, 8-entry matrix, so all entries group under a single summary box | |
| # in the run graph (previously split into 3 category-group jobs calling a | |
| # reusable workflow - reverted since that lost the combined summary and, | |
| # worse, made all calls share the reusable workflow's name, which triggers | |
| # GitHub's default same-name concurrency cap of 2 and throttled the shards | |
| # to 2-at-a-time instead of running in parallel). | |
| opal-dsbase: | |
| name: Opal tests (${{ matrix.category }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| # smk and perf are split by the first letter of the function name | |
| # (after any "ds." prefix) rather than an enumerated file list, so | |
| # newly added test files fall into a bucket automatically. dsdanger is | |
| # a separate, smaller suite (steps below are gated on | |
| # matrix.category == 'dsdanger') folded in here so it groups under this | |
| # job's summary box instead of a standalone job. | |
| matrix: | |
| include: | |
| - category: smk-1 | |
| filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)' | |
| - category: smk-2 | |
| filter: 'smk-ds.[m-vM-V]' | |
| - category: arg | |
| filter: 'arg-' | |
| - category: perf-1 | |
| filter: 'perf-ds.[a-cA-C]' | |
| - category: perf-2 | |
| filter: 'perf-ds.[d-mD-M]' | |
| - category: perf-3 | |
| filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)' | |
| - category: misc | |
| filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-' | |
| - category: dsdanger | |
| env: | |
| PROJECT_NAME: dsBaseClient | |
| DS_DRIVER: OpalDriver | |
| DSDANGER_REF: '6.3.4' | |
| steps: | |
| - name: Checkout dsBaseClient | |
| uses: actions/checkout@v5 | |
| with: | |
| path: dsBaseClient | |
| - uses: ./dsBaseClient/.github/actions/setup-opal-with-dsbase | |
| with: | |
| dsbase-ref: ${{ env.DSBASE_REF }} | |
| - name: Install dsDangerClient | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| R -q -e " | |
| ref <- Sys.getenv('BRANCH_NAME') | |
| ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE) | |
| if (!ok) pak::pkg_install('github::datashield/dsDangerClient')" | |
| - name: Install dsDanger package on Opal server | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" | |
| R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsDanger', username = 'datashield', ref = '${{ env.DSDANGER_REF }}'); opal.logout(opal)" | |
| working-directory: dsBaseClient | |
| - name: Run dsBase tests with JUnit report | |
| if: matrix.category != 'dsdanger' | |
| run: | | |
| R -q -e ' | |
| devtools::load_all(quiet = TRUE); | |
| library(testthat); | |
| output_file <- file("test_console_output_dsbase.txt"); | |
| sink(output_file, split = TRUE); | |
| junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); | |
| progress_rep <- ProgressReporter$new(max_failures = 999999); | |
| multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); | |
| options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); | |
| test_dir("tests/testthat", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? | |
| cat test_console_output_dsbase.txt | |
| n_tests=$(grep -c '<testcase' test_results_dsbase.xml || true) | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_tests:-0}" -eq 0 ]; then | |
| echo "0 tests actually ran (no <testcase> entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass." | |
| R_EXIT=1 | |
| fi | |
| n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsbase.xml | wc -l | tr -d ' ') | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then | |
| echo "$n_failed test failure(s)/error(s) in test_results_dsbase.xml - failing this shard." | |
| R_EXIT=1 | |
| fi | |
| exit "${R_EXIT:-0}" | |
| working-directory: dsBaseClient | |
| - name: Run dsDanger tests with JUnit report | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| R -q -e ' | |
| devtools::load_all(quiet = TRUE); | |
| library(testthat); | |
| output_file <- file("test_console_output_dsdanger.txt"); | |
| sink(output_file, split = TRUE); | |
| junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml")); | |
| progress_rep <- ProgressReporter$new(max_failures = 999999); | |
| multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); | |
| options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); | |
| test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? | |
| cat test_console_output_dsdanger.txt | |
| n_tests=$(grep -c '<testcase' test_results_dsdanger.xml || true) | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_tests:-0}" -eq 0 ]; then | |
| echo "0 tests actually ran (no <testcase> entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass." | |
| R_EXIT=1 | |
| fi | |
| n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsdanger.xml | wc -l | tr -d ' ') | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then | |
| echo "$n_failed test failure(s)/error(s) in test_results_dsdanger.xml - failing this shard." | |
| R_EXIT=1 | |
| fi | |
| exit "${R_EXIT:-0}" | |
| working-directory: dsBaseClient | |
| # Written regardless of outcome so opal-report can tell a shard that | |
| # never reported (crashed in setup, before any test XML existed) apart | |
| # from one that reported 0 failures - job.status already reflects the | |
| # test step's exit code by this point. mkdir -p so this still succeeds | |
| # even if checkout itself is what failed. Written under dsBaseClient/, | |
| # alongside everything else in the same upload below, so upload-artifact | |
| # doesn't widen its common-ancestor computation to the workspace root | |
| # and shift every other file down an extra directory level. | |
| - name: Record shard outcome | |
| if: always() | |
| run: | | |
| mkdir -p dsBaseClient | |
| echo "${{ job.status }}" > dsBaseClient/shard_status.txt | |
| - name: Upload shard results | |
| if: always() && matrix.category != 'dsdanger' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: opal-dsbase-${{ matrix.category }} | |
| path: | | |
| dsBaseClient/test_results_dsbase.xml | |
| dsBaseClient/test_console_output_dsbase.txt | |
| dsBaseClient/tests/testthat/data_files/dsbase_version.txt | |
| dsBaseClient/shard_status.txt | |
| - name: Upload dsDanger results | |
| if: always() && matrix.category == 'dsdanger' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: opal-dsdanger | |
| path: | | |
| dsBaseClient/test_results_dsdanger.xml | |
| dsBaseClient/test_console_output_dsdanger.txt | |
| dsBaseClient/shard_status.txt | |
| ################################################################################ | |
| # Opal - merge all matrix entry results and publish the report. | |
| ################################################################################ | |
| opal-report: | |
| name: Opal report | |
| needs: [opal-dsbase] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Checkout dsBaseClient | |
| uses: actions/checkout@v5 | |
| with: | |
| path: dsBaseClient | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: release | |
| use-public-rspm: true | |
| - uses: r-lib/actions/setup-r-dependencies@v2 | |
| env: | |
| PKG_INCLUDE_LINKINGTO: true | |
| with: | |
| working-directory: dsBaseClient | |
| dependencies: 'c("Depends", "Imports", "LinkingTo")' | |
| extra-packages: | | |
| cran::xml2 | |
| - name: Download shard/dsdanger results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: 'opal-*' | |
| path: dsBaseClient/artifacts | |
| - name: Merge JUnit results | |
| run: | | |
| mkdir -p logs | |
| cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt | |
| Rscript -e ' | |
| xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE) | |
| docs <- lapply(xml_files, xml2::read_xml) | |
| root <- xml2::xml_new_root("testsuites") | |
| for (doc in docs) { | |
| for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s) | |
| } | |
| xml2::write_xml(root, "logs/test_results.xml") | |
| ' | |
| working-directory: dsBaseClient | |
| - name: Upload merged results | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: opal-report-results | |
| path: | | |
| dsBaseClient/logs/test_results.xml | |
| dsBaseClient/logs/test_console_output.txt | |
| - name: Compute results & write summary | |
| id: results | |
| env: | |
| OPAL_DSBASE_RESULT: ${{ needs.opal-dsbase.result }} | |
| run: | | |
| Rscript -e ' | |
| source(".github/scripts/summarise-junit.R") | |
| res <- summarise_junit("logs/test_results.xml", "Opal", "artifacts") | |
| writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY")) | |
| cat(res$summary, sep = "\n") | |
| version <- find_dsbase_version("artifacts") | |
| # needs.opal-dsbase.result is "failure" if ANY matrix shard did not | |
| # succeed (even if the shards that DID upload results show 0 | |
| # failures) - a shard that never reported must not look like a pass. | |
| shard_ok <- Sys.getenv("OPAL_DSBASE_RESULT") == "success" | |
| ok <- res$ok && shard_ok | |
| # Only a fallback: the usual case (a shard crashed/reported 0 | |
| # tests) is already named above via res$shard_problems - this | |
| # covers the rare gap where a shard job failed without leaving | |
| # any trace summarise_junit() could identify. | |
| if (!shard_ok && length(res$shard_problems) == 0) { | |
| message("One or more Opal dsbase/dsdanger matrix entries did not succeed, but none could be identified from their uploaded results.") | |
| } | |
| out <- Sys.getenv("GITHUB_OUTPUT") | |
| cat( | |
| sprintf("ok=%s\n", tolower(ok)), | |
| sprintf("tally=%s\n", res$tally), | |
| sprintf("version=%s\n", version), | |
| file = out, append = TRUE, sep = "" | |
| ) | |
| if (!ok) message("Opal tests failed.") | |
| quit(save = "no", status = if (ok) 0 else 1) | |
| ' | |
| working-directory: dsBaseClient | |
| - name: Post PR comment | |
| if: always() | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| // steps.results.outputs.* come back empty (not "true"/"false") if | |
| // that step never got far enough to write them - e.g. it errored | |
| // or was skipped outright because an earlier step failed. Fall | |
| // back to something informative rather than a blank tally. | |
| const ok = '${{ steps.results.outputs.ok }}' === 'true'; | |
| const tally = '${{ steps.results.outputs.tally }}' || 'error - see log'; | |
| const version = '${{ steps.results.outputs.version }}' || 'unknown'; | |
| // #summary-<job_id> scrolls straight to this job's summary card | |
| // (the pass/fail tally written via GITHUB_STEP_SUMMARY) instead | |
| // of just the top of the run page. | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}#summary-${{ job.check_run_id }}`; | |
| const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js'); | |
| await postCiComment({ github, context, updates: { | |
| 'row:tests-opal': `<tr><td>Opal unit tests</td><td>${ok ? '✅' : '❌'} <code>${tally}</code></td></tr>`, | |
| 'ver:opal': `\`${version}\``, | |
| 'log:tests-opal': `<a href="${runUrl}" target="_blank" rel="noopener noreferrer">Opal unit tests</a>` | |
| }}); | |
| ################################################################################ | |
| # Armadillo - dsBase suite, sharded 4 ways. Each shard downloads and runs its | |
| # own Armadillo jar instance. | |
| # | |
| # Runs as a plain `java -jar` process instead of docker-compose: Armadillo | |
| # self-manages its own Rock container over the host Docker socket | |
| # (docker-management-enabled: true / docker-run-in-container: false), which | |
| # avoids building/pulling the old custom armadillo_citest image and its | |
| # dockerised Armadillo layer. The latest GitHub release jar is downloaded at | |
| # run time. No process is restarted after installing dsBase - install then | |
| # whitelist directly. | |
| ################################################################################ | |
| # One job, 8-entry matrix, so all entries group under a single summary box | |
| # in the run graph (previously split into 3 category-group jobs calling a | |
| # reusable workflow - reverted since that lost the combined summary and, | |
| # worse, made all calls share the reusable workflow's name, which triggers | |
| # GitHub's default same-name concurrency cap of 2 and throttled the shards | |
| # to 2-at-a-time instead of running in parallel). | |
| armadillo-dsbase: | |
| name: Armadillo tests (${{ matrix.category }}) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| # smk and perf are split by the first letter of the function name | |
| # (after any "ds." prefix) rather than an enumerated file list, so | |
| # newly added test files fall into a bucket automatically. dsdanger is | |
| # a separate, smaller suite (steps below are gated on | |
| # matrix.category == 'dsdanger') folded in here so it groups under this | |
| # job's summary box instead of a standalone job. | |
| matrix: | |
| include: | |
| - category: smk-1 | |
| filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)' | |
| - category: smk-2 | |
| filter: 'smk-ds.[m-vM-V]' | |
| - category: arg | |
| filter: 'arg-' | |
| - category: perf-1 | |
| filter: 'perf-ds.[a-cA-C]' | |
| - category: perf-2 | |
| filter: 'perf-ds.[d-mD-M]' | |
| - category: perf-3 | |
| filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)' | |
| - category: misc | |
| filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-' | |
| - category: dsdanger | |
| env: | |
| PROJECT_NAME: dsBaseClient | |
| DS_DRIVER: ArmadilloDriver | |
| DSDANGER_TARBALL: dsDanger_6.3.4.tar.gz | |
| steps: | |
| - name: Checkout dsBaseClient | |
| uses: actions/checkout@v5 | |
| with: | |
| path: dsBaseClient | |
| - uses: ./dsBaseClient/.github/actions/setup-armadillo-with-dsbase | |
| with: | |
| dsbase-ref: ${{ env.DSBASE_REF }} | |
| - name: Install dsDangerClient | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| R -q -e " | |
| ref <- Sys.getenv('BRANCH_NAME') | |
| ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE) | |
| if (!ok) pak::pkg_install('github::datashield/dsDangerClient')" | |
| - name: Install dsDanger package on Armadillo server | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| curl -u admin:admin http://localhost:8080/whitelist | |
| install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSDANGER_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) | |
| if [ "$install_status" != "200" ]; then | |
| echo "dsDanger install request failed with HTTP status $install_status" | |
| exit 1 | |
| fi | |
| for i in $(seq 1 30); do | |
| packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) | |
| if echo "$packages_json" | jq -e '.[] | select(.name == "dsDanger")' >/dev/null 2>&1; then | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| curl -u admin:admin -X POST http://localhost:8080/whitelist/dsDanger | |
| curl -u admin:admin http://localhost:8080/whitelist | |
| working-directory: dsBaseClient | |
| - name: Run dsBase tests with coverage & JUnit report | |
| if: matrix.category != 'dsdanger' | |
| run: | | |
| R -q -e "devtools::load_all();" | |
| R -q -e ' | |
| cov <- covr::package_coverage( | |
| type = c("none"), | |
| code = c('"'"' | |
| output_file <- file("test_console_output_dsbase.txt"); | |
| sink(output_file, split = TRUE); | |
| junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); | |
| progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); | |
| multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); | |
| options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); | |
| testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' | |
| ) | |
| ) | |
| saveRDS(cov, "coverage.rds") | |
| write.csv(covr::coverage_to_list(cov), "coveragelist.csv") | |
| covr::to_cobertura(cov, "cobertura.xml")' || R_EXIT=$? | |
| cat test_console_output_dsbase.txt | |
| n_tests=$(grep -c '<testcase' test_results_dsbase.xml || true) | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_tests:-0}" -eq 0 ]; then | |
| echo "0 tests actually ran (no <testcase> entries in test_results_dsbase.xml) - treating as a failure rather than a silent pass." | |
| R_EXIT=1 | |
| fi | |
| n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsbase.xml | wc -l | tr -d ' ') | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then | |
| echo "$n_failed test failure(s)/error(s) in test_results_dsbase.xml - failing this shard." | |
| R_EXIT=1 | |
| fi | |
| exit "${R_EXIT:-0}" | |
| working-directory: dsBaseClient | |
| - name: Run dsDanger tests with JUnit report | |
| if: matrix.category == 'dsdanger' | |
| run: | | |
| R -q -e ' | |
| devtools::load_all(quiet = TRUE); | |
| library(testthat); | |
| output_file <- file("test_console_output_dsdanger.txt"); | |
| sink(output_file, split = TRUE); | |
| junit_rep <- JunitReporter$new(file = file.path(getwd(), "test_results_dsdanger.xml")); | |
| progress_rep <- ProgressReporter$new(max_failures = 999999); | |
| multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); | |
| options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); | |
| test_dir("tests/testthat", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$? | |
| cat test_console_output_dsdanger.txt | |
| n_tests=$(grep -c '<testcase' test_results_dsdanger.xml || true) | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_tests:-0}" -eq 0 ]; then | |
| echo "0 tests actually ran (no <testcase> entries in test_results_dsdanger.xml) - treating as a failure rather than a silent pass." | |
| R_EXIT=1 | |
| fi | |
| n_failed=$(grep -oE '<(failure|error)[ >]' test_results_dsdanger.xml | wc -l | tr -d ' ') | |
| if [ "${R_EXIT:-0}" -eq 0 ] && [ "${n_failed:-0}" -gt 0 ]; then | |
| echo "$n_failed test failure(s)/error(s) in test_results_dsdanger.xml - failing this shard." | |
| R_EXIT=1 | |
| fi | |
| exit "${R_EXIT:-0}" | |
| working-directory: dsBaseClient | |
| # Written regardless of outcome so armadillo-report can tell a shard | |
| # that never reported (crashed in setup, before any test XML existed) | |
| # apart from one that reported 0 failures, and so the coverage step | |
| # below knows how many Codecov sessions to actually expect. mkdir -p so | |
| # this still succeeds even if checkout itself is what failed. Written | |
| # under dsBaseClient/, alongside everything else in the same upload | |
| # below, so upload-artifact doesn't widen its common-ancestor | |
| # computation to the workspace root and shift every other file down an | |
| # extra directory level. | |
| - name: Record shard outcome | |
| if: always() | |
| run: | | |
| mkdir -p dsBaseClient | |
| echo "${{ job.status }}" > dsBaseClient/shard_status.txt | |
| - name: Upload shard results | |
| if: always() && matrix.category != 'dsdanger' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: armadillo-dsbase-${{ matrix.category }} | |
| path: | | |
| dsBaseClient/test_results_dsbase.xml | |
| dsBaseClient/test_console_output_dsbase.txt | |
| dsBaseClient/coveragelist.csv | |
| dsBaseClient/coverage.rds | |
| dsBaseClient/dsbase_version.txt | |
| dsBaseClient/shard_status.txt | |
| - name: Upload dsDanger results | |
| if: always() && matrix.category == 'dsdanger' | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: armadillo-dsdanger | |
| path: | | |
| dsBaseClient/test_results_dsdanger.xml | |
| dsBaseClient/test_console_output_dsdanger.txt | |
| dsBaseClient/shard_status.txt | |
| - name: Upload coverage to Codecov | |
| if: always() && matrix.category != 'dsdanger' | |
| uses: codecov/codecov-action@v6 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: dsBaseClient/cobertura.xml | |
| flags: armadillo-${{ matrix.category }} | |
| fail_ci_if_error: false | |
| ################################################################################ | |
| # Armadillo - merge all matrix entry results and publish the report. | |
| ################################################################################ | |
| armadillo-report: | |
| name: Armadillo report | |
| needs: [armadillo-dsbase] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Checkout dsBaseClient | |
| uses: actions/checkout@v5 | |
| with: | |
| path: dsBaseClient | |
| - uses: r-lib/actions/setup-r@v2 | |
| with: | |
| r-version: release | |
| use-public-rspm: true | |
| - uses: r-lib/actions/setup-r-dependencies@v2 | |
| env: | |
| PKG_INCLUDE_LINKINGTO: true | |
| with: | |
| working-directory: dsBaseClient | |
| dependencies: 'c("Depends", "Imports", "LinkingTo")' | |
| extra-packages: | | |
| cran::xml2 | |
| - name: Download shard/dsdanger results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: 'armadillo-*' | |
| path: dsBaseClient/artifacts | |
| - name: Merge JUnit results | |
| run: | | |
| mkdir -p logs | |
| cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt | |
| Rscript -e ' | |
| xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE) | |
| docs <- lapply(xml_files, xml2::read_xml) | |
| root <- xml2::xml_new_root("testsuites") | |
| for (doc in docs) { | |
| for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s) | |
| } | |
| xml2::write_xml(root, "logs/test_results.xml") | |
| ' | |
| working-directory: dsBaseClient | |
| - name: Upload merged results | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: armadillo-report-results | |
| path: | | |
| dsBaseClient/logs/test_results.xml | |
| dsBaseClient/logs/test_console_output.txt | |
| - name: Compute coverage | |
| id: coverage | |
| # Codecov already computes patch (diff) coverage for this commit from | |
| # the cobertura.xml uploaded by each armadillo-dsbase shard, gated at | |
| # 80% in codecov.yml - that is the branch-level figure we want here. | |
| # We used to poll the "codecov/patch" GitHub commit status for it, | |
| # but this repo's Codecov integration never posts that status/check | |
| # (confirmed via the GitHub API - only the PR comment feature is | |
| # active), so we poll Codecov's own public API instead, which has | |
| # the same data regardless of GitHub status-posting being enabled. | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| let prNumber = context.payload.pull_request?.number; | |
| if (!prNumber) { | |
| const branch = context.ref.replace('refs/heads/', ''); | |
| const prs = await github.rest.pulls.list({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| head: `${context.repo.owner}:${branch}`, state: 'open' | |
| }); | |
| prNumber = prs.data[0]?.number; | |
| } | |
| // One session per armadillo-dsbase shard that uploads coverage | |
| // (dsdanger is excluded - see the matrix above); Codecov's report | |
| // isn't final until all of them have landed. Counted from each | |
| // shard's own shard_status.txt (written in armadillo-dsbase, | |
| // downloaded above) rather than hardcoded, so a shard that never | |
| // got as far as generating cobertura.xml isn't waited on forever, | |
| // and this doesn't need updating if the matrix is resharded. | |
| const fs = require('fs'); | |
| const artifactsDir = 'dsBaseClient/artifacts'; | |
| let expectedSessions = 0; | |
| if (fs.existsSync(artifactsDir)) { | |
| for (const dir of fs.readdirSync(artifactsDir)) { | |
| if (dir.includes('dsdanger')) continue; | |
| const statusFile = `${artifactsDir}/${dir}/shard_status.txt`; | |
| if (fs.existsSync(statusFile) && fs.readFileSync(statusFile, 'utf8').trim() === 'success') { | |
| expectedSessions++; | |
| } | |
| } | |
| } | |
| const EXPECTED_SESSIONS = Math.max(expectedSessions, 1); | |
| const THRESHOLD = 80; | |
| let icon = '❓'; | |
| let text = 'no PR found to look up patch coverage for'; | |
| let project = 'unknown'; | |
| let url = `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`; | |
| if (prNumber) { | |
| url = `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}`; | |
| icon = '❓'; | |
| text = 'no codecov patch data found after polling'; | |
| for (let i = 0; i < 18; i++) { | |
| if (i > 0) await new Promise(r => setTimeout(r, 10000)); | |
| const res = await fetch(`https://api.codecov.io/api/v2/github/${context.repo.owner}/repos/${context.repo.repo}/pulls/${prNumber}/`); | |
| if (!res.ok) continue; | |
| const data = await res.json(); | |
| // Whole-project total (informational only per codecov.yml) - | |
| // report it as soon as it's available, independent of the | |
| // patch-readiness gate below. | |
| if (typeof data.head_totals?.coverage === 'number') { | |
| project = `${data.head_totals.coverage.toFixed(1)}%`; | |
| } | |
| if ((data.head_totals?.sessions ?? 0) < EXPECTED_SESSIONS) continue; | |
| // Codecov returns patch: null, rather than a zeroed object, | |
| // when the PR touches no covr-instrumented line at all. | |
| const { hits, misses, partials, coverage } = data.patch ?? { hits: 0, misses: 0, partials: 0 }; | |
| if (hits + misses + partials === 0) { | |
| icon = 'ℹ️'; | |
| text = 'no coverable lines changed'; | |
| } else { | |
| icon = coverage >= THRESHOLD ? '✅' : '❌'; | |
| text = `${coverage.toFixed(1)}% vs ${THRESHOLD}% target`; | |
| } | |
| break; | |
| } | |
| } | |
| core.setOutput('icon', icon); | |
| core.setOutput('text', text); | |
| core.setOutput('project', project); | |
| core.setOutput('url', url); | |
| - name: Compute results & write summary | |
| id: results | |
| env: | |
| ARMADILLO_DSBASE_RESULT: ${{ needs.armadillo-dsbase.result }} | |
| run: | | |
| Rscript -e ' | |
| source(".github/scripts/summarise-junit.R") | |
| res <- summarise_junit("logs/test_results.xml", "Armadillo", "artifacts") | |
| writeLines(res$summary, Sys.getenv("GITHUB_STEP_SUMMARY")) | |
| cat(res$summary, sep = "\n") | |
| version <- find_dsbase_version("artifacts") | |
| # needs.armadillo-dsbase.result is "failure" if ANY matrix shard did | |
| # not succeed (even if the shards that DID upload results show 0 | |
| # failures) - a shard that never reported must not look like a pass. | |
| shard_ok <- Sys.getenv("ARMADILLO_DSBASE_RESULT") == "success" | |
| ok <- res$ok && shard_ok | |
| # Only a fallback: the usual case (a shard crashed/reported 0 | |
| # tests) is already named above via res$shard_problems - this | |
| # covers the rare gap where a shard job failed without leaving | |
| # any trace summarise_junit() could identify. | |
| if (!shard_ok && length(res$shard_problems) == 0) { | |
| message("One or more Armadillo dsbase/dsdanger matrix entries did not succeed, but none could be identified from their uploaded results.") | |
| } | |
| out <- Sys.getenv("GITHUB_OUTPUT") | |
| cat( | |
| sprintf("ok=%s\n", tolower(ok)), | |
| sprintf("tally=%s\n", res$tally), | |
| sprintf("version=%s\n", version), | |
| file = out, append = TRUE, sep = "" | |
| ) | |
| if (!ok) message("Armadillo tests failed.") | |
| quit(save = "no", status = if (ok) 0 else 1) | |
| ' | |
| working-directory: dsBaseClient | |
| - name: Post PR comment | |
| if: always() | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| // steps.results.outputs.* come back empty (not "true"/"false") if | |
| // that step never got far enough to write them - e.g. it errored | |
| // or was skipped outright because an earlier step failed. Fall | |
| // back to something informative rather than a blank tally. | |
| const ok = '${{ steps.results.outputs.ok }}' === 'true'; | |
| const tally = '${{ steps.results.outputs.tally }}' || 'error - see log'; | |
| const version = '${{ steps.results.outputs.version }}' || 'unknown'; | |
| const coverageIcon = '${{ steps.coverage.outputs.icon }}' || '❓'; | |
| const coverageText = '${{ steps.coverage.outputs.text }}' || 'error - see log'; | |
| const coverageProject = '${{ steps.coverage.outputs.project }}' || 'unknown'; | |
| // #summary-<job_id> scrolls straight to this job's summary card | |
| // (the pass/fail tally written via GITHUB_STEP_SUMMARY) instead | |
| // of just the top of the run page. | |
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}#summary-${{ job.check_run_id }}`; | |
| const codecovUrl = '${{ steps.coverage.outputs.url }}' || `https://app.codecov.io/gh/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}`; | |
| const postCiComment = require('${{ github.workspace }}/dsBaseClient/.github/scripts/post-ci-comment.js'); | |
| await postCiComment({ github, context, updates: { | |
| 'row:tests-armadillo': `<tr><td>Armadillo unit tests</td><td>${ok ? '✅' : '❌'} <code>${tally}</code></td></tr>`, | |
| 'row:coverage': `<tr><td>Test coverage</td><td>${coverageIcon} ${coverageText} (project: ${coverageProject})</td></tr>`, | |
| 'ver:armadillo': `\`${version}\``, | |
| 'log:tests-armadillo': `<a href="${runUrl}" target="_blank" rel="noopener noreferrer">Armadillo unit tests</a>`, | |
| 'log:coverage': `<a href="${codecovUrl}" target="_blank" rel="noopener noreferrer">Codecov</a>` | |
| }}); | |