Skip to content
dsBaseClient tests' suite

revert back to single github flow #113

Sign in to view logs

revert back to single github flow

revert back to single github flow #113

Workflow file for this run

.github/workflows/dsBaseClient_test_suite.yaml at cf14b02
################################################################################
# trigger CI
# DataSHIELD GHA test suite - dsBaseClient
# Replaces azure-pipelines.yml / opal_azure-pipelines.yml / armadillo_azure-pipelines.yml.
#
# This is one of three separate workflow files, each its own named check on a
# PR/run: this one (all the actual test execution), check.yaml (doc-sync +
# R CMD check), lint.yaml (lintr). Split so each shows as its own category
# rather than one graph mixing test execution with static checks.
#
# Structure (all jobs below run in parallel except where "needs" says otherwise):
# opal-dsbase (matrix x7) - dsBase suite against Opal, one shard per
# category, all grouped under one summary box.
# opal-dsdanger - dsDanger suite against Opal (small, not sharded).
# opal-report - needs opal-dsbase + opal-dsdanger; merges results.
# armadillo-dsbase (matrix x7) - dsBase suite against Armadillo, same split.
# armadillo-dsdanger - dsDanger suite against Armadillo.
# armadillo-report - needs armadillo-dsbase + armadillo-dsdanger; merges results.
# test-summary - needs opal-report + armadillo-report; one combined
# coverage figure + pass/fail table + collapsible
# failure details, written to the job summary.
#
# The dsBase suite (matching TEST_FILTER_DSBASE - same 292 files the Azure
# pipelines run) is split into 7 shards - smk (116 files, 2 shards), perf (51
# fixed 30s-loop benchmarks, 3 shards - by far the slowest per-file), arg (1
# shard), and misc (1 shard) - each shard with its own testthat filter
# substring. smk/perf are split by the first letter of the function name
# (after any "ds." prefix) rather than an enumerated file list, so newly
# added test files fall into a bucket automatically. This is one job with a
# 7-entry matrix (not split into separate job definitions per category) so
# all shards share one summary box in the run graph, and so they don't share
# a reusable workflow name - GitHub's default same-name concurrency cap of 2
# would otherwise throttle them to 2-at-a-time. Setup (checkout through
# installing dsBase) still lives in a shared composite action
# (setup-armadillo-with-dsbase / setup-opal-with-dsbase), reused by the
# dsdanger jobs too, so that part stays DRY without needing separate jobs.
#
# Each dsbase/dsdanger job spins up its OWN backend instance (isolated - no
# shared server state / concurrency risk between categories running at once).
#
# Opal runs via docker-compose (docker-compose_opal.yml).
# Armadillo runs as a plain `java -jar` process (not docker-compose): Armadillo
# self-manages its Rock container over the host Docker socket
# (docker-management-enabled: true / docker-run-in-container: false), which skips
# building/pulling the old custom armadillo_citest image. See
# molgenis-service-armadillo's application.template.yml for that flag pairing.
#
# Every job starts its backend as the very first step so it boots in the
# background while R dependencies install, instead of paying for both serially.
#
# As of Sept. 2025 the single-backend, dsBase-only, unsharded version of this
# took ~ 95 mins; the full (Opal+Armadillo, dsBase+dsDanger) unsharded run is
# well over an hour.
################################################################################
name: dsBaseClient tests' suite
on:
push:
workflow_dispatch:
schedule:
- cron: '0 0 * * 0' # Weekly
- cron: '0 1 * * *' # Nightly
permissions:
contents: read
env:
_r_check_system_clock_: 0
PROJECT_NAME: dsBaseClient
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
R_KEEP_PKG_SOURCE: yes
# These should all be constant, except the two TEST_FILTER_* values. These can
# be used to test subsets of test files in the testthat directory. Options are
# like: '*' <- run all tests, '*_smk_*' <- run all the smoke tests.
TEST_FILTER_DSBASE: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-'
TEST_FILTER_DSDANGER: '__dgr-|datachk_dgr-|smk_dgr-|arg_dgr-|disc_dgr-|smk_expt_dgr-|expt_dgr-|math_dgr-'
jobs:
################################################################################
# Opal - dsBase suite, sharded 4 ways. Each shard is a fully isolated job
# with its own Opal instance.
################################################################################
# One job, 7-entry matrix, so all shards group under a single summary box
# in the run graph (previously split into 3 category-group jobs calling a
# reusable workflow - reverted since that lost the combined summary and,
# worse, made all 7 calls share the reusable workflow's name, which
# triggers GitHub's default same-name concurrency cap of 2 and throttled
# the shards to 2-at-a-time instead of running in parallel).
opal-dsbase:
name: Opal dsBase tests (${{ matrix.category }})
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
# smk and perf are split by the first letter of the function name
# (after any "ds." prefix) rather than an enumerated file list, so
# newly added test files fall into a bucket automatically.
matrix:
include:
- category: smk-1
filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)'
- category: smk-2
filter: 'smk-ds.[m-vM-V]'
- category: arg
filter: 'arg-'
- category: perf-1
filter: 'perf-ds.[a-cA-C]'
- category: perf-2
filter: 'perf-ds.[d-mD-M]'
- category: perf-3
filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)'
- category: misc
filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-'
env:
PROJECT_NAME: dsBaseClient
DS_DRIVER: OpalDriver
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: ./dsBaseClient/.github/actions/setup-opal-with-dsbase
- name: Run dsBase tests with coverage & JUnit report
run: |
R -q -e "devtools::load_all();"
R -q -e '
cov <- covr::package_coverage(
type = c("none"),
code = c('"'"'
output_file <- file("test_console_output_dsbase.txt");
sink(output_file, split = TRUE);
junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml"));
progress_rep <- testthat::ProgressReporter$new(max_failures = 999999);
multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep));
options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"'
)
)
saveRDS(cov, "coverage.rds")
write.csv(covr::coverage_to_list(cov), "coveragelist.csv")' || R_EXIT=$?
cat test_console_output_dsbase.txt
exit "${R_EXIT:-0}"
working-directory: dsBaseClient
- name: Upload shard results
uses: actions/upload-artifact@v4
with:
name: opal-dsbase-${{ matrix.category }}
path: |
dsBaseClient/test_results_dsbase.xml
dsBaseClient/test_console_output_dsbase.txt
dsBaseClient/coveragelist.csv
dsBaseClient/coverage.rds
dsBaseClient/tests/testthat/data_files/dsbase_version.txt
################################################################################
# Opal - dsDanger suite. Small (~15 files), so it stays a single job rather
# than sharding further.
################################################################################
opal-dsdanger:
name: Opal dsDanger tests
runs-on: ubuntu-latest
timeout-minutes: 60
env:
DS_DRIVER: OpalDriver
DSDANGER_REF: '6.3.4'
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: ./dsBaseClient/.github/actions/setup-opal-with-dsbase
with:
dsbase-ref: v7.0-dev
- name: Install dsDangerClient
run: |
R -q -e "
ref <- Sys.getenv('BRANCH_NAME')
ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE)
if (!ok) pak::pkg_install('github::datashield/dsDangerClient')"
- name: Install dsDanger package on Opal server
run: |
R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)"
R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsDanger', username = 'datashield', ref = '${{ env.DSDANGER_REF }}'); opal.logout(opal)"
working-directory: dsBaseClient
- name: Run dsDanger tests with JUnit report
run: |
R -q -e '
library(testthat);
output_file <- file("test_console_output_dsdanger.txt");
sink(output_file, split = TRUE);
junit_rep <- JunitReporter$new(file = "test_results_dsdanger.xml");
progress_rep <- ProgressReporter$new(max_failures = 999999);
multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$?
cat test_console_output_dsdanger.txt
exit "${R_EXIT:-0}"
working-directory: dsBaseClient
- name: Upload dsDanger results
uses: actions/upload-artifact@v4
with:
name: opal-dsdanger
path: |
dsBaseClient/test_results_dsdanger.xml
dsBaseClient/test_console_output_dsdanger.txt
################################################################################
# Opal - merge all shard + dsDanger results and publish the report.
################################################################################
opal-report:
name: Opal report
needs: [opal-dsbase, opal-dsdanger]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: r-lib/actions/setup-r@v2
with:
r-version: release
use-public-rspm: true
- uses: r-lib/actions/setup-r-dependencies@v2
with:
dependencies: 'c("Depends", "Imports", "LinkingTo")'
extra-packages: |
cran::xml2
github::datashield/dsDangerClient
- name: Download shard/dsdanger results
uses: actions/download-artifact@v4
with:
pattern: 'opal-*'
path: dsBaseClient/artifacts
- name: Merge JUnit results
run: |
mkdir -p logs
cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt
Rscript -e '
xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE)
docs <- lapply(xml_files, xml2::read_xml)
root <- xml2::xml_new_root("testsuites")
for (doc in docs) {
for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s)
}
xml2::write_xml(root, "logs/test_results.xml")
'
working-directory: dsBaseClient
- name: Upload merged results
uses: actions/upload-artifact@v4
with:
name: opal-report-results
path: |
dsBaseClient/logs/test_results.xml
dsBaseClient/logs/test_console_output.txt
################################################################################
# Armadillo - dsBase suite, sharded 4 ways. Each shard downloads and runs its
# own Armadillo jar instance.
#
# Runs as a plain `java -jar` process instead of docker-compose: Armadillo
# self-manages its own Rock container over the host Docker socket
# (docker-management-enabled: true / docker-run-in-container: false), which
# avoids building/pulling the old custom armadillo_citest image and its
# dockerised Armadillo layer. The latest GitHub release jar is downloaded at
# run time. No process is restarted after installing dsBase - install then
# whitelist directly.
################################################################################
# One job, 7-entry matrix, so all shards group under a single summary box
# in the run graph (previously split into 3 category-group jobs calling a
# reusable workflow - reverted since that lost the combined summary and,
# worse, made all 7 calls share the reusable workflow's name, which
# triggers GitHub's default same-name concurrency cap of 2 and throttled
# the shards to 2-at-a-time instead of running in parallel).
armadillo-dsbase:
name: Armadillo dsBase tests (${{ matrix.category }})
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
# smk and perf are split by the first letter of the function name
# (after any "ds." prefix) rather than an enumerated file list, so
# newly added test files fall into a bucket automatically.
matrix:
include:
- category: smk-1
filter: 'smk-ds.[a-lA-L]|smk-(checkClass|isDefined)'
- category: smk-2
filter: 'smk-ds.[m-vM-V]'
- category: arg
filter: 'arg-'
- category: perf-1
filter: 'perf-ds.[a-cA-C]'
- category: perf-2
filter: 'perf-ds.[d-mD-M]'
- category: perf-3
filter: 'perf-ds.[n-vN-V]|perf-(conndisconn|void)'
- category: misc
filter: 'datachk-|disc-|expt-|smk_expt-|math-|_-'
env:
PROJECT_NAME: dsBaseClient
DS_DRIVER: ArmadilloDriver
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: ./dsBaseClient/.github/actions/setup-armadillo-with-dsbase
- name: Run dsBase tests with coverage & JUnit report
run: |
R -q -e "devtools::load_all();"
R -q -e '
cov <- covr::package_coverage(
type = c("none"),
code = c('"'"'
output_file <- file("test_console_output_dsbase.txt");
sink(output_file, split = TRUE);
junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml"));
progress_rep <- testthat::ProgressReporter$new(max_failures = 999999);
multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep));
options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ matrix.filter }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"'
)
)
saveRDS(cov, "coverage.rds")
write.csv(covr::coverage_to_list(cov), "coveragelist.csv")' || R_EXIT=$?
cat test_console_output_dsbase.txt
exit "${R_EXIT:-0}"
working-directory: dsBaseClient
- name: Upload shard results
uses: actions/upload-artifact@v4
with:
name: armadillo-dsbase-${{ matrix.category }}
path: |
dsBaseClient/test_results_dsbase.xml
dsBaseClient/test_console_output_dsbase.txt
dsBaseClient/coveragelist.csv
dsBaseClient/coverage.rds
dsBaseClient/dsbase_version.txt
################################################################################
# Armadillo - dsDanger suite. Small, so it stays a single job.
################################################################################
armadillo-dsdanger:
name: Armadillo dsDanger tests
runs-on: ubuntu-latest
timeout-minutes: 60
env:
DS_DRIVER: ArmadilloDriver
DSDANGER_TARBALL: dsDanger_6.3.4.tar.gz
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: ./dsBaseClient/.github/actions/setup-armadillo-with-dsbase
- name: Install dsDangerClient
run: |
R -q -e "
ref <- Sys.getenv('BRANCH_NAME')
ok <- tryCatch({ pak::pkg_install(sprintf('github::datashield/dsDangerClient@%s', ref)); TRUE }, error = function(e) FALSE)
if (!ok) pak::pkg_install('github::datashield/dsDangerClient')"
- name: Install dsDanger package on Armadillo server
run: |
curl -u admin:admin http://localhost:8080/whitelist
install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSDANGER_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package)
if [ "$install_status" != "200" ]; then
echo "dsDanger install request failed with HTTP status $install_status"
exit 1
fi
for i in $(seq 1 30); do
packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true)
if echo "$packages_json" | jq -e '.[] | select(.name == "dsDanger")' >/dev/null 2>&1; then
break
fi
sleep 10
done
curl -u admin:admin -X POST http://localhost:8080/whitelist/dsDanger
curl -u admin:admin http://localhost:8080/whitelist
working-directory: dsBaseClient
- name: Run dsDanger tests with JUnit report
run: |
R -q -e '
library(testthat);
output_file <- file("test_console_output_dsdanger.txt");
sink(output_file, split = TRUE);
junit_rep <- JunitReporter$new(file = "test_results_dsdanger.xml");
progress_rep <- ProgressReporter$new(max_failures = 999999);
multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep));
options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}");
testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' || R_EXIT=$?
cat test_console_output_dsdanger.txt
exit "${R_EXIT:-0}"
working-directory: dsBaseClient
- name: Upload dsDanger results
uses: actions/upload-artifact@v4
with:
name: armadillo-dsdanger
path: |
dsBaseClient/test_results_dsdanger.xml
dsBaseClient/test_console_output_dsdanger.txt
################################################################################
# Armadillo - merge all shard + dsDanger results and publish the report.
################################################################################
armadillo-report:
name: Armadillo report
needs: [armadillo-dsbase, armadillo-dsdanger]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout dsBaseClient
uses: actions/checkout@v4
with:
path: dsBaseClient
- uses: r-lib/actions/setup-r@v2
with:
r-version: release
use-public-rspm: true
- uses: r-lib/actions/setup-r-dependencies@v2
with:
dependencies: 'c("Depends", "Imports", "LinkingTo")'
extra-packages: |
cran::xml2
github::datashield/dsDangerClient
- name: Download shard/dsdanger results
uses: actions/download-artifact@v4
with:
pattern: 'armadillo-*'
path: dsBaseClient/artifacts
- name: Merge JUnit results
run: |
mkdir -p logs
cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt
Rscript -e '
xml_files <- list.files("artifacts", pattern = "^test_results_.*\\.xml$", recursive = TRUE, full.names = TRUE)
docs <- lapply(xml_files, xml2::read_xml)
root <- xml2::xml_new_root("testsuites")
for (doc in docs) {
for (s in xml2::xml_find_all(doc, ".//testsuite")) xml2::xml_add_child(root, s)
}
xml2::write_xml(root, "logs/test_results.xml")
'
working-directory: dsBaseClient
- name: Upload merged results
uses: actions/upload-artifact@v4
with:
name: armadillo-report-results
path: |
dsBaseClient/logs/test_results.xml
dsBaseClient/logs/test_console_output.txt
################################################################################
# Single consolidated summary across both backends: one combined coverage
# figure (merged from every shard's raw coverage.rds, both backends
# together - coverage measures client-side code paths, which are largely
# backend-agnostic, so one figure is more meaningful than two near-duplicate
# ones), a pass/fail table per backend, and failures tucked into collapsible
# <details> blocks rather than cluttering the main summary.
################################################################################
test-summary:
name: Test summary
needs: [opal-report, armadillo-report]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: r-lib/actions/setup-r@v2
with:
r-version: release
use-public-rspm: true
- uses: r-lib/actions/setup-r-dependencies@v2
with:
dependencies: 'c("Depends", "Imports", "LinkingTo")'
packages: 'any::sessioninfo'
extra-packages: |
cran::xml2
cran::covr
- name: Download merged report results
uses: actions/download-artifact@v4
with:
pattern: '*-report-results'
path: artifacts/reports
- name: Download shard coverage and versions
uses: actions/download-artifact@v4
with:
pattern: '*-dsbase-*'
path: artifacts/shards
- name: Write combined job summary
run: |
Rscript -e '
find_version <- function(prefix) {
files <- list.files("artifacts/shards", pattern = "dsbase_version\\.txt$", recursive = TRUE, full.names = TRUE)
files <- files[grepl(prefix, files)]
if (length(files) == 0) return("unknown")
trimws(readLines(files[1], warn = FALSE)[1])
}
find_results_xml <- function(artifact_dir) {
files <- list.files(artifact_dir, pattern = "^test_results\\.xml$", recursive = TRUE, full.names = TRUE)
if (length(files) == 0) stop(sprintf("No test_results.xml found under %s", artifact_dir))
files[1]
}
summarise_backend <- function(name, artifact_dir, version) {
doc <- xml2::read_xml(find_results_xml(artifact_dir))
suites <- xml2::xml_find_all(doc, ".//testsuite")
n_tests <- sum(as.integer(xml2::xml_attr(suites, "tests")), na.rm = TRUE)
n_failures <- sum(as.integer(xml2::xml_attr(suites, "failures")), na.rm = TRUE)
n_errors <- sum(as.integer(xml2::xml_attr(suites, "errors")), na.rm = TRUE)
n_skipped <- sum(as.integer(xml2::xml_attr(suites, "skipped")), na.rm = TRUE)
failed <- xml2::xml_find_all(doc, ".//testcase[failure or error]")
fail_lines <- if (length(failed) > 0) {
msgs <- vapply(failed, function(tc) {
node <- xml2::xml_find_first(tc, "failure|error")
m <- xml2::xml_attr(node, "message")
if (is.na(m) || !nzchar(m)) m <- trimws(xml2::xml_text(node))
m
}, character(1))
paste0("- `", xml2::xml_attr(failed, "classname"), "::", xml2::xml_attr(failed, "name"), "` - ", msgs)
} else {
"All tests passed."
}
list(
row = sprintf("| %s | %s | %d | %d | %d | %d |", name, version, n_tests, n_failures, n_errors, n_skipped),
details = c(
sprintf("<details><summary>Failed tests (%s)</summary>", name),
"",
fail_lines,
"",
"</details>"
)
)
}
armadillo <- summarise_backend("Armadillo", "artifacts/reports/armadillo-report-results", find_version("armadillo-dsbase"))
opal <- summarise_backend("Opal", "artifacts/reports/opal-report-results", find_version("opal-dsbase"))
rds_files <- list.files("artifacts/shards", pattern = "coverage\\.rds$", recursive = TRUE, full.names = TRUE)
if (length(rds_files) > 0) {
tallies <- lapply(rds_files, function(f) covr::tally_coverage(readRDS(f)))
agg <- aggregate(value ~ filename + line, data = do.call(rbind, tallies), FUN = sum)
totalcoverage <- round(sum(agg$value > 0) / nrow(agg) * 100, 1)
cov_line <- sprintf("**Combined coverage: %.1f%%**", totalcoverage)
} else {
cov_line <- "No coverage data found."
}
summary <- c(
"## dsBaseClient test suite results",
"",
cov_line,
"",
"| Backend | dsBase version | Tests | Failures | Errors | Skipped |",
"|---|---|---|---|---|---|",
armadillo$row,
opal$row,
"",
armadillo$details,
"",
opal$details
)
writeLines(summary, Sys.getenv("GITHUB_STEP_SUMMARY"))
'