diff --git a/.gitignore b/.gitignore index 58dcb2a7f32..51274ad373f 100644 --- a/.gitignore +++ b/.gitignore @@ -389,6 +389,8 @@ Release test/__pycache__ test/*/__pycache__ test/*/*/__pycache__ +__pycache__/ +*.pyc # make check check diff --git a/test/README.pytest b/test/README.pytest index 0b5cfd57a4a..3c59abfc810 100644 --- a/test/README.pytest +++ b/test/README.pytest @@ -93,6 +93,52 @@ For example, most tests in test/modules/md require an installation of 'pebble', an ACME test server, and look for it in $PATH. +Strict optional dependencies +---------------------------- +A skip is convenient locally and dangerous in CI: a run that was meant +to cover SSL, HTTP/2 and mod_md stays green while quietly covering none +of them, because the machine was misconfigured. Strict mode turns that +silence into a failure. + +> pytest --strict-optional +> PYHTTPD_STRICT_OPTIONAL=1 pytest + +Both are off by default, so nothing changes unless you ask for it. The +env var is the one to set in CI; the flag is the one to use by hand. + +Not every skip becomes a failure. Skips come in two kinds: + + - "does not apply here" -- wrong MPM, httpd too old, STRESS_TEST not + set, $SSL selects a different TLS module. These always skip, in + strict mode too, and a test gated on both kinds skips rather than + fails. + + - "something we expected to be installed is missing" -- curl, nghttp, + h2load, the 'websockets' and 'multipart' python packages, mod_ssl, + mod_tls, mod_md, a2md, the ACME test server, an OCSP responder. + These fail under strict mode. + +The OCSP responder is the one judgement call in that split. pebble +serves no OCSP, boulder does, so with ACME=pebble the stapling tests +have nothing to talk to. That is a property of the environment rather +than of this machine's packages, and a pipeline that is meant to cover +OCSP stapling and runs pebble has lost that coverage silently -- which +is the thing strict mode exists to catch. So it fails, and the message +names ACME=pebble as what was detected. Run ACME=boulder to cover +stapling, or leave strict mode off. + +A strict failure is raised during test setup, so pytest reports it as +ERROR rather than FAILED; the exit code is non-zero either way. The +message is prefixed '[strict-optional]' so it greps cleanly out of a CI +log, and names the dependency, what was expected, what was found +instead, and how to switch strict mode back off. + +curl is the one dependency gated for the suite as a whole rather than +test by test: apache_restart() proves the server came up by making a +curl request, so without curl nothing under test/modules/ can run. That +gate lives in the root conftest.py. + + Workings -------- All tests start httpd on their own and try hard to shut it down @@ -168,6 +214,39 @@ simplest example. 'test/modules/http2' shows how you load other modules. 'test/modules/md' checks and starts external processes (an ACME test server). +If your test needs something that may not be installed, say +so with the decorator from 'pyhttpd.depends'. It takes the +condition itself, so it works on a method, a class or a +module-level 'pytestmark': + + from pyhttpd.depends import needs_dependency + + @needs_dependency("a2md", MDTestEnv.has_a2md(), + reason="no a2md available") + class TestStore: + +For the tools the suite gates on most often there are ready +made versions of that decorator, which probe the tool once +and cache the answer: + + @H2TestEnv.needs_nghttp() # and (assets=True) + @H2TestEnv.needs_h2load("1.41.0") + @HttpdTestEnv.needs_multipart() + +For a check that can only be made once the test is running, +'env.require()' does the same job from inside the body: + + env.require(env.curl_is_at_least('8.0.0'), + 'need at least curl v8.0.0 for this', + dep='curl', + detected=f"curl {env.curl_version_str()}") + +Both skip normally and fail under strict mode; see "Strict +optional dependencies" above. When the reason is instead +"this test does not apply here", keep using a plain +'pytest.mark.skipif' or 'pytest.skip()' -- those never +become failures. + Infrastructure -------------- diff --git a/test/conftest.py b/test/conftest.py index 5242f724049..1af75404ee3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,6 +10,15 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '.')) from pyhttpd.env import HttpdTestEnv +from pyhttpd.depends import apply_gate, strict_optional_enabled + +# pyhttpd.depends provides --strict-optional and the needs_dependency marker. +# pytest_plugins is only honoured in the rootdir conftest, and there is no +# pytest.ini in this tree, so this is the only place it can be declared. +pytest_plugins = ["pyhttpd.depends"] + +# Everything below test/modules/ drives a real httpd. +MODULES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'modules') def _has_ipv6(): @@ -148,6 +157,29 @@ def pytest_configure(config): _generate_config_ini(config) +def pytest_collection_modifyitems(config, items): + """Gate the integration suite on the clients the framework itself needs. + + curl is not an optional dependency of the individual test cases that + call curl_get(): apache_restart() proves the server came up with + is_live(), and is_live() is a curl request. Without curl every test + that starts an httpd dies in fixture setup, which is neither a skip nor + a useful failure. So it is gated once here for the whole of + test/modules/ rather than test by test. + + Tests outside that directory - the strict-mode self-test - run no httpd + and are left alone. + """ + env = HttpdTestEnv() + if env.has_curl(): + return + missing = [("curl", "no curl available", f"tried to run {env.curl}")] + strict = strict_optional_enabled(config) + for item in items: + if str(item.path).startswith(MODULES_DIR): + apply_gate(item, missing, strict) + + def pytest_generate_tests(metafunc): if "repeat" in metafunc.fixturenames: count = int(metafunc.config.getoption("repeat")) @@ -199,6 +231,6 @@ def _package_scope(env, request): warnings.warn("--archive option was empty, skipping archiving") if archive_dir: fspath = str(request.fspath) - parts = fspath.split('modules/') - package_name = parts[1].split('/')[0] - env.archive_logs(package_name, archive_dir) + if 'modules/' in fspath: + package_name = fspath.split('modules/')[1].split('/')[0] + env.archive_logs(package_name, archive_dir) diff --git a/test/modules/arch/linux/conftest.py b/test/modules/arch/linux/conftest.py index 7b1a0d66cd7..72f1df8f8a2 100644 --- a/test/modules/arch/linux/conftest.py +++ b/test/modules/arch/linux/conftest.py @@ -24,8 +24,9 @@ def env(pytestconfig) -> SystemdTestEnv: logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(level=level) env = SystemdTestEnv(pytestconfig=pytestconfig) - if not env.has_systemd_module: - pytest.skip("mod_systemd is not built, configure with --enable-systemd") + env.require(env.has_systemd_module, "mod_systemd is not built, configure with --enable-systemd", + dep="mod_systemd", + detected=f"not found in {env.libexec_dir}") env.setup_httpd() env.apache_access_log_clear() env.httpd_error_log.clear_log() diff --git a/test/modules/arch/linux/test_005_service.py b/test/modules/arch/linux/test_005_service.py index ef220426302..41d59171a0f 100644 --- a/test/modules/arch/linux/test_005_service.py +++ b/test/modules/arch/linux/test_005_service.py @@ -2,11 +2,14 @@ import pytest +from pyhttpd.depends import needs_dependency + from .env import MONITOR_STATUS, MONITOR_TIMEOUT, TransientService, http_responds -pytestmark = pytest.mark.skipif( - not TransientService.is_available(), - reason="no per-user systemd manager to run a transient service under") +pytestmark = needs_dependency( + "systemd --user manager", TransientService.is_available(), + reason="no per-user systemd manager to run a transient service under", + detected="systemd-run --user is not usable here") class TestSystemdService: diff --git a/test/modules/arch/linux/test_006_watchdog.py b/test/modules/arch/linux/test_006_watchdog.py index d4fa8653bf6..8b7daade5e8 100644 --- a/test/modules/arch/linux/test_006_watchdog.py +++ b/test/modules/arch/linux/test_006_watchdog.py @@ -4,6 +4,8 @@ import pytest +from pyhttpd.depends import needs_dependency + from .env import (ForegroundServer, TransientService, MIN_WATCHDOG_SEC, NO_MONITOR_TIMEOUT, WATCHDOG_TIMEOUT, is_watchdog_ping) @@ -187,8 +189,9 @@ def test_systemd_006_09_workable_timeout_not_warned(self, env, env.httpd_error_log.scan_recent( re.compile(r'.*AH10621: .*'), timeout=1) - @pytest.mark.skipif(not TransientService.is_available(), - reason="no per-user systemd manager") + @needs_dependency("systemd --user manager", TransientService.is_available(), + reason="no per-user systemd manager", + detected="systemd-run --user is not usable here") def test_systemd_006_10_service_keeps_watchdog_alive(self, env): """The real thing: systemd records each keep-alive it receives, and the unit stays active rather than failing with Result=watchdog.""" diff --git a/test/modules/arch/linux/test_007_notify_reload.py b/test/modules/arch/linux/test_007_notify_reload.py index 93833299c63..01bd684afa0 100644 --- a/test/modules/arch/linux/test_007_notify_reload.py +++ b/test/modules/arch/linux/test_007_notify_reload.py @@ -3,13 +3,17 @@ import pytest +from pyhttpd.depends import needs_dependency + from .env import (NOTIFY_RELOAD_VERSION, TransientService, error_log_since, error_log_size, http_responds, systemd_version) pytestmark = [ - pytest.mark.skipif( - not TransientService.is_available(), - reason="no per-user systemd manager to run a transient service under"), + needs_dependency( + "systemd --user manager", TransientService.is_available(), + reason="no per-user systemd manager to run a transient service under", + detected="systemd-run --user is not usable here"), + # a version gate, not a missing dependency: keeps skipping when strict pytest.mark.skipif( systemd_version() < NOTIFY_RELOAD_VERSION, reason=f"Type=notify-reload needs systemd {NOTIFY_RELOAD_VERSION}"), diff --git a/test/modules/core/test_002_restarts.py b/test/modules/core/test_002_restarts.py index cf203bc82db..621a0cf0431 100644 --- a/test/modules/core/test_002_restarts.py +++ b/test/modules/core/test_002_restarts.py @@ -88,8 +88,8 @@ def _rtime(self, s: str) -> timedelta: @pytest.mark.skipif(condition='STRESS_TEST' not in os.environ, reason="STRESS_TEST not set in env") -@pytest.mark.skipif(condition=not CoreTestEnv().h2load_is_at_least('1.41.0'), - reason="h2load unavailable or misses --connect-to option") +@CoreTestEnv.needs_h2load("1.41.0", + reason="h2load unavailable or misses --connect-to option") class TestRestarts: def test_core_002_01(self, env): diff --git a/test/modules/generators/test_001_cgid.py b/test/modules/generators/test_001_cgid.py index afaeb3234ad..65bde24c481 100644 --- a/test/modules/generators/test_001_cgid.py +++ b/test/modules/generators/test_001_cgid.py @@ -18,8 +18,9 @@ class TestCgid: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_cgid_module: - pytest.skip("mod_cgid not built") + env.require(env.has_cgid_module, "mod_cgid not built", + dep="mod_cgid", + detected=f"not found in {env.libexec_dir}") conf = HttpdConf(env, extras={ 'base': textwrap.dedent(f""" LogLevel cgid:trace1 diff --git a/test/modules/http1/test_004_post.py b/test/modules/http1/test_004_post.py index 005a8c2fbcf..e4bf43e09a7 100644 --- a/test/modules/http1/test_004_post.py +++ b/test/modules/http1/test_004_post.py @@ -7,9 +7,12 @@ import pytest +from pyhttpd.env import HttpdTestEnv + from .env import H1Conf +@HttpdTestEnv.needs_multipart() class TestPost: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http1/test_005_trailers.py b/test/modules/http1/test_005_trailers.py index ca717a0fad5..9f99a3862b9 100644 --- a/test/modules/http1/test_005_trailers.py +++ b/test/modules/http1/test_005_trailers.py @@ -1,11 +1,13 @@ import os import pytest +from pyhttpd.env import HttpdTestEnv + from .env import H1Conf -# The trailer tests depend on "nghttp" as no other client seems to be able to send those -# rare things. +# These upload to the cgi handler, which parses the multipart body. +@HttpdTestEnv.needs_multipart() class TestTrailers: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/conftest.py b/test/modules/http2/conftest.py index 90cd6a7c853..c99472fdbb8 100644 --- a/test/modules/http2/conftest.py +++ b/test/modules/http2/conftest.py @@ -23,6 +23,10 @@ def env(pytestconfig) -> H2TestEnv: logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(level=level) env = H2TestEnv(pytestconfig=pytestconfig) + # the h2 vhosts are TLS, so setup_httpd() hard-requires mod_ssl and + # raises if it was not built. Report that as the missing dependency. + env.require(env.has_mod_ssl(), "no mod_ssl available", dep="mod_ssl", + detected=f"no mod_ssl.so in {env.libexec_dir}") env.setup_httpd() env.apache_access_log_clear() env.httpd_error_log.clear_log() diff --git a/test/modules/http2/htdocs/cgi/echo.py b/test/modules/http2/htdocs/cgi/echo.py index c9083e1d53f..2eb55fd482b 100644 --- a/test/modules/http2/htdocs/cgi/echo.py +++ b/test/modules/http2/htdocs/cgi/echo.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os, sys -import multipart status = '200 Ok' diff --git a/test/modules/http2/test_003_get.py b/test/modules/http2/test_003_get.py index 29a9459a0ab..5cc8e435a49 100644 --- a/test/modules/http2/test_003_get.py +++ b/test/modules/http2/test_003_get.py @@ -5,7 +5,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestGet: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_004_post.py b/test/modules/http2/test_004_post.py index 36b6bc3a46b..bc2b83bab21 100644 --- a/test/modules/http2/test_004_post.py +++ b/test/modules/http2/test_004_post.py @@ -13,7 +13,8 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() +@H2TestEnv.needs_multipart() class TestPost: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_006_assets.py b/test/modules/http2/test_006_assets.py index d7dbb7963b7..a0a965984ac 100644 --- a/test/modules/http2/test_006_assets.py +++ b/test/modules/http2/test_006_assets.py @@ -4,7 +4,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp(assets=True) class TestAssets: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_009_timing.py b/test/modules/http2/test_009_timing.py index 94895cc35fe..31a0b66fee3 100644 --- a/test/modules/http2/test_009_timing.py +++ b/test/modules/http2/test_009_timing.py @@ -7,7 +7,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().h2load_is_at_least('1.41.0'), reason="h2load misses --connect-to option") +@H2TestEnv.needs_h2load("1.41.0", reason="h2load misses --connect-to option") class TestTiming: LOGFILE = "" diff --git a/test/modules/http2/test_101_ssl_reneg.py b/test/modules/http2/test_101_ssl_reneg.py index c3579b87827..70a459fdbec 100644 --- a/test/modules/http2/test_101_ssl_reneg.py +++ b/test/modules/http2/test_101_ssl_reneg.py @@ -65,8 +65,9 @@ def test_h2_101_01(self, env): # try to renegotiate the cipher, should fail with correct code def test_h2_101_02(self, env): - if not (env.curl_is_at_least('8.2.0') or env.curl_is_less_than('8.1.0')): - pytest.skip("need curl != 8.1.x version") + env.require(not env.curl_is_8_1_x(), "need curl != 8.1.x version", + dep="curl != 8.1.x", + detected=f"curl {env.curl_version_str()}") url = env.mkurl("https", "ssl", "/renegotiate/cipher/") r = env.curl_get(url, options=[ "-vvv", "--tlsv1.2", "--tls-max", "1.2", "--ciphers", "ECDHE-RSA-AES256-GCM-SHA384" @@ -88,8 +89,9 @@ def test_h2_101_02(self, env): # try to renegotiate a client certificate from Location # needs to fail with correct code def test_h2_101_03(self, env): - if not (env.curl_is_at_least('8.2.0') or env.curl_is_less_than('8.1.0')): - pytest.skip("need curl != 8.1.x version") + env.require(not env.curl_is_8_1_x(), "need curl != 8.1.x version", + dep="curl != 8.1.x", + detected=f"curl {env.curl_version_str()}") url = env.mkurl("https", "ssl", "/renegotiate/verify/") r = env.curl_get(url, options=["-vvv", "--tlsv1.2", "--tls-max", "1.2"]) assert 0 != r.exit_code @@ -109,8 +111,9 @@ def test_h2_101_03(self, env): # try to renegotiate a client certificate from Directory # needs to fail with correct code def test_h2_101_04(self, env): - if not (env.curl_is_at_least('8.2.0') or env.curl_is_less_than('8.1.0')): - pytest.skip("need curl != 8.1.x version") + env.require(not env.curl_is_8_1_x(), "need curl != 8.1.x version", + dep="curl != 8.1.x", + detected=f"curl {env.curl_version_str()}") url = env.mkurl("https", "ssl", "/ssl-client-verify/index.html") r = env.curl_get(url, options=["-vvv", "--tlsv1.2", "--tls-max", "1.2"]) assert 0 != r.exit_code, f"{r}" @@ -130,8 +133,8 @@ def test_h2_101_04(self, env): # make 10 requests on the same connection, none should produce a status code # reported by erki@example.ee def test_h2_101_05(self, env): - if not env.has_h2load(): - pytest.skip("h2load not available") + env.require(env.has_h2load(), "h2load not available", dep="h2load", + detected=f"tried to run {env.h2load}") r = env.run([env.h2load, "-n", "10", "-c", "1", "-m", "1", "-vvvv", f"{env.https_base_url}/ssl-client-verify/index.html"]) assert 0 == r.exit_code @@ -165,8 +168,9 @@ def test_h2_101_10b(self, env): # Check that status works with ErrorDoc, see pull #174, fixes #172 def test_h2_101_11(self, env): - if not (env.curl_is_at_least('8.2.0') or env.curl_is_less_than('8.1.0')): - pytest.skip("need curl != 8.1.x version") + env.require(not env.curl_is_8_1_x(), "need curl != 8.1.x version", + dep="curl != 8.1.x", + detected=f"curl {env.curl_version_str()}") url = env.mkurl("https", "ssl", "/renegotiate/err-doc-cipher") r = env.curl_get(url, options=[ "-vvv", "--tlsv1.2", "--tls-max", "1.2", "--ciphers", "ECDHE-RSA-AES256-GCM-SHA384" diff --git a/test/modules/http2/test_103_upgrade.py b/test/modules/http2/test_103_upgrade.py index ec9a568b839..df0bf99fb71 100644 --- a/test/modules/http2/test_103_upgrade.py +++ b/test/modules/http2/test_103_upgrade.py @@ -4,7 +4,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestUpgrade: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_104_padding.py b/test/modules/http2/test_104_padding.py index 0c65fc7e240..b21bce7a59f 100644 --- a/test/modules/http2/test_104_padding.py +++ b/test/modules/http2/test_104_padding.py @@ -9,7 +9,7 @@ def frame_padding(payload, padbits): @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestPadding: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_105_timeout.py b/test/modules/http2/test_105_timeout.py index 5c759fcc8dd..088f2e1c6d0 100644 --- a/test/modules/http2/test_105_timeout.py +++ b/test/modules/http2/test_105_timeout.py @@ -142,8 +142,10 @@ def test_h2_105_11(self, env): def test_h2_105_12(self, env): # long connection timeout, short stream timeout # sending a slow POST - if not env.curl_is_at_least('8.0.0'): - pytest.skip(f'need at least curl v8.0.0 for this') + env.require(env.curl_is_at_least('8.0.0'), + 'need at least curl v8.0.0 for this', + dep='curl >= 8.0.0', + detected=f"curl {env.curl_version_str()}") if not env.httpd_is_at_least("2.5.0"): pytest.skip(f'need at least httpd 2.5.0 for this') conf = H2Conf(env) diff --git a/test/modules/http2/test_107_frame_lengths.py b/test/modules/http2/test_107_frame_lengths.py index dc21fd9fd6f..1956c565ee2 100644 --- a/test/modules/http2/test_107_frame_lengths.py +++ b/test/modules/http2/test_107_frame_lengths.py @@ -16,7 +16,7 @@ def mk_text_file(fpath: str, lines: int): @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestFrameLengths: URI_PATHS = [] diff --git a/test/modules/http2/test_200_header_invalid.py b/test/modules/http2/test_200_header_invalid.py index 6d852afdd51..8cf82f1f691 100644 --- a/test/modules/http2/test_200_header_invalid.py +++ b/test/modules/http2/test_200_header_invalid.py @@ -5,7 +5,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestInvalidHeaders: @pytest.fixture(autouse=True, scope='class') @@ -216,8 +216,12 @@ def test_h2_200_15(self, env): # invalid chars in method def test_h2_200_16(self, env): - if not env.h2load_is_at_least('1.45.0'): - pytest.skip(f'nhttp2 version too old') + # this drives nghttp, not h2load: gate on the version of the client + # actually used, so an absent h2load does not skip a runnable test + env.require(env.nghttp_is_at_least('1.45.0'), + 'nghttp2 version too old', + dep='nghttp >= 1.45.0', + detected=f"nghttp {env.nghttp_version_str()}") conf = H2Conf(env) conf.add_vhost_cgi() conf.install() diff --git a/test/modules/http2/test_202_trailer.py b/test/modules/http2/test_202_trailer.py index 31804176481..dc8ae2161a5 100644 --- a/test/modules/http2/test_202_trailer.py +++ b/test/modules/http2/test_202_trailer.py @@ -14,7 +14,7 @@ def setup_data(env): # The trailer tests depend on "nghttp" as no other client seems to be able to send those # rare things. @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestTrailers: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_400_push.py b/test/modules/http2/test_400_push.py index 739101fa28e..b1a828a5806 100644 --- a/test/modules/http2/test_400_push.py +++ b/test/modules/http2/test_400_push.py @@ -6,7 +6,7 @@ # The push tests depend on "nghttp" @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestPush: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_401_early_hints.py b/test/modules/http2/test_401_early_hints.py index 566dfcb16bb..a2a8e3c2a31 100644 --- a/test/modules/http2/test_401_early_hints.py +++ b/test/modules/http2/test_401_early_hints.py @@ -5,7 +5,7 @@ # The push tests depend on "nghttp" @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestEarlyHints: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_500_proxy.py b/test/modules/http2/test_500_proxy.py index 436adf6eb34..321cc7098a6 100644 --- a/test/modules/http2/test_500_proxy.py +++ b/test/modules/http2/test_500_proxy.py @@ -7,7 +7,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_nghttp(), reason="nghttp not available") +@H2TestEnv.needs_nghttp() class TestProxy: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_503_proxy_fwd.py b/test/modules/http2/test_503_proxy_fwd.py index e007325f7ef..0bf49fd0478 100644 --- a/test/modules/http2/test_503_proxy_fwd.py +++ b/test/modules/http2/test_503_proxy_fwd.py @@ -46,8 +46,10 @@ def test_h2_503_01_proxy_fwd_h1(self, env): assert r.json['port'] == f'{env.http_port}' def test_h2_503_02_fwd_proxy_h2_off(self, env): - if not env.curl_is_at_least('8.1.0'): - pytest.skip(f'need at least curl v8.1.0 for this') + env.require(env.curl_is_at_least('8.1.0'), + 'need at least curl v8.1.0 for this', + dep='curl >= 8.1.0', + detected=f"curl {env.curl_version_str()}") url = f'http://localhost:{env.http_port}/hello.py' proxy_host = f'test1.{env.http_tld}' options = [ @@ -62,8 +64,10 @@ def test_h2_503_02_fwd_proxy_h2_off(self, env): # test the HTTP/2 setup working def test_h2_503_03_proxy_fwd_h2_on(self, env): - if not env.curl_is_at_least('8.1.0'): - pytest.skip(f'need at least curl v8.1.0 for this') + env.require(env.curl_is_at_least('8.1.0'), + 'need at least curl v8.1.0 for this', + dep='curl >= 8.1.0', + detected=f"curl {env.curl_version_str()}") self.config_fwd_proxy(env, h2_enabled=True) url = f'http://localhost:{env.http_port}/hello.py' proxy_host = f'test1.{env.http_tld}' diff --git a/test/modules/http2/test_601_h2proxy_twisted.py b/test/modules/http2/test_601_h2proxy_twisted.py index 90ffc0390a9..b25305df3ca 100644 --- a/test/modules/http2/test_601_h2proxy_twisted.py +++ b/test/modules/http2/test_601_h2proxy_twisted.py @@ -75,8 +75,10 @@ def test_h2_601_05_echo_fail_many(self, env): pytest.skip(f'needs httpd 2.4.58') if sys.platform == "win32": pytest.skip("command line too long for Windows (200 parallel curl requests)") - if not env.curl_is_at_least('8.0.0'): - pytest.skip(f'need at least curl v8.0.0 for this') + env.require(env.curl_is_at_least('8.0.0'), + 'need at least curl v8.0.0 for this', + dep='curl >= 8.0.0', + detected=f"curl {env.curl_version_str()}") count = 200 fpath = os.path.join(env.gen_dir, "data-100k") args = [env.curl, '--parallel', '--parallel-max', '20'] diff --git a/test/modules/http2/test_700_load_get.py b/test/modules/http2/test_700_load_get.py index b5a6b73cb72..f6e2c2a3d30 100644 --- a/test/modules/http2/test_700_load_get.py +++ b/test/modules/http2/test_700_load_get.py @@ -4,8 +4,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().h2load_is_at_least('1.41.0'), - reason="h2load misses --connect-to option") +@H2TestEnv.needs_h2load("1.41.0", reason="h2load misses --connect-to option") class TestLoadGet: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_710_load_post_static.py b/test/modules/http2/test_710_load_post_static.py index cd593fe9abd..0f715457fe7 100644 --- a/test/modules/http2/test_710_load_post_static.py +++ b/test/modules/http2/test_710_load_post_static.py @@ -5,7 +5,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_h2load(), reason="h2load not available") +@H2TestEnv.needs_h2load() class TestLoadPostStatic: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_711_load_post_cgi.py b/test/modules/http2/test_711_load_post_cgi.py index c68a8fb4239..65064b1c8b5 100644 --- a/test/modules/http2/test_711_load_post_cgi.py +++ b/test/modules/http2/test_711_load_post_cgi.py @@ -5,7 +5,7 @@ @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") -@pytest.mark.skipif(not H2TestEnv().has_h2load(), reason="h2load not available") +@H2TestEnv.needs_h2load() class TestLoadCgi: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/http2/test_800_websockets.py b/test/modules/http2/test_800_websockets.py index 5c49ded6565..9765726082d 100644 --- a/test/modules/http2/test_800_websockets.py +++ b/test/modules/http2/test_800_websockets.py @@ -9,7 +9,7 @@ import packaging.version import pytest -import websockets +from pyhttpd.depends import needs_dependency from pyhttpd.result import ExecResult from pyhttpd.ws_util import WsFrameReader, WsFrame @@ -18,9 +18,19 @@ log = logging.getLogger(__name__) -ws_version = packaging.version.parse(websockets.version.version) ws_version_min = packaging.version.Version('10.4') +# ws_server.py, which these tests run as a subprocess, needs this too. The +# import is guarded so that an absent package is reported as the missing +# optional dependency it is, rather than failing collection of the whole +# module - which took the rest of the http2 suite's reporting with it. +try: + import websockets + ws_version = packaging.version.parse(websockets.version.version) +except ImportError: + websockets = None + ws_version = None + def ws_run(env: H2TestEnv, path, authority=None, do_input=None, inbytes=None, send_close=True, timeout=5, scenario='ws-stdin', @@ -86,8 +96,13 @@ def ws_run(env: H2TestEnv, path, authority=None, do_input=None, inbytes=None, @pytest.mark.skipif(condition=H2TestEnv.is_unsupported(), reason="mod_http2 not supported here") @pytest.mark.skipif(condition=not H2TestEnv().httpd_is_at_least("2.4.60"), reason=f'need at least httpd 2.4.60 for this') -@pytest.mark.skipif(condition=ws_version < ws_version_min, - reason=f'websockets is {ws_version}, need at least {ws_version_min}') +@needs_dependency(f"websockets >= {ws_version_min}", + ws_version is not None and ws_version >= ws_version_min, + reason=(f'websockets is {ws_version}, need at least {ws_version_min}' + if ws_version is not None + else 'no websockets python package available'), + detected=(f"websockets {ws_version}" if ws_version is not None + else "python package 'websockets' not installed")) class TestWebSockets: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/md_env.py b/test/modules/md/md_env.py index 304dfad4da3..a4d78be6384 100755 --- a/test/modules/md/md_env.py +++ b/test/modules/md/md_env.py @@ -213,16 +213,21 @@ def a2md(self, args, raw=False) -> ExecResult: def check_acme(self): if self._acme_server_ok: return True - if self._acme_server_down: - pytest.skip(msg="ACME server not running") - return False - if self.is_live(self.acme_url, timeout=timedelta(seconds=0.5)): - self._acme_server_ok = True - return True - else: + if not self._acme_server_down: + if self.is_live(self.acme_url, timeout=timedelta(seconds=0.5)): + self._acme_server_ok = True + return True self._acme_server_down = True - pytest.fail(msg="ACME server not running", pytrace=False) - return False + # Both the first test to notice and every one after it report the + # same way: a configured-but-unreachable ACME server is a missing + # dependency, so skip, or fail under --strict-optional. Previously + # the first failed and the rest skipped, and both calls passed the + # 'msg' keyword that pytest removed in 8.0, raising TypeError. + self.require(False, "ACME server not running", + dep="ACME test server", + detected=f"ACME={self.get_acme_server()}, " + f"no response from {self.acme_url}") + return False def get_ca_pem_file(self, hostname: str) -> Optional[str]: pem_file = super().get_ca_pem_file(hostname) diff --git a/test/modules/md/test_001_store.py b/test/modules/md/test_001_store.py index a315529aa4d..2748a124d1c 100644 --- a/test/modules/md/test_001_store.py +++ b/test/modules/md/test_001_store.py @@ -4,6 +4,8 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_env import MDTestEnv @@ -11,7 +13,7 @@ def md_name(md): return md['name'] -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") class TestStore: @pytest.fixture(autouse=True, scope='function') diff --git a/test/modules/md/test_010_store_migrate.py b/test/modules/md/test_010_store_migrate.py index 1872f471077..9bf66a81633 100644 --- a/test/modules/md/test_010_store_migrate.py +++ b/test/modules/md/test_010_store_migrate.py @@ -3,11 +3,13 @@ import os import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") class TestStoreMigrate: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_100_reg_add.py b/test/modules/md/test_100_reg_add.py index 1a6d3fe838e..95b3784577f 100644 --- a/test/modules/md/test_100_reg_add.py +++ b/test/modules/md/test_100_reg_add.py @@ -2,12 +2,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestRegAdd: @pytest.fixture(autouse=True, scope='function') diff --git a/test/modules/md/test_110_reg_update.py b/test/modules/md/test_110_reg_update.py index 71b50f8ea31..44a7a342138 100644 --- a/test/modules/md/test_110_reg_update.py +++ b/test/modules/md/test_110_reg_update.py @@ -2,12 +2,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestRegUpdate: NAME1 = "greenbytes2.de" diff --git a/test/modules/md/test_120_reg_list.py b/test/modules/md/test_120_reg_list.py index 82e109f723e..d0f8c3dd535 100644 --- a/test/modules/md/test_120_reg_list.py +++ b/test/modules/md/test_120_reg_list.py @@ -4,12 +4,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestRegAdd: @pytest.fixture(autouse=True, scope='function') diff --git a/test/modules/md/test_202_acmev2_regs.py b/test/modules/md/test_202_acmev2_regs.py index b99c745f2a5..4a2ff87a239 100644 --- a/test/modules/md/test_202_acmev2_regs.py +++ b/test/modules/md/test_202_acmev2_regs.py @@ -4,13 +4,16 @@ import json import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestAcmeAcc: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_300_conf_validate.py b/test/modules/md/test_300_conf_validate.py index dc65aa20690..26461c20990 100644 --- a/test/modules/md/test_300_conf_validate.py +++ b/test/modules/md/test_300_conf_validate.py @@ -7,12 +7,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestConf: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_310_conf_store.py b/test/modules/md/test_310_conf_store.py index 542fbb10840..d57108f9fa9 100644 --- a/test/modules/md/test_310_conf_store.py +++ b/test/modules/md/test_310_conf_store.py @@ -2,6 +2,8 @@ import time import pytest + +from pyhttpd.depends import needs_dependency import os from .md_conf import MDConf @@ -12,9 +14,10 @@ NS_PER_DAY = MS_PER_DAY * 1000 -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestConf: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_502_acmev2_drive.py b/test/modules/md/test_502_acmev2_drive.py index d1b47e8720f..4420c16896d 100644 --- a/test/modules/md/test_502_acmev2_drive.py +++ b/test/modules/md/test_502_acmev2_drive.py @@ -7,6 +7,8 @@ from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from .md_conf import MDConf @@ -14,9 +16,10 @@ from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestDrivev2: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_602_roundtrip.py b/test/modules/md/test_602_roundtrip.py index 05bf34bbd84..7dd497fc1a8 100644 --- a/test/modules/md/test_602_roundtrip.py +++ b/test/modules/md/test_602_roundtrip.py @@ -4,13 +4,16 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_a2md(), reason="no a2md available") -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestRoundtripv2: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_702_auto.py b/test/modules/md/test_702_auto.py index b4d7a0b0517..40ece6c790a 100644 --- a/test/modules/md/test_702_auto.py +++ b/test/modules/md/test_702_auto.py @@ -3,6 +3,8 @@ from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from pyhttpd.env import HttpdTestEnv @@ -11,8 +13,9 @@ from .md_conf import MDConf -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestAutov2: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_710_profiles.py b/test/modules/md/test_710_profiles.py index a7faad64f0b..4b66564252e 100644 --- a/test/modules/md/test_710_profiles.py +++ b/test/modules/md/test_710_profiles.py @@ -4,6 +4,8 @@ from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from pyhttpd.env import HttpdTestEnv @@ -12,8 +14,9 @@ from .md_conf import MDConf -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestProfiles: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_720_wildcard.py b/test/modules/md/test_720_wildcard.py index 1930f72fba8..5efaad5857c 100644 --- a/test/modules/md/test_720_wildcard.py +++ b/test/modules/md/test_720_wildcard.py @@ -3,12 +3,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf, MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestWildcard: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_721_dns01.py b/test/modules/md/test_721_dns01.py index 583eeba7663..ecf5eceac4e 100644 --- a/test/modules/md/test_721_dns01.py +++ b/test/modules/md/test_721_dns01.py @@ -3,12 +3,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestDns01: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_730_static.py b/test/modules/md/test_730_static.py index bc3f20ce6b3..55a32db56d2 100644 --- a/test/modules/md/test_730_static.py +++ b/test/modules/md/test_730_static.py @@ -2,14 +2,17 @@ from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestStatic: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_740_acme_errors.py b/test/modules/md/test_740_acme_errors.py index 130723ba11b..d909cc525c3 100644 --- a/test/modules/md/test_740_acme_errors.py +++ b/test/modules/md/test_740_acme_errors.py @@ -1,12 +1,15 @@ # test ACME error responses and their processing import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestAcmeErrors: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_741_setup_errors.py b/test/modules/md/test_741_setup_errors.py index d041e970772..2fbfb38866b 100644 --- a/test/modules/md/test_741_setup_errors.py +++ b/test/modules/md/test_741_setup_errors.py @@ -3,12 +3,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestSetupErrors: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_790_failover.py b/test/modules/md/test_790_failover.py index 5b9d4e2fcc6..9f0d6e2f761 100644 --- a/test/modules/md/test_790_failover.py +++ b/test/modules/md/test_790_failover.py @@ -1,11 +1,14 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_env import MDTestEnv from .md_conf import MDConf -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestFailover: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_800_must_staple.py b/test/modules/md/test_800_must_staple.py index d2d8225dd62..3b3ac2471d8 100644 --- a/test/modules/md/test_800_must_staple.py +++ b/test/modules/md/test_800_must_staple.py @@ -2,13 +2,16 @@ import time import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_cert_util import MDCertUtil from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestMustStaple: domain = None @@ -53,7 +56,9 @@ def test_md_800_002(self, env): assert 'ocsp' not in stat or stat['ocsp'] == "no response sent" # MD that must staple and toggle off again - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_800_003(self, env): self.configure_httpd(env, self.domain, "MDMustStaple on") assert env.apache_restart() == 0, f'{env.apachectl_stderr}' @@ -69,7 +74,9 @@ def test_md_800_003(self, env): assert not cert1.get_must_staple() # MD that must staple - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") @pytest.mark.skipif(MDTestEnv.get_ssl_module() != "mod_ssl", reason="only for mod_ssl") def test_md_800_004(self, env): # mod_ssl stapling is off, expect no stapling diff --git a/test/modules/md/test_801_stapling.py b/test/modules/md/test_801_stapling.py index 33485725061..e9868a70993 100644 --- a/test/modules/md/test_801_stapling.py +++ b/test/modules/md/test_801_stapling.py @@ -4,14 +4,17 @@ import time from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestStapling: @pytest.fixture(autouse=True, scope='class') @@ -113,7 +116,9 @@ def test_md_801_001(self, env): # MD with stapling on/off and mod_ssl stapling on # expect to see stapling response in all cases - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_002(self, env): md = self.mdA self.configure_httpd(env, md, ssl_stapling=True).install() @@ -183,7 +188,9 @@ def test_md_801_003(self, env): assert not stat["stapling"] # 2 MDs, md stapling on+off, ssl stapling on - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_004(self, env): md_a = self.mdA md_b = self.mdB @@ -217,7 +224,9 @@ def test_md_801_004(self, env): # MD, check that restart leaves response unchanged, reconfigure keep interval, # should remove the file on restart and get a new one - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_005(self, env): # TODO: mod_watchdog seems to have problems sometimes with fast restarts # turn stapling on, wait for it to appear in connections @@ -263,7 +272,9 @@ def test_md_801_005(self, env): # MD, check that stapling renew window works. Set a large window # that causes response to be retrieved all the time. - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_006(self, env): # turn stapling on, wait for it to appear in connections md = self.mdA @@ -302,7 +313,9 @@ def test_md_801_006(self, env): assert mtime1 != mtime3 # MD, make a MDomain with static files, check that stapling works - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_007(self, env): # turn stapling on, wait for it to appear in connections md = self.mdA @@ -331,7 +344,9 @@ def test_md_801_007(self, env): assert ocsp_file # Use certificate files in direct config, check that stapling works - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_008(self, env): # turn stapling on, wait for it to appear in connections md = self.mdA @@ -357,7 +372,9 @@ def test_md_801_008(self, env): # Turn on stapling for a certificate without OCSP responder and issuer # (certificates without issuer prevent mod_ssl asking around for stapling) - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_009(self, env): md = self.mdA domains = [md] @@ -387,7 +404,9 @@ def test_md_801_009(self, env): # Turn on stapling for an MDomain not used in any virtualhost # There was a crash in server-status in this case - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_010(self, env): env.clear_ocsp_store() md = self.mdA @@ -407,7 +426,9 @@ def test_md_801_010(self, env): # scheduling. # This checks the mistaken assert() reported in # - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_801_011(self, env): domains = [ f'test-801-011-{i}-{env.DOMAIN_SUFFIX}' for i in range(7)] self.configure_httpd(env, domains, """ diff --git a/test/modules/md/test_810_ec.py b/test/modules/md/test_810_ec.py index dba1080f63b..87487cdcfe8 100644 --- a/test/modules/md/test_810_ec.py +++ b/test/modules/md/test_810_ec.py @@ -3,12 +3,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestAutov2: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_820_locks.py b/test/modules/md/test_820_locks.py index fc55914d2d4..542e710a71e 100644 --- a/test/modules/md/test_820_locks.py +++ b/test/modules/md/test_820_locks.py @@ -1,6 +1,8 @@ import os import pytest + +from pyhttpd.depends import needs_dependency from filelock import Timeout, FileLock from .md_cert_util import MDCertUtil @@ -8,8 +10,9 @@ from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestLocks: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_900_notify.py b/test/modules/md/test_900_notify.py index 15521c5964d..4231a0b133f 100644 --- a/test/modules/md/test_900_notify.py +++ b/test/modules/md/test_900_notify.py @@ -5,12 +5,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf, MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestNotify: notify_cmd = None notify_log = None diff --git a/test/modules/md/test_901_message.py b/test/modules/md/test_901_message.py index 86475fc5717..752eeb6c9f7 100644 --- a/test/modules/md/test_901_message.py +++ b/test/modules/md/test_901_message.py @@ -5,14 +5,17 @@ import time from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestMessage: @pytest.fixture(autouse=True, scope='class') @@ -214,7 +217,9 @@ def test_md_901_011(self, env): assert nlines[0].strip() == f"['{self.mcmd}', '{self.mlog}', 'expiring', '{domain}']" # MD, check messages from stapling - @pytest.mark.skipif(MDTestEnv.lacks_ocsp(), reason="no OCSP responder") + @needs_dependency("OCSP responder", not MDTestEnv.lacks_ocsp(), + reason="no OCSP responder", + detected=f"ACME={MDTestEnv.get_acme_server()} serves no OCSP") def test_md_901_020(self, env): domain = self.test_domain domains = [domain] diff --git a/test/modules/md/test_910_cleanups.py b/test/modules/md/test_910_cleanups.py index 08578c156f1..a3a5932939a 100644 --- a/test/modules/md/test_910_cleanups.py +++ b/test/modules/md/test_910_cleanups.py @@ -4,12 +4,15 @@ import pytest +from pyhttpd.depends import needs_dependency + from .md_conf import MDConf from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestCleanups: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/md/test_920_status.py b/test/modules/md/test_920_status.py index f7dc63174a4..66a597faf79 100644 --- a/test/modules/md/test_920_status.py +++ b/test/modules/md/test_920_status.py @@ -5,6 +5,8 @@ from datetime import timedelta import pytest + +from pyhttpd.depends import needs_dependency from pyhttpd.certs import CertificateSpec from .md_conf import MDConf @@ -13,8 +15,9 @@ from .md_env import MDTestEnv -@pytest.mark.skipif(condition=not MDTestEnv.has_acme_server(), - reason="no ACME test server configured") +@needs_dependency("ACME test server", MDTestEnv.has_acme_server(), + reason="no ACME test server configured", + detected=f"ACME={MDTestEnv.get_acme_server()}") class TestStatus: @pytest.fixture(autouse=True, scope='class') diff --git a/test/modules/metadata/test_002_mime_libmagic.py b/test/modules/metadata/test_002_mime_libmagic.py index 8ba335fb7cf..a622d1ebb12 100644 --- a/test/modules/metadata/test_002_mime_libmagic.py +++ b/test/modules/metadata/test_002_mime_libmagic.py @@ -18,8 +18,9 @@ class TestMimeLibmagic: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_libmagic_module: - pytest.skip("mod_mime_libmagic is not built") + env.require(env.has_libmagic_module, "mod_mime_libmagic is not built", + dep="mod_libmagic", + detected=f"not found in {env.libexec_dir}") write_samples(os.path.join(env.server_dir, "htdocs", "test1", "libmagic")) conf = HttpdConf(env, extras={ 'base': "MimeLibmagic On", @@ -62,8 +63,9 @@ class TestMimeLibmagicCharset: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_libmagic_module: - pytest.skip("mod_mime_libmagic is not built") + env.require(env.has_libmagic_module, "mod_mime_libmagic is not built", + dep="mod_libmagic", + detected=f"not found in {env.libexec_dir}") write_samples(os.path.join(env.server_dir, "htdocs", "test1", "libmagic")) conf = HttpdConf(env, extras={ 'base': """ diff --git a/test/modules/metadata/test_003_compare.py b/test/modules/metadata/test_003_compare.py index cdc48c44f9b..93821a95805 100644 --- a/test/modules/metadata/test_003_compare.py +++ b/test/modules/metadata/test_003_compare.py @@ -46,8 +46,9 @@ class TestCompare: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_libmagic_module: - pytest.skip("mod_mime_libmagic is not built") + env.require(env.has_libmagic_module, "mod_mime_libmagic is not built", + dep="mod_libmagic", + detected=f"not found in {env.libexec_dir}") for vhost in ["test1", "test2"]: write_samples(os.path.join(env.server_dir, "htdocs", vhost, "cmp")) # No MimeMagicFile in the main server, so test2 inherits no diff --git a/test/modules/metadata/test_004_remoteip.py b/test/modules/metadata/test_004_remoteip.py index de2ce41cb8f..80f9e695ea8 100644 --- a/test/modules/metadata/test_004_remoteip.py +++ b/test/modules/metadata/test_004_remoteip.py @@ -3,12 +3,14 @@ import pytest +from pyhttpd.depends import needs_dependency + from pyhttpd.conf import HttpdConf from .env import MetadataTestEnv -@pytest.mark.skipif(condition=not MetadataTestEnv.has_shared_module("remoteip"), - reason="mod_remoteip not available") +@needs_dependency("mod_remoteip", MetadataTestEnv.has_shared_module("remoteip"), + reason="mod_remoteip not available") class TestRemoteIp: LOG_FILE = "test_remoteip.log" PEER = "127.0.0.1" diff --git a/test/modules/proxy/test_01_http.py b/test/modules/proxy/test_01_http.py index ef71b16e47f..617997f6ae1 100644 --- a/test/modules/proxy/test_01_http.py +++ b/test/modules/proxy/test_01_http.py @@ -59,8 +59,10 @@ def test_proxy_01_002(self, env, via, seen): # check that we see the document we expect there (host matching worked) # we need to explicitly provide a Host: header since mod_proxy cannot # resolve the name via DNS. - if not env.curl_is_at_least('8.0.0'): - pytest.skip(f'need at least curl v8.0.0 for this') + env.require(env.curl_is_at_least('8.0.0'), + 'need at least curl v8.0.0 for this', + dep='curl >= 8.0.0', + detected=f"curl {env.curl_version_str()}") domain = f"{via}.{env.http_tld}" r = env.curl_get(f"http://127.0.0.1:{env.http_port}/alive.json", 5, options=[ '-H', f"Host: {domain}", diff --git a/test/modules/proxy/test_02_unix.py b/test/modules/proxy/test_02_unix.py index 0c39bc9c12c..6a2fce78bc9 100644 --- a/test/modules/proxy/test_02_unix.py +++ b/test/modules/proxy/test_02_unix.py @@ -110,8 +110,10 @@ def test_proxy_02_002(self, env, via, seen): # check that we see the document we expect there (host matching worked) # we need to explicitly provide a Host: header since mod_proxy cannot # resolve the name via DNS. - if not env.curl_is_at_least('8.0.0'): - pytest.skip(f'need at least curl v8.0.0 for this') + env.require(env.curl_is_at_least('8.0.0'), + 'need at least curl v8.0.0 for this', + dep='curl >= 8.0.0', + detected=f"curl {env.curl_version_str()}") domain = f"{via}.{env.http_tld}" r = env.curl_get(f"http://127.0.0.1:{env.http_port}/alive.json", 5, options=[ '-H', f"Host: {domain}", diff --git a/test/modules/proxy/test_03_response.py b/test/modules/proxy/test_03_response.py index cce0d622357..5a51d1d7e8b 100644 --- a/test/modules/proxy/test_03_response.py +++ b/test/modules/proxy/test_03_response.py @@ -143,8 +143,9 @@ def test_proxy_03_004(self, env): # a trailing empty RemoteIPHeader token must not underflow the trim pointer def test_proxy_03_005(self, env): - if not env.has_shared_module("remoteip"): - pytest.skip("need mod_remoteip for this") + env.require(env.has_shared_module("remoteip"), "need mod_remoteip for this", + dep="mod_remoteip", + detected=f"no mod_remoteip.so in {env.libexec_dir}") r = env.curl_get(env.mkurl("http", "test1", "/forwarded"), options=[ '-H', 'X-Forwarded-For: 192.0.2.1,', diff --git a/test/modules/proxy/test_04_ajp.py b/test/modules/proxy/test_04_ajp.py index fabc8ecc838..20e42b3a0a4 100644 --- a/test/modules/proxy/test_04_ajp.py +++ b/test/modules/proxy/test_04_ajp.py @@ -55,8 +55,9 @@ class TestProxyAjp: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_shared_module("proxy_ajp"): - pytest.skip("mod_proxy_ajp not available") + env.require(env.has_shared_module("proxy_ajp"), "mod_proxy_ajp not available", + dep="mod_proxy_ajp", + detected=f"no mod_proxy_ajp.so in {env.libexec_dir}") faker = _AJPFaker("127.0.0.1", env.http_port2) faker.start() conf = HttpdConf(env) diff --git a/test/modules/proxy/test_05_uwsgi.py b/test/modules/proxy/test_05_uwsgi.py index 0b360b89ff6..04bfb12609c 100644 --- a/test/modules/proxy/test_05_uwsgi.py +++ b/test/modules/proxy/test_05_uwsgi.py @@ -33,8 +33,9 @@ class TestProxyUwsgi: @pytest.fixture(autouse=True, scope='class') def _class_scope(self, env): - if not env.has_shared_module("proxy_uwsgi"): - pytest.skip("mod_proxy_uwsgi not available") + env.require(env.has_shared_module("proxy_uwsgi"), "mod_proxy_uwsgi not available", + dep="mod_proxy_uwsgi", + detected=f"no mod_proxy_uwsgi.so in {env.libexec_dir}") faker = _UWSGIFaker("127.0.0.1", env.http_port2) faker.start() conf = HttpdConf(env) diff --git a/test/modules/ssl/conftest.py b/test/modules/ssl/conftest.py index e6b836a3747..46f8f3074bf 100644 --- a/test/modules/ssl/conftest.py +++ b/test/modules/ssl/conftest.py @@ -22,6 +22,11 @@ def env(pytestconfig) -> SSLTestEnv: logging.getLogger('').addHandler(console) logging.getLogger('').setLevel(level=level) env = SSLTestEnv(pytestconfig=pytestconfig) + # setup_httpd() raises outright when a required module is not built, + # which is neither a skip nor a useful failure. mod_ssl being absent is + # a missing dependency, so report it as one before we get there. + env.require(env.has_mod_ssl(), "no mod_ssl available", dep="mod_ssl", + detected=f"no mod_ssl.so in {env.libexec_dir}") env.setup_httpd() env.apache_access_log_clear() env.httpd_error_log.clear_log() @@ -30,8 +35,8 @@ def env(pytestconfig) -> SSLTestEnv: @pytest.fixture(autouse=True, scope="package") def require_openssl(env): - if not env.has_tool("openssl"): - pytest.skip("openssl not installed") + env.require(env.has_tool("openssl"), "openssl not installed", + dep="openssl", detected="no openssl on $PATH") @pytest.fixture(autouse=True, scope="package") diff --git a/test/modules/ssl/test_004_socache_redis.py b/test/modules/ssl/test_004_socache_redis.py index d4e3f849537..7826331b80f 100644 --- a/test/modules/ssl/test_004_socache_redis.py +++ b/test/modules/ssl/test_004_socache_redis.py @@ -4,6 +4,8 @@ import pytest +from pyhttpd.depends import needs_dependency + from pyhttpd.conf import HttpdConf from .env import SSLTestEnv @@ -43,8 +45,8 @@ def stop(self): self._socket.close() -@pytest.mark.skipif(condition=not SSLTestEnv.has_shared_module("socache_redis"), - reason="mod_socache_redis not available") +@needs_dependency("mod_socache_redis", SSLTestEnv.has_shared_module("socache_redis"), + reason="mod_socache_redis not available") class TestSocacheRedis: @pytest.fixture(autouse=True, scope='class') diff --git a/test/pyhttpd/depends.py b/test/pyhttpd/depends.py new file mode 100644 index 00000000000..c51197a3ae8 --- /dev/null +++ b/test/pyhttpd/depends.py @@ -0,0 +1,255 @@ +"""Optional-dependency gating for the httpd integration test suite. + +Many test cases need something that is not part of httpd itself: the 'curl' +or 'nghttp' clients, the 'websockets' python package, an ACME test server, +a shared module that was not built, ... By default such a test is skipped +when its dependency is missing, which is the right thing for a developer +working on an unrelated part of the tree. + +In a CI environment that is *supposed* to provide those dependencies, a skip +instead hides a broken configuration: the run stays green while the coverage +silently disappears. Strict mode exists for those environments. Enable it +with --strict-optional or PYHTTPD_STRICT_OPTIONAL=1 and a missing dependency +becomes a failure naming what was expected and what was found instead. + +Only *missing dependencies* are affected. A test skipped because it does not +apply here - wrong mpm, httpd too old, STRESS_TEST not set - keeps skipping +in strict mode. Those keep using a plain pytest.mark.skipif, and when a test +carries both kinds of gate the "does not apply" one wins. + +Usage: + + from pyhttpd.depends import needs_dependency + + @needs_dependency("a2md", MDTestEnv.has_a2md(), reason="no a2md available") + class TestStore: + ... + +Note the polarity: skipif takes "skip when True", needs_dependency takes +"available when True". + +This module is also a pytest plugin, loaded from test/conftest.py. It +deliberately imports nothing from pyhttpd.env so that it can be loaded on +its own with '-p pyhttpd.depends' by a test that has no httpd installed. +""" + +import os + +import pytest + +STRICT_ENV_VAR = "PYHTTPD_STRICT_OPTIONAL" +MARKER_NAME = "needs_dependency" + +_FALSEY = ("", "0", "false", "no", "off") + +_HOWTO = (f"To go back to skipping missing optional dependencies, drop " + f"--strict-optional and unset {STRICT_ENV_VAR}.") + +# set at collection time on items that must hard-fail in strict mode +strict_fail_key = pytest.StashKey[str]() + + +def strict_optional_enabled(config=None) -> bool: + """Whether a missing optional dependency must fail instead of skip. + + *config* may be a pytest Config, or None for an HttpdTestEnv built + outside a pytest run. The environment variable wins when set to anything + truthy, so that CI can enable strict mode without a command line. + """ + if os.environ.get(STRICT_ENV_VAR, "").strip().lower() not in _FALSEY: + return True + if config is None: + return False + # passing a default keeps this quiet when the plugin is not registered + return bool(config.getoption("strict_optional", False)) + + +def strict_failure_message(dep, expected=None, detected=None, where=None) -> str: + """Format a strict-optional failure. The one place that does so. + + The dependency name carries what was expected ("h2load >= 1.41.0"), so + *expected* - the reason the test would otherwise have skipped - is + labelled for what it is. Those reasons are worded as an absence ("no + a2md available"), which under an "expected:" label read backwards. + """ + lines = [f"[strict-optional] missing optional dependency: {dep}"] + if expected: + lines.append(f" reason: {expected}") + if detected is not None: + lines.append(f" detected: {detected}") + if where: + lines.append(f" test: {where}") + lines.append(f" {_HOWTO}") + return "\n".join(lines) + + +def needs_dependency(dep, available, reason=None, detected=None): + """Declare that a test, class or module needs optional dependency *dep*. + + :param dep: short name for reports, e.g. "a2md", "h2load >= 1.41.0" + :param available: truthy if the dependency was found. May be a zero-arg + callable, evaluated once at collection rather than at import. + :param reason: skip reason used in non-strict mode. Keep the wording of + the skipif being replaced, so that -rs output does not change. + :param detected: what was found instead, e.g. "curl 7.88.1". Only used + in the strict-mode message. May also be a zero-arg callable. + """ + return pytest.mark.needs_dependency( + dep=dep, available=available, reason=reason, detected=detected, + ) + + +def _resolve(mark): + """Return (dep, available, reason, detected) for one marker. + + Positional arguments are tolerated, so a bare + pytest.mark.needs_dependency("x", cond) works as well. + """ + if "dep" in mark.kwargs: + dep = mark.kwargs["dep"] + elif mark.args: + dep = mark.args[0] + else: + dep = "" + + if "available" in mark.kwargs: + available = mark.kwargs["available"] + elif len(mark.args) > 1: + available = mark.args[1] + else: + # a marker carrying only a name is metadata, it never gates + available = True + if callable(available): + available = available() + + detected = mark.kwargs.get("detected") + if callable(detected): + detected = detected() + + reason = mark.kwargs.get("reason") or f"missing optional dependency: {dep}" + return dep, bool(available), reason, detected + + +def apply_gate(item, missing, strict): + """Skip *item*, or arm it to fail, for the dependencies in *missing*. + + *missing* is a list of (dep, reason, detected) in the order the gates + were declared. Shared by the needs_dependency marker below and by the + whole-suite gates in test/conftest.py, so that a dependency of the + framework itself reports exactly like a dependency of a single test, + with the same "does not apply here" precedence. + + Calling this twice for one item accumulates, so a test missing two + different dependencies is told about both. + """ + if not missing: + return + skip_reason = "; ".join(r for _, r, _ in missing) + + if not strict: + item.add_marker(pytest.mark.skip(reason=skip_reason)) + return + + gate = _other_gate(item) + if gate is _GATE_ACTIVE: + # does not apply here, let pytest skip it with its own reason + return + if gate is _GATE_UNKNOWN: + # a string condition we cannot evaluate, keep the old behaviour + item.add_marker(pytest.mark.skip(reason=skip_reason)) + return + + msg = "\n\n".join( + strict_failure_message(dep, expected=reason, detected=detected, + where=item.nodeid) + for dep, reason, detected in missing) + previous = item.stash.get(strict_fail_key, None) + item.stash[strict_fail_key] = ( + msg if previous is None else f"{previous}\n\n{msg}") + + +_GATE_ACTIVE = "active" +_GATE_UNKNOWN = "unknown" + + +def _other_gate(item): + """Whether this item is already skipped by something else. + + That is the "does not apply here" case: an httpd version gate, an mpm + gate, a STRESS_TEST gate, an unconditional skip. Those keep skipping in + strict mode, so they take precedence over a missing dependency. + + Returns _GATE_ACTIVE when a skip is certain, _GATE_UNKNOWN for a string + condition only pytest can eval() in the module namespace, and None when + nothing else gates the item. This covers the marker forms the suite + actually uses, without reaching into pytest's private skipping API. + + Must be called before we inject a skip marker of our own. + """ + for mark in item.iter_markers(): + if mark.name == "skip": + return _GATE_ACTIVE + if mark.name != "skipif": + continue + if "condition" in mark.kwargs: + conditions = [mark.kwargs["condition"]] + else: + conditions = list(mark.args) + for cond in conditions: + if isinstance(cond, str): + return _GATE_UNKNOWN + if cond: + return _GATE_ACTIVE + return None + + +def pytest_addoption(parser): + group = parser.getgroup("httpd", "apache httpd test suite") + group.addoption( + "--strict-optional", + dest="strict_optional", + action="store_true", + default=False, + help="Fail instead of skip when an optional dependency declared with " + "@needs_dependency() or env.require() is missing. Also enabled " + f"by {STRICT_ENV_VAR}=1. Tests that do not apply here (mpm, " + "httpd version, STRESS_TEST) keep skipping.", + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "needs_dependency(dep, available, reason=None, detected=None): " + "declare an optional dependency. Missing means skip by default and " + "fail under --strict-optional. Use the needs_dependency() helper " + "from pyhttpd.depends rather than this marker directly.", + ) + + +def pytest_collection_modifyitems(config, items): + strict = strict_optional_enabled(config) + for item in items: + missing = [(dep, reason, detected) + for dep, ok, reason, detected + in (_resolve(m) for m in item.iter_markers(name=MARKER_NAME)) + if not ok] + if not missing: + continue + # markers arrive closest-first (function, class, module), report + # them in the order they are written instead + missing.reverse() + apply_gate(item, missing, strict) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + """Fail before any fixture runs, so that a test about to fail for a + missing dependency does not first go and start an httpd. + + Only ever armed for items carrying no active skip of their own, see + pytest_collection_modifyitems, so this cannot race pytest's own skipping. + """ + msg = item.stash.get(strict_fail_key, None) + if msg is not None: + pytest.fail(msg, pytrace=False) diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index 521c16124d7..6441ff7be46 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -305,6 +305,93 @@ def has_shared_module(cls, name): path = os.path.join(cls.LIBEXEC_DIR, f"mod_{name}.so") return os.path.isfile(path) + @classmethod + def has_mod_ssl(cls): + """Whether mod_ssl is installed. + + Not the same question as get_ssl_module(), which only reports which + TLS module $SSL selected for this run. A test that is "only for + mod_ssl" is not applicable under SSL=mod_tls and should skip; a test + that needs mod_ssl to be installed at all has a real dependency. + """ + return cls.has_shared_module("ssl") + + _nghttp_gate = {} + + @classmethod + def needs_nghttp(cls, assets: bool = False): + """A marker gating a test, class or module on the nghttp client. + + nghttp is the only client that can drive PUSH, trailers and explicit + frame padding, so the tests that use it have no fallback. It is named + unconditionally in config.ini, which says nothing about whether it is + installed, so the binary has to be run to find out. + + :param assets: also require the '-a' asset-fetching support, which + nghttp only has when built against libxml2 + """ + from .depends import needs_dependency + if assets not in cls._nghttp_gate: + env = HttpdTestEnv() + ok = env.has_nghttp_get_assets() if assets else env.has_nghttp() + if assets: + reason = "nghttp not available or misses -a (needs libxml2)" + else: + reason = "nghttp not available" + cls._nghttp_gate[assets] = needs_dependency( + "nghttp", ok, reason=reason, + detected=f"tried to run {env._nghttp}") + return cls._nghttp_gate[assets] + + _h2load_gate = {} + + @classmethod + def needs_h2load(cls, minv: str = None, reason: str = None): + """A marker gating a test, class or module on the h2load client. + + :param minv: also require at least this h2load version. 1.41.0 is + the one that matters here, it brought --connect-to. + :param reason: skip reason, when the default wording is too vague + about what the version is needed for. + """ + from .depends import needs_dependency + key = (minv, reason) + if key not in cls._h2load_gate: + env = HttpdTestEnv() + if minv is None: + ok = env.has_h2load() + dep = "h2load" + default_reason = "h2load not available" + else: + ok = env.h2load_is_at_least(minv) + dep = f"h2load >= {minv}" + default_reason = f"h2load unavailable or older than {minv}" + found = env.h2load_version_str() + cls._h2load_gate[key] = needs_dependency( + dep, ok, reason=reason or default_reason, + detected=(f"h2load {found}" if found is not None + else f"tried to run {env._h2load}")) + return cls._h2load_gate[key] + + _multipart_gate = None + + @classmethod + def needs_multipart(cls): + """A marker gating a test, class or module on the multipart package. + + The CGI helpers in htdocs/cgi/ parse file uploads with it. They + import it only for a multipart/ body, so this belongs on the tests + that upload a file, not on everything that touches a CGI. + """ + from .depends import needs_dependency + if cls._multipart_gate is None: + cls._multipart_gate = needs_dependency( + "multipart", cls.has_python_package('multipart'), + reason="no multipart python package available", + detected="python package 'multipart' not installed " + "(pip install python-multipart)") + return cls._multipart_gate + def __init__(self, pytestconfig=None): self._our_dir = os.path.dirname(inspect.getfile(Dummy)) self._config_ini = os.getenv("PYHTTPD_CONFIG", @@ -375,6 +462,10 @@ def __init__(self, pytestconfig=None): self._https_base = f"https://{self._httpd_addr}:{self.https_port}" self._verbosity = pytestconfig.option.verbose if pytestconfig is not None else 0 + # imported here, not at module scope, to keep env.py free of a + # top-level pytest dependency: it is also built outside pytest runs + from .depends import strict_optional_enabled + self._strict_optional = strict_optional_enabled(pytestconfig) self._test_conf = os.path.join(self._server_conf_dir, "test.conf") self._httpd_base_conf = [] self._httpd_env = {} @@ -394,6 +485,7 @@ def __init__(self, pytestconfig=None): self._curl_headerfiles_n = 0 self._curl_version = None self._h2load_version = None + self._nghttp_version = None self._current_test = None def add_httpd_conf(self, lines: List[str]): @@ -649,6 +741,31 @@ def current_test_name(self) -> str: def set_current_test_name(self, val) -> None: self._current_test = val + def require(self, condition: bool, reason: str, + dep: str = None, detected=None) -> None: + """Skip, or fail under strict-optional mode, on a missing dependency. + + The in-test counterpart of the @needs_dependency decorator, for + checks that can only be made once a test is running. Use it instead + of a bare pytest.skip() whenever the reason is "something we expected + to be installed is not", and a plain pytest.skip() when the reason is + "this test does not apply here". + + :param condition: truthy if the dependency is present + :param reason: skip reason, also the "reason" line when strict + :param dep: short dependency name for the strict message + :param detected: what was found instead, for the strict message + """ + import pytest + if condition: + return + if self._strict_optional: + from .depends import strict_failure_message + pytest.fail(strict_failure_message( + dep or reason, expected=reason, detected=detected, + where=self.current_test_name), pytrace=False) + pytest.skip(reason) + @property def apachectl_stderr(self) -> str: return self._apachectl_stderr @@ -678,76 +795,118 @@ def httpd_is_at_least(self, minv): hv = self._versiontuple(self.get_httpd_version()) return hv >= self._versiontuple(minv) + @staticmethod + def _run_version(args): + """Run a --version style command, returning its stdout or None. + + None means the binary is not there or would not run, which callers + must distinguish from "ran fine but is too old". config.ini/default + may just be the bare command name ("h2load"), not a verified path, + so catching OSError here is what keeps an uninstalled tool from + crashing collection with FileNotFoundError. + """ + try: + p = subprocess.run(args, capture_output=True, text=True) + except OSError: + return None + return p.stdout if p.returncode == 0 else None + + def _probe_h2load(self): + """The h2load version tuple, or None when h2load is unusable.""" + if self._h2load_version is None: + self._h2load_version = False # probed, not found + out = self._run_version([self._h2load, '--version']) if self._h2load else None + if out is not None: + m = re.match(r'h2load nghttp2/(\S+)', out.strip()) + if m: + self._h2load_version = self._versiontuple(m.group(1)) + return self._h2load_version or None + def has_h2load(self): - if self._h2load == "": - return False - # config.ini/default may just be the bare command name ("h2load"), - # not a verified path -- confirm it actually resolves so - # h2load_is_at_least() below doesn't crash with FileNotFoundError - # (breaking test collection) when the tool isn't installed. - if os.path.dirname(self._h2load): - return os.path.isfile(self._h2load) and os.access(self._h2load, os.X_OK) - return self.has_tool(self._h2load) + return self._probe_h2load() is not None + + def h2load_version_str(self): + """'1.41.0', or None when h2load is absent. For strict messages.""" + v = self._probe_h2load() + return None if v is None else ".".join(str(n) for n in v) def h2load_is_at_least(self, minv): - if not self.has_h2load(): - return False - if self._h2load_version is None: - p = subprocess.run([self._h2load, '--version'], capture_output=True, text=True) - if p.returncode != 0: - return False - s = p.stdout.strip() - m = re.match(r'h2load nghttp2/(\S+)', s) - if m: - self._h2load_version = self._versiontuple(m.group(1)) - if self._h2load_version is not None: - return self._h2load_version >= self._versiontuple(minv) - return False + v = self._probe_h2load() + return v is not None and v >= self._versiontuple(minv) - def curl_is_at_least(self, minv): + def _probe_curl(self): + """The curl version tuple, or None when curl is unusable.""" if self._curl_version is None: - p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) - if p.returncode != 0: - return False - for l in p.stdout.splitlines(): - m = re.match(r'curl ([0-9.]+)[- ].*', l) - if m: - self._curl_version = self._versiontuple(m.group(1)) - break - if self._curl_version is not None: - return self._curl_version >= self._versiontuple(minv) - return False + self._curl_version = False # probed, not found + out = self._run_version([self._curl, '-V']) if self._curl else None + if out is not None: + for l in out.splitlines(): + m = re.match(r'curl ([0-9.]+)[- ].*', l) + if m: + self._curl_version = self._versiontuple(m.group(1)) + break + return self._curl_version or None + + def has_curl(self): + return self._probe_curl() is not None + + def curl_version_str(self): + """'8.4.0', or None when curl is absent. For strict messages.""" + v = self._probe_curl() + return None if v is None else ".".join(str(n) for n in v) + + def curl_is_at_least(self, minv): + v = self._probe_curl() + return v is not None and v >= self._versiontuple(minv) def curl_is_less_than(self, version): - if self._curl_version is None: - p = subprocess.run([self._curl, '-V'], capture_output=True, text=True) - if p.returncode != 0: - return False - for l in p.stdout.splitlines(): - m = re.match(r'curl ([0-9.]+)[- ].*', l) + v = self._probe_curl() + return v is not None and v < self._versiontuple(version) + + def curl_is_8_1_x(self): + """Whether curl is one of the 8.1.x releases. + + Those mishandle TLS renegotiation; 8.0.x and 8.2.0+ are both fine. + Returns False when curl is absent, so combine it with has_curl() when + the test also needs curl to exist at all. + """ + v = self._probe_curl() + return (v is not None + and self._versiontuple('8.1.0') <= v < self._versiontuple('8.2.0')) + + def _probe_nghttp(self): + """The nghttp version tuple, or None when nghttp is unusable. + + config.ini always names a binary, so the name alone proves nothing + about whether nghttp is installed: it has to be run. 'nghttp + --version' prints 'nghttp2/1.64.0', the nghttp2 library version. + """ + if self._nghttp_version is None: + self._nghttp_version = False # probed, not found + out = self._run_version([self._nghttp, '--version']) if self._nghttp else None + if out is not None: + m = re.search(r'nghttp2/(\S+)', out.strip()) if m: - self._curl_version = self._versiontuple(m.group(1)) - break - if self._curl_version is not None: - return self._curl_version < self._versiontuple(version) - return False + self._nghttp_version = self._versiontuple(m.group(1)) + return self._nghttp_version or None def has_nghttp(self): - if self._nghttp == "": - return False - if os.path.dirname(self._nghttp): - return os.path.isfile(self._nghttp) and os.access(self._nghttp, os.X_OK) - return self.has_tool(self._nghttp) + return self._probe_nghttp() is not None + + def nghttp_version_str(self): + """'1.64.0', or None when nghttp is absent. For strict messages.""" + v = self._probe_nghttp() + return None if v is None else ".".join(str(n) for n in v) + + def nghttp_is_at_least(self, minv): + v = self._probe_nghttp() + return v is not None and v >= self._versiontuple(minv) def has_nghttp_get_assets(self): if not self.has_nghttp(): return False - args = [self._nghttp, "-a"] - p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - rv = p.returncode - if rv != 0: - return False - return p.stderr == "" + p = subprocess.run([self._nghttp, "-a"], capture_output=True, text=True) + return p.returncode == 0 and p.stderr == "" def get_apxs_var(self, name: str) -> str: p = subprocess.run([self._apxs, "-q", name], capture_output=True, text=True) diff --git a/test/test_strict_optional.py b/test/test_strict_optional.py new file mode 100644 index 00000000000..ba059697ebe --- /dev/null +++ b/test/test_strict_optional.py @@ -0,0 +1,308 @@ +"""Self-test for the optional-dependency gating in pyhttpd/depends.py. + +Runs a synthetic test file in a pytester subprocess, with and without +--strict-optional, and asserts skipped-vs-failed outcomes. The subprocess +loads the real plugin with '-p pyhttpd.depends' and nothing else, so this +needs no built httpd, no config.ini and no network. +""" +import os + +import pytest + +from pyhttpd.env import HttpdTestEnv + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + +pytest_plugins = ["pytester"] + + +class _StubErrorLog: + """Enough of HttpdErrorLog for the autouse _package_scope fixture.""" + + def clear_ignored_matches(self): + pass + + def clear_ignored_lognos(self): + pass + + def add_ignored_lognos(self, lognos): + pass + + +class _StubEnv: + """Satisfies the autouse fixtures in test/conftest.py. + + Those are written for a test that drives a real httpd; this module + drives a pytester subprocess instead, so every one of them is a no-op + here. apache_stop() must report success, _package_scope asserts on it. + """ + + httpd_error_log = _StubErrorLog() + + def set_current_test_name(self, name): + pass + + def check_error_log(self): + pass + + def apache_stop(self): + return 0 + + +@pytest.fixture(scope="package") +def env(): + return _StubEnv() + + +SYNTHETIC = """ +import pytest +from pyhttpd.depends import needs_dependency + +# dependency missing: skip by default, fail when strict +@needs_dependency("frobnicator", False, + reason="frobnicator >= 2.0 not installed", + detected="frobnicator 1.0 at /usr/bin/frobnicator") +def test_dep_missing(): + assert False, "must never run" + +# dependency present: always runs +@needs_dependency("widget", True, reason="widget not installed") +def test_dep_present(): + assert True + +# not applicable: skips in both modes +@pytest.mark.skipif(True, reason="httpd too old") +def test_not_applicable(): + assert False, "must never run" + +# both kinds of gate on one test: "not applicable" wins, even when strict +@pytest.mark.skipif(True, reason="STRESS_TEST not set in env") +@needs_dependency("frobnicator", False, reason="frobnicator not installed") +def test_not_applicable_beats_missing_dep(): + assert False, "must never run" + +# an inactive skipif must not shield a missing dependency +@pytest.mark.skipif(False, reason="httpd too old") +@needs_dependency("frobnicator", False, reason="frobnicator not installed") +def test_inactive_skipif_does_not_shield(): + assert False, "must never run" + +# two dependencies missing at once, both should be reported +@needs_dependency("OCSP responder", False, reason="no OCSP responder") +@needs_dependency("mod_ssl", False, reason="no mod_ssl available") +class TestStacked: + def test_stacked(self): + assert False, "must never run" + +# a class-level marker must reach the method +@needs_dependency("a2md", False, reason="no a2md available") +class TestClassLevel: + def test_class_level(self): + assert False, "must never run" +""" + +MODULE_LEVEL = """ +import pytest +from pyhttpd.depends import needs_dependency + +# module-level pytestmark, which a per-decorator scan would not see at all +pytestmark = [needs_dependency("nghttp", False, reason="nghttp not available")] + +def test_module_gated(): + assert False, "must never run" +""" + +LAZY = """ +from pyhttpd.depends import needs_dependency + +CALLS = [] + +def probe(): + CALLS.append(1) + return False + +@needs_dependency("lazy", probe, reason="lazy dep missing") +def test_lazy(): + assert False, "must never run" + +def test_probe_ran_once(): + assert CALLS == [1] +""" + +# test_dep_present is the only one that runs; of the rest, test_dep_missing, +# TestStacked and TestClassLevel are dependency gates and the other three +# are "not applicable" or shielded. +ALL_SKIPPED = dict(passed=1, skipped=6) +STRICT_SPLIT = dict(passed=1, skipped=2, errors=4) + + +@pytest.fixture +def synth(pytester, monkeypatch): + # never inherit strict mode from the outer run + monkeypatch.delenv("PYHTTPD_STRICT_OPTIONAL", raising=False) + # let the subprocess import pyhttpd.depends + monkeypatch.setenv("PYTHONPATH", os.pathsep.join( + [TEST_DIR, os.environ.get("PYTHONPATH", "")]).rstrip(os.pathsep)) + pytester.makepyfile(test_synthetic=SYNTHETIC, test_modlevel=MODULE_LEVEL, + test_lazy=LAZY) + return pytester + + +def _run(pytester, *args): + return pytester.runpytest_subprocess("-p", "pyhttpd.depends", + "test_synthetic.py", *args) + + +def test_flag_is_advertised(synth): + r = synth.runpytest_subprocess("-p", "pyhttpd.depends", "--help") + r.stdout.fnmatch_lines(["*--strict-optional*"]) + + +def test_default_is_off_and_missing_dep_skips(synth): + r = _run(synth) + r.assert_outcomes(**ALL_SKIPPED) + assert r.ret == 0 + + +def test_default_skip_keeps_the_original_reason(synth): + r = _run(synth, "-rs") + r.stdout.fnmatch_lines(["*frobnicator >= 2.0 not installed*"]) + r.stdout.fnmatch_lines(["*no a2md available*"]) + + +def test_strict_flag_fails_on_missing_dep(synth): + r = _run(synth, "--strict-optional") + r.assert_outcomes(**STRICT_SPLIT) + assert r.ret != 0 + r.stdout.fnmatch_lines([ + "*[[]strict-optional[]] missing optional dependency: frobnicator*", + "*reason:*frobnicator >= 2.0 not installed*", + "*detected: frobnicator 1.0 at /usr/bin/frobnicator*", + "*drop*--strict-optional*", + ]) + + +def test_strict_reports_every_missing_dep(synth): + r = _run(synth, "--strict-optional") + r.stdout.fnmatch_lines(["*missing optional dependency: OCSP responder*"]) + r.stdout.fnmatch_lines(["*missing optional dependency: mod_ssl*"]) + + +def test_not_applicable_still_skips_when_strict(synth): + r = _run(synth, "--strict-optional", "-rs") + # a plain skipif carrying no dependency marker at all + r.stdout.fnmatch_lines(["*httpd too old*"]) + # and one stacked on a missing dependency: the skipif wins, and the + # reason reported is its own, not the dependency's + r.stdout.fnmatch_lines(["*STRESS_TEST not set in env*"]) + + +def test_env_var_enables_strict(synth, monkeypatch): + monkeypatch.setenv("PYHTTPD_STRICT_OPTIONAL", "1") + r = _run(synth) + r.assert_outcomes(**STRICT_SPLIT) + assert r.ret != 0 + + +@pytest.mark.parametrize("val", ["", "0", "false", "no", "off"]) +def test_env_var_falsey_values_keep_strict_off(synth, monkeypatch, val): + monkeypatch.setenv("PYHTTPD_STRICT_OPTIONAL", val) + r = _run(synth) + r.assert_outcomes(**ALL_SKIPPED) + assert r.ret == 0 + + +def test_module_level_pytestmark_is_honoured(synth): + r = synth.runpytest_subprocess("-p", "pyhttpd.depends", "test_modlevel.py") + r.assert_outcomes(skipped=1) + r = synth.runpytest_subprocess("-p", "pyhttpd.depends", "test_modlevel.py", + "--strict-optional") + r.assert_outcomes(errors=1) + + +def test_callable_available_is_evaluated_once_at_collection(synth): + r = synth.runpytest_subprocess("-p", "pyhttpd.depends", "test_lazy.py") + r.assert_outcomes(passed=1, skipped=1) + + +# --- env.require(), the in-test counterpart of the decorator --------------- +# +# Exercised against a stand-in rather than a real HttpdTestEnv, which would +# need a built httpd and a running server just to reach the one branch that +# matters here. + +class _RequireEnv: + require = HttpdTestEnv.require + + def __init__(self, strict): + self._strict_optional = strict + self.current_test_name = "unit::require" + + +def test_require_is_a_noop_when_the_dependency_is_present(): + _RequireEnv(strict=False).require(True, "need curl") + _RequireEnv(strict=True).require(True, "need curl") + + +def test_require_skips_when_not_strict(): + with pytest.raises(BaseException) as exc: + _RequireEnv(strict=False).require( + False, "need at least curl v8.0.0 for this", + dep="curl", detected="curl 7.88.1") + assert exc.value.__class__.__name__ == "Skipped" + assert "need at least curl v8.0.0 for this" in str(exc.value.args[0]) + + +def test_require_reports_the_dependency_name_not_the_reason(): + """dep is what is missing; reason is why the test wanted it.""" + with pytest.raises(BaseException) as exc: + _RequireEnv(strict=True).require(False, "no mod_ssl available") + # no dep= given, so the reason stands in for the name + assert "missing optional dependency: no mod_ssl available" in str(exc.value.args[0]) + + +# --- the tool probes ------------------------------------------------------- +# +# config.ini always names curl/nghttp/h2load, so the name alone says nothing +# about whether the tool is installed. Probing used to raise FileNotFoundError +# out of collection; it must report "absent" instead. + +class _ProbeEnv: + """Drives the curl probe without a config.ini or a built httpd.""" + + _run_version = staticmethod(HttpdTestEnv._run_version) + _versiontuple = HttpdTestEnv._versiontuple + _probe_curl = HttpdTestEnv._probe_curl + has_curl = HttpdTestEnv.has_curl + curl_version_str = HttpdTestEnv.curl_version_str + curl_is_at_least = HttpdTestEnv.curl_is_at_least + curl_is_less_than = HttpdTestEnv.curl_is_less_than + curl_is_8_1_x = HttpdTestEnv.curl_is_8_1_x + + def __init__(self, curl): + self._curl = curl + self._curl_version = None + + +@pytest.mark.parametrize("curl", ["/nonexistent/curl", "curl-does-not-exist", ""]) +def test_absent_curl_probes_as_missing_rather_than_raising(curl): + env = _ProbeEnv(curl) + assert env.has_curl() is False + assert env.curl_version_str() is None + # every comparison answers False rather than blowing up collection + assert env.curl_is_at_least('8.0.0') is False + assert env.curl_is_less_than('8.0.0') is False + assert env.curl_is_8_1_x() is False + + +def test_require_fails_when_strict_and_names_the_dependency(): + with pytest.raises(BaseException) as exc: + _RequireEnv(strict=True).require( + False, "need at least curl v8.0.0 for this", + dep="curl", detected="curl 7.88.1") + assert exc.value.__class__.__name__ == "Failed" + msg = str(exc.value.args[0]) + assert "[strict-optional] missing optional dependency: curl" in msg + assert "reason: need at least curl v8.0.0 for this" in msg + assert "detected: curl 7.88.1" in msg + assert "unit::require" in msg