Add test runner telemetry and local image build

.dockerignore

@@ -0,0 +1,13 @@
+.git
+.venv*
+.tox
+.pytest_cache
+**/__pycache__
+**/*.pyc
+**/*.egg-info
+.coverage
+artifacts
+logs
+build
+dist
+.lock-verify

.git_archival.txt

@@ -0,0 +1,4 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
+ref-names: $Format:%D$

.gitattributes

@@ -0,0 +1 @@
+.git_archival.txt export-subst

.github/workflows/main.yml

@@ -48,6 +48,21 @@ jobs:
       - name: Build a binary wheel and a source tarball
         run: python3 -m build
 
+  docker-test:
+    # The container image must build and satisfy its contract
+    name: Test container image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+      - name: Build the image
+        run: make docker-build
+      - name: Test the image
+        run: make docker-test
+
   verify-lock:
     # Ensure requirements.lock stays consistent with pyproject.toml.
     name: Verify dependency lock

Dockerfile

@@ -0,0 +1,32 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM ghcr.io/eiffel-community/etos-base-test-runner:ubuntu-noble
+
+ARG ETR_REVISION
+LABEL org.opencontainers.image.revision="${ETR_REVISION}"
+
+COPY --chown=3705:3705 . /home/etos/runner-source/
+RUN test -n "$ETR_REVISION" && \
+    grep -Fx "node: $ETR_REVISION" /home/etos/runner-source/.git_archival.txt && \
+    UV_CACHE_DIR=/home/etos/.cache/uv \
+    TMPDIR=/home/etos/runner-source \
+    uv pip install --python /home/etos/.venv/bin/python \
+        --constraint /home/etos/runner-source/requirements.lock \
+        /home/etos/runner-source && \
+    rm -rf /home/etos/runner-source /home/etos/.cache/uv
+
+# Bypass the base installer; DEV and ETR_VERSION must not replace the baked package.
+ENTRYPOINT ["/home/etos/.venv/bin/python", "-m", "etos_test_runner.etr"]

Makefile

@@ -8,7 +8,19 @@ LOCK := requirements.lock
 # Scratch directory for the throwaway venvs used by lock-verify.
 LOCK_VERIFY_DIR := .lock-verify
 
-.PHONY: lock lock-verify
+.PHONY: lock lock-verify docker-build docker-test
+
+ETR_REVISION := $(shell git rev-parse HEAD)
+IMG ?= etos-test-runner:local-$(shell git rev-parse --short=12 HEAD)
+
+# Archive HEAD rather than copying a worktree's invalid .git pointer or dirty files.
+docker-build:
+	git archive --format=tar HEAD | docker build \
+		--build-arg ETR_REVISION=$(ETR_REVISION) --tag $(IMG) -
+
+docker-test:
+	ETOS_ENABLE_SENDING_LOGS=false DISABLE_EVENT_PUBLISHING=true \
+		ETR_IMAGE=$(IMG) python -m unittest tests.test_container
 
 # Regenerate the dependency lock from pyproject.toml.
 #

README.rst

@@ -21,6 +21,53 @@ Installation
    pip install etos_test_runner
 
 
+Prebuilt local container
+=======================
+
+Commit the intended changes, then run ``make docker-build``. This builds only
+``HEAD`` (including in a Git worktree), not staged or uncommitted files.
+The default image tag is ``etos-test-runner:local-<12-character-commit>``;
+override it with ``IMG=etos-test-runner:<unique-non-latest-tag>`` if needed.
+``make docker-test`` validates that image locally without an ETOS service or
+Kubernetes cluster. It requires Docker and Python, but no Python test dependencies.
+
+The Dockerfile inherits the Ubuntu Noble base runner's Python 3.13 virtual
+environment, etos user (UID/GID 3705), tools, artifact paths and separate test
+framework virtual environment. Dependencies are installed at build time using
+``requirements.lock`` as constraints. Startup runs ``etos_test_runner.etr``
+directly, forwarding arguments and ignoring runtime ``DEV`` / ``ETR_VERSION``
+installation overrides. The package's ``lib/executor.sh`` still executes tests.
+
+Git archive substitutes ``.git_archival.txt`` using ``.gitattributes`` so
+setuptools-scm derives the package version from the committed revision without
+copying ``.git``. The full commit is also recorded in the image's
+``org.opencontainers.image.revision`` label. Use ``make docker-build`` rather
+than ``docker build .``; the latter deliberately fails with unsubstituted metadata.
+Builds need access to the base image and Python dependency indexes; runner
+startup does not install the runner package. Test commands may still need
+their own dependencies and network access.
+
+Load the image with ``kind load docker-image <image> --name <cluster>`` and
+select it in each suite execution's ``environment.testRunner``. See the ETOS
+local installation guide for the complete suite workflow.
+
+
+Telemetry
+=========
+
+Set ``OTEL_EXPORTER_OTLP_ENDPOINT`` to enable tracing and correlated logging.
+Like the suite runner, initialization uses OpenTelemetry environment and process
+resource detectors with the test runner's service name and installed version. 
+
+Suite-level lifecycle spans and bounded phase/outcome metrics are retained;
+individual test executions do not emit tracing events. Metrics use the
+``OTEL_EXPORTER_OTLP_ENDPOINT`` endpoint or ``OTEL_EXPORTER_OTLP_METRICS_ENDPOINT``
+and expose no local scrape endpoint.
+
+Without an endpoint, or with ``OTEL_SDK_DISABLED=true``, tracing, metrics and
+logs are no-op.
+
+
 Contribute
 ==========
 

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "opentelemetry-exporter-otlp==1.45.0",
     "opentelemetry-semantic-conventions==0.66b0",
     "opentelemetry-instrumentation-logging==0.66b0",
+    "opentelemetry-instrumentation-system-metrics==0.66b0",
 ]
 
 [options]

requirements.lock

@@ -703,6 +703,9 @@ opentelemetry-instrumentation==0.66b0 \
 opentelemetry-instrumentation-logging==0.66b0 \
     --hash=sha256:3b580f15f8b7874e58c835230571f1a46cc156399bedd77279d7e7510f76df43 \
     --hash=sha256:e3516fd5dbff7a93bb159e72802bb8f96ab92645c5bd81488d240b6f23a6df59
+opentelemetry-instrumentation-system-metrics==0.66b0 \
+    --hash=sha256:31a395c44078dd328adf8f2acdf6f76a5510341512cacdf9f9bb523d9a98d5c2 \
+    --hash=sha256:40e83de3d09310b2d5754ee89ce4ffc75c6d8d3bda64a2df79176430651ae9dd
 opentelemetry-proto==1.45.0 \
     --hash=sha256:96ee414f24bc3f61ea8e17dc56b4348d4049d73db3eb17c6b3edf75b5b403300 \
     --hash=sha256:9731566359d7b8e1ee1e149d8e4ad6865bab965e657ad3387ec2784d16927666
@@ -852,6 +855,28 @@ protobuf==7.36.0 \
     --hash=sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37 \
     --hash=sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44 \
     --hash=sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea
+psutil==7.2.2 \
+    --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \
+    --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \
+    --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \
+    --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \
+    --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \
+    --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \
+    --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \
+    --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \
+    --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \
+    --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \
+    --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \
+    --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \
+    --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \
+    --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \
+    --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \
+    --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \
+    --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \
+    --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \
+    --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \
+    --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \
+    --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8
 pyasn1==0.6.4 \
     --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \
     --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b

src/etos_test_runner/__init__.py

@@ -22,6 +22,7 @@
 from etos_lib.logging.logger import setup_logging
 
 from etos_test_runner.lib.decrypt import decrypt
+from etos_test_runner.lib.tracing import configure_tracing
 
 try:
     VERSION = version("etos_test_runner")
@@ -35,7 +36,8 @@
 
 DEV = os.getenv("DEV", "false").lower() == "true"
 ENVIRONMENT = "development" if DEV else "production"
-setup_logging("ETOS Test Runner", VERSION, ENVIRONMENT)
+OTEL_RESOURCE = configure_tracing(VERSION)
+setup_logging("ETOS Test Runner", VERSION, ENVIRONMENT, otel_resource=OTEL_RESOURCE)
 
 # JSONTas would print all passwords as they are decrypted,
 # which is not safe, so we disable propagation on the loggers.

src/etos_test_runner/etr.py

@@ -24,6 +24,7 @@
 import pkgutil
 import signal
 import sys
+import time
 from collections import OrderedDict
 from importlib.metadata import version
 from pprint import pprint
@@ -48,6 +49,13 @@
 from etos_test_runner.lib.events import EventPublisher
 from etos_test_runner.lib.iut import Iut
 from etos_test_runner.lib.testrunner import TestRunner
+from etos_test_runner.lib.tracing import (
+    extract_launch_context,
+    flush,
+    record_error,
+    record_milestone,
+    tracer,
+)
 
 VERSION = version("etos_test_runner")
 
@@ -64,6 +72,7 @@ class ETR:
 
     def __init__(self) -> None:
         """Initialize ETOS library and configure the internal message bus."""
+        self.trace_context = extract_launch_context()
         self.etos = ETOS("ETOS Test Runner", os.getenv("HOSTNAME"), "ETOS Test Runner")
         # The internal message bus (ETOS_RABBITMQ_*) is used already before the sub suite
         # has been downloaded, e.g. for status events, so it is configured from the
@@ -243,22 +252,37 @@ def run_etr(self) -> Union[int, dict]:
                 )
             ),
         )
+        startup = tracer().start_span("etos.runner.startup", context=self.trace_context)
+        startup_started = time.monotonic()
         try:
-            _LOGGER.info("Starting ETR.")
-            sub_suite_url = self.environment_url
-            if sub_suite_url is None:
-                sub_suite_url = self.get_sub_suite_url(self.environment_id)
+            startup.add_event("runner.startup.started")
+            with tracer().start_as_current_span(
+                "etos.suite.admission", context=self.trace_context
+            ) as admission:
+                admission.add_event("suite.admitted")
+                record_milestone("suite_admission", "started")
+                _LOGGER.info("Starting ETR.")
+                sub_suite_url = self.environment_url
                 if sub_suite_url is None:
-                    raise TimeoutError(
-                        f"Could not get sub suite environment event with id {self.environment_id!r}"
-                    )
-            suite = self.download_and_load(sub_suite_url)
+                    sub_suite_url = self.get_sub_suite_url(self.environment_id)
+                    if sub_suite_url is None:
+                        raise TimeoutError("Could not get sub suite environment event.")
+            with tracer().start_as_current_span(
+                "etos.suite.load", context=self.trace_context
+            ) as load:
+                suite = self.download_and_load(sub_suite_url)
+                load.add_event("suite.loaded")
+                record_milestone("suite_load", "successful")
             self.configure_eiffel_publisher()
             FORMAT_CONFIG.identifier = self.etos.config.get("suite_id")
             self.load_plugins()
             iut = Iut(suite.iut)
-            test_runner = TestRunner(iut, self.etos)
+            test_runner = TestRunner(iut, self.etos, trace_context=self.trace_context)
+            startup.add_event("runner.startup.finished")
+            record_milestone("startup", "successful", time.monotonic() - startup_started)
         except Exception as exception:  # pylint:disable=broad-except
+            record_error(startup, exception)
+            record_milestone("startup", "error", time.monotonic() - startup_started)
             _LOGGER.exception("ETR failed to start.")
             _LOGGER.info("Publishing status event: ETOS Test Runner failed to start.")
             event_publisher.publish_v2(
@@ -273,10 +297,13 @@ def run_etr(self) -> Union[int, dict]:
                 ),
             )
             raise
+        finally:
+            startup.end()
         # test_runner.execute() will publish TestSuiteStarted and TestSuiteFinished events to manage
         # if there are any failures, no need for status events after this point.
         result = test_runner.execute()
         _LOGGER.info("ETR finished.")
+        flush()
         return result