test(api): resolve #977 #972 #971 #968 — testing & observability hardening

.github/workflows/test.yml

@@ -831,6 +831,95 @@ jobs:
           path: frontend/coverage/
           retention-days: 14
 
+  # ── #971: API integration tests with Docker Compose ─────────────────────────
+  api-integration-tests:
+    name: API Integration Tests (Compose)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo dependencies
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-api-integration-${{ hashFiles('services/api/Cargo.lock') }}
+
+      - name: Start test services
+        run: docker compose -f docker-compose.test.yml up -d --wait
+
+      - name: Run integration tests
+        working-directory: services/api
+        env:
+          TEST_DATABASE_URL: postgres://predictiq_test:predictiq_test@localhost:5433/predictiq_test
+          TEST_REDIS_URL: redis://localhost:6380
+          STELLAR_RPC_URL: http://localhost:8080
+        run: cargo test --test '*' -- --test-threads=1
+
+      - name: Tear down test services
+        if: always()
+        run: docker compose -f docker-compose.test.yml down -v
+
+  # ── #972: test-order independence ────────────────────────────────────────────
+  api-test-order-independence:
+    name: API Tests — Order Independence
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo dependencies
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-api-order-${{ hashFiles('services/api/Cargo.lock') }}
+
+      - name: Install cargo-nextest
+        run: cargo install cargo-nextest --locked
+
+      - name: Run tests with seed 1 (unit + doc tests)
+        working-directory: services/api
+        run: cargo nextest run --test-threads=1 --rand-rng-seed=1
+
+      - name: Run tests with seed 2 (different order)
+        working-directory: services/api
+        run: cargo nextest run --test-threads=1 --rand-rng-seed=2
+
+  # ── #977: property-based tests ───────────────────────────────────────────────
+  api-property-tests:
+    name: API Property-Based Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Cache cargo dependencies
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-api-proptest-${{ hashFiles('services/api/Cargo.lock') }}
+
+      - name: Run property tests (1000 cases each)
+        working-directory: services/api
+        env:
+          PROPTEST_CASES: "1000"
+        run: cargo test prop_
+
   all-tests-passed:
     name: All Tests Passed
     needs:
@@ -855,6 +944,9 @@ jobs:
       - validate-migration-rollbacks
       - api-criterion-benchmarks
       - frontend-unit-coverage
+      - api-integration-tests
+      - api-test-order-independence
+      - api-property-tests
     runs-on: ubuntu-latest
     steps:
       - name: Success

CONTRIBUTING.md

@@ -148,6 +148,63 @@ cd services/api
 cargo test
 ```
 
+### Integration Tests (API — requires backing services)
+
+Integration tests require PostgreSQL, Redis, and a Stellar RPC node.
+The easiest way to start them is via the provided Makefile target, which
+starts a Docker Compose stack, runs the tests, and tears the stack down:
+
+```bash
+make test-integration
+```
+
+You can also manage the services manually:
+
+```bash
+# Start services
+docker compose -f docker-compose.test.yml up -d --wait
+
+# Run tests
+cd services/api
+TEST_DATABASE_URL=postgres://predictiq_test:predictiq_test@localhost:5433/predictiq_test \
+TEST_REDIS_URL=redis://localhost:6380 \
+STELLAR_RPC_URL=http://localhost:8080 \
+cargo test --test '*' -- --test-threads=1
+
+# Tear down (always run, even on failure)
+docker compose -f docker-compose.test.yml down -v
+```
+
+If a previous run left the stack running, clean it up first:
+
+```bash
+make test-integration-down
+```
+
+#### Database fixture — transaction rollback
+
+Each integration test that touches the database should use the
+`with_test_transaction` helper from `tests/common/db_fixture.rs`.
+It wraps the test body in a database transaction that is **rolled back**
+at the end, so no test leaves rows that can affect subsequent tests:
+
+```rust
+use common::db_fixture::with_test_transaction;
+
+#[tokio::test]
+async fn my_test() {
+    let pool = common::db_fixture::test_pool().await;
+    with_test_transaction(&pool, |mut conn| async move {
+        // use conn for all DB operations in this test
+        sqlx::query("INSERT INTO ...").execute(&mut *conn).await.unwrap();
+        // transaction is automatically rolled back when this closure returns
+    }).await;
+}
+```
+
+Do **not** commit within the closure — the rollback guarantees a clean slate
+for the next test regardless of execution order.
+
 ### Frontend (Next.js)
 
 ```bash
@@ -223,6 +280,31 @@ make test
 
 ---
 
+### Property-Based Tests
+
+Validation logic in `src/validation.rs` is covered by property-based tests
+using [`proptest`](https://github.com/proptest-rs/proptest).  These run as
+part of `cargo test` and are gated in CI with at least **1 000 cases per
+property** via `PROPTEST_CASES=1000`.
+
+To run them locally with the same case count:
+
+```bash
+cd services/api
+PROPTEST_CASES=1000 cargo test prop_
+```
+
+When adding a new validation function, add a corresponding `proptest!` block
+that at minimum covers:
+
+- Zero-length input
+- Input longer than `MAX_LEN + 1`
+- All-whitespace strings
+- Strings containing null bytes (`\0`)
+- Strings that must pass unchanged (valid inputs)
+
+---
+
 ## Code Style
 
 ### Rust

Makefile

@@ -0,0 +1,32 @@
+.PHONY: test-integration test-integration-down test-unit help
+
+COMPOSE_TEST = docker compose -f docker-compose.test.yml
+
+TEST_DATABASE_URL = postgres://predictiq_test:predictiq_test@localhost:5433/predictiq_test
+TEST_REDIS_URL    = redis://localhost:6380
+STELLAR_RPC_URL   = http://localhost:8080
+
+##@ Testing
+
+help: ## Show this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+test-integration: ## Start backing services, run API integration tests, then tear down
+	@echo "==> Starting test services..."
+	$(COMPOSE_TEST) up -d --wait
+	@echo "==> Running integration tests..."
+	@cd services/api && \
+		TEST_DATABASE_URL=$(TEST_DATABASE_URL) \
+		TEST_REDIS_URL=$(TEST_REDIS_URL) \
+		STELLAR_RPC_URL=$(STELLAR_RPC_URL) \
+		cargo test --test '*' -- --test-threads=1; \
+	STATUS=$$?; \
+	echo "==> Tearing down test services..."; \
+	cd ../.. && $(COMPOSE_TEST) down -v; \
+	exit $$STATUS
+
+test-integration-down: ## Tear down test services (cleanup after a failed run)
+	$(COMPOSE_TEST) down -v
+
+test-unit: ## Run unit tests (no external services needed)
+	cargo test --lib --workspace

docker-compose.test.yml

@@ -0,0 +1,65 @@
+version: '3.8'
+
+# Test-only compose stack.
+# Provides isolated PostgreSQL, Redis, and a Stellar standalone network
+# for integration tests.  Do NOT use this file in production.
+#
+# Usage:
+#   docker compose -f docker-compose.test.yml up -d
+#   # ... run tests ...
+#   docker compose -f docker-compose.test.yml down -v
+
+services:
+  postgres-test:
+    image: postgres:15-alpine
+    container_name: predictiq-test-postgres
+    environment:
+      POSTGRES_USER: predictiq_test
+      POSTGRES_PASSWORD: predictiq_test
+      POSTGRES_DB: predictiq_test
+    ports:
+      - "5433:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U predictiq_test -d predictiq_test"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    tmpfs:
+      - /var/lib/postgresql/data
+    networks:
+      - predictiq-test
+
+  redis-test:
+    image: redis:7-alpine
+    container_name: predictiq-test-redis
+    ports:
+      - "6380:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+    networks:
+      - predictiq-test
+
+  stellar-rpc-stub:
+    image: stellar/quickstart:latest
+    container_name: predictiq-test-stellar
+    command: --standalone --enable-soroban-rpc
+    environment:
+      - ENABLE_SOROBAN_RPC=true
+    ports:
+      - "8000:8000"    # Horizon + Friendbot
+      - "8080:8080"    # Stellar RPC
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:8000/health || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 20
+      start_period: 30s
+    networks:
+      - predictiq-test
+
+networks:
+  predictiq-test:
+    driver: bridge

services/api/Cargo.toml

@@ -26,7 +26,8 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] }
-tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "time", "sync"] }
+tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal", "time", "sync", "net"] }
+url = "2"
 tokio-util = { version = "0.7", features = ["rt"] }
 tower = { version = "0.5", features = ["util"] }
 tower-http = { version = "0.6", features = ["trace", "cors", "compression-gzip", "compression-br"] }
@@ -52,6 +53,7 @@ ipnet = "2"
 criterion = { version = "0.5", features = ["html_reports"] }
 testcontainers = { version = "0.23", features = ["tokio"] }
 testcontainers-modules = { version = "0.11", features = ["redis", "tokio"] }
+proptest = "1"
 
 [[bench]]
 name = "api_key_auth"

services/api/TRACING.md

@@ -10,13 +10,36 @@ This service implements distributed tracing using OpenTelemetry to track request
 - Automatic span creation at service boundaries
 - Request correlation with trace IDs
 
+## Endpoint Validation
+
+`OTEL_EXPORTER_OTLP_ENDPOINT` is validated as a parseable URL at startup.
+An invalid value (e.g. `not-a-url`) causes the service to **exit immediately**
+with a clear error rather than failing silently during the first export.
+
+After the tracing subscriber is initialised, a **TCP connectivity check**
+(2-second timeout) is attempted against the configured endpoint.
+If the endpoint is unreachable the service **continues to start** but:
+
+- Logs a `WARN` message referencing the endpoint and error.
+- Increments `otel_export_errors_total{reason="unreachable"}`.
+
+Monitor that counter to detect collector outages before they cause silent
+data loss in production.
+
+## Prometheus Metrics
+
+| Metric | Labels | Description |
+|---|---|---|
+| `otel_export_errors_total` | `reason` | Export failures — `unreachable` (startup TCP check), `export_failed` (runtime) |
+
 ## Configuration
 
 Configure tracing via environment variables:
 
 ```bash
 # OTLP endpoint (leave unset to disable trace export)
-OTLP_ENDPOINT=http://localhost:4317
+# Must be a valid URL — invalid values fail the service at startup.
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
 # ── Sampling (OTel standard env vars — preferred) ────────────────────────────
 # OTEL_TRACES_SAMPLER selects the sampler strategy.
@@ -219,17 +242,20 @@ Tracing adds minimal overhead:
 
 ### Traces not appearing
 
-1. Check OTLP endpoint is reachable:
+1. Check `otel_export_errors_total` in `/metrics` — any non-zero value means
+   the startup connectivity check failed.
+
+2. Check OTLP endpoint is reachable:
 ```bash
 curl http://localhost:4317
 ```
 
-2. Check API logs for tracing initialization:
+3. Check API logs for tracing initialization:
 ```
 Distributed tracing initialized service_name="predictiq-api"
 ```
 
-3. Verify sampling rate is > 0:
+4. Verify sampling rate is > 0:
 ```bash
 echo $OTEL_TRACES_SAMPLER_ARG   # OTel standard
 echo $TRACE_SAMPLE_RATE          # legacy fallback

services/api/src/main.rs

@@ -95,6 +95,15 @@ async fn main() -> anyhow::Result<()> {
     config.validate()?;
 
     let metrics = Metrics::new()?;
+
+    // Warn at startup if the OTLP endpoint is unreachable so operators know
+    // that traces are being dropped before any export attempt is made.
+    if let Some(ref endpoint) = config.otlp_endpoint {
+        if !tracing_config::check_otlp_connectivity(endpoint).await {
+            metrics.observe_otel_export_error("unreachable");
+        }
+    }
+
     let cache = RedisCache::new(&config.redis_url).await?;
     let db = Database::new(&config.database_url, cache.clone(), metrics.clone(), &config.db_pool).await?;
     let blockchain = BlockchainClient::new(&config, cache.clone(), metrics.clone())?;