speed up integration tests

.github/workflows/ci.yaml

@@ -15,6 +15,8 @@ jobs:
   clippy:
     name: Clippy Check
     runs-on: ubuntu-latest
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -27,6 +29,14 @@ jobs:
         with:
           components: clippy
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
+
       - name: Clippy check
         run: |
           cargo clippy --manifest-path=stratum-apps/Cargo.toml --all-features -- -D warnings

.github/workflows/integration-tests.yaml

@@ -8,6 +8,9 @@ on:
 
 name: Integration Tests
 
+env:
+  CARGO_TARGET_DIR: ${{ github.workspace }}/target
+
 jobs:
   ci:
     runs-on: ${{ matrix.os }}
@@ -27,6 +30,14 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@1.88
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
+
       - name: Install capnp dependencies (Ubuntu)
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt-get install -y capnproto libcapnp-dev
@@ -48,6 +59,9 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         run: cargo binstall --no-confirm --disable-telemetry --disable-strategies quick-install --locked cargo-nextest@0.9.100
 
+      # Failure triage: re-run a single test with captured logs shown live:
+      #   RUST_LOG=debug cargo nextest run --manifest-path=integration-tests/Cargo.toml \
+      #     -E 'test(<name>)' --no-capture
       - name: Integration Tests
         run: |
-         RUST_BACKTRACE=1 RUST_LOG=debug cargo nextest run --manifest-path=integration-tests/Cargo.toml --nocapture
+          RUST_BACKTRACE=1 RUST_LOG=debug cargo nextest run --manifest-path=integration-tests/Cargo.toml

.github/workflows/lockfiles.yaml

@@ -11,6 +11,8 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
     steps:
       - uses: actions/checkout@v4
@@ -21,6 +23,14 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
+
       - name: Build with locked dependencies
         run: |
           cargo build --manifest-path=stratum-apps/Cargo.toml --all-features --locked

.github/workflows/msrv.yaml

@@ -13,12 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CARGO_INCREMENTAL: 0
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
       RUSTFLAGS: -C debuginfo=0
 
     steps:
       - uses: actions/checkout@v4
-      - uses: Swatinem/rust-cache@v1.2.0
       - uses: dtolnay/rust-toolchain@1.88
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
 
       - name: Install capnp dependencies
         run: sudo apt-get install -y capnproto libcapnp-dev

.github/workflows/semver-check.yaml

@@ -11,6 +11,8 @@ on:
 jobs:
   semver-check:
     runs-on: ubuntu-latest
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
     steps:
       - name: Checkout repository
@@ -22,21 +24,13 @@ jobs:
       - name: Install capnp dependencies
         run: sudo apt-get install -y capnproto libcapnp-dev
 
-      - name: Cache Cargo registry
-        uses: actions/cache@v4
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
         with:
-          path: ~/.cargo/registry
-          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-registry-
-
-      - name: Cache Cargo index
-        uses: actions/cache@v4
-        with:
-          path: ~/.cargo/git
-          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-index-
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y cmake

.github/workflows/tests.yaml

@@ -8,6 +8,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
 jobs:
   test:
@@ -25,23 +26,15 @@ jobs:
       - name: Install capnp dependencies (macOS)
         if: matrix.os == 'macos-latest'
         run: brew install capnp
-      - name: Cache cargo registry
-        uses: actions/cache@v4
-        with:
-          path: ~/.cargo/registry
-          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-      - name: Cache cargo index
-        uses: actions/cache@v4
-        with:
-          path: ~/.cargo/git
-          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
-      - name: Cache cargo build
-        uses: actions/cache@v4
-        with:
-          path: target
-          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
       - name: Install Rust
         uses: dtolnay/rust-toolchain@1.88
+      - name: Cache Rust build
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-bin: "false"
+          # Hash every Cargo workspace without registering the shared target more than once.
+          key: cargo-inputs-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
+          workspaces: integration-tests -> ../target
       - name: Test stratum-apps workspace
         run: cargo test --manifest-path=stratum-apps/Cargo.toml --all-features
       - name: Test pool

integration-tests/.config/nextest.toml

@@ -5,8 +5,14 @@
 # and that's a reliable indication that we shouldn't merge this PR
 retries = { backoff = "fixed", count = 3, delay = "2s" }
 
-# only run one test at a time, which allows a human-friendly experience for inspecting logs
-test-threads = 1
+# Number of test processes to run concurrently.
+#
+# "num-cpus" matches the runner (4 vCPU on ubuntu-latest, 3 vCPU on
+# macos-latest — both free-tier public repos per GitHub docs).
+#
+# For sequential human-readable logs (one test at a time), override locally:
+#     NEXTEST_TEST_THREADS=1 cargo nextest run ...
+test-threads = "num-cpus"
 
 # label as slow if a test runs for more than 60s
 # kill it after 120s

integration-tests/Cargo.toml

@@ -31,6 +31,7 @@ hex = "0.4.3"
 clap = { version = "^4.5.4", features = ["derive"] }
 serde_json = "1"
 serde = { version = "1", features = ["derive"] }
+libc = "0.2"
 
 # Direct dependencies kept only for the embedded `mining_device` module.
 # Remove this block when removing:

integration-tests/lib/mining_device/mod.rs

@@ -120,10 +120,10 @@ pub async fn connect(
                 Ok(socket) => break socket,
                 Err(e) => {
                     error!(
-                        "Failed to connect to Upstream role at {}, retrying in 5s: {}",
+                        "Failed to connect to Upstream role at {}, retrying: {}",
                         address, e
                     );
-                    tokio::time::sleep(Duration::from_secs(5)).await;
+                    tokio::time::sleep(crate::utils::CONNECT_RETRY_INTERVAL).await;
                 }
             },
             Err(_) => {

integration-tests/lib/mock_roles.rs

@@ -67,10 +67,8 @@ impl MockDownstream {
             match TcpStream::connect(upstream_address).await {
                 Ok(stream) => break stream,
                 Err(_) => {
-                    tracing::warn!(
-                        "MockDownstream: unable to connect to upstream, retrying after 1 second"
-                    );
-                    tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+                    tracing::warn!("MockDownstream: unable to connect to upstream, retrying");
+                    tokio::time::sleep(crate::utils::CONNECT_RETRY_INTERVAL).await;
                 }
             }
         })

integration-tests/lib/mod.rs

@@ -9,7 +9,6 @@ use pool_sv2::PoolSv2;
 use std::{
     convert::TryFrom,
     net::{Ipv4Addr, SocketAddr},
-    time::Duration,
 };
 use stratum_apps::{
     bitcoin_core_sv2::runtime_api::BitcoinCoreVersion,
@@ -20,7 +19,7 @@ use stratum_apps::{
 use tracing::Level;
 use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};
 use translator_sv2::TranslatorSv2;
-use utils::get_available_address;
+use utils::{ROLE_STARTUP_DELAY, get_available_address};
 
 pub mod interceptor;
 pub mod message_aggregator;
@@ -172,7 +171,7 @@ pub async fn start_pool(
     tokio::spawn(async move {
         _ = pool_clone.start().await;
     });
-    tokio::time::sleep(Duration::from_secs(1)).await;
+    tokio::time::sleep(ROLE_STARTUP_DELAY).await;
     (pool, listening_address, monitoring_address)
 }
 
@@ -360,7 +359,7 @@ pub async fn start_pool_with_jds(
     tokio::spawn(async move {
         _ = pool_clone.start().await;
     });
-    tokio::time::sleep(Duration::from_secs(1)).await;
+    tokio::time::sleep(ROLE_STARTUP_DELAY).await;
     (pool, pool_address, jds_address, monitoring_address)
 }
 
@@ -556,6 +555,9 @@ pub fn start_mining_device_sv2(
     nominal_hashrate_multiplier: Option<f32>,
     single_submit: bool,
 ) {
+    // Pin to one thread so concurrent test processes do not oversubscribe the CI runner's CPUs
+    // and make share-rate assertions flaky.
+    crate::mining_device::set_cores(1);
     tokio::spawn(async move {
         crate::mining_device::connect(
             upstream.to_string(),

integration-tests/lib/prometheus_metrics_assertions.rs

@@ -166,13 +166,16 @@ impl MonitoringApi {
 
     /// Poll `path` until the response deserialises into `T` and `predicate` returns true.
     ///
-    /// Retries every 500 ms until `timeout`. Non-2xx responses and
+    /// Retries every [`crate::utils::POLL_INTERVAL`] until `timeout`. Non-2xx responses and
     /// deserialisation failures are tolerated (the endpoint may not be ready
     /// yet — for example a `/api/v1/clients/{id}` route returns 404 until the
     /// snapshot cache first populates). On timeout, the panic message includes
     /// the path, target type, last status, and last body so CI failures are
     /// debuggable without re-running with extra logging.
-    pub async fn poll_until<T, F>(&self, path: &'static str, timeout: Duration, predicate: F) -> T
+    ///
+    /// `path` is a plain `&str` so callers can poll dynamic routes such as
+    /// `/api/v1/clients/{id}/channels`, whose state settles after the client itself appears.
+    pub async fn poll_until<T, F>(&self, path: &str, timeout: Duration, predicate: F) -> T
     where
         T: serde::de::DeserializeOwned,
         F: Fn(&T) -> bool,
@@ -205,7 +208,7 @@ impl MonitoringApi {
                     last_body,
                 );
             }
-            tokio::time::sleep(Duration::from_millis(500)).await;
+            tokio::time::sleep(crate::utils::POLL_INTERVAL).await;
         }
     }
 

integration-tests/lib/sniffer.rs

@@ -3,8 +3,8 @@ use crate::{
     message_aggregator::MessagesAggregator,
     types::MsgType,
     utils::{
-        create_downstream, create_upstream, recv_from_down_send_to_up, recv_from_up_send_to_down,
-        wait_for_client,
+        CONNECT_RETRY_INTERVAL, POLL_INTERVAL, create_downstream, create_upstream,
+        recv_from_down_send_to_up, recv_from_up_send_to_down, wait_for_client,
     },
 };
 use std::{
@@ -96,11 +96,12 @@ impl<'a> Sniffer<'a> {
                     Ok(stream) => break stream,
                     Err(_) => {
                         tracing::warn!(
-                            "Sniffer {}: unable to connect to upstream {}, retrying after 1 second",
+                            "Sniffer {}: unable to connect to upstream {}, retrying in {:?}",
                             identifier,
-                            upstream_address
+                            upstream_address,
+                            CONNECT_RETRY_INTERVAL
                         );
-                        tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
+                        tokio::time::sleep(CONNECT_RETRY_INTERVAL).await;
                     }
                 }
             })
@@ -193,8 +194,7 @@ impl<'a> Sniffer<'a> {
                 );
             }
 
-            // sleep to reduce async lock contention
-            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            tokio::time::sleep(POLL_INTERVAL).await;
         }
     }
 
@@ -257,8 +257,7 @@ impl<'a> Sniffer<'a> {
                 );
             }
 
-            // sleep to reduce async lock contention
-            tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+            tokio::time::sleep(POLL_INTERVAL).await;
         }
     }
 

integration-tests/lib/sv1_minerd/error.rs

@@ -19,6 +19,8 @@ pub enum MinerdError {
     InvalidConfiguration(String),
     /// Failed to parse hashrate from minerd benchmark output
     HashrateParseError,
+    /// Timed out before minerd produced a benchmark hashrate
+    HashrateMeasurementTimeout(std::time::Duration),
     /// Mutex was poisoned
     MutexPoisoned,
     /// OS or Architecture not supported
@@ -38,6 +40,12 @@ impl fmt::Display for MinerdError {
             MinerdError::HashrateParseError => {
                 write!(f, "Failed to parse hashrate from minerd benchmark output")
             }
+            MinerdError::HashrateMeasurementTimeout(timeout) => {
+                write!(
+                    f,
+                    "Timed out after {timeout:?} waiting for minerd benchmark output"
+                )
+            }
             MinerdError::MutexPoisoned => write!(f, "Mutex was poisoned"),
             MinerdError::OsArchNotSupported(msg) => {
                 write!(f, "OS or architecture not supported: {msg}")
@@ -57,6 +65,7 @@ impl std::error::Error for MinerdError {
             | MinerdError::ProcessNotRunning
             | MinerdError::InvalidConfiguration(_)
             | MinerdError::HashrateParseError
+            | MinerdError::HashrateMeasurementTimeout(_)
             | MinerdError::MutexPoisoned => None,
             MinerdError::OsArchNotSupported(_) => None,
         }