RELOPS-2402: fleetbench PSU/firmware-throttle detector (hardware fleet)

data/os/Windows.yaml

@@ -48,6 +48,14 @@ windows:
   ext_pkg_src: "https://roninpuppetassets.blob.core.windows.net/binaries"
   ext_gpg_src: "https://roninpuppetassets.blob.core.windows.net/binaries/gpg"
 
+  # Fleetbench - fleet hardware-health benchmarking collector (RELOPS-2402)
+  # Collector binary is pulled from GitHub releases; pin the version here.
+  fleetbench:
+    version: '0.4.0'
+    download_url: "https://github.com/mozilla-platform-ops/fleetbench/releases/download"
+    install_dir: "%{facts.custom_win_systemdrive}\\fleetbench"
+    results_dir: "%{facts.custom_win_systemdrive}\\fleetbench\\results"
+
   # Taskcluster resources
   taskcluster:
     root_url: "https://firefox-ci-tc.services.mozilla.com"

modules/roles_profiles/manifests/profiles/hardware_observability.pp

@@ -9,6 +9,12 @@
             server    => lookup('windows.datacenter.marlin.ip'),
             server_pw => lookup('marlin_pw')
         }
+        class { win_fleetbench::init:
+            version      => lookup('windows.fleetbench.version'),
+            download_url => lookup('windows.fleetbench.download_url'),
+            install_dir  => lookup('windows.fleetbench.install_dir'),
+            results_dir  => lookup('windows.fleetbench.results_dir'),
+        }
     }
     default: {
       fail("${$facts['os']['name']} not supported")

modules/win_fleetbench/files/fleetbench_baselines.json

@@ -0,0 +1,26 @@
+{
+  "schema_version": 1,
+  "_comment": "Known-good fleetbench `cpu` benchmark ranges per hardware type (RELOPS-2402). Deployed alongside the collector by the win_fleetbench module and read by maintainsystem-hw.ps1. To support a NEW hardware type, add an entry keyed by a short type name with a `model_match` glob (matched against Win32_ComputerSystem.Model) and its thresholds. All frequency values are % of the CPU base clock; tput_cv_pct is the prime-sieve throughput coefficient of variation (%). Collected with: fleetbench cpu --mode quick --duration 900s --json.",
+  "hardware_types": {
+    "nuc13": {
+      "model_match": "NUC13*",
+      "source": "healthy NUC13ANHi5 perf-debug nodes (i5-1340P, base 1900 MHz) at 900s, 2026-06-10; see RELOPS-2402",
+      "thresholds": {
+        "min_floor_pct": { "good_min": 75, "bad_below": 50 },
+        "mean_pct":      { "good_min": 100, "bad_below": 95 },
+        "tput_cv_pct":   { "good_max": 25, "bad_above": 40 }
+      },
+      "reference": {
+        "min_floor_pct": 84,
+        "mean_pct": 108,
+        "tput_cv_pct": 17,
+        "iterations": 221600
+      },
+      "drop_off": {
+        "mean_pct_drop": 5,
+        "min_floor_pct_drop": 10,
+        "iterations_drop_pct": 10
+      }
+    }
+  }
+}

modules/win_fleetbench/files/run_fleetbench.ps1

@@ -0,0 +1,92 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+<#
+.SYNOPSIS
+    Wrapper that runs the fleetbench collector and saves its JSON output to a
+    known results location.
+
+.DESCRIPTION
+    Invokes the fleetbench collector binary for the given suite/mode, captures
+    its JSON envelope from stdout, and writes it atomically (via a .partial
+    file + rename) into the results directory using a sortable, host-scoped
+    filename: <UTC-timestamp>_<host>_<suite>.json
+
+    Deployed by the win_fleetbench Puppet module (RELOPS-2402). Intended to be
+    invoked by a scheduled task / worker-startup hook in a later step.
+
+.PARAMETER BinaryPath
+    Full path to the fleetbench collector executable.
+
+.PARAMETER ResultsDir
+    Directory where result envelopes are written.
+
+.PARAMETER Suite
+    Collector subcommand to run (default: cpu).
+
+.PARAMETER Mode
+    Benchmark mode passed to the collector (quick|normal|long; default: normal).
+
+.PARAMETER ExtraArgs
+    Additional arguments forwarded verbatim to the collector (e.g. --duration 10m).
+#>
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory = $true)]
+    [string]$BinaryPath,
+
+    [Parameter(Mandatory = $true)]
+    [string]$ResultsDir,
+
+    [string]$Suite = 'cpu',
+
+    [string]$Mode = 'normal',
+
+    [string[]]$ExtraArgs = @()
+)
+
+$ErrorActionPreference = 'Stop'
+
+if (-not (Test-Path -LiteralPath $BinaryPath)) {
+    Write-Error "fleetbench binary not found: $BinaryPath"
+    exit 1
+}
+
+# Ensure the known results location exists.
+if (-not (Test-Path -LiteralPath $ResultsDir)) {
+    New-Item -ItemType Directory -Path $ResultsDir -Force | Out-Null
+}
+
+$timestamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH-mm-ssZ')
+$hostName  = $env:COMPUTERNAME
+$baseName  = "${timestamp}_${hostName}_${Suite}"
+$finalPath = Join-Path $ResultsDir "$baseName.json"
+$partPath  = "$finalPath.partial"
+$errPath   = Join-Path $ResultsDir "$baseName.stderr.log"
+
+$collectorArgs = @($Suite, '--mode', $Mode, '--json') + $ExtraArgs
+
+Write-Host "Running: $BinaryPath $($collectorArgs -join ' ')"
+
+# Run the collector, capturing stdout to the .partial file and stderr to a log.
+& $BinaryPath @collectorArgs 1> $partPath 2> $errPath
+$exitCode = $LASTEXITCODE
+
+if ($exitCode -eq 0 -and (Test-Path -LiteralPath $partPath) -and (Get-Item -LiteralPath $partPath).Length -gt 0) {
+    Move-Item -LiteralPath $partPath -Destination $finalPath -Force
+    # No stderr content worth keeping on success.
+    if ((Test-Path -LiteralPath $errPath) -and (Get-Item -LiteralPath $errPath).Length -eq 0) {
+        Remove-Item -LiteralPath $errPath -Force
+    }
+    Write-Host "fleetbench result written: $finalPath"
+    exit 0
+}
+else {
+    # Preserve the partial output for diagnostics on failure.
+    if (Test-Path -LiteralPath $partPath) {
+        Move-Item -LiteralPath $partPath -Destination "$finalPath.failed" -Force
+    }
+    Write-Error "fleetbench collector failed (exit $exitCode); see $errPath"
+    exit $exitCode
+}

modules/win_fleetbench/manifests/init.pp

@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Installs the fleetbench hardware-health benchmarking collector on Windows
+# hardware nodes and deploys a wrapper script that runs it and saves the
+# results to a known location. Wired in via the hardware_observability profile.
+# See https://github.com/mozilla-platform-ops/fleetbench (RELOPS-2402).
+class win_fleetbench::init (
+  String $version,
+  String $download_url,
+  String $install_dir,
+  String $results_dir,
+) {
+  $pkg       = "fleetbench-v${version}-windows-x86_64.exe"
+  $url       = "${download_url}/v${version}/${pkg}"
+  $binary    = "${install_dir}\\fleetbench-${version}.exe"
+  $wrapper   = "${install_dir}\\run_fleetbench.ps1"
+  $baselines = "${install_dir}\\fleetbench_baselines.json"
+
+  file { $install_dir:
+    ensure => directory,
+  }
+
+  # Known location where benchmark result envelopes are written.
+  file { $results_dir:
+    ensure  => directory,
+    require => File[$install_dir],
+  }
+
+  # Pull the pinned collector binary from GitHub releases. The on-disk name is
+  # version-stamped so a hiera version bump triggers a fresh download.
+  archive { 'fleetbench-collector':
+    ensure  => present,
+    source  => $url,
+    path    => $binary,
+    creates => $binary,
+    cleanup => false,
+    extract => false,
+    require => File[$install_dir],
+  }
+
+  # Wrapper that invokes the collector and writes results to $results_dir.
+  file { $wrapper:
+    ensure  => file,
+    content => file('win_fleetbench/run_fleetbench.ps1'),
+    require => File[$install_dir],
+  }
+
+  # Known-good per-hardware-type baselines, co-located with the collector. The
+  # maintain-system fleetbench check reads this for its good/bad determination.
+  file { $baselines:
+    ensure  => file,
+    content => file('win_fleetbench/fleetbench_baselines.json'),
+    require => File[$install_dir],
+  }
+}

modules/win_nsclient/files/check_fleetbench.ps1

@@ -0,0 +1,65 @@
+# scripts\check_fleetbench.ps1
+# NSClient++ external check that surfaces the latest fleetbench hardware-health
+# verdict to Marlin (Icinga2). It does NOT run the benchmark — it reads the status
+# file written by the maintain-system fleetbench check (RELOPS-2402):
+#   C:\fleetbench\results\fleetbench_status.json
+# Nagios contract: print one line "STATE - text | perfdata" and exit 0/1/2/3.
+
+$ErrorActionPreference = 'Stop'
+
+function Exit-With([int]$code, [string]$msg) {
+    Write-Output $msg
+    exit $code
+}
+
+$statusFile  = 'C:\fleetbench\results\fleetbench_status.json'
+$maxAgeHours = 168   # status older than 7 days is considered stale
+
+if (-not (Test-Path -LiteralPath $statusFile)) {
+    Exit-With 3 "UNKNOWN - no fleetbench status yet (no benchmark has run)"
+}
+
+try {
+    $s = Get-Content -LiteralPath $statusFile -Raw | ConvertFrom-Json
+}
+catch {
+    Exit-With 3 "UNKNOWN - cannot read fleetbench status: $($_.Exception.Message)"
+}
+
+# Age of the last run
+$ageH = $null
+try {
+    $ageH = [math]::Round(((Get-Date).ToUniversalTime() - ([datetime]$s.timestamp_utc).ToUniversalTime()).TotalHours, 1)
+}
+catch { }
+
+# Coerce metrics (may be null for UNKNOWN/unparseable)
+$minPct  = if ($null -ne $s.min_pct)    { [double]$s.min_pct }    else { 0 }
+$meanPct = if ($null -ne $s.mean_pct)   { [double]$s.mean_pct }   else { 0 }
+$cv      = if ($null -ne $s.tput_cv)    { [double]$s.tput_cv }    else { 0 }
+$iters   = if ($null -ne $s.iterations) { [int]$s.iterations }    else { 0 }
+
+# Numeric health code for graphing in Grafana/InfluxDB
+$health = switch ($s.verdict) { 'GOOD' { 0 } 'MARGINAL' { 1 } 'BAD' { 2 } default { 3 } }
+
+$ageVal = if ($null -ne $ageH) { $ageH } else { 0 }
+$perf = "health=$health;1;2;0;3 min_pct=$minPct;;;0;200 mean_pct=$meanPct;;;0;200 tput_cv=$cv;;;0;100 iters=$iters age_h=$ageVal"
+
+$summary = "verdict=$($s.verdict) hw=$($s.hw_type) min%base=$([math]::Round($minPct)) mean%base=$([math]::Round($meanPct)) tputCV=$([math]::Round($cv))% iters=$iters"
+if ($s.drift) { $summary += " DROP-OFF($($s.drift_note))" }
+if ($null -ne $ageH) { $summary += " age=${ageH}h" }
+
+# Stale status -> WARN regardless of last verdict
+if ($null -ne $ageH -and $ageH -gt $maxAgeHours) {
+    Exit-With 1 "WARN - fleetbench status stale - $summary | $perf"
+}
+
+switch ($s.verdict) {
+    'GOOD' {
+        if ($s.drift) { Exit-With 1 "WARN - $summary | $perf" }
+        else { Exit-With 0 "OK - $summary | $perf" }
+    }
+    'MARGINAL'    { Exit-With 1 "WARN - $summary | $perf" }
+    'BAD'         { Exit-With 2 "CRITICAL - $summary | $perf" }
+    default       { Exit-With 3 "UNKNOWN - $summary | $perf" }
+}

modules/win_nsclient/files/check_fleetbench_variance.ps1

@@ -0,0 +1,62 @@
+# scripts\check_fleetbench_variance.ps1
+# NSClient++ external check that surfaces fleetbench run-to-run VARIANCE to Marlin:
+# the latest run compared to the node's FIRST recorded run (its initial baseline).
+# Complements check_fleetbench.ps1 (which reports the absolute GOOD/BAD verdict).
+# Reads the status file written by the maintain-system fleetbench check (RELOPS-2402):
+#   C:\fleetbench\results\fleetbench_status.json
+# Nagios contract: print one line "STATE - text | perfdata" and exit 0/1/2/3.
+
+$ErrorActionPreference = 'Stop'
+
+function Exit-With([int]$code, [string]$msg) {
+    Write-Output $msg
+    exit $code
+}
+
+$statusFile  = 'C:\fleetbench\results\fleetbench_status.json'
+$maxAgeHours = 168   # status older than 7 days is considered stale
+
+if (-not (Test-Path -LiteralPath $statusFile)) {
+    Exit-With 3 "UNKNOWN - no fleetbench status yet (no benchmark has run)"
+}
+
+try {
+    $s = Get-Content -LiteralPath $statusFile -Raw | ConvertFrom-Json
+}
+catch {
+    Exit-With 3 "UNKNOWN - cannot read fleetbench status: $($_.Exception.Message)"
+}
+
+# No variance available (first run / unknown hardware / unparseable)
+if ($null -eq $s.var_min_delta) {
+    $why = if ($s.drift_note) { $s.drift_note } else { 'no_baseline' }
+    Exit-With 3 "UNKNOWN - no variance data ($why)"
+}
+
+$dMin  = [double]$s.var_min_delta
+$dMean = [double]$s.var_mean_delta
+$dIter = [double]$s.var_iter_pct
+
+$ageH = $null
+try {
+    $ageH = [math]::Round(((Get-Date).ToUniversalTime() - ([datetime]$s.timestamp_utc).ToUniversalTime()).TotalHours, 1)
+}
+catch { }
+
+$driftCode = if ($s.drift) { 1 } else { 0 }
+$ageVal = if ($null -ne $ageH) { $ageH } else { 0 }
+# Deltas are (last - first); negative = degraded vs first run.
+$perf = "variance_drift=$driftCode;1;;0;1 var_min_delta=$dMin var_mean_delta=$dMean var_iter_pct=$dIter age_h=$ageVal"
+
+$summary = "vs first run ($($s.first_run)): min%base d=$dMin mean%base d=$dMean iters d=$dIter%"
+if ($null -ne $ageH) { $summary += " age=${ageH}h" }
+
+# Stale status -> WARN
+if ($null -ne $ageH -and $ageH -gt $maxAgeHours) {
+    Exit-With 1 "WARN - fleetbench variance status stale - $summary | $perf"
+}
+
+if ($s.drift) {
+    Exit-With 1 "WARN - fleetbench variance drift - $summary | $perf"
+}
+Exit-With 0 "OK - fleetbench variance within range - $summary | $perf"

modules/win_nsclient/manifests/init.pp

@@ -64,6 +64,20 @@
     notify  => Service['nscp'],
   }
 
+  # Surfaces the latest fleetbench hardware-health verdict to Marlin (RELOPS-2402).
+  file { "${scripts_dir}\\check_fleetbench.ps1":
+    content => file('win_nsclient/check_fleetbench.ps1'),
+    require => File[$scripts_dir],
+    notify  => Service['nscp'],
+  }
+
+  # Surfaces fleetbench run-to-run variance (latest vs first run) to Marlin.
+  file { "${scripts_dir}\\check_fleetbench_variance.ps1":
+    content => file('win_nsclient/check_fleetbench_variance.ps1'),
+    require => File[$scripts_dir],
+    notify  => Service['nscp'],
+  }
+
   service { 'nscp':
     ensure  => running,
     enable  => true,

modules/win_nsclient/templates/nsclient.ini.epp

@@ -38,6 +38,8 @@ worker_bootstrap_stage = powershell.exe -ExecutionPolicy RemoteSigned -File scri
 thermal = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_thermal.ps1
 thermal_hp = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_thermal_hp.ps1
 worker_pool_id = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\worker_pool_id.ps1
+fleetbench = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_fleetbench.ps1
+fleetbench_variance = powershell.exe -NoProfile -ExecutionPolicy RemoteSigned -File scripts\check_fleetbench_variance.ps1
 
 [/settings/external scripts/alias/cpu_5s]
 alias = cpu_5s

modules/win_scheduled_tasks/files/gw_exe_check.ps1

@@ -94,6 +94,31 @@ if ($uptimeMinutes -lt 15) {
     exit
 }
 
+# RELOPS-2402: fleetbench runs a ~15-min CPU benchmark BEFORE worker-runner starts
+# (maintainsystem-hw.ps1 :: Invoke-FleetbenchCheck). During that run generic-worker is
+# intentionally not started yet and uptime can pass the 15-min grace above, which would
+# otherwise trip this watchdog into a needless reboot/PXE. Skip while a fresh benchmark
+# marker is present. Read at Machine scope so we get the value live from the registry, not
+# the (possibly stale) process env block. The marker is the run's UTC start time; a marker
+# older than the max expected run is treated as stale so a crashed/rebooted run mid-benchmark
+# cannot silently disable this check forever.
+$fleetbenchMarker = [Environment]::GetEnvironmentVariable('MOZ_FLEETBENCH_RUNNING', 'Machine')
+if ($fleetbenchMarker) {
+    $fleetbenchMaxRunMinutes = 30   # 900s benchmark + warmup/startup slack
+    [datetimeoffset] $fleetbenchStarted = [datetimeoffset]::MinValue
+    if ([datetimeoffset]::TryParse($fleetbenchMarker, [ref] $fleetbenchStarted)) {
+        $fleetbenchRunMinutes = ([datetimeoffset]::UtcNow - $fleetbenchStarted).TotalMinutes
+        if ($fleetbenchRunMinutes -ge 0 -and $fleetbenchRunMinutes -lt $fleetbenchMaxRunMinutes) {
+            Write-Log -message "Fleetbench benchmark in progress (started $($fleetbenchStarted.UtcDateTime.ToString('o')), $([math]::Round($fleetbenchRunMinutes, 1)) min ago). Skipping generic-worker check until it completes." -severity 'DEBUG'
+            exit
+        }
+        Write-Log -message "Fleetbench marker present but stale ($([math]::Round($fleetbenchRunMinutes, 1)) min old, max $fleetbenchMaxRunMinutes). Ignoring and continuing generic-worker check." -severity 'WARN'
+    }
+    else {
+        Write-Log -message "Fleetbench marker present but unparseable ('$fleetbenchMarker'). Ignoring and continuing generic-worker check." -severity 'WARN'
+    }
+}
+
 # Var set by the maintain system script
 # Check gw_initiated env var
 if ($env:gw_initiated -ne 'true') {