Detect wedged vGPU VFs from the guest and report to the health store

cmd/api/main.go

@@ -654,6 +654,12 @@ func run() error {
 			return app.HealthCheckController.Run(gctx)
 		})
 	}
+	if app.VGPUSentinelController != nil {
+		grp.Go(func() error {
+			logger.Info("starting vGPU sentinel controller")
+			return app.VGPUSentinelController.Run(gctx)
+		})
+	}
 	if restartController, ok := app.InstanceManager.(interface {
 		StartRestartPolicyController(context.Context) error
 	}); ok {

cmd/api/wire.go

@@ -29,26 +29,27 @@ import (
 
 // application struct to hold initialized components
 type application struct {
-	Ctx                   context.Context
-	Logger                *slog.Logger
-	Config                *config.Config
-	ImageManager          images.Manager
-	SystemManager         system.Manager
-	NetworkManager        network.Manager
-	DeviceManager         devices.Manager
-	InstanceManager       instances.Manager
-	VolumeManager         volumes.Manager
-	BuilderManager        builders.Manager
-	IngressManager        ingress.Manager
-	BuildManager          builds.Manager
-	PushManager           imagepush.Manager
-	ResourceManager       *resources.Manager
-	GuestMemoryController guestmemory.Controller
-	AutoStandbyController *autostandby.Controller
-	HealthCheckController *instances.HealthCheckController
-	VMMetricsManager      *vm_metrics.Manager
-	Registry              *registry.Registry
-	ApiService            *api.ApiService
+	Ctx                    context.Context
+	Logger                 *slog.Logger
+	Config                 *config.Config
+	ImageManager           images.Manager
+	SystemManager          system.Manager
+	NetworkManager         network.Manager
+	DeviceManager          devices.Manager
+	InstanceManager        instances.Manager
+	VolumeManager          volumes.Manager
+	BuilderManager         builders.Manager
+	IngressManager         ingress.Manager
+	BuildManager           builds.Manager
+	PushManager            imagepush.Manager
+	ResourceManager        *resources.Manager
+	GuestMemoryController  guestmemory.Controller
+	AutoStandbyController  *autostandby.Controller
+	HealthCheckController  *instances.HealthCheckController
+	VGPUSentinelController *instances.VGPUSentinelController
+	VMMetricsManager       *vm_metrics.Manager
+	Registry               *registry.Registry
+	ApiService             *api.ApiService
 }
 
 // initializeApp is the injector function
@@ -72,6 +73,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideGuestMemoryController,
 		providers.ProvideAutoStandbyController,
 		providers.ProvideHealthCheckController,
+		providers.ProvideVGPUSentinelController,
 		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,

lib/devices/GPU.md

@@ -99,7 +99,7 @@ Instance Create → Assign profile to VF → Attach VF to VM → Instance Runnin
 Instance Stop/Delete → Release profile → VF available again
 ```
 
-Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free.
+Hypeman reconciles orphaned assignments with a periodic fail-closed pass: once at startup and every minute afterward (skipped entirely on hosts without GPUs). Each pass releases assignments whose owning instance is no longer live and clears their metadata, then sweeps device-level leftovers with no live metadata claim. Devices held open by a running VMM and assignments younger than five minutes are preserved, so a release that fails during stop or delete (typically because a GPU-busy VMM's kernel-side VFIO teardown outlives the force-kill wait) is simply retried on later passes until the device is free. An ambiguous hypervisor ownership check also preserves the assignment, logs a warning, and increments `hypeman_instances_vgpu_reconcile_liveness_uncertain_total`.
 
 ### Hypervisor Support
 
@@ -290,29 +290,68 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)
 (0x65 = timeout; the guest's init requests are never answered, and
 `/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle).
 
-Hypeman tracks these failures in `<data-dir>/gpu/vf-health.json` (it survives
-restarts): each reported init failure is tallied per instance assignment, and
-once failures accumulate from `gpu.vf_quarantine_threshold` distinct
-assignments (default 2), the VF is quarantined: excluded from placement and
-from advertised profile availability, and its parent GPU becomes
-overflow-only — deprioritized for new placements. Selection among a card's
-equivalent free VFs is randomized so a wedged VF cannot capture every
-placement. A reported init success clears failures only when that exact
-assignment has a recorded failure, removing the match and older tallies; if
-that assignment crossed the threshold, its later success also rescinds the
-quarantine. If the state file exists but cannot be loaded, or the last write
-to it failed, placement and advertised availability fail closed until a load
-or write succeeds. Recorded tallies are re-evaluated against the configured
-threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs
-whose persisted failures already meet the new value.
+Hypeman detects this automatically: the guest agent watches the guest kernel
+log (`/dev/kmsg`) for that line and records it as its GPU init state, which
+the vGPU sentinel controller polls over vsock (`GetGPUInitStatus`) for every
+vendor VFIO instance whose VMM is up (control socket present). Stopped and
+standby instances are not polled, even when a failed release retains their
+assignment metadata — QEMU vsock dials by guest CID alone, and a stale CID
+could since have been reused by an unrelated instance. A guest-reported
+failure records one init failure against the VF in
+`<data-dir>/gpu/vf-health.json` (it survives restarts),
+tallied per instance assignment; once failures accumulate from
+`gpu.vf_quarantine_threshold` distinct assignments (default 2), the VF is
+quarantined: excluded from placement and from advertised profile availability,
+and its parent GPU becomes overflow-only — deprioritized for new placements.
+Selection among a card's equivalent free VFs is randomized so a wedged VF
+cannot capture every placement. The guest agent also probes driver init at boot
+with `nvidia-smi -L` (when present in the image): the device open runs
+RmInitAdapter, so on a wedged VF the probe itself triggers the failure line
+without waiting for the workload to touch the GPU. On success the reported
+state becomes a terminal OK, suppressing later failure reports. An OK state
+clears failures only when that exact assignment has a recorded failure,
+removing the match and older tallies. If that assignment crossed the threshold,
+its later success also rescinds the quarantine. A success with no exact match
+clears nothing; other quarantines require manual recovery. If the state file
+exists but cannot be loaded, or the last write to it failed, placement and
+advertised availability fail closed until a load or write succeeds. The
+sentinel makes one repair attempt before each poll; individual guest reports do
+not retry the full-store write. Recorded tallies are re-evaluated against the
+configured threshold at load, so lowering
+`gpu.vf_quarantine_threshold` quarantines VFs whose persisted failures already
+meet the new value.
 
 `used_slots` includes quarantined VFs still held by running instances, so it
 can overlap `quarantined_slots`; use `allocatable_slots` for admission.
-
-Quarantine only removes capacity — it never touches a running instance.
-
-The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
-crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
+Quarantine only removes a VF from future placement: it never detaches the VF
+or otherwise affects a running instance.
+
+Below-threshold failures log at warn and increment
+`hypeman_instances_vgpu_sentinel_init_failures_total`; quarantines log at
+error and increment `hypeman_instances_vgpu_sentinel_quarantines_total`.
+`hypeman_instances_vgpu_sentinel_checks_total` records checks by result
+(`ok`, `failed`, `unknown`, `rpc_error`, `unsupported_agent`, or `list_error`)
+so hosts that lose sentinel coverage are visible. `unsupported_agent` means a
+running instance has a guest agent from before the status RPC was introduced;
+it is expected while those instances drain during an upgrade.
+`hypeman_instances_vgpu_quarantined_vfs` gauges the current count.
+`hypeman_instances_vgpu_vf_health_store_unavailable` is 1 while persisted
+health state cannot be loaded or the last write failed (and placement is
+therefore disabled), and 0 otherwise. A systemic guest/host driver mismatch
+can still quarantine every VF, so validate driver changes on a test host and
+alert on the failure counter.
+
+Detection requires the hypeman guest agent and a running instance: the state
+lives in the agent, so a wedge whose instance stops before the next poll (5s)
+is detected on the next boot that lands on the VF. The state travels only
+over the vsock control channel — the serial console is shared with workload
+output, so nothing a workload prints can influence the tally.
+
+The wedge-creating kill itself leaves no host-side log: no kernel error, no
+XID, no plugin crash. Detection therefore happens on the next boot that lands
+on the VF, whose guest driver starts failing ~27s after spawn.
+
+The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
 still initializing the VF (roughly the first seconds after process start):
 a single hard kill in that window wedges the VF near-deterministically,
 while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM —
@@ -360,7 +399,9 @@ systemctl start nvidia-dcgm nvidia-dcgm-exporter
 ```
 
 After the cycle, remove the card's entries from `vf-health.json`, restart,
-and boot a GPU instance to verify recovery.
+and boot a GPU instance to verify recovery. If the cycle did not work, the
+sentinel quarantines the VF again after the configured number of fresh
+assignment failures.
 
 Do not unbind/rebind the VF from the nvidia driver — it breaks the
 nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`)

lib/devices/vf_health.go

@@ -270,6 +270,20 @@ func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) {
 	return vfHealth.reportSuccess(report)
 }
 
+// RepairVFHealthStore retries a failed load or persist. It serializes with
+// vendor-VFIO placement and health mutations.
+func RepairVFHealthStore() error {
+	vendorVFIOMu.Lock()
+	defer vendorVFIOMu.Unlock()
+
+	vfHealth.mu.Lock()
+	defer vfHealth.mu.Unlock()
+	if err := vfHealth.ensureLoadedLocked(); err != nil {
+		return err
+	}
+	return vfHealth.retryPersistLocked()
+}
+
 // VFHealthStoreUnavailable reports whether persisted state failed to load or
 // the last write failed.
 func VFHealthStoreUnavailable() bool {
@@ -309,8 +323,8 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul
 	if !vfHealthAddressPattern.MatchString(report.VFAddress) {
 		return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress)
 	}
-	if err := s.retryPersistLocked(); err != nil {
-		return VFReportResult{}, err
+	if s.persistErr != nil {
+		return VFReportResult{}, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr)
 	}
 
 	previous, existed := s.records[report.VFAddress]
@@ -367,9 +381,6 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu
 	if !vfHealthAddressPattern.MatchString(report.VFAddress) {
 		return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress)
 	}
-	if err := s.retryPersistLocked(); err != nil {
-		return VFSuccessResult{}, err
-	}
 	previous, ok := s.records[report.VFAddress]
 	if !ok || len(previous.Failures) == 0 {
 		return VFSuccessResult{}, nil
@@ -385,6 +396,9 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu
 	if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) {
 		return VFSuccessResult{}, nil
 	}
+	if s.persistErr != nil {
+		return VFSuccessResult{}, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr)
+	}
 
 	remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...)
 	result := VFSuccessResult{