From 5cc0932fbcd6b9f2c8df19a86a9ed5935e617bf7 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:23:35 +0000 Subject: [PATCH 01/21] Quarantine unhealthy vGPU VFs via a persisted health store Add a VF health store persisted at /gpu/vf-health.json: init failures reported against a VF are tallied per instance assignment, and once failures accumulate from gpu.vf_quarantine_threshold distinct assignments (default 2) the VF is quarantined. Quarantined VFs are excluded from placement and advertised profile availability, cards with quarantined VFs are deprioritized, and selection among equivalent free VFs is randomized. An exact-assignment success report clears the match and older tallies and rescinds that assignment's quarantine. An unreadable or invalid state file fails closed: mutations are refused, placement and advertised availability are disabled, and loads are retried after repair. Writes fsync before and after the rename. GET /resources reports allocatable_slots and quarantined_slots, and GPU admission gates on the allocatable count. GPU.md documents the store semantics, draining the parent GPU, the SR-IOV recovery cycle, and clearing quarantine state. (cherry picked from commit f0b0e11d0199158215c5b90d4fe690295c8cc973) --- cmd/api/api/resources.go | 8 +- cmd/api/config/config.go | 9 +- cmd/api/config/config_test.go | 12 + cmd/api/main.go | 1 + config.example.yaml | 6 + lib/devices/GPU.md | 65 +++- lib/devices/manager.go | 4 + lib/devices/vendor_vfio_linux.go | 32 +- lib/devices/vendor_vfio_linux_test.go | 42 +++ lib/devices/vf_health.go | 416 +++++++++++++++++++++ lib/devices/vf_health_test.go | 408 +++++++++++++++++++++ lib/oapi/oapi.go | 503 +++++++++++++------------- lib/paths/paths.go | 5 + lib/resources/gpu.go | 53 +-- lib/resources/gpu_test.go | 88 +++++ lib/resources/monitoring_test.go | 4 +- lib/resources/resource.go | 17 +- openapi.yaml | 12 +- 18 files changed, 1382 insertions(+), 303 deletions(-) create mode 100644 lib/devices/vf_health.go create mode 100644 lib/devices/vf_health_test.go create mode 100644 lib/resources/gpu_test.go diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index 6084ed52f..3b0ec54e1 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index f06af1983..bb7b3f320 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,10 +288,25 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment crossed the threshold, its later success also rescinds the +quarantine. If the state file exists but cannot be loaded, placement and +advertised availability fail closed until it is repaired or removed. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Quarantine only removes capacity — it never touches a running instance. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is @@ -303,18 +324,44 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 7b047d5d9..a871ecacf 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -12,7 +12,6 @@ import ( "sort" "strconv" "strings" - "sync" "syscall" "github.com/kernel/hypeman/lib/logger" @@ -30,14 +29,11 @@ type vendorVFIOSysfs struct { openVFIOPathsFunc func() (map[string]struct{}, error) } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -99,6 +95,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // available_instances. This is a best-effort snapshot because creating on one // VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } profilesByType := make(map[string]VGPUProfileType) creatableVFs := make(map[string]int) profilesByVF, err := s.profileTypes(vfs) @@ -106,9 +106,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro return nil, err } for _, vf := range vfs { + _, bad := quarantined[vf.PCIAddress] for _, profile := range profilesByVF[vf.PCIAddress] { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -159,6 +160,17 @@ func (s vendorVFIOSysfs) configure(ctx context.Context, vfAddress, profileType s if profileType == "" || profileType == "0" { return fmt.Errorf("invalid vendor VFIO vGPU profile type %q", profileType) } + // Placement filters quarantined VFs from a snapshot taken outside this + // lock. Re-checking here, under the lock quarantine mutations take, + // closes the window where a VF is quarantined between selection and + // configuration. + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return err + } + if _, bad := quarantined[vfAddress]; bad { + return fmt.Errorf("vendor VFIO vGPU on VF %s is quarantined", vfAddress) + } currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") currentType, err := readCurrentVGPUType(currentTypePath) if err != nil { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index dc88951de..647002bf9 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -178,3 +178,45 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOConfigureRefusesQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes) + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorContains(t, err, "is quarantined") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOConfigureFailsClosedWhenVFHealthUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "0", testCreatableTypes) + + err := sysfs.configure(context.Background(), vfAddress, "1148") + require.ErrorContains(t, err, "VF health state unavailable") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..b47ff11a5 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,416 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so placement + // cannot select a VF while it is being quarantined. + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + if err := s.persistLocked(); err != nil { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + return VFSuccessResult{}, nil + } + + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + if err := s.persistLocked(); err != nil { + s.records[report.VFAddress] = previous + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) persistLocked() error { + if s.path == "" { + return nil + } + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return fmt.Errorf("marshal VF health state: %w", err) + } + if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return fmt.Errorf("rename VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + dir, err := os.Open(dirPath) + if err != nil { + slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) + return nil + } + if err := dir.Sync(); err != nil { + slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + } + _ = dir.Close() + return nil +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..dd286c296 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,408 @@ +package devices + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { + resetVFHealthStore(t) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + + for i, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, i+1, result.Failures) + assert.Equal(t, 3, result.Threshold) + } + + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + addresses, err := vfHealth.checkedAddresses() + require.NoError(t, err) + assert.Contains(t, addresses, "0000:e3:00.4") +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 7242d5503..afd5e03de 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1019,6 +1019,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1028,10 +1031,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -19370,252 +19376,255 @@ var swaggerSpec = []string{ "bf/VRM1r8gOJjTITPZTyuFbjyb7f7iJ6dfrutGlMRdUl5I9uaU4r4H2Wo10mOTN1nzybCrb1GC3imsHj", "MZ310Y9Eqh6ZTLhQ+zaQBRxSXs1XvECCxLmmBx+ajDMk6TiBM1r0qgV+/YueEACBjPPJhIgq6sCLkCzt", "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", - "hQzYch/9WhTW0kstS1SlfsBSWEfua0gkO50tJI1wYlo0VTEo8w18cBpaS+in5YfWFBqQ08P12NzJQxvz", - "aZbDuT972zt+834rjcm8WxkTBPLNeEL0uDc99jR3aAZl1luFK82bLC2GMGTbE+utVcEyWi+SxyACq6O4", - "wslIJjwUM3SuHyJ4iDbe/2TSkfUIuiirbKX+3Yca8un7afDEAIJ9Q7dn0GHdZFs54EHdtQ6B1qlOr9Jp", - "6KiYVKplGWu5GCK/rG40v1xfgM800tzvocv2qhnVrSCLTFKYwWt0YRDIfGqs5tY0I0mGBVYkWdTiMauA", - "6mTZo02uSXSDdLOX+vVPpohGLshIzQSRM55U4yB2u8uFWCXEHM+JrT1l5uQZ/hVHKRaXcBM7QR7lzKxA", - "NWR9dx2+zEyp7AaT+vn8/NRo94qIOU7qSQ9yycN/RBK8QGOirghhbipYIuzHvNaTRmVD/R+hRhkRlFfX", - "sLMb6PfMxEGjqcARQeYrV8bZbomE+KK2S2l7CSAWRhGRsmF/t1ftr/10kift9jg0rO21Vc+jm2zw+eGp", - "q2NT1Al2y7yzvMqnRPTMkXMFg1dv7Y5cXVHJdcUMEO2SwDAmUGLJZpr4iaAuvglKSOnPK8mXHnOQfrqg", - "7QdOgVmqrjnnH1pJl/XjHnLkp5jFoXrLJsHApOZPAa8LopxFDqIfjU3wk0ELMHqAnychCI4pI1LW8pqj", - "XCSdbqc3sbPa39pKeIQTgCzc291+vrU6jHRl/LANlxrFdJV+6YKqTNiNy6I1eG8w6SpJbOEsa2GBM+u4", - "5n4A9rQcrwg1lvXd5kl4LoB9MFhK0b7GkXJV4cAkV3G5Yv/YArBwZT7QYGoKvWhR+4V/PgdBH3iG1axK", - "/ltLtA9xMRDTqGnEmDtq62iI/GNQouIiVPqLC2Uz/sbEBT0W96ELKXQQuhXn3OC5P8unT57sPlnHh4DZ", - "1I65PXeBqZq3qznNxFtue35tAxCeVZU53JFeXb1Ur8samtIccYmkVi8ozwi70Xo+2dvdudl6tp3IsQsL", - "q/GlECTM4cmRkYkizhSmjAiUEoVjrHCVyYAtS3MZqC2DSQppQZPvV7OWhvgJH+PltoWxvpT3vaFG3lsH", - "CZ1iRieaIds3/Z7lDO88ebpvKnnGZLL35Gm/378p8sXLEuqi1VZsmSA9DwSjL2eftw93AHDRZi5/dE4P", - "zn/WjCyXwlxaW3JM2b737+Kf5QP4w/xzTFkYGKNN8Vc6WSr6Wo1Hyy3yMIn3UVnf28k9beKDGozREJ0M", - "aDxBmLlKlObd4ckVNE6rZQNukBu8IldWiytvWLJoXJtb13EtK5wrr36rn23WopYr/bjav+7MXfCO7dMA", - "UBdlbpc967cqVCxX1nJcKvWVEVZUb0wS81fEGQD7hko5Vq5I96xFJTC4RmzJr6JL/8eid+/HQ38g3u+u", - "kpj3k63p+OGGITErBdK/Lcuh67mQE0fXHOaw7bG4FdrWz7U4dMFY8Ae+C28TNlbt/c30v37/P/L02d+3", - "f3/9/v1/z1/919Gv9L/fJ6dvPgvjZDUE4YPiCH4x6EBTE97HD2xLSidYRQEbnVb/GlbYPjEWBxXNoOIn", - "GpP9Ieuh11QRYerH1ZIfhx20QUBTgq+0uAvFcUze2ab++NR4NPXHfzgx+FO9jdgmpQu7IQXWiMzHMU8x", - "ZZtDNmS2LeQmIkEv0H/FKMKZKQJHGdL67wKNBVTssy6msvMu+gNn2afNIbNV+g2adoah6NmkyPpizkFs", - "R2XCYO3rpICdMBmJQ1bc1gUGn/Ez9kvAfkqSes5Qw6Ks1t+s5vR8EEIrhHwWvZFQlAZUkIKyNRkViTbo", - "+WBzWZ9bo2MUNLSC/Kwj3iQ8HuQhc3FTkuQRiWkEfMXlCc5sJmmRomkozRrxMsGvF7A3b03yWoxwrmaa", - "F0U2sT7i/JKSLmxpF9xhEPkBXxp//oxnvfGiN+NZAbKAhYl2wcYjXlWy/0/PTrT3ngg6sT0Fc+U1iQSE", - "TjgydmYm5bCwLixN7NzU82Ba9JkT+7opDCNNLRATyq1ywVy5CgJFKwHNo6A+EpLJv0dRQsHqJGc8T2I0", - "A8A9pZsJYeZ1BkXKDB5HMZnU/10Nadh58hQ0WPfv3Z3WGatm6VZRWZ4EdNrUsb4WHNuwSRiAEQ9GzhC+", - "JghJ34DWjwt2CsXhv2fINVSeuIKRGO+UyaGTtvpBIr3suc1gmpM9BhYyZITtaWpzHy2dwkrWVIsWTHQC", - "fJa0ACt5abI1z1+fIUVE6vLnNyK9O3BKDDJFj0qZ20JcB4cnLzf7nSDQUsVVBVu1MquqOugA1oKNVmgK", - "wihtNDglXXR8BNmy9lopdTFIb/iJC5SYW7G8jPYBrKNq7sGmhN/xkRVAk0UZ8mDElmFn07WY1a+3ffS2", - "UAFxMZQi77GkLddkeZlAszYAzuReLLVeS5MF/5hV/+x9DJkWUN/Q8GIAi2y8v9rbHB30lL6oahayG19I", - "fhRKo/3L2/svDav85WX03ZvJ6NYHPMpmWIaoe+Z7NeGlpX333chVdi+aY4Yq/Y4kDZ6tv7kqL941pIi+", - "5yqfh3BEn/S2t8+3925uvrspIm4VCsuDyStAcduj2d4FKmwA45WqUWNwOdKPbSi5s4u8P0EzLNl3Ch7W", - "rCPbu8/aGCWg17Zh2X5ANp+YIRVcyuFqFeHEBmHskiaJEWAknTKcoBdo4+z41S/Hr19voh568+akvhWr", - "vgjuzy3AceEWgHU0GWYBaKUKJAAqcgfPz1/D4UoIpF8YOfzy9pC5a02LLSB03eBenb4Dxz+WIxe42Zyr", - "iMt8X3JNpZLLqGqt4p8/B7LXfNquyL+bpGmjrPW/Gvf35wowbRAmb/MOAHtd8PrScj4Alu1DJgl+fTi6", - "K5FvPxe+1toZ7gi9tvFKCyG/1hARnjTdbrfHob2T4VQAZUJsy5dwXAb3rYFfux0ayF49kPriITE6Pi0r", - "PZXOCNd8bU4vdvrbT59DsdLtQRvGnuJoRd8nB4ftOx/smFtmH4/3o3gfFPbb+qwsYRsVBCdXeAHF/szS", - "DjvmwvS0W+/YWkWyVXzNMr7u7eB062JcA2AuiLMucEmO0pW1RlqkJ9ZB09LcQkKmNEmoJBFnsazKyDMs", - "kcwMEqqpuVFI8EMGA+yiovQxSCkIR5HIS9Ojla6tvJ9nlu6h7mfGmdYBAPj/F7KQKKXgBC26h9BHiYos", - "mHjINoTLmCpSo6DkZ6x/gPyDro1sj/XQqILaIvqDIZOzXGkmttlHh5zJPCXCWmXRmILHaBPJ3Ki0MF5Y", - "jYVmmJLGRAyZfi2AtfpHoZ7sPx0MBoNup9DkdvW/ByFqulPnZ99iCZucX0D7YxZVGGAERc5QzmIiivrf", - "xJBDPUTuho7TzwQRdp+3E6/s56VcFT6Y6zCH24EJfy6CKwy1QT+H6NBbKOdPbi+it8o5cvKrzTayX41u", - "EsFAUMTzJNYa31jfdsYgR2JrhpREGe5s3qUSvTO1N6tTt6HHiqPfcyIW6P3JSSXsQZCJ5gHtJg5comEf", - "eHajbdhZYyNZO5qbuJc9vNv7wLitSyqehPjFEW19D6NLgjYUWjFsVRTnVfY1rVUGs0goM/ukiWbFBEMp", - "MqMyMNJvVS7kxOI7WlEaO3nI4E0X8A+lU18u5NY4l1tZRLds/s0WYHM8B2yOvWDydEzmozwPqUb6kQNj", - "effu+AhtwC+ALQsplFUCxvjp9vPB8xe95+Ptp729eLDdw9u7T3s7T/Bgshs9293e2V2RCNMim+72CXJB", - "jTkQx1xErY9c9HwoqLkpd6Emm9h47CvKYn5Vuf6CAbJ+7zb4dl33y6H1rYcQTMhJsFTGfNHAyU7gkieR", - "btsEpNuMzaLUUtjQ+fR8sP251h8YXMMdcS5yZtyqBmOgcCGk3oD9zaqO83YsHwbkEl/WrZbfeftFG+w/", - "ebH/5HMXzSVvrBtjnZzucXObIsIcBnMtO8RlKHp2JGeg7FiZyFj1bTJJp9sp8l3gbxAGarHUxeNWSVxN", - "B7YbZiOrrpWG5Onjir4CkSoGgy/e15KK00egokORoq9FoMOE5zHybHEGkgz8cMee7qKbAbeYNdEZiFGT", - "jKF1HMC0hsoRlGlGDP5H3YjNtN5Hr+BdeIRTo9bZQZj6Jb7rDccLEy+jz5fr2ihZq4d8ZvUr+EYrW0j/", - "C6atl8GabFc3YaSzffQrh28KbY/xuu3XvA5q1vLrdTvxhoXrdsgZ0JkVNffRT4V4WQioViDdkMT+ObIM", - "qwSs2azABtgd72hqKXfOS4HvdsyKdrodt1CQKr+cNP+upPql8+eTYiiQjOAEznKZI5wrmliYbpgJlYpG", - "0iaP6M1tEntsaSUSj4zy1BSTahJPrYJVfOSkqvcnaAOQGP+CrGFb/2uziF+t3HU7L/ZePH228+JpK7yl", - "coDrReNDSIteHtxaOTnK8pG1jTRN/fD0nbF9RMaqUMS+vD/x4S0ywTXr0TN3Dfqdv+i/8GGmYp6PE8+x", - "aDHpDKotbFgQSa3gRQ1xkL/TZE4nE/b7x+hy5++CptvXT+XOeLsBPtd0FDa7HfvBBUs2ajLumTJJYSQg", - "ICghG8Gy3hIJM0BnRCGgnx7CEag3RTazJTkHqWVXPEhYe7u7u8+fPdlpRVd2dN7BGYERLnAp2xF4Rwze", - "RBtvz87Qlkdwpk2HKQEI58yqvuFzhmyN40FVIO1vD3ZDVNJwcZdUY9uep41L/t6qj3ZSdtEhKbtQLZdO", - "eXC1d3cHz/aePH/S7hhb8/BIXK/mMC5lySyPBeL3d34DpMnzg1MECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjY82dPn+zt7my3Q30LBZ1YPMPKga3yrsChCxBFYDcCS7HMertNt0VInDIE9pZECabpQeRSLGq3jwF5", - "HwnzWrkJbS4Gq4EvXVwtvm1l3CpMViZBx4gGXKCcFaVF+utdsl/Es9rMtc31sJ6rh9JymF49C09kSqjd", - "YikzQeaU5/ILNMSVyZmdJJyLG33bpLC8JTJPlLHZUInen3wHPEXTGpKKZFUdylLjChCnW07uRue5QiJh", - "Im9arFa70WbrV02423Bqu6sANSrcoBE6LdacK2frgz8PcRLlUEwHF/upZwUYYAAJkGXJwsT2JwnnDEUz", - "zMBJIjzEIzTjSdwPRsLqJ6NJMKqCX6GEG9DnS0IyW2fGDEJ/pkUYOidow6+wZkipVvf0SWqYjK0kUqXG", - "J2m4gCOWoWS1IhVerydW3MMjNp9ULKEJn0pQChVkLfTrMPgZFiYZATNTN2meGl0yEHAdGGKNmYduVHOT", - "8olVcK3IAYnmZiVxJLiUiCR0CjV63p/U8pdX5LwVWczrAzqrg21BusahGbjKDBpW6/JqofsxkM/zOTck", - "0DDkDK4IlXTGyRSzHCrPeIRsDfH91uGQMy7VqMCluuFgpRpBOYlckBItr8i6L+xB7p3gvehY222Wy8Yd", - "3+rrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyMtdKMLASXawOJXUTsLqy/gCV0Cr1YMvQBuS8eGzJg6DbbBMk", - "E1ZZdT9L2qqtDvp6b3DWFtZtNYrbKVazYzbhAeyPG3hOnSXaRqtmRKQUCqqgmDBKYqdLFi5Ua+qChPFE", - "EhTnxK6ckU8FtguOzfEGnxVzNjLKpjVeX++wjXnYjGF1tQno177YJtxJhhNqz0UOa2XiFSXCZWptqyBQ", - "Kkdhd9Zyw4JM8wQLZAEZ2wxZLtKEsss2rctFOuYJjZD+oO4Xn/Ak4Vcj/Uj+AHPZbDU7/cGoqTTRmRmc", - "zQs0G1Lrt5zCD3qWm7WsZLDEbJnvt8Ax2iZ6LBgp/hNNiEX3e8fotUfoVTj2vZ1BU7Z8Q6OVPPllZMib", - "cm5LssETn8tAbuFKKccVVSKxxcg3Yk+WS1PfpcWt5EBYnQvwdh6dauLI50GTHBp+XQMmQWMCeT9uastc", - "owVbbDOVYGmJXM7Q3/m4ahBtG/YbKFi2wUqIDEEmwfh+2NGVBmnzxtKaeLt7EwwKYKt6ovDRDaEd1pV2", - "K+OrmvjJ26UqZzNil4y6OZqKZy0qeLj4jwK+wPbaHsegXo8uEK8MKDVSLaCyK5TTWXhFFiUacyEAgVpL", - "OJy52QDsipZ59Fo73Ct0PiMLJEiKKRsyygojKYCpEcTInAgvS5YLrWRNSdxHf/NUPMDsTjO1sGDwYDz/", - "TiJ+xYoxDpk/SN14LnU7B8xYFkWeqUq5SN0saH2aUCBrGZxgSkD9RKpmaCKInPlzD9XM1DLeFRdxYzGi", - "BXKvQI0b8LEixS8J81lZ0UxQNTQNjcxXy1F8puAtPLX6J6rUoEX1GrOr+8slEWEhsZhS8Uqr0BXvqHjK", - "iQGBAUQUqC1o/zIsvkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6Jgxk+MGbboAk2Mmk8tWCfqidtbagKpMGt", - "QrNZ9iWgDRdC7Qq0VCUBr1BKq3uyXSZePVnAjWZLkqja+97zJ8+etqxU81nOOoPe9aVdc/N0hUuuYadO", - "2vh9nj95/uLF7t6TFzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6", - "oPcn//rHP92obj2gEKNZRghv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDTKZEDCDjsy69crB1GBA", - "2knBOMMRVYsAI8dXJtq9eKWGdN3GHVQdbEjkNW1bVFTNuWQ+LpNON1zn6D+Nb7hGC89bF7yS+bjJD/2m", - "3qvxQpdeCz/GoUWIgSxqrS8buIv5XGFZCejWf0eQd+EyzJazbcwbq1F366kQEMVi67p5oYAhtPaaPGk/", - "8re/tp2e37Ji1qmv+IcV57D5CN7I6hu4kQNG32h9am2NP9gL8HZfjcZ+KbqVtf4qdevKW/fm/bbIHl6u", - "k1DcYDfvz0uYvMmHdUxgoEc7BrvkZdvdCkk0UJOXCxMwoPGE9IpAPZsog2RuPIL6zFuY+UAGZ3TJJ5Mq", - "1u2TZmx0gP2BZC/XC1ZKayZdRK6dzaIOrG0wfoadJ3LY0SrAsLOdDjs1t1UwfTLF1yPbQRXbZbAKrLxM", - "f68NUroZjBMeXZoqa1C8u48GKCWYSZQzOPw1r9r2YLV3qNvJvL0poMGJCXFaYlswpjGZ4TmFihTWpzKt", - "BGKSa6okBIxCO/so5gbtqVJi1s5Qv2aSG/fLScOlg9nCNqwb1O9x5iJay3fBwDeBwrbsIxG8a8EKNMd+", - "8+akawIYIPTQDKwS3+gmakagGWTRRa28Qvl7OH54nJARjLsO158ur6Ofkw6eVUEkUdLid5fkUCMCFPGc", - "qTqOf9pOkaumlS1fSTmDYD8b/gG4bLZ3QyAoJhGcSLl8FquEfgviruUN2JUOJQ7shkgYDgX4ksK+4rfW", - "IVwfgDE2eNWhTTt+XLfxEo6k4racWHGqR+Q6IiSuA36GX2kbK2+/DMbKv8YWI6go3Gzfhnjn5dn17y7B", - "C8batNp+TD/jrAfoJG5LLZKIgQa0WDVVQqtAj3uQFqMQvGrohTYZ1+R69Vr/Sq4V4KPHeWJA78Kka1mV", - "vYzWrfitMxubDjQXZG15vjsoW2fizW9VuM6Gqj9E7Tr71p3Uq1vanTOi3Ltnlowad6ha6KXi0nIB/+6V", - "aoyNIaUushc82k43ayS4NwtbRSwob8scTYZTMsoEmdDrFcRjXjCKcRXWpDxIRQaDwRfdSPE12nuGohkW", - "sjZ2RqczlSyqATh7ASylzyrqKIgizBkK2+x8uZvuw+VoN7udfush4fjMgwZaKmliRdLRKtzsw9LbZq3z", - "GV6AFafRSfhsd28w2N0Z3Ao42w3rBst1WH5iSyBW22lKqfO+s47+SpSq30KRZL1cV/dKUMjVLpZJKkFw", - "ug+JNxmOCErIBEDyioTW9Z7FeterB28FKptFW9C/2yi7b84HXy2ZU3RlMcfdNDrOuVjFIPKfr3GINrCZ", - "aAlSL5Bzt9sbPD3f3t1/8nR/e/suwK6LRWrK9nj2cfvqWbKDJ3vJ88Wz37dnz6Y76W5QD7ukpjJQG1r9", - "Rb/bGGVTXpJVLKMKS0Mbdg4ZEfWCyfVC45IklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjirTtIXIbAq", - "F6dCWQ+Dv2Uns9J3UZ/N8dHqWdwqA6k+kDC91YcC5NVuMFChYrvzmcgMOWt5Db3zXmx9Ea3Milt3FYU8", - "7HDSg7vcsOIh8q4BM3izXnWBL19yAdvplAuqZunq26J4rYARh7jpj1LFVbynPjqeMqiW7v9chMn5SpT+", - "uNPtJB/3qmfG/t4e+csiEBcEaLfalwpahJFBMf7VqwCvlIqHMJHsWlfXY/5hu7f9AuIQko97Pwx6L6oR", - "B12zWv7ybbu3K78O2qyhXwLQlY7afnGjiGu3nqso6BcaKmBX3ssWm9jSeFmb2l0dLuG2ssHl46U9riH5", - "NAqgnyvp2ctt5AtNMUnwIoRN7xlqZU179IkMjcmUMtnGbrs7KAy3T9Jhp48OLEA46LKKF/34zUMNeo9O", - "aJqSmGoZ06j+zRkMOy1tcXVd4ma1SdxXAWmtHxbXXqyHSFiXcLXumux/Rj7uZ2m/7TTeVegdYFdzKipg", - "iMGLXUQnCLNagVLK5jihsU2kh8RIiFfbd0BtJclaHiBLOdDZSbpoyhUqU+hb2tty1mwXLMZPrsHeugIz", - "wxDEzhcBRCkAxOgq9nV8hDLB4zwq80cTGHSJ+CHyGkTbCiF/fUjuXdo3IDF7wgVab99oMmi0s0827XfN", - "NqkJtnmrtwfrt/pOjCLdTp7F63mYeakdB7sRcvuaFMSAiaa67DVJ0JvMhxYc/a2/gss6r7ElR1okyjPn", - "YNE0tUxJAXcLuBhCcb1HJCH6mlpuBPEkLrMkqCy56HqWuv30+azJxQkeqeWB/EJIpnUVwD+C/lLMFsGB", - "ubKjxV2yMXBo39I4vHqmXJFdrergnq2VxBq3yjfhNpVQMFy+ZvM2eCmXnvm7wPj2RbNlBBTH8CtC2tvm", - "EgD2Sxf21mg/vguz3EMKaW+s66EG2+pAhQt0dNd/GQusxboq8e6F3PMhsjjHU6iVt8aP0kgqobqS4OTA", - "SGFb3AoKbOp/IqgiKh1gixMQ4HOX0WuLDSKqEJ7ieuWIprKUxhO9PoaoudDLuTURNGHy1tNhffP7Qe9/", - "jLkdjfr7Wz/85f/uffjPoNm9ZkCQRPRiMoGIq0uy6JkqTApPq1foH6YEhNYqpvbMEJyCMQ3Q3i1X8sf7", - "ZFBwz8WvOF2aAoSqeSWUttdO6C//0Rzo5S3jO7gw1p7dz66QcheVZBV39/JGSsTUBdW7jLrN/pBBkt4l", - "WUjkFWazsp07sd/J4hMvFB9dGHLvEza/QGMKlS7lkGn1HkcRybRaZWv9UFOunQMbFgQnfju2QJw7L9Yz", - "awIrCHp/sgRn/Obd+Y9v3v16NHpz+vLXg+PRLy//G2Jdrnqmh7inaW/vyVNbpN1fye1goZCb17vooxOb", - "r2BjHiY5aPYAWCZRmqscomPIdZTkks6dp1Qlt69ssZy1fPtKEZ8JhaxUEgrPsJDdCZ0QCHCAe9VGF1Hp", - "iJFKqG5vrTyUoWXRxRDOsANXiuJOkgjWFdFbEV7tcmOri/501u4uMSCxgcMOqb/6sAWsqK+pBOAOFwTj", - "vYw2IIXGleB1GcSbNwOtPSgaDIZgfuFKS4MXX6Ia6ruV5U/nPOlpPa+hZETQrG7WIphCAE2Z1IxOk/dl", - "Og4oM9bGPaVTHHC4hBwrX6RqqRvQ2tSxpf1vLN8WTug4qtfTMMfSLFWt/kPNWiJVrznfI9XifQMCMUAs", - "myRe6gUZVjN2U6a2bHXhEHBIzAHVfVXadnnKHExkDz5an428UsH0ZuaNpHlvTpweVdP0VizQqV6aqxkR", - "xNsI+KCsU3DDJbMJSi3gaEx1xoyIMnjXZTdp8Rz87hJtFCYwtwRF2vWyX2B1HYoTfF30AD4lLJccsTCP", - "sg7W9qsfoWbAW1f7k05cEzCMmpYbRsivUtGqNXFUtbwZPlUtz9u8Hzx4llet4H5NZ6tGnGUfFdIM0ePf", - "MFU/cQF6cTP4y50D7cPlHxMBYHh1GP1WGPQ0JfGI52r1+devaenRXPlFfdiyvrCzAWAg4qiS19zECxw8", - "STmG5ZXWy0GiXFC1ONPrZaPaIR/UFfWFhYSO4OeyYyik+ukTWM8ngcyZV4QRQSMoU6vPY4oZaEzo/YlX", - "rdAUrlwCrgUR6M3hsbW7OOxj0KOpAtJzAagHp8edbmdOhLE9dAb93f4ADnNGGM5oZ7+z29/uDzqgVc1g", - "ilvjnCaxTSS3GnWhwR/HVhL60b2kvxQ4JQq++C0AiQABmPZ1UEHw1FMiM0yF1SKzBKAaDMFQ/TXUXXAX", - "6r65lbtm2VsbjyHfGtKASPbGbu4HEJTh7MA0dwYDi/Cu7PULSUwmc2Lr7zaMtuy3lVRnlyhQhmBJzXOy", - "ZbH0n7qdvcH2jca0aihwdkMdv2PYZjMT0M6f3HAhbtXpMTP5iTbb3MaF+ScOCMk/a7990Hsm8zTFYuEW", - "zF+tjMsmwZhIhN27Ro9TEkWaVUCxpD56w4h5jrBC2IRwi5xBjWn3oabQ6ikwbbtNLtCafuTx4ostYaUP", - "Z6P4VGVn+rh8WqLnL0c7BRkvb6R95KDGDdXeAwH9iIsC6Q92UvYGL+6+00POJgmNFOoVBGwDs6mE2KcE", - "gNMdCBMX6PecK4yKvIZHdKStzDouyK1bXkVbf9D4kzneCQn5A06JSDEzWSLmnTWHfuk4G99MeZxX3mqO", - "8I+POvamcmhE5qICQa56RP1rqy4MLl9HewEoCtunmV78gIS/dw8n3E62qJH7kEcOKpOiXJLHdJysr3Fc", - "CiFBWe4VUV8LzQ/u88qy1RT+hKfosRDwK1JIeOVuLV0KW5nImVGAgxLg2zJz0373XVX4Oy+feOFC4NfQ", - "TUNdD2UczDhe9JFbU6P0qwVgTQkC84yXr5VTPbyv5YTt3McJgxkXnqJv19S3a2rVKTfU4qYAB9M75S1s", - "EDeyQPz57A83tj58sz20tz20sjwwcmWtC3/n4z6yobkRjwmSM54nMRoTZICfXBCOwqI//YiwiGZ0TgDd", - "D6rV5YmiGRYQYpOiGCtsfOiNhomVZomiuS3dXM8FZJYLXAf0kGQEITCjJiDOMhSTMkZipD+xUTMlruJS", - "XXVz9oMG9qLB8mpEVzMuSQFsyJR3m0OetzTaMTTbH7Jzi3irFxCiyh2vkSQB3N4V9h/OEB4y+8H3joW4", - "iDiJ05JzYQHgidRAdJptWc7x0yMdyYiHQIfOCcNM9WRGIjqhkZ3WJVnYwNZgg60KUOkBu3G+PykyV9DO", - "Zhi4DkKXwijFR8UzZCmp6r9hEA0eJXlcOrkclhIWY5wkwQol04SPcTIy63NJAj7BV/CGXZTS4VJ6kxiP", - "iamlny3UjDPzdz7OmcrN32PBryQRw85mf8ggI8WuNYm7pYCIrqCiXZpxfc4ET02fW2aIW39cksWn/pAd", - "xClljiLgE5xIjsg1fAdxYwAeYrhXAz2Y0xT2gx/mUvHUh4B1dGeGyXOV5cqm1kiiuiH40yFTHP3hQC4/", - "bf1R9vgJnMUEx5pOvFfMlEC2bhq1HGE9+xG8GnC3E1iAYUdfpCbMYyowUwa/tEDpRFN/SzeKMhFQOra+", - "whFmKOOZKbEBRDXDmuQqbQBoBU4SpOAouW+14A472TAfi0GYjhsBCA1iXO0YUYZOfvQO02Dvefg8SRIJ", - "Eooo+a+zN78iuJX1HpjXynAtk9vCtMCA4hxcp46nvcTRDBlHFVRVHHZoPOwU7tx4E8aaSxsu0+uBT/EH", - "PbQfTDddGv/Q7+umjLtyH/32h2llX5+lLDWAqMPOpy7yHkypmuXj4tmH8II24bidVRgB2jDX3CZwEkwB", - "cse78c0ViVmMuL0FkgXCqORAfuDKmDIsFqsyKgNLb1eQT0wko7cYfwwhcnHY2R+62MVhpzvsEDaH32yA", - "47DzKbwC1mvZXMIP7rPCuVkQ0dPBYHM9JLhd34DPsoVj4AvrgI1aUVF/VO+gxaP9c/kH/q31z8L1g5nu", - "vMRoMoq/M74/QgeEJ7H7mmjABVETuzGLSOLE7vWGnvt3HujNikiS3DeBPhR5Fu6xomTBoyJH2KzyGK00", - "3z8wxQ3u61KpmO0fhn4fnf08YD23tnMyd6HO4YItAMZjVWlkXkZYojMYU+9MK98v4de+/a/T/QBc8iLh", - "04t9o7qjhE9RQpnNB/AClbV4YNcSPjJ4PMV3Fp7HVcvbMJLEv/7xTxgUZdN//eOfFuT+X//4Jxz3LYMz", - "B8W2L2YECzUmWF3so18IyXo4oXPiJgPlcMmciAXaHVibPzxCXs1/K6XJIRuyt0Tlgnl5E6ZwnbQNWleB", - "ng9lOZEWzwjShCa2qo6JbQzYbdxZNkt5rye6G8CFhBl4E9C3oqMBANWjpuK41UQ7YZOpmXPFaFoP01wK", - "1lvPXxS5VoZ6e2aAN2QwsMShcwcP7KTRxtnZy80+Am3LUAVUTgLdoWzGqhH9bzxpPU8yHKXKUGCVDW+K", - "cIbHNKHO5NhQ9sUcwRRHM8pIGV9cgK67JvbdSDWPOTg9RjYQsguvDtmbsy0wsSoSqVyQruUEwkKtlnXh", - "uM1zgR6Af1EF0WE9++6QTQiGPKHjI8MEPDTyIjGyaJgBognEuFJVKUHXHTIDqWshnPXBS3lMEvgI+p9i", - "Ra7woouKor+uTEyClVaIZVe/PGQm1dCuQQ8wW5A3zD7wMzOknovktTlbgkwSrRpDBL6pfw59b0y4QDbC", - "uVvmlbruTLapGZZetBRHb870/KagCXJjD4SW3py53djsIslRlFCghgizIZtCIJBDMeassqtFQtkMi7gX", - "cX0J+KhWl4xfJSSeNvHYQ5/I7lCSqfQTOE4/18n1sQkXs+UJ6ENskPpWe+6O7DvtXHe2xT+T785WxLyB", - "885YcInhN2Z1vznyWjjywuvmnHohz9qRg6K8u4hf08UDBfw62ltec/PEW7KHsOihDYfxA14RLtDp4THC", - "cSyIlJv/3vY+PVNDpaX8p+9HzYofIvTEjoULi35o7S1VAnks7OCtHTXCbl71QsP+/bZVqULUeNMVBYnK", - "K+/ub49apze5Rkqht6S1bzfJ2mBbKiMO9RZLaumBaJSQQnwpzqlPReusyiaMt7hyVopLlj0fH7kDeX/2", - "Zdt1zup3wz0wxaMaQ3xARlhNtfbLhz8man5X7KKD3V5hfv66SHNwf1LQfZuiQ2T+mNTFuLZsmgsaoJPG", - "C/QVUQbe5C71dNtDYOJnRLhTbQa6MLMupmU+RQanBSYElpjVuu+xeaWd6mva+zNpvrA8N5FY7JJ/E1Fa", - "KLvlWq1ScI9tLey7028rSG73HLZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkV", - "etZ9yXUHzC+8rmU6q9dShiYJnc6sEyCmE4jVU34hcxjlzj2MsigYLrAiNkTxMeb9nupFtl7gOREeUqN/", - "pW79AUGr61Ulx7xW3q7v3r7uERbxuHCeNMuk9skXVpgM/Vdyee//1D3CfFbqxIMmgfEz9t8Ek6MCuvN/", - "7fxkwTv/185POMkoI/9r9yDBiki1eWfEMrivm+6+FZhHTHxaf6HVRVtiTVsKT/081CplOvjam1BmBXbW", - "GbuLRJovRn1fXsirY/W2kvDuje61JnXfiCXHriyB2VIuHHpVDZL431voO6sQ9L3b6yrdUwkjAMHuEUYp", - "c+FQV7GFraaOuC1vYlPA9V5jjCjeammPcO//qUwSZtI3MkoU6/rNLtHGLuEv10rThN2KOzVOmD4eyPte", - "EFtoteHRN7ite3DoWIr04LYqHu4ScGvGpYJHjw97wV4ltKA4/9po6ZksD+TK68OR7vFRFxYSyv5D2SGb", - "2nxPfko3jntXvG2/9y/0HKRjOs15Lv2s6RSraEakRRRISJUBPzaTQHk9NxoFvmIqHdzn1XHvOv83ur8j", - "a0R9Qw3zNsEG62R+91Zbmd++r2V+g7ZsURdsbaSuq5u32ZAE4vCW25JxBZZ6OTklNK6QLoLeaUWlVBcQ", - "aBD7Q/a/tf7xmyI4/fCDS+/OB4Odp/A7YfMPP7gMb3biSIUwJagtc3rw6xFEeEwhiB8qoZZgEvVxoDSX", - "ypCeK3nyb6cglUEu7TUkR4XfNKRWGpK3XKs1JLsXd6siVcsm3buO5OgttOC23sGfU0v6k7tuKxqczCcT", - "GlHCoPgUGP/kUqyy0eS+eW1vCZbAbKyEF+hYkURaq5EF11ojoZeF/79kJGG3sQYFR1gpkmYKTQWOyCRP", - "TNUWJGe5ivkVcyUpYIKuuhkt5xO63l1TI9dIOEE2XKK9raZblGW8b1XXdvxIM1R5ZiuMW+WyFG2atcuH", - "Jd671SlbXLX3r1U+ZhIz6tvy0mVaQwiUWDNunjQ36bzFlyU6Yx+dn792qbtaPRGuYJ/irkqfq+Q8ZH6V", - "vj56WZY/NC+4FrT6QGKb6g8JzbbuXUxwnFBGINeByFCWbbW25oMeiy8vAYcLh963i7vNsbQ1sR9OAn4w", - "VnAvsmbh01fWIVscTa+maHFanLwJp+ZR8SvLgAKMJyTrbeFc8Z4FA9iacYMQGQbJPU1wBBi5+jUD32jx", - "Vwxeq98UgKoIniREGFjOLFdO3BqyYnCUKV4UnraS2YVufpQzRZOLrgk1BGwliTBbWGy6Iat0ZmU+wEgA", - "/A8YoSCZGXGtiq4eNOW5hLcAzsDvEuHkCi/kkFlUBfM5lF4XJDIItknSRz9zALQx1aQ9xmtKuX4nh+yC", - "xgkZWTyaC0QlkjMuFGEkRimfE1ntl2CRUCJgEodYr5xEKV4AMKTByDXrwzNiwBcrqDdc/xuzmEJRUN1z", - "MeX9IcNoZzBAKcFMWgwLiSdw4dg2EAyiMqDvEUZ7gxf2q9q+AXi5W/4NfZqEIHMe4XGyQERTsSm2vQkb", - "mNoivabqu96+CRXS7Fdh37TVFysbS6WrORt3Uc5KlA6w9eesANXQ26VywWCe1gtIqCiuQQtMNCYR1uvJ", - "eLUfgITlUZSL0AWpt9qrFv3vKDh60zuDpQpjYCRgMohIDHvOuJrBmeZwlDa/b6Cqkqj+HBdN8JBwgTDy", - "6Lq0aJAoB9a4ARCqF2XpU+ZKmV9sfu/Ojj6+lhG4429ATB/L/QRExCeTygFcfzWZA7wq92yZhP+s5/TQ", - "1bz2WVxM8ZRxqWjkmKFDqPaV5m8KYSuFcPXKBql5wsVlc8DxT1xcttXAXGDk41LE/Bl+hY4IPTwAwX94", - "fwRYw42yoonm3pW0On0VpxSELqpkERKMEs6m+hSVVvl7dxv4Wt2GAbTUl6kwzu4CfkwrISP7oymb7YV1", - "g4shsq0+NC/Svd+DM+pXrhBNs4SkBMpq9wyx6c0uoerGCwulVwC23YxX6lPl4yoYXVCa+IOuE4eArtyG", - "bYD0vrxdQaaa8Ol6QNSic4f+GUBEHbJ30pQquDCupwtU8GAt0JryI+hqRqMZoKOC3qrbN+CpOMsuCmD4", - "zX30Cg6yj48PnW+YoiOa1iRPiAE9nafpxf5y4ej3JyfwkQFGNSWiL/aRKxZd3B9Sv+WjnepZJFgq9KvF", - "cN0olHHY0QuFtb5ZzG/T4qCWwP1DFsJEZeTKNkgn6MKDR71owO5z/PY1n8qvxlVUllsxc1EcWdURaJOw", - "uNMU5EGTsONnezAIVQFoidJqhnHHIK1Lg3nNp0Wplwop4yxrS752mEDF8zRdQcNow4N8lCrmufqLVDER", - "Aj621N1E3GgDR7bMH77UhGoBPt3B3gTyC4YymdoLwaXSTLXT7RCWp5393+y/5mna6XbsePR3VypLvdoN", - "NxDy16De1htcDr3RO+RB234Tz28CWltl+h5qbe0GsWp1s2T+1rzwp/caOtvdA5IhyAk1Y+7XJIp6460a", - "fhgv0HdhZC/uY2QA0YuihEtScfQ8HoA/a/CqyY7NBiO3xj09vDh3FdHaRLKc2U/P3JdfgQ6+LmbEjRm5", - "6d578MjyCB4zWIFcms2Eizoq3Lqokq+ekL7clixNtQ2FfKPNm1sbWxGm1heWWYT9IDYVMnGueIoVjaA6", - "WzTjXHpkX0C4mzqK1ohcUCaYWIy2azMJLjSpXlhz9IVVJ/at6Qxh/5Htow+f2/yD8BfuUfnFT551oOD4", - "XacCQAUTiTAaC0omKMO5JFqqy1OCokWkuaIpx0dwNEMRzlQuCFQaJSiljKZ56mPz6x2bY8ARuthOL7po", - "nCuUYDEF7cw8dEE3EU9TwmICdrohmxE8p1q1FCjBirBo0ZMEKpTPCbri4jLhOAZTQxZj8PhAhVNBNAVC", - "oYOUKBxjhUHQudAnfmSSmS6KouVGvWfkuqSGeMhEzr43VVd0sxduoBeIQFkBKmdFcdsIx4RFQbj9s6+b", - "jX15m/QZUfWJPlCE0K146UOGDPm2VzecryOa6NFCQ7Rg8yuEXtmswlazQBwZ/XseaTNXN8cHcjQVS7zq", - "FH8dHqaC6L4aL9PDu5G4QHFuuvNOJZD5n9U3VDAUP+gKMkzNNt7WQVRU8SyW+UY8b+sP9+fxLWx5Xwkn", - "7DYq9k314spJfw0s167qrXjuAxkxrS3Jt8k9HAt2kV0PJj5x4XG5x2JsrSC0FXzb505KYNC+OPvGtuts", - "2wY+3JZtO9vskmvfY+SU9SBWNMzBrRm3kVVb08G/aVZKbXYey3xwFll6Lu4dbtGxxgwvEo7jP0Ow8Ar/", - "UcSFMDAYAKzxmCCiPauhnyYAtrmyEGXXZW2+PznZbOISQq3kEUI9Yg7hpeboz9J42YD7Zk6EoLGDwTw8", - "ObJhu1QikbM+epNShRRHl4RkZWYLZBf29fwcIEht2HXkj26HMCUWGadMrR1F+erdDKb8gVtclK9QlLQ1", - "B765w1u7w8Gy//jYGXAZyN0wE1itmSqs1tZCpmzCRWrkMjzmuW5d8yC9THo/DWLBhCZELqQiqYlOnOQJ", - "HDeoX2NrlNvvzC53ITZXnxyTNpcRkVIpKWdyyGzOSEaE7lt/rtv3Aq2CDgGFC/56apjk1xHEpwdj4taw", - "alo1gG6C2sed/c4WzrKtGCvcEChmh/cZQ/oJovKQXKRjntAIJZRdSrSR0EujnqC5RIn+Y3NlWN8IvvvS", - "Fdhvf7L0Sh+zCQ/WtzQ0WxDznyq7y7I155h8dGztFfEPi+M/sNFhtra+xrsgOOlBzXQH4INyRRP60bA6", - "3QiVikYm9QgXa/f+pGCq/SE7IUrodzCkuCWJQTYA7XIrEzzaGuaDwW6UUUCB2yUwOGB4zY9T6PHw9J1J", - "RyUpF4vukOl/QMPnB6fGuzvB1prgDdQWd0fHW2/WBDqfwTL9G0cImgmuRDEIbvg3l+DNsUYaz5BsOKI8", - "W6Uq8exPH8JqJbhvdoXHaVcAsKdiNhsFwJdD5QrbEOY8yVP9D/PH8Tp8M4Wj2Xt49auRds1w1nbjJvgo", - "DqWdU0xM/d0HcXqYBXusMat64dwUQIipRAMGb4ED9Wek7i9vvvfX8St0d9oVdbWtv5qzdd83nx2DQ9rw", - "1+OxHHNDaW4miq+2Pl1h2mx9+jHh0aW0kCy+2VDrbYCzrn8scbGtixDEBMgQRRbKyABmEdkdspoB0iD/", - "SISRIiKlDCdbMGfTCCB8OysWnnMKidoR5Kn0JI0BOykBGG+AwdOzAUOVa8Dz6Epb/c9/x3dGKo7GJOIp", - "cajnmyHV7W+Yqp+4qEKYfy188dxbf4AGxBTs7WtQ25t7/CwU9xN8DaHScW4dym5EG694+aMxBXUR7M2w", - "szuQw04XDTs76bCjd+AQgwkVK/QEpZTlisg+OjL2LUjFfTpAkkScxdKBrzsL3u5ANiXmGrJsyPJ8Ct/d", - "p9hjqQqW8q3tJMQe9HtIfw9JO2jDP3D2TMZdOHQx4rky5n57ruxbMVFgHtm8d1+td0a+6fZtOPnf7PGt", - "8CjYZc0uva03nD3L5Yw0m9xem4JGuRoDqLcrgCxn6O98LLuIkStjDRdS9Zf4nv761HRwHwUHdFc3KTZg", - "5/6t0kCLSgPlWoVBG02Apb6SHXUY5EZynXGhAM3R5twbGgJNAhAkoEzhm8PjIYs0KzIQg4KkHLiTxUU3", - "t/DB387Qy8O3XXQExXjRz/l4s4/esGRha9hbH82QGUnMMK8IMzQ2VEvi0PVsxg7Uc5fB4rqDB6pub05G", - "wLPi9soFiXc7M4JjkEj+6LzmprMA+vDb1/oAAQCw+bLY9s5K4aPzliix6B1MFBHLzZ7YPClWYGfYS9pB", - "0VnBzQBg6g6lQ2Ar+zSygYHI2N3pBBAzPn0r/nD3RZzvx0tm4kRM2b1xrh5t/VY4iAVzDLFA/7ouyic0", - "ZQlbXrZSwYAumyK/vyKT+0reVcGY/3c9XTDTR+toyir7pIm4KLuy1tPrkoNnBhbZOqoinOGIqkUX4SSx", - "d5S9CYqIlF4h/o4FwZcxv2L9IXtbFHyxCb3o8PRd1zlqUUzlpWnB+mL76M2cCJmPi8EhOGjGawxrTuIh", - "UxxFOInyRIsbZDIhEeTiQh0X2eDLLYbSucOzU3YSLDrjRbXnj67WXZgmYPdKsqhT3JbZ6i1BogTTtBmE", - "3ApqEHAIoQZj3ShniLJJYkOqIsGlRLapHknolI4TGyAk++h8RpDEKRmyLMGMEYFyaaLi9dB7mSBS5ibB", - "WzcAYL2GorqoBBjMBFc2NCHhXEgTTaAp/P0JkopkK8jsrWn5BOZ8R7Ktadz29EBG6toYmk0h9hWkN8RQ", - "illwTUd54gIY7zUU3QzooaXEx3LwzwWdTonQpwIbJmvC8cyxdstpDn0lY7mx7uVZ8Va7updFq15Wopex", - "txIgblRibsedm0X9BTq/pI0YgvbRzbKIf9Eftey7mq0aHoR99JmzDJXw/HeslnnmJQm2NWCVFP7YzEne", - "yCtHtZJoux5Wq3Vm7V1murbGz3ow2KzHjJaFK+mzTQrv10cIg/tFebjvYmuPm7YqaFcV3bQh5X89qv5X", - "QYF3A6f/wCgnt4DT/6ry7gHv/OHwT4IH9aHy6Cu+Z1d090+PiH9X6fMGFh/g2JrS5w3Xs8GrKxWl9/ad", - "dmqSbfHPJMHbeMcbyO9u2b9p/S1UBm+x1rmgNcGTNFMLF9BmfZVl0JmkH0m/wRFcxK3enSv4FiGdX448", - "HJ02BnT+OWvkP0jMqC0hSCU6PgoUn39kGIP+matcLFv61ulhEc3onDQb3asn2C5RJkgv4xk4V2KzYHY9", - "3F2msOhPPyLbvMVctf+CGpQA1U9iFFNBIpUsTD1QzRFMH99JJLjWBOA5F4vmKBFzRH4SPD2ws1lzH9oz", - "ZY1hZZxhuujFWOHe3HGbFSa0z4judPGUmuEhytCrH9EGuVbCVLpAE635IDoplpRcR4TEEmhy0x/w9qDB", - "skk/ktF03GaUK2qWvLE1YVCUS8VTt/fHR2gDaqBNCdN7oUX9CUiymeBzGpO4MsbOnCdmVbcbFvSmdlct", - "VBQF7JxyYQb3IDJMmwtp+pFmVbZQhMSMKcMwuLVVQapnyiTx6/4wZS4Ax+6RG8W3K8xqfhtO2dGUCPU4", - "7SIqzg3E8+a3a+4xX3N+MpS70yq3nQvPWW28bpcf1TJt6S4KPxS5c/drtn7/9aT0UPkos3ms6XxeKKRN", - "ZvOviwQH93c/3Le5/P0jTgF9RZzy7ZnKoQHdYohgXkNMd0zmJOFZCnXR4d1Ot5OLpLPfmSmV7W9tQez3", - "jEu1v/fi2W7n04dP/38AAAD//+NCURtO9gEA", + "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", + "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", + "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLjLU", + "LduyHW+tCr7XepE8LhdYHY8Kmgju/U9VYjEXj0lVteZayqgC6C3AW94oBmkQovvoBC+sVp8BBzddeQio", + "eDIxAJQFZxMkIVhCNWqJ3v/UXxsmqLjCSdMczvVDe2o29IT0nuphdlFWIUo4SR7yk9/t0yADK+cTcHpA", + "h3ULeh8ds/AhNLEogN1mYJikpFOmNQ1pw1EizIqVXNq7KicPGinqWHed6sJVptMNsKMQxYR4p8mtWxa6", + "l6tj8svqoeGX6ysymkaa+z106X81L4vVbBx/gyxBFxeDzKfGjWJtdZJkWGBFkkUtQLeKsE+WQxzINYlu", + "kH/4Ur/+yVRVyQUZqZkgcsaTamDMbne5Mq+EIPQ5scXIzJw8T5DiKMXiEk6Z0+xQzswKVHMYdtcBDs2U", + "ym4wqZ/Pz0+NuUcRMcdJPQtGLoV8HJEEL9CYqCtCmJsKlgj7QdD1LGLZUBBKqFFGBOXVNezsBvo9M4Hx", + "aCpwRJD5ytX1LtiaPnptl9L2EoCwjCIiZcP+bq/aX/vpJE/a7XFoWNtry+BHN9ng88NTV9ioKBztlnln", + "eZVPieiZI+cqSK/e2h25usSW64oZZOIlCXJMoOaWTT3yM4NdwBvUFNOfV7JxPeYg/fxR2w+cArNUXXPO", + "P7RSN+rHPRTZkWIWhwpwm4wTg9UwBQA3CHsXOegCNDbRcOZO9u5nmzgjCI4pI1LWEt2jXCSdbqc3sbPa", + "39rS/D4BDMu93e3nW6vjilcGlNv4uVFMVxkcXJSdicNyadUGABAmXSWJLZxlLUyyZh3X3A/AnpYDWKHo", + "tr7bPJHfZTQMBks5+9c4Uq5MINhoKz547B9bQJquCjK6wdRU/tG61wv/fA6CQREZVrMq+W8t0T4ESkGQ", + "q6YRY/+qraMh8o9B6ZSLUC04LpRNAR0TFwVb3IcuxtRhKle8tYPn/iyfPnmy+2QdHwJmUzvm9twFpmre", + "ria5E2+57fm1DUC8XlXmcEd6dTlbvS5raEpzxCWSWr2gPCPsRuv5ZG9352br2XYixy5OsMaXQhhBhydH", + "RibSmiWmjAiUEoVjrHCVyYBxU3MZKDaESQp5YpPvV7OWhoAaH/TntpXSvlQ4RkPRxLcOIzzFjE5ASTJv", + "+j3LGd558nTflHaNyWTvydN+v39TKJSXJfZJq63YMlGbHipKX84+bx/uAPGkzVz+6JwenP+sGVkuhbm0", + "tuSYsn3v38U/ywfwh/nnmLIwUkqbasB0slQFuBqgmFsoahLvo7Lgu5N72gSMNXgnIFwd4JmCuIOVsN27", + "AxgsaJxW60jcIFl8RfK0FlfesGTRuDa3LuxblrxXXkFf367Qorgv/bg64MLZP+Ed26exdBR1j5dDLW5V", + "uVquLO65VPstI6wo55kk5q+IM0B6DtX2rFyR7lmL0nBwjdgacEWX/o9F796Ph/5AvN9daTnvJ1vk88MN", + "Y6RWCqR/W5ZD13MhJ46uOcxhY3RxK7QtqGyBCYPJAQ98F94mjrDa+5vpf/3+f+Tps79v//76/fv/nr/6", + "r6Nf6X+/T07ffBbozWpMygcFlvxiWJIQPFcBlGxLSidYRQEbnVb/GlbYPjEWBxXNoAQsGpP9Ieuh11QR", + "YQoK1rJhhx20QUBTgq+0uAvVkkwi4qb++NS4uPXHfzgx+FO9jdiiFAi7IQX4jMzHMU8xZZtDNmS2LeQm", + "IkEv0H/FKMKZqQpIGdL67wKNBZRwtD7HsvMu+gNn2afNIQOrLLk28OoZhip4kyINkLmIATsqExdtXycF", + "DolJUR2y4rYuQBmN47lfVnCgJKknkTUsymr9zWpOzwch+EpIcNIbCVWKQAUpKFuTUZF5hZ4PNpf1uTU6", + "RkFDK8jPRmaYDNiDPGQubsqaPSIxjYCvuMTRmU0tLnJ2DaVZI14m+PUC9uatyWaMEc7VTPOiyCItRJxf", + "UtKFLe2CfxRCgeBLE+Ax41lvvOjNeFagbmBhwp+wCZGoKtn/p2cn2ntPBJ3YnoLgCZpEAkInHBk7M5OD", + "WlgXliZ2bgq8MC36zIl93VQKkqY4jIntV7lgrn4JgSqmAO9SUB8JyeTfoyihYHWSM54nMZoBAqPSzYRA", + "FDuDIocKj6OYTOr/rsa47Dx5Chqs+/fuTusUZrN0q6gsTwI6bepYXwuObdgkDMCIByNnCF8TlaZvQOvY", + "BzuF4vDfM+QaKk9cwUiMp88kVUpbDiORXjrlZjDvzR4DiyEzwvY0tbmPlk5hJY2uRQsmXAU+S1qg17w0", + "6bvnr8+QIiJ1gAobkd4dOCUGqqRHpcxtZbaDw5OXm/1OEHmr4tKCrVqZZlcddAB8w4avNEXllDYanJIu", + "Oj6C9Gl7rZS6GOS7/MQFSsytWF5G+4DeUjX3YFPT8fjICqDJooyBMWLLsLPpWszq19s+eluogLgYSpEI", + "W9KWa7K8TKBZGxFpknGWWq/lTYN/zKp/9j6G1BsoeGl4MaCHNt5f7W2ODotMX1Q1C9mNLyQ/LKnR/uXt", + "/ZfG2f7yMvruzWR064UeZTMsQ9Q9872a8NLSvvuO7Cq7F81BZJV+R5IGz9bfXNkf7xpSRN9zlc9DwLJP", + "etvb59t7Nzff3RQiuYqN5uEmFijJ7eGN7wImOAD6S9WoMdsA6cc2t8DZRd6foBmW7DsFD2vWke3dZ22M", + "EtBr2zh9P0KfT8yQCi7lgNaK+HIDOXdJk8QIMJJOGU7QC7Rxdvzql+PXrzdRD715c1LfilVfBPfnFmjJ", + "cAvAOpqUwwDWVgUjAhXJpOfnr+FwJQTycYwcfnl7DOW1psUWmMpucK9O34HjH8uRi+RtTl7FZQI4uaZS", + "yWWYvVYB8Z+D4Ww+LQ1jbSZp2rAhfmuBoH+uIBUHcRM37wDB2WUzLC3nA4AbP2TW6NcHrLwSCvlz8Yyt", + "neGO4Iwbr7QQFHANIuNJ0+12e2DiOxlOBWEoxLZ8Ccel9N8aCbjboYF05gMbx4eOT8vSX6UzwjVfm9OL", + "nf720+dQvXZ70Iaxpzha0ffJwWH7zgc75pbZx+P9KN4Hhf22PitL2EYFwckVXkD1R7O0w465MD3t1ju2", + "VpFsFV+zDLh8O3zluhjXgKAM4qwLXJKjdGXxmRb5qnUUvTS3GKEpTRIqScRZLKsy8gxLJDMDjWuKsBQS", + "/JDBALuoqIUNUgrCUSTy0vRopWsr7+eZpXsoBJtxpnUAqATxC1lIlFJwghbdQ+ijREVaVDxkG8Kl0BW5", + "clADNtY/QEJK16Y6xF2IGoZiM/qDIZOzXGkmttlHh5zJPCXCWmXRmILHaBPJ3Ki0MF5YjYVmmJLGRAyZ", + "fi0AvvtHoZ7sPx0MBoNup9DkdvW/ByFqulPnZ9+CS5skcIB/ZBZmGnAlRc5QzmIiioLwxJBDPUTuho7T", + "z0SVdp+3E6/s56VcFT6Y60Co26FLfy6kLwy1QT+H6NBbKOdPbi+it0pCc/KrTT+zX41uEsFAUMTzJNYa", + "31jfdsYgR2JrhpREGe5cZIK8M8VYq1O3oceKo99zIhbo/clJJexBkInmAe0mDlyiYR94dqNt2FljI1k7", + "mpu4lz0A5PsAPa5LKp6E+MUhjn0Po8uKNxRaMWxVFOdV9jWtVQYzcigz+6SJZsUEa/V4DYZIGRjptyoX", + "cmIBP60o7XIqLAB5gQdSOvXlQm6Nc7mVRXTL5jJtAVjLcwBr2Qtm08dkPsrzkGqkHzl0nnfvjo/QBvwC", + "YMMmQabSPcZPt58Pnr/oPR9vP+3txYPtHt7efdrbeYIHk93o2e72zu6KpKIW6ZW3z5gMasyBOOYian3k", + "oudDQc1NuQs12cTGY19RFvOryvUXDJD1e7fBt+u6Xw6tbz2EYEpQgqUy5osGTnYClzyJdNsmIN2m8Ba1", + "t8KGzqfng+3Ptf7A4BruiHORM+NWNaAThQsh9Qbsb1Z1nLdj+TAgl/iybrX8ztsv2mD/yYv9J5+7aC55", + "Y90Y6+R0j5vbFBHmQLlr2SEuZdWzIzkDZcfKRMaqb5NJOt1Oke8Cf4MwUIulLh63SuJqOrDdMBtZda00", + "ZNMfV/QViFQxoIzxvpZUnD4CJT4KzAYtAh0mPI+RZ4szGHXghzv2dBfdDLjFrInOpNGaZAzIjKTS5vNR", + "phkx+B91Izb1fh+9gnfhEU6NWmcHYQra+K43HC9MvIw+X65ro2StHvKZ1a/gG61sIf0vmLZeBmuyXd2E", + "kc720a8cvim0Pcbrtl/zOqhZy6/X7cQbFr/dQalAZ1bU3Ec/FeJlIaBagXRDEvvnyDKsEsFos4IjYXe8", + "o6ml3DkPE6HbMSva6XbcQgF2wjKKwruS6pfOn0+KoUAyghM4y2XSeK5oYnHbYSZUKhpJmzyiN7dJ7LG5", + "mSQeGeWpKSbVpL5aBav4yElV70/QBkBz/gVZw7b+12YRv1q563Ze7L14+mznxdNWAFzlANeLxoeQJ788", + "uLVycpTlI2sbaZr64ek7Y/uIjFWhiH15f+LjnWSCa9ajZ+4a9Dt/0X/h447FPB8nnmPRghQamGPYsCC0", + "XsGLGuIgf6fJnE4m7PeP0eXO3wVNt6+fyp3xdgOesukobHY79oMLlmzUZNwzdbPC0FBAUEI2oqe9JRJm", + "gM6IQkA/PYQjUG+KfGpLcg5jza54kLD2dnd3nz97stOKruzovIMzAiNc4FK2I/COGLyJNt6enaEtj+BM", + "mw5kBCDvmVV9w+cM2aLXg6pA2t8e7IaopOHiLqnGtj1PG5f8vVUf7aTsokPydqFaLp3y4Grv7g6e7T15", + "/qTdMbbm4ZG4Xs1hXMqSWR5bmcHf+Q2QJs8PThEkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCeP3v6ZG93", + "Z7sdDGAo6MQCXFYObJV3BQ5dgCgCuxFYimXW2226LULilCGwtyRKME0PIpdiUbt9DOr/SJjXyk1oczFY", + "DXzp4mrxbSvjVmGyMgk6RjTgAuWsqDXTX++S/SKe1Wauba6H9Vw9lJbD9OpZvCpTU+8WS5kJMqc8l1+g", + "Ia5Mzuwk4Vzc6NsmheUtkXmijM2GSvT+5DvgKZrWkFQkq+pQlhpXoHrdcnI3Os8VEgkTedNitdqNNlu/", + "asLdhlPbXQWoUeEGjVh6seZcOVsf/HmIkyiH6kq42E89KwCFA0iALEsWJrY/SThnKJphBk4S4UFgoRlP", + "4n4wElY/GU2CURX8CiXcoIBfEpLZwkNmEPozLcLQOUEbfsk9Q0q1QrhPUsNkbGmZKjU+ScMVPbEMJasV", + "qfB6PbHiHkC1+aRiCU34VIJSqCBroV+vi5BhYZIRMDOFtOap0SUDAdeBIdaYeehGNTcpn1gF14ockGhu", + "VhJHgksPnOr9SS1/eUXOW5HFvD6gszrYFqRrHJqBq8zAo7Wutxe6HwP5PJ9zQwINQ87gilBJZ5xMMcuh", + "FJFHyNYQ328dDjnjUo0KkK8bDlaqEdQXyQUp4ROLrPvCHuTeCd6LjrXdZrls3PGtvl6iqnBTTQNs5qnB", + "FQ2vVregwRAZL6OcrQRWK+Hm6mBWN0EvLAtSUAmtUg/HDm1AzovHljxMws02QTJhlVX3s6St2nKxr/cG", + "Z21x/lbD+p1iNTtmEx7A/riB59RZom20akaEg9qLCaMkdrpk4UK1pi5IGE8kQXFO7MoZ+VRgu+DYHG/w", + "WTFnI6NsWuP19Q7bmIfNGFaXH4F+7Yttwp1kOKH2XOSwViZeUSJcpta2CgKlchR2Zy03LMg0T7BAFqGz", + "zZDlIk0ou2zTulykY57QCOkP6n7xCU8SfjXSj+QPMJfNVrPTH4yaalWdmcHZvECzIbV+yyn8oGe5WctK", + "BkvMlvl+CxyjbaLHgpHiP9GEWKTEd4xee4Rexeff2xk0Zcs3NFrJk1+GCr0p57YkGzzxuQzkFq6UclyV", + "LRLboglG7MlyaQr+tLiVHCqvcwHezqNTTRz5PGiSQ8Ova8AkaEwg78dNbZlrtGCLbaYSrDWSyxn6Ox9X", + "DaJtw34DFew2WAmRIcgkGN8PO7rSIG3eWFoTb3dvgkEBbFVPFD66IbTDulp/ZXxVEz95u1T2bkbsklE3", + "R1MCr0VJFxf/UcAX2F7b4xjUCxQG4pUBpUaqBZT6hfpKC6/qpkRjLgRAkmsJhzM3G4Bd0TKPXmuHe4XO", + "Z2SBBEkxZUNGWWEkBTA1ghiZE+FlyXKhlawpifvob56KByDuaaYWtjoAGM+/k4hfsWKMQ+YPUjeeS93O", + "ATOWRZFnqlI/VDcLWp8mFMhaBieYElBQk6oZmggiZ/7cQ0VUtYx3xUXcWJ1qgdwrUPQIfKxI8UvCfFZW", + "NBNUDU1DI/PVchSfqYAMT63+iSpFiVG96PDq/nJJRFhILKZUvNIqdMU7Kp5yYkBgABEFik3avwyLL1BQ", + "WmCelM3/1TVZ/nRaNF79rfaah2vicKcPjNk2aIKNTBpPLdin6klbG6oCaXCr0GyWfQlow4VQu4o9VUnA", + "q5zT6p5sl4lXTxZwo9mSJKr2vvf8ybOnLUsXfZazzqB3fWnX3Dxd4ZJr2KmTNn6f50+ev3ixu/fkxc6N", + "PCwur6Rhf5pyS/z9QRvkWunDmvzrH/98f1Lz+jyBGOzBjQZlMkvCQ2rILqkO6P3Jv/7xTzeqWw8oxGiW", + "IeMb/PaNUTqJv5MuUKDqwmvnJFuh3x9UjAS4YDNogwAUN52TkVm3XjmYGgxIOykYZziiahFg5PjKRLsX", + "r9Swttu4g6qDDYm8pm2Liqo5l8zHZdLphusc/afxDddo4XnrCmgyHzf5od/UezVe6NJr4cc4tAgxkEXx", + "/WUDdzGfKywrAd367wjyLlyG2XK2jXljNepuPRUColhsoT8vFDCEfF+TJ+1H/vbXttPzW1bMOvUV/7Di", + "HDYfwRtZfQM3csDoG61Pra3xB3sB3u6r0divTbiy+GOlkGF569683xbZw8uFM4ob7Ob9eQmTN/mwjgkM", + "9GjHYJe8bLtbIYkGavJyYQIGNJ6Qnle8wBR+krnxCOozb2HmAxmc0SWfTKpYt0+asdEB9geSvVwvWCmt", + "mXQRuXY2izqwtsH4GXaeyGFHqwDDznY67NTcVsH0yRRfj2wHVWyXwSqw8jL9vTZI6WYwTnh0acruQTX3", + "PhqglGAmUc7g8Ne8atuD1d6hbifz9qaABicmxGmJbcGYxmSG5xSqe1ifyrQSiEmuqZIQMArt7KOYG7Sn", + "Ss1hO0P9mklu3C8nDZcOZgvbsG5Qv8eZi2gt3wUD3wQqHbOPRPCuBSvQHPvNm5OuCWCA0EMzsEp8o5uo", + "GYFmkEUXtfIK5e/h+OFxQkYw7jpcf7q8jn5OOnhWBZFESYvfXZJDjQhQxHOm6jj+aTtFrppWtnwl5QyC", + "/Wz4B+Cy2d4NgaCYRHAi5fJZrBL6LYi7ljdgVzqUOLAbImE4FOBLCvuK31qHcH0AxtjglQs37fhx3cZL", + "OJKK2/pyxakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKKSt30b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", + "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtfUa", + "76COoYk3v1UlQxuq/hDFDO1bd1LAcGl3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", + "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmKZljI2tgZnc5UsqgG4OwFsJQ+", + "q8qnIIowZyhss/PlbroPl6Pd7Hb6rYeE4zMPGmippIkVSUercLMPS2+btc5neAFWnEYn4bPdvcFgd2dw", + "K+BsN6wbLNdh+YmtiVltpymlzvvOOvorUap+C0WS9XKh5StBIVe7WCapBMHpPiTeZDgiKCETAMkrElrX", + "exbrXa8evBWobBZtQf9uo+y+OR98tWRO0ZXFHHfT6DjnYhWDyH++xiHawGaiJUi9QM7dbm/w9Hx7d//J", + "0/3t7bsAuy4WqSnb49nH7atnyQ6e7CXPF89+3549m+6ku0E97JKaykBtaPUX/W5jlE15SVaxjCosDW3Y", + "OWRE1Cto1yvPS5JQRnqyyJBan6a4ghcY//va838zO7+ZwUrZ4aw6SV+EwKpcnAplPQz+lp3MSt9FfTbH", + "R6tncasMpPpAwvRWHwqQV7vBQIWK7c5nIjPkrOU19M57sfVFtDIrbt1VFPKww0kP7nLDiofIuwbM4M16", + "1QW+fMkFbKdTLqiapatvi+K1AkYc4qY/ShVX8Z766HjKoHy+/3MRJucrUfrjTreTfNyrnhn7e3vkL4tA", + "XBCg3WpfKmgRRpaQOUlWrwK8UioewkSya11dj/mH7d72C4hDSD7u/TDovahGHHTNavnLt+3ervw6aLOG", + "fglAVzpq+8WNIq7deq6ioF9oqIBdeS9bbGJL42Wxcnd1uITbygaXj5f2uIbk0yiAfq6kZy+3kS80xSTB", + "ixA2vWeolTXt0ScyNCZTymQbu+3uoDDcPkmHnT46sADhoMsqXvTjN69pxacTmqYkplrGNKp/cwbDTktb", + "XF2XuFltEvdVQFrrh8W1F+shEtYlXK27JvufkY/7WdpvO413FXoH2NWcigoYYvBiF9EJwqxWoJSyOU5o", + "bBPpITES4tX2HVBbSbKWB8hSDnR2ki6acoXKFPqW9racNdsFi/GTa7C3rsDMMASx80UAUQoAMbqKfR0f", + "oUzwOI/K/NEEBl0ifoi8BtG2QshfH5J7l/YNSMyecIHW2zeaDBrt7JNN+12zTWqCbd7q7cH6rb4To0i3", + "k2fxeh5mXmrHwW6E3L4mBTFgoqkue00S9CbzoQVHf+uv4LLOa2zJkRaJ8sw5WDRNLVNSwN0CLoZQXO8R", + "SYi+ppYbQTyJyywJKksuup6lbj99PmtycYJHankgvxCSaV0F8I+gvxSzRXBgruxocZdsDBzatzQOr54p", + "V2RXqzq4Z2slscat8k24TSUUDJev2bwNXsqlZ/4uML590WwZAcUx/IqQ9ra5BID90oW9NdqP78Is95BC", + "2hvreqjBtjpQ4QId3fVfxgJrsa5KvHsh93yILM7xFGrlrfGjNJJKqK4kODkwUtgWt4ICm/qfCKqISgfY", + "4gQE+Nxl9Npig4gqhKe4XjmiqSyl8USvjyFqLvRybk0ETZi89XRY3/x+0PsfY25Ho/7+1g9/+b97H/4z", + "aHavGRAkEb2YTCDi6pIseqYKk8LT6hX6hykBobWKqT0zBKdgTAO0d8uV/PE+GRTcc/ErTpemAKFqXgml", + "7bUT+st/NAd6ecv4Di6MtWf3syuk3EUlWcXdvbyREjF1QfUuo26zP2SQpHdJFhJ5hdmsbOdO7Hey+MQL", + "xUcXhtz7hM0v0JhCpUs5ZFq9x1FEMq1W2Vo/1JRr58CGBcGJ344tEOfOi/XMmsAKgt6fLMEZv3l3/uOb", + "d78ejd6cvvz14Hj0y8v/hliXq57pIe5p2tt78tQWafdXcjtYKOTm9S766MTmK9iYh0kOmj0AlkmU5iqH", + "6BhyHSW5pHPnKVXJ7StbLGct375SxGdCISuVhMIzLGR3QicEAhzgXrXRRVQ6YqQSqttbKw9laFl0MYQz", + "7MCVoriTJIJ1RfRWhFe73Njqoj+dtbtLDEhs4LBD6q8+bAEr6msqAbjDBcF4L6MNSKFxJXhdBvHmzUBr", + "D4oGgyGYX7jS0uDFl6iG+m5l+dM5T3paz2soGRE0q5u1CKYQQFMmNaPT5H2ZjgPKjLVxT+kUBxwuIcfK", + "F6la6ga0NnVsaf8by7eFEzqO6vU0zLE0S1Wr/1CzlkjVa873SLV434BADBDLJomXekGG1YzdlKktW104", + "BBwSc0B1X5W2XZ4yBxPZg4/WZyOvVDC9mXkjad6bE6dH1TS9FQt0qpfmakYE8TYCPijrFNxwyWyCUgs4", + "GlOdMSOiDN512U1aPAe/u0QbhQnMLUGRdr3sF1hdh+IEXxc9gE8JyyVHLMyjrIO1/epHqBnw1tX+pBPX", + "BAyjpuWGEfKrVLRqTRxVLW+GT1XL8zbvBw+e5VUruF/T2aoRZ9lHhTRD9Pg3TNVPXIBe3Az+cudA+3D5", + "x0QAGF4dRr8VBj1NSTziuVp9/vVrWno0V35RH7asL+xsABiIOKrkNTfxAgdPUo5heaX1cpAoF1QtzvR6", + "2ah2yAd1RX1hIaEj+LnsGAqpfvoE1vNJIHPmFWFE0AjK1OrzmGIGGhN6f+JVKzSFK5eAa0EEenN4bO0u", + "DvsY9GiqgPRcAOrB6XGn25kTYWwPnUF/tz+Aw5wRhjPa2e/s9rf7gw5oVTOY4tY4p0lsE8mtRl1o8Mex", + "lYR+dC/pLwVOiYIvfgtAIkAApn0dVBA89ZTIDFNhtcgsAagGQzBUfw11F9yFum9u5a5Z9tbGY8i3hjQg", + "kr2xm/sBBGU4OzDNncHAIrwre/1CEpPJnNj6uw2jLfttJdXZJQqUIVhS85xsWSz9p25nb7B9ozGtGgqc", + "3VDH7xi22cwEtPMnN1yIW3V6zEx+os02t3Fh/okDQvLP2m8f9J7JPE2xWLgF81cr47JJMCYSYfeu0eOU", + "RJFmFVAsqY/eMGKeI6wQNiHcImdQY9p9qCm0egpM226TC7SmH3m8+GJLWOnD2Sg+VdmZPi6fluj5y9FO", + "QcbLG2kfOahxQ7X3QEA/4qJA+oOdlL3Bi7vv9JCzSUIjhXoFAdvAbCoh9ikB4HQHwsQF+j3nCqMir+ER", + "HWkrs44LcuuWV9HWHzT+ZI53QkL+gFMiUsxMloh5Z82hXzrOxjdTHueVt5oj/OOjjr2pHBqRuahAkKse", + "Uf/aqguDy9fRXgCKwvZpphc/IOHv3cMJt5MtauQ+5JGDyqQol+QxHSfraxyXQkhQlntF1NdC84P7vLJs", + "NYU/4Sl6LAT8ihQSXrlbS5fCViZyZhTgoAT4tszctN99VxX+zssnXrgQ+DV001DXQxkHM44XfeTW1Cj9", + "agFYU4LAPOPla+VUD+9rOWE793HCYMaFp+jbNfXtmlp1yg21uCnAwfROeQsbxI0sEH8++8ONrQ/fbA/t", + "bQ+tLA+MXFnrwt/5uI9saG7EY4LkjOdJjMYEGeAnF4SjsOhPPyIsohmdE0D3g2p1eaJohgWE2KQoxgob", + "H3qjYWKlWaJobks313MBmeUC1wE9JBlBCMyoCYizDMWkjJEY6U9s1EyJq7hUV92c/aCBvWiwvBrR1YxL", + "UgAbMuXd5pDnLY12DM32h+zcIt7qBYSocsdrJEkAt3eF/YczhIfMfvC9YyEuIk7itORcWAB4IjUQnWZb", + "lnP89EhHMuIh0KFzwjBTPZmRiE5oZKd1SRY2sDXYYKsCVHrAbpzvT4rMFbSzGQaug9ClMErxUfEMWUqq", + "+m8YRINHSR6XTi6HpYTFGCdJsELJNOFjnIzM+lySgE/wFbxhF6V0uJTeJMZjYmrpZws148z8nY9zpnLz", + "91jwK0nEsLPZHzLISLFrTeJuKSCiK6hol2ZcnzPBU9Pnlhni1h+XZPGpP2QHcUqZowj4BCeSI3IN30Hc", + "GICHGO7VQA/mNIX94Ie5VDz1IWAd3Zlh8lxlubKpNZKobgj+dMgUR384kMtPW3+UPX4CZzHBsaYT7xUz", + "JZCtm0YtR1jPfgSvBtztBBZg2NEXqQnzmArMlMEvLVA60dTf0o2iTASUjq2vcIQZynhmSmwAUc2wJrlK", + "GwBagZMEKThK7lstuMNONszHYhCm40YAQoMYVztGlKGTH73DNNh7Hj5PkkSChCJK/uvsza8IbmW9B+a1", + "MlzL5LYwLTCgOAfXqeNpL3E0Q8ZRBVUVhx0aDzuFOzfehLHm0obL9HrgU/xBD+0H002Xxj/0+7op467c", + "R7/9YVrZ12cpSw0g6rDzqYu8B1OqZvm4ePYhvKBNOG5nFUaANsw1twmcBFOA3PFufHNFYhYjbm+BZIEw", + "KjmQH7gypgyLxaqMysDS2xXkExPJ6C3GH0OIXBx29ocudnHY6Q47hM3hNxvgOOx8Cq+A9Vo2l/CD+6xw", + "bhZE9HQw2FwPCW7XN+CzbOEY+MI6YKNWVNQf1Tto8Wj/XP6Bf2v9s3D9YKY7LzGajOLvjO+P0AHhSey+", + "JhpwQdTEbswikjixe72h5/6dB3qzIpIk902gD0WehXusKFnwqMgRNqs8RivN9w9McYP7ulQqZvuHod9H", + "Zz8PWM+t7ZzMXahzuGALgPFYVRqZlxGW6AzG1DvTyvdL+LVv/+t0PwCXvEj49GLfqO4o4VOUUGbzAbxA", + "ZS0e2LWEjwweT/Gdhedx1fI2jCTxr3/8EwZF2fRf//inBbn/1z/+Ccd9y+DMQbHtixnBQo0JVhf76BdC", + "sh5O6Jy4yUA5XDInYoF2B9bmD4+QV/PfSmlyyIbsLVG5YF7ehClcJ22D1lWg50NZTqTFM4I0oYmtqmNi", + "GwN2G3eWzVLe64nuBnAhYQbeBPSt6GgAQPWoqThuNdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkM", + "LHHo3MEDO2m0cXb2crOPQNsyVAGVk0B3KJuxakT/G09az5MMR6kyFFhlw5sinOExTagzOTaUfTFHMMXR", + "jDJSxhcXoOuuiX03Us1jDk6PkQ2E7MKrQ/bmbAtMrIpEKhekazmBsFCrZV04bvNcoAfgX1RBdFjPvjtk", + "E4IhT+j4yDABD428SIwsGmaAaAIxrlRVStB1h8xA6loIZ33wUh6TBD6C/qdYkSu86KKi6K8rE5NgpRVi", + "2dUvD5lJNbRr0APMFuQNsw/8zAyp5yJ5bc6WIJNEq8YQgW/qn0PfGxMukI1w7pZ5pa47k21qhqUXLcXR", + "mzM9vylogtzYA6GlN2duNza7SHIUJRSoIcJsyKYQCORQjDmr7GqRUDbDIu5FXF8CPqrVJeNXCYmnTTz2", + "0CeyO5RkKv0EjtPPdXJ9bMLFbHkC+hAbpL7Vnrsj+047151t8c/ku7MVMW/gvDMWXGL4jVndb468Fo68", + "8Lo5p17Is3bkoCjvLuLXdPFAAb+O9pbX3DzxluwhLHpow2H8gFeEC3R6eIxwHAsi5ea/t71Pz9RQaSn/", + "6ftRs+KHCD2xY+HCoh9ae0uVQB4LO3hrR42wm1e90LB/v21VqhA13nRFQaLyyrv726PW6U2ukVLoLWnt", + "202yNtiWyohDvcWSWnogGiWkEF+Kc+pT0TqrsgnjLa6cleKSZc/HR+5A3p992Xads/rdcA9M8ajGEB+Q", + "EVZTrf3y4Y+Jmt8Vu+hgt1eYn78u0hzcnxR036boEJk/JnUxri2b5oIG6KTxAn1FlIE3uUs93fYQmPgZ", + "Ee5Um4EuzKyLaZlPkcFpgQmBJWa17ntsXmmn+pr2/kyaLyzPTSQWu+TfRJQWym65VqsU3GNbC/vu9NsK", + "kts9h61YAgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0B8wuva5nO6rWU", + "oUlCpzPrBIjpBGL1lF/IHEa5cw+jLAqGC6yIDVF8jHm/p3qRrRd4ToSH1OhfqVt/QNDqelXJMa+Vt+u7", + "t697hEU8LpwnzTKpffKFFSZD/5Vc3vs/dY8wn5U68aBJYPyM/TfB5KiA7vxfOz9Z8M7/tfMTTjLKyP/a", + "PUiwIlJt3hmxDO7rprtvBeYRE5/WX2h10ZZY05bCUz8PtUqZDr72JpRZgZ11xu4ikeaLUd+XF/LqWL2t", + "JLx7o3utSd03YsmxK0tgtpQLh15VgyT+9xb6zioEfe/2ukr3VMIIQLB7hFHKXDjUVWxhq6kjbsub2BRw", + "vdcYI4q3Wtoj3Pt/KpOEmfSNjBLFun6zS7SxS/jLtdI0YbfiTo0Tpo8H8r4XxBZabXj0DW7rHhw6liI9", + "uK2Kh7sE3JpxqeDR48NesFcJLSjOvzZaeibLA7ny+nCke3zUhYWEsv9QdsimNt+Tn9KN494Vb9vv/Qs9", + "B+mYTnOeSz9rOsUqmhFpEQUSUmXAj80kUF7PjUaBr5hKB/d5ddy7zv+N7u/IGlHfUMO8TbDBOpnfvdVW", + "5rfva5nfoC1b1AVbG6nr6uZtNiSBOLzltmRcgaVeTk4JjSuki6B3WlEp1QUEGsT+kP1vrX/8pghOP/zg", + "0rvzwWDnKfxO2PzDDy7Dm504UiFMCWrLnB78egQRHlMI4odKqCWYRH0cKM2lMqTnSp782ylIZZBLew3J", + "UeE3DamVhuQt12oNye7F3apI1bJJ964jOXoLLbitd/Dn1JL+5K7bigYn88mERpQwKD4Fxj+5FKtsNLlv", + "XttbgiUwGyvhBTpWJJHWamTBtdZI6GXh/y8ZSdhtrEHBEVaKpJlCU4EjMskTU7UFyVmuYn7FXEkKmKCr", + "bkbL+YSud9fUyDUSTpANl2hvq+kWZRnvW9W1HT/SDFWe2QrjVrksRZtm7fJhifdudcoWV+39a5WPmcSM", + "+ra8dJnWEAIl1oybJ81NOm/xZYnO2Efn569d6q5WT4Qr2Ke4q9LnKjkPmV+lr49eluUPzQuuBa0+kNim", + "+kNCs617FxMcJ5QRyHUgMpRlW62t+aDH4stLwOHCofft4m5zLG1N7IeTgB+MFdyLrFn49JV1yBZH06sp", + "WpwWJ2/CqXlU/MoyoADjCcl6WzhXvGfBALZm3CBEhkFyTxMcAUaufs3AN1r8FYPX6jcFoCqCJwkRBpYz", + "y5UTt4asGBxliheFp61kdqGbH+VM0eSia0INAVtJIswWFptuyCqdWZkPMBIA/wNGKEhmRlyroqsHTXku", + "4S2AM/C7RDi5wgs5ZBZVwXwOpdcFiQyCbZL00c8cAG1MNWmP8ZpSrt/JIbugcUJGFo/mAlGJ5IwLRRiJ", + "UcrnRFb7JVgklAiYxCHWKydRihcADGkwcs368IwY8MUK6g3X/8YsplAUVPdcTHl/yDDaGQxQSjCTFsNC", + "4glcOLYNBIOoDOh7hNHe4IX9qrZvAF7uln9DnyYhyJxHeJwsENFUbIptb8IGprZIr6n6rrdvQoU0+1XY", + "N231xcrGUulqzsZdlLMSpQNs/TkrQDX0dqlcMJin9QISKopr0AITjUmE9XoyXu0HIGF5FOUidEHqrfaq", + "Rf87Co7e9M5gqcIYGAmYDCISw54zrmZwpjkcpc3vG6iqJKo/x0UTPCRcIIw8ui4tGiTKgTVuAITqRVn6", + "lLlS5heb37uzo4+vZQTu+BsQ08dyPwER8cmkcgDXX03mAK/KPVsm4T/rOT10Na99FhdTPGVcKho5ZugQ", + "qn2l+ZtC2EohXL2yQWqecHHZHHD8ExeXbTUwFxj5uBQxf4ZfoSNCDw9A8B/eHwHWcKOsaKK5dyWtTl/F", + "KQWhiypZhASjhLOpPkWlVf7e3Qa+VrdhAC31ZSqMs7uAH9NKyMj+aMpme2Hd4GKIbKsPzYt07/fgjPqV", + "K0TTLCEpgbLaPUNserNLqLrxwkLpFYBtN+OV+lT5uApGF5Qm/qDrxCGgK7dhGyC9L29XkKkmfLoeELXo", + "3KF/BhBRh+ydNKUKLozr6QIVPFgLtKb8CLqa0WgG6Kigt+r2DXgqzrKLAhh+cx+9goPs4+ND5xum6Iim", + "NckTYkBP52l6sb9cOPr9yQl8ZIBRTYnoi33kikUX94fUb/lop3oWCZYK/WoxXDcKZRx29EJhrW8W89u0", + "OKglcP+QhTBRGbmyDdIJuvDgUS8asPscv33Np/KrcRWV5VbMXBRHVnUE2iQs7jQFedAk7PjZHgxCVQBa", + "orSaYdwxSOvSYF7zaVHqpULKOMvakq8dJlDxPE1X0DDa8CAfpYp5rv4iVUyEgI8tdTcRN9rAkS3zhy81", + "oVqAT3ewN4H8gqFMpvZCcKk0U+10O4TlaWf/N/uveZp2uh07Hv3dlcpSr3bDDYT8Nai39QaXQ2/0DnnQ", + "tt/E85uA1laZvodaW7tBrFrdLJm/NS/86b2Gznb3gGQIckLNmPs1iaLeeKuGH8YL9F0Y2Yv7GBlA9KIo", + "4ZJUHD2PB+DPGrxqsmOzwcitcU8PL85dRbQ2kSxn9tMz9+VXoIOvixlxY0ZuuvcePLI8gscMViCXZjPh", + "oo4Kty6q5KsnpC+3JUtTbUMh32jz5tbGVoSp9YVlFmE/iE2FTJwrnmJFI6jOFs04lx7ZFxDupo6iNSIX", + "lAkmFqPt2kyCC02qF9YcfWHViX1rOkPYf2T76MPnNv8g/IV7VH7xk2cdKDh+16kAUMFEIozGgpIJynAu", + "iZbq8pSgaBFprmjK8REczVCEM5ULApVGCUopo2me+tj8esfmGHCELrbTiy4a5wolWExBOzMPXdBNxNOU", + "sJiAnW7IZgTPqVYtBUqwIixa9CSBCuVzgq64uEw4jsHUkMUYPD5Q4VQQTYFQ6CAlCsdYYRB0LvSJH5lk", + "pouiaLlR7xm5LqkhHjKRs+9N1RXd7IUb6AUiUFaAyllR3DbCMWFREG7/7OtmY1/eJn1GVH2iDxQhdCte", + "+pAhQ77t1Q3n64gmerTQEC3Y/AqhVzarsNUsEEdG/55H2szVzfGBHE3FEq86xV+Hh6kguq/Gy/TwbiQu", + "UJyb7rxTCWT+Z/UNFQzFD7qCDFOzjbd1EBVVPItlvhHP2/rD/Xl8C1veV8IJu42KfVO9uHLSXwPLtat6", + "K577QEZMa0vybXIPx4JdZNeDiU9ceFzusRhbKwhtBd/2uZMSGLQvzr6x7TrbtoEPt2Xbzja75Nr3GDll", + "PYgVDXNwa8ZtZNXWdPBvmpVSm53HMh+cRZaei3uHW3SsMcOLhOP4zxAsvMJ/FHEhDAwGAGs8Johoz2ro", + "pwmAba4sRNl1WZvvT042m7iEUCt5hFCPmEN4qTn6szReNuC+mRMhaOxgMA9PjmzYLpVI5KyP3qRUIcXR", + "JSFZmdkC2YV9PT8HCFIbdh35o9shTIlFxilTa0dRvno3gyl/4BYX5SsUJW3NgW/u8NbucLDsPz52BlwG", + "cjfMBFZrpgqrtbWQKZtwkRq5DI95rlvXPEgvk95Pg1gwoQmRC6lIaqITJ3kCxw3q19ga5fY7s8tdiM3V", + "J8ekzWVEpFRKypkcMpszkhGh+9af6/a9QKugQ0Dhgr+eGib5dQTx6cGYuDWsmlYNoJug9nFnv7OFs2wr", + "xgo3BIrZ4X3GkH6CqDwkF+mYJzRCCWWXEm0k9NKoJ2guUaL/2FwZ1jeC7750Bfbbnyy90sdswoP1LQ3N", + "FsT8p8rusmzNOSYfHVt7RfzD4vgPbHSYra2v8S4ITnpQM90B+KBc0YR+NKxON0KlopFJPcLF2r0/KZhq", + "f8hOiBL6HQwpbklikA1Au9zKBI+2hvlgsBtlFFDgdgkMDhhe8+MUejw8fWfSUUnKxaI7ZPof0PD5wanx", + "7k6wtSZ4A7XF3dHx1ps1gc5nsEz/xhGCZoIrUQyCG/7NJXhzrJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", + "2hUA7KmYzUYB8OVQucI2hDlP8lT/w/xxvA7fTOFo9h5e/WqkXTOctd24CT6KQ2nnFBNTf/dBnB5mwR5r", + "zKpeODcFEGIq0YDBW+BA/Rmp+8ub7/11/ArdnXZFXW3rr+Zs3ffNZ8fgkDb89Xgsx9xQmpuJ4qutT1eY", + "Nluffkx4dCktJItvNtR6G+Cs6x9LXGzrIgQxATJEkYUyMoBZRHaHrGaANMg/EmGkiEgpw8kWzNk0Agjf", + "zoqF55xConYEeSo9SWPATkoAxhtg8PRswFDlGvA8utJW//Pf8Z2RiqMxiXhKHOr5Zkh1+xum6icuqhDm", + "XwtfPPfWH6ABMQV7+xrU9uYePwvF/QRfQ6h0nFuHshvRxite/mhMQV0EezPs7A7ksNNFw85OOuzoHTjE", + "YELFCj1BKWW5IrKPjox9C1Jxnw6QJBFnsXTg686CtzuQTYm5hiwbsjyfwnf3KfZYqoKlfGs7CbEH/R7S", + "30PSDtrwD5w9k3EXDl2MeK6Mud+eK/tWTBSYRzbv3VfrnZFvun0bTv43e3wrPAp2WbNLb+sNZ89yOSPN", + "JrfXpqBRrsYA6u0KIMsZ+jsfyy5i5MpYw4VU/SW+p78+NR3cR8EB3dVNig3YuX+rNNCi0kC5VmHQRhNg", + "qa9kRx0GuZFcZ1woQHO0OfeGhkCTAAQJKFP45vB4yCLNigzEoCApB+5kcdHNLXzwtzP08vBtFx1BMV70", + "cz7e7KM3LFnYGvbWRzNkRhIzzCvCDI0N1ZI4dD2bsQP13GWwuO7ggarbm5MR8Ky4vXJB4t3OjOAYJJI/", + "Oq+56SyAPvz2tT5AAABsviy2vbNS+Oi8JUosegcTRcRysyc2T4oV2Bn2knZQdFZwMwCYukPpENjKPo1s", + "YCAydnc6AcSMT9+KP9x9Eef78ZKZOBFTdm+cq0dbvxUOYsEcQyzQv66L8glNWcKWl61UMKDLpsjvr8jk", + "vpJ3VTDm/11PF8z00Tqasso+aSIuyq6s9fS65OCZgUW2jqoIZziiatFFOEnsHWVvgiIipVeIv2NB8GXM", + "r1h/yN4WBV9sQi86PH3XdY5aFFN5aVqwvtg+ejMnQubjYnAIDprxGsOak3jIFEcRTqI80eIGmUxIBLm4", + "UMdFNvhyi6F07vDslJ0Ei854Ue35o6t1F6YJ2L2SLOoUt2W2ekuQKME0bQYht4IaBBxCqMFYN8oZomyS", + "2JCqSHApkW2qRxI6pePEBgjJPjqfESRxSoYsSzBjRKBcmqh4PfReJoiUuUnw1g0AWK+hqC4qAQYzwZUN", + "TUg4F9JEE2gKf3+CpCLZCjJ7a1o+gTnfkWxrGrc9PZCRujaGZlOIfQXpDTGUYhZc01GeuADGew1FNwN6", + "aCnxsRz8c0GnUyL0qcCGyZpwPHOs3XKaQ1/JWG6se3lWvNWu7mXRqpeV6GXsrQSIG5WY23HnZlF/gc4v", + "aSOGoH10syziX/RHLfuuZquGB2EffeYsQyU8/x2rZZ55SYJtDVglhT82c5I38spRrSTarofVap1Ze5eZ", + "rq3xsx4MNusxo2XhSvpsk8L79RHC4H5RHu672Nrjpq0K2lVFN21I+V+Pqv9VUODdwOk/MMrJLeD0v6q8", + "e8A7fzj8k+BBfag8+orv2RXd/dMj4t9V+ryBxQc4tqb0ecP1bPDqSkXpvX2nnZpkW/wzSfA23vEG8rtb", + "9m9afwuVwVusdS5oTfAkzdTCBbRZX2UZdCbpR9JvcAQXcat35wq+RUjnlyMPR6eNAZ1/zhr5DxIzaksI", + "UomOjwLF5x8ZxqB/5ioXy5a+dXpYRDM6J81G9+oJtkuUCdLLeAbOldgsmF0Pd5cpLPrTj8g2bzFX7b+g", + "BiVA9ZMYxVSQSCULUw9UcwTTx3cSCa41AXjOxaI5SsQckZ8ETw/sbNbch/ZMWWNYGWeYLnoxVrg3d9xm", + "hQntM6I7XTylZniIMvTqR7RBrpUwlS7QRGs+iE6KJSXXESGxBJrc9Ae8PWiwbNKPZDQdtxnlipolb2xN", + "GBTlUvHU7f3xEdqAGmhTwvReaFF/ApJsJvicxiSujLEz54lZ1e2GBb2p3VULFUUBO6dcmME9iAzT5kKa", + "fqRZlS0UITFjyjAMbm1VkOqZMkn8uj9MmQvAsXvkRvHtCrOa34ZTdjQlQj1Ou4iKcwPxvPntmnvM15yf", + "DOXutMpt58JzVhuv2+VHtUxbuovCD0Xu3P2ard9/PSk9VD7KbB5rOp8XCmmT2fzrIsHB/d0P920uf/+I", + "U0BfEad8e6ZyaEC3GCKY1xDTHZM5SXiWQl10eLfT7eQi6ex3Zkpl+1tbEPs941Lt7714ttv59OHT/x8A", + "AP//G3o07F/4AQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index a6218cadd..fc3f221eb 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -354,6 +354,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..2c7843532 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,16 +10,19 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) *GPUResourceStatus { +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. It returns nil if no GPU is +// available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -27,16 +30,16 @@ func GetGPUStatus(ctx context.Context) *GPUResourceStatus { // expose the PFs/VFs as available passthrough slots while active vGPU // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return nil, nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) } - return getPassthroughStatus() + return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -51,13 +54,20 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 } + return status, err } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +102,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..825a20260 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,88 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return status, errors.New("VF health state unavailable: read failed") + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index bef0740dc..39166856e 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -207,7 +207,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..9c28fda0f 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,15 +691,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/openapi.yaml b/openapi.yaml index edcc53a3c..ffa6d7e51 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1809,7 +1809,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1822,8 +1822,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode) From 16b7d91e402a03bc79b0f5b5e1011ac42f6dd7f2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:56:12 +0000 Subject: [PATCH 02/21] Fail vGPU placement closed on VF health persist failures A failed state write previously rolled memory back and left the store reporting healthy, so a VF whose threshold-crossing failure could not be persisted stayed allocatable. Latch write failures and refuse placement until a later write succeeds; re-reported markers retry the write. Also make acknowledged reports crash-durable (fsync the parent when the state dir is first created, treat directory sync failures as persist failures instead of logging success), and re-evaluate persisted tallies against the configured threshold at load and on threshold changes so a lowered gpu.vf_quarantine_threshold applies to existing failures. (cherry picked from commit def61def247337d0340e4340ea0a4fe69a5538ce) --- lib/devices/GPU.md | 7 ++- lib/devices/vf_health.go | 85 +++++++++++++++++++++++++++-------- lib/devices/vf_health_test.go | 73 ++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 21 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index bb7b3f320..44432898a 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -300,8 +300,11 @@ equivalent free VFs is randomized so a wedged VF cannot capture every placement. A reported init success clears failures only when that exact assignment has a recorded failure, removing the match and older tallies; if that assignment crossed the threshold, its later success also rescinds the -quarantine. If the state file exists but cannot be loaded, placement and -advertised availability fail closed until it is repaired or removed. +quarantine. If the state file exists but cannot be loaded, or the last write +to it failed, placement and advertised availability fail closed until a load +or write succeeds. Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. `used_slots` includes quarantined VFs still held by running instances, so it can overlap `quarantined_slots`; use `allocatable_slots` for admission. diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b47ff11a5..b0870e212 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,11 +75,12 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) @@ -97,16 +98,39 @@ func initVFHealth(path string) error { } // SetVFQuarantineThreshold configures the number of failed assignments -// required to quarantine a VF. +// required to quarantine a VF. Already-recorded tallies are re-evaluated so a +// lowered threshold applies to failures persisted before the change. func SetVFQuarantineThreshold(n int) { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.threshold = n + vfHealth.requarantineLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +func (s *vfHealthStore) requarantineLocked() { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if changed { + if err := s.persistLocked(); err != nil { + slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) + } + } } func (s *vfHealthStore) loadLocked() error { s.records = make(map[string]vfHealthRecord) s.loadErr = nil + s.persistErr = nil data, err := os.ReadFile(s.path) if err != nil { @@ -163,6 +187,7 @@ func (s *vfHealthStore) loadLocked() error { loaded[record.VFAddress] = record } s.records = loaded + s.requarantineLocked() return nil } @@ -179,6 +204,9 @@ func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { if err := s.ensureLoadedLocked(); err != nil { return nil, fmt.Errorf("VF health state unavailable: %w", err) } + if s.persistErr != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + } addresses := make(map[string]struct{}, len(s.records)) for address, record := range s.records { if record.QuarantinedAt != nil { @@ -240,11 +268,12 @@ func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { return vfHealth.reportSuccess(report) } -// VFHealthStoreUnavailable reports whether persisted state failed to load. +// VFHealthStoreUnavailable reports whether persisted state failed to load or +// the last write failed. func VFHealthStoreUnavailable() bool { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() - return vfHealth.loadErr != nil + return vfHealth.loadErr != nil || vfHealth.persistErr != nil } // TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. @@ -365,10 +394,19 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu return result, nil } +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a later write succeeds, because in-memory +// rollback alone would leave a reported-unhealthy VF allocatable. func (s *vfHealthStore) persistLocked() error { if s.path == "" { return nil } + err := s.writeStateLocked() + s.persistErr = err + return err +} + +func (s *vfHealthStore) writeStateLocked() error { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), @@ -376,8 +414,15 @@ func (s *vfHealthStore) persistLocked() error { if err != nil { return fmt.Errorf("marshal VF health state: %w", err) } - if err := os.MkdirAll(filepath.Dir(s.path), 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + dirPath := filepath.Dir(s.path) + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + if err := os.MkdirAll(dirPath, 0755); err != nil { + return fmt.Errorf("create VF health state dir: %w", err) + } + // Make the new directory entry itself durable. + if err := syncDir(filepath.Dir(dirPath)); err != nil { + return fmt.Errorf("sync VF health state parent dir: %w", err) + } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) @@ -402,15 +447,17 @@ func (s *vfHealthStore) persistLocked() error { os.Remove(tmp) return fmt.Errorf("rename VF health state: %w", err) } - dirPath := filepath.Dir(s.path) - dir, err := os.Open(dirPath) - if err != nil { - slog.Default().Warn("failed to open VF health state directory for sync", "path", dirPath, "error", err) - return nil - } - if err := dir.Sync(); err != nil { - slog.Default().Warn("failed to sync VF health state directory", "path", dirPath, "error", err) + if err := syncDir(dirPath); err != nil { + return fmt.Errorf("sync VF health state dir: %w", err) } - _ = dir.Close() return nil } + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index dd286c296..75d8fbabf 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -1,6 +1,7 @@ package devices import ( + "encoding/json" "os" "path/filepath" "testing" @@ -20,6 +21,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.records = make(map[string]vfHealthRecord) vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil + vfHealth.persistErr = nil }) return path } @@ -92,6 +94,77 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { assert.Zero(t, quarantined) } +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, VFHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + SetVFQuarantineThreshold(2) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart where the threshold is configured lower before the + // persisted tallies are loaded. + vfHealth.mu.Lock() + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = 2 + vfHealth.mu.Unlock() + require.NoError(t, initVFHealth(path)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { path := resetVFHealthStore(t) From d049a804ce0ae62fc566d47a5d9bb0b57e6ef8fa Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 19:51:36 +0000 Subject: [PATCH 03/21] Preserve quarantines across sync failures (cherry picked from commit 4527e647c5284d711467503f06ad7b94d3c0334d) --- lib/devices/vf_health.go | 93 ++++++++++++++++++++++------------- lib/devices/vf_health_test.go | 75 ++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 34 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index b0870e212..479696661 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -75,18 +75,23 @@ type VFSuccessResult struct { } type vfHealthStore struct { - mu sync.Mutex - path string - records map[string]vfHealthRecord - threshold int - loadErr error - persistErr error + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error } var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) var ( - vfHealth = &vfHealthStore{records: make(map[string]vfHealthRecord), threshold: defaultVFQuarantineThreshold} + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } vendorVFIOMu sync.Mutex ) @@ -121,7 +126,7 @@ func (s *vfHealthStore) requarantineLocked() { changed = true } if changed { - if err := s.persistLocked(); err != nil { + if _, err := s.persistLocked(); err != nil { slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) } } @@ -307,6 +312,9 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } previous, existed := s.records[report.VFAddress] result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} @@ -335,11 +343,14 @@ func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResul result.Outcome = VFReportQuarantined } s.records[report.VFAddress] = record - if err := s.persistLocked(); err != nil { - if existed { - s.records[report.VFAddress] = previous - } else { - delete(s.records, report.VFAddress) + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } } return VFReportResult{}, err } @@ -359,6 +370,9 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu if !vfHealthAddressPattern.MatchString(report.VFAddress) { return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } previous, ok := s.records[report.VFAddress] if !ok || len(previous.Failures) == 0 { return VFSuccessResult{}, nil @@ -387,70 +401,81 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu record.Failures = remaining s.records[report.VFAddress] = record } - if err := s.persistLocked(); err != nil { - s.records[report.VFAddress] = previous + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } return VFSuccessResult{}, err } return result, nil } +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + // persistLocked writes the current records to disk. A failure is latched and -// fails placement closed until a later write succeeds, because in-memory -// rollback alone would leave a reported-unhealthy VF allocatable. -func (s *vfHealthStore) persistLocked() error { +// fails placement closed until a later write succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { if s.path == "" { - return nil + return false, nil } - err := s.writeStateLocked() + renamed, err := s.writeStateLocked() s.persistErr = err - return err + return renamed, err } -func (s *vfHealthStore) writeStateLocked() error { +func (s *vfHealthStore) writeStateLocked() (bool, error) { data, err := json.MarshalIndent(vfHealthFile{ Version: vfHealthFileVersion, Records: s.sortedRecordsLocked(), }, "", " ") if err != nil { - return fmt.Errorf("marshal VF health state: %w", err) + return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) if _, err := os.Stat(dirPath); os.IsNotExist(err) { if err := os.MkdirAll(dirPath, 0755); err != nil { - return fmt.Errorf("create VF health state dir: %w", err) + return false, fmt.Errorf("create VF health state dir: %w", err) } // Make the new directory entry itself durable. - if err := syncDir(filepath.Dir(dirPath)); err != nil { - return fmt.Errorf("sync VF health state parent dir: %w", err) + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { - return fmt.Errorf("create VF health state: %w", err) + return false, fmt.Errorf("create VF health state: %w", err) } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("write VF health state: %w", err) + return false, fmt.Errorf("write VF health state: %w", err) } if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) - return fmt.Errorf("sync VF health state: %w", err) + return false, fmt.Errorf("sync VF health state: %w", err) } if err := f.Close(); err != nil { os.Remove(tmp) - return fmt.Errorf("close VF health state: %w", err) + return false, fmt.Errorf("close VF health state: %w", err) } if err := os.Rename(tmp, s.path); err != nil { os.Remove(tmp) - return fmt.Errorf("rename VF health state: %w", err) + return false, fmt.Errorf("rename VF health state: %w", err) } - if err := syncDir(dirPath); err != nil { - return fmt.Errorf("sync VF health state dir: %w", err) + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) } - return nil + return true, nil } func syncDir(path string) error { diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 75d8fbabf..5166a76d2 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -2,6 +2,7 @@ package devices import ( "encoding/json" + "errors" "os" "path/filepath" "testing" @@ -22,6 +23,7 @@ func resetVFHealthStore(t *testing.T) string { vfHealth.threshold = defaultVFQuarantineThreshold vfHealth.loadErr = nil vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir }) return path } @@ -362,6 +364,79 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + SetVFQuarantineThreshold(2) + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { resetVFHealthStore(t) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) From ce327afb454edc63f2311ee89a8f77879a538329 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:41:07 +0000 Subject: [PATCH 04/21] Retry VF health parent directory sync (cherry picked from commit 54f03c429cf938c0a1a3418b888f1e576dc81c3a) --- lib/devices/vf_health.go | 13 +++++------ lib/devices/vf_health_test.go | 41 ++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 479696661..843060ca9 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -440,14 +440,11 @@ func (s *vfHealthStore) writeStateLocked() (bool, error) { return false, fmt.Errorf("marshal VF health state: %w", err) } dirPath := filepath.Dir(s.path) - if _, err := os.Stat(dirPath); os.IsNotExist(err) { - if err := os.MkdirAll(dirPath, 0755); err != nil { - return false, fmt.Errorf("create VF health state dir: %w", err) - } - // Make the new directory entry itself durable. - if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { - return false, fmt.Errorf("sync VF health state parent dir: %w", err) - } + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) } tmp := s.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 5166a76d2..260c5cc83 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -364,13 +364,52 @@ func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { assert.False(t, exists, "a failure whose persist failed must be retried by the next report") } +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, VFHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, VFHealthStoreUnavailable()) +} + func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) require.NoError(t, err) - vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) require.ErrorContains(t, err, "sync VF health state dir") From 2d48d8a937d1a91f62862578592e87f115804145 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 05/21] Deduplicate VF health lock-order comment (cherry picked from commit 294405abddcf728fb49970a2be15b21d477916a7) --- lib/devices/vf_health.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 843060ca9..3ae8d02c0 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -92,6 +92,9 @@ var ( threshold: defaultVFQuarantineThreshold, syncDirFunc: syncDir, } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so + // placement cannot select a VF while it is being quarantined. vendorVFIOMu sync.Mutex ) @@ -254,9 +257,6 @@ func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { // ReportVFInitFailure records a guest-reported driver init failure and // quarantines the VF once failures from enough distinct assignments accumulate. func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportFailure(report) @@ -265,9 +265,6 @@ func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { // ReportVFInitSuccess clears failures through an exactly matched successful // assignment. A quarantine is rescinded only when that assignment triggered it. func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { - // Lock order is vendorVFIOMu before vfHealth.mu. This serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so placement - // cannot select a VF while it is being quarantined. vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() return vfHealth.reportSuccess(report) From 04165f6be6d49bd3f9fad2b271bfecf2dfb46f0c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:44:17 +0000 Subject: [PATCH 06/21] Collapse redundant VF health test cases Fold the below-threshold placement assertion into TestVGPUAvailability and the repaired-state recovery assertion into TestVGPUAvailabilityFailsWhenStoreUnavailable, exercising both through the public API. Drop TestReportVFInitFailureRespectsConfiguredThreshold and TestCheckedAddressesFailsClosedOnUnloadedState, whose remaining coverage is subsumed by the threshold re-evaluation and invalid-record tests. (cherry picked from commit bf0fd622f06d393cd51db2db5b3d19c7387052ed) --- lib/devices/vf_health_test.go | 58 +++++++---------------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 260c5cc83..00f14dbf7 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -53,6 +53,9 @@ func quarantineVF(t *testing.T, address string) { func TestVGPUAvailability(t *testing.T) { resetVFHealthStore(t) quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) vfs := []VirtualFunction{ {PCIAddress: "0000:82:00.4"}, {PCIAddress: "0000:82:00.5", Allocated: true}, @@ -61,7 +64,7 @@ func TestVGPUAvailability(t *testing.T) { available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) require.NoError(t, err) - assert.Equal(t, 1, available) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") assert.Equal(t, 1, quarantined) available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) @@ -70,18 +73,6 @@ func TestVGPUAvailability(t *testing.T) { assert.Zero(t, quarantined) } -func TestVGPUAvailabilityExcludesOnlyQuarantinedVFs(t *testing.T) { - resetVFHealthStore(t) - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: "instance-1"}) - require.NoError(t, err) - require.Equal(t, VFReportRecorded, result.Outcome) - - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) - require.NoError(t, err) - assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") - assert.Zero(t, quarantined) -} - func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) @@ -94,6 +85,13 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, available) assert.Zero(t, quarantined) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) } func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { @@ -228,23 +226,6 @@ func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") } -func TestReportVFInitFailureRespectsConfiguredThreshold(t *testing.T) { - resetVFHealthStore(t) - SetVFQuarantineThreshold(3) - - for i, instance := range []string{"instance-1", "instance-2"} { - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) - require.NoError(t, err) - assert.Equal(t, VFReportRecorded, result.Outcome) - assert.Equal(t, i+1, result.Failures) - assert.Equal(t, 3, result.Threshold) - } - - result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) - require.NoError(t, err) - assert.Equal(t, VFReportQuarantined, result.Outcome) -} - func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { path := resetVFHealthStore(t) report := VFInitFailureReport{ @@ -497,23 +478,6 @@ func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { assert.Len(t, record.Failures, 1) } -func TestCheckedAddressesFailsClosedOnUnloadedState(t *testing.T) { - path := resetVFHealthStore(t) - quarantineVF(t, "0000:e3:00.4") - - require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, initVFHealth(path)) - - _, err := vfHealth.checkedAddresses() - require.Error(t, err) - - restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` - require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) - addresses, err := vfHealth.checkedAddresses() - require.NoError(t, err) - assert.Contains(t, addresses, "0000:e3:00.4") -} - func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { tests := []struct { name string From 6712e0ffd6c13a1edcfa19124a78f8b5521af71f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:30:47 +0000 Subject: [PATCH 07/21] Exclude quarantined VFs from claim-first vGPU placement Placement for vendor VFIO vGPUs lives in the instance manager's claim path, so quarantine exclusion moves there: selectVendorVFIOVF drops quarantined VFs from the candidate set, counts them against their parent GPU so placement drifts away from cards carrying a wedged VF, and picks uniformly at random among equally ranked clean VFs instead of always taking the lowest PCI address. Claims fail closed while the VF health store is unavailable. The quarantine set is read under the allocation lock but mutated under the devices lock, so configure re-checks it under that lock before touching the VF. This replaces the selection-time lock coupling the original design had. Each persisted claim now records GPUClaimedAt. The health store keys failure and success reports on (instance, assignment), and this field is the assignment identity the detection path will report. --- lib/devices/vf_health.go | 10 +++- lib/instances/manager.go | 2 + lib/instances/snapshot.go | 1 + lib/instances/types.go | 3 +- lib/instances/vgpu.go | 52 +++++++++++++++++-- lib/instances/vgpu_linux_test.go | 56 ++++++++++++++++++++ lib/instances/vgpu_test.go | 89 ++++++++++++++++++++++++++++++-- 7 files changed, 202 insertions(+), 11 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 3ae8d02c0..a84367a32 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -93,8 +93,8 @@ var ( syncDirFunc: syncDir, } // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine - // mutations with vendor-VFIO create, destroy, and reconciliation so - // placement cannot select a VF while it is being quarantined. + // mutations with vendor-VFIO configure and destroy so a VF cannot be + // configured for a new claim while it is being quarantined. vendorVFIOMu sync.Mutex ) @@ -224,6 +224,12 @@ func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { return addresses, nil } +// QuarantinedVFAddresses returns the PCI addresses of quarantined VFs. It +// fails while the health store is unavailable so placement fails closed. +func QuarantinedVFAddresses() (map[string]struct{}, error) { + return vfHealth.checkedAddresses() +} + // VGPUAvailability returns free allocatable and quarantined VF counts. func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { if framework != VGPUFrameworkVendorVFIO { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 2faf2c382..de5cbf8ed 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -187,6 +187,8 @@ type manager struct { createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) configureVGPU func(context.Context, string, string) error vendorVFIOProfiles func([]devices.VirtualFunction) (map[string][]devices.VGPUProfileType, error) + quarantinedVFs func() (map[string]struct{}, error) + pickVFIndex func(n int) int destroyVGPU func(context.Context, devices.VGPUAssignment) error reconcileVGPUDevices func(context.Context, map[string]struct{}) error vgpuAllocationMu sync.Mutex diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index f68f6f6dd..1058af131 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -311,6 +311,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUClaimedAt = sourceMeta.GPUClaimedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/types.go b/lib/instances/types.go index 6aac15985..efa057b31 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUClaimedAt *time.Time // when the vendor VFIO claim was persisted; identifies this assignment in VF health reports // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index f036161f9..98d4a9d13 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -3,6 +3,7 @@ package instances import ( "context" "fmt" + "math/rand/v2" "path/filepath" "sort" @@ -56,7 +57,14 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } - vfAddress, profileType, err := selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName) + // Quarantine is read under the allocation lock but mutated under the + // devices lock, so this is a snapshot. configure re-checks it under the + // devices lock before the VF is touched. + quarantined, err := m.quarantinedVFAddresses() + if err != nil { + return nil, err + } + vfAddress, profileType, err := selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex) if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the @@ -70,7 +78,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if profilesByVF, err = listProfiles(vfs); err != nil { return nil, err } - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName); err != nil { + if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex); err != nil { return nil, err } } @@ -97,7 +105,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str } log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) vfs = withoutVF(vfs, vf.PCIAddress) - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, profileName); err != nil { + if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex); err != nil { return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) } } @@ -109,6 +117,8 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str SysfsPath: filepath.Clean(devices.GetDeviceSysfsPath(vfAddress)), } setStoredVGPUDevice(&meta.StoredMetadata, device) + claimedAt := m.nowUTC() + meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { clearStoredVGPUDevice(&meta.StoredMetadata) return nil, fmt.Errorf("save vGPU claim: %w", err) @@ -150,7 +160,20 @@ func (m *manager) resetDirtyUnclaimedVFs(ctx context.Context, vfs []devices.Virt return reset } -func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType, allMetadata []StoredMetadata, profileName string) (string, string, error) { +func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { + quarantined := m.quarantinedVFs + if quarantined == nil { + quarantined = devices.QuarantinedVFAddresses + } + return quarantined() +} + +// selectVendorVFIOVF picks the VF to claim for profileName. Quarantined VFs +// are never candidates and count against their parent GPU, so placement +// drifts away from cards carrying a wedged VF. Among equally ranked +// candidates on the chosen GPU, pick selects the index (nil is uniform +// random), so a single VF cannot capture every placement on an idle host. +func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType, allMetadata []StoredMetadata, quarantined map[string]struct{}, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) advertises := make(map[string]map[string]struct{}, len(profilesByVF)) for vfAddress, profiles := range profilesByVF { @@ -200,8 +223,13 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ parentAdvertises[vf.ParentGPU] = true } } + quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]devices.VirtualFunction) for _, vf := range vfs { + if _, bad := quarantined[vf.PCIAddress]; bad { + quarantinedByGPU[vf.ParentGPU]++ + continue + } if _, ok := claimed[vf.PCIAddress]; ok { continue } @@ -227,6 +255,9 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] { + return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]] + } if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { return !unknownUsageByGPU[gpus[i]] } @@ -238,7 +269,17 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ if len(gpus) == 0 { return "", "", fmt.Errorf("no available VF for profile %q", profileName) } - return freeByGPU[gpus[0]][0].PCIAddress, requested.TypeName, nil + // Candidates are sorted clean-first, so the leading run with the same + // Allocated state is the set of equally ranked VFs. + candidates := freeByGPU[gpus[0]] + n := 1 + for n < len(candidates) && candidates[n].Allocated == candidates[0].Allocated { + n++ + } + if pick == nil { + pick = rand.IntN + } + return candidates[pick(n)].PCIAddress, requested.TypeName, nil } func (m *manager) configureClaimedVGPU(ctx context.Context, device *devices.VGPUDevice) error { @@ -270,6 +311,7 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUClaimedAt = nil } // vgpuCleanupGuard checks whether the VF can be safely released: the VMM diff --git a/lib/instances/vgpu_linux_test.go b/lib/instances/vgpu_linux_test.go index dcfb08bd2..c93e4b2f5 100644 --- a/lib/instances/vgpu_linux_test.go +++ b/lib/instances/vgpu_linux_test.go @@ -7,6 +7,7 @@ import ( "errors" "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/paths" @@ -72,6 +73,59 @@ func TestConcurrentVGPUClaimsUseDistinctVFs(t *testing.T) { assert.Len(t, claims, 2) } +func TestVGPUClaimSkipsQuarantinedVF(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + m := newVGPUAllocationManager(t, vfs) + m.quarantinedVFs = func() (map[string]struct{}, error) { + return map[string]struct{}{"0000:82:00.4": {}}, nil + } + m.pickVFIndex = pickFirst + meta := saveTestVGPUInstance(t, m, "new") + + device, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVGPUClaimFailsClosedWhenVFHealthUnavailable(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + m.quarantinedVFs = func() (map[string]struct{}, error) { + return nil, errors.New("VF health state unavailable: read failed") + } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.ErrorContains(t, err, "VF health state unavailable") + stored, loadErr := m.loadMetadata("new") + require.NoError(t, loadErr) + assert.Empty(t, stored.GPUDevicePath, "placement must not claim while quarantine state is unknown") +} + +func TestVGPUClaimRecordsClaimTime(t *testing.T) { + vfs := []devices.VirtualFunction{{PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}} + m := newVGPUAllocationManager(t, vfs) + claimedAt := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + m.now = func() time.Time { return claimedAt } + m.destroyVGPU = func(context.Context, devices.VGPUAssignment) error { return nil } + meta := saveTestVGPUInstance(t, m, "new") + + _, err := m.claimVGPU(context.Background(), meta, testVGPUProfile) + require.NoError(t, err) + stored, err := m.loadMetadata("new") + require.NoError(t, err) + require.NotNil(t, stored.GPUClaimedAt) + assert.True(t, claimedAt.Equal(*stored.GPUClaimedAt)) + + require.NoError(t, m.releaseStoredVGPUPersisted(context.Background(), stored)) + stored, err = m.loadMetadata("new") + require.NoError(t, err) + assert.Nil(t, stored.GPUClaimedAt, "release must clear the assignment identity with the claim") +} + func TestVGPUClaimUsesLeastLoadedGPU(t *testing.T) { vfs := []devices.VirtualFunction{ {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, @@ -244,6 +298,8 @@ func TestVGPUClaimFallsBackWhenDirtyVFRefusesReset(t *testing.T) { {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, } m := newVGPUAllocationManager(t, vfs) + // Both VFs are equally ranked; pin the tiebreak so the fallback order is fixed. + m.pickVFIndex = pickFirst var resets []string m.destroyVGPU = func(_ context.Context, assignment devices.VGPUAssignment) error { resets = append(resets, assignment.DevicePath) diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index de697b8b8..ea35b15fa 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -122,10 +122,93 @@ func TestSelectVendorVFIOVFFailsClosedOnClaimedVF(t *testing.T) { } _, _, err := selectVendorVFIOVF(vfs, profiles, []StoredMetadata{{ GPUDevicePath: testVFDevicePath, - }}, testVGPUProfile) + }}, nil, testVGPUProfile, nil) require.ErrorContains(t, err, "no available VF") } +func testVFProfiles(addresses ...string) map[string][]devices.VGPUProfileType { + profiles := make(map[string][]devices.VGPUProfileType, len(addresses)) + for _, address := range addresses { + profiles[address] = []devices.VGPUProfileType{{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}} + } + return profiles +} + +func pickFirst(int) int { return 0 } + +func pickLast(n int) int { return n - 1 } + +func TestSelectVendorVFIOVFSkipsQuarantinedVF(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + quarantined := map[string]struct{}{"0000:82:00.4": {}} + + vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5"), nil, quarantined, testVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", vf) + + _, _, err = selectVendorVFIOVF(vfs[:1], testVFProfiles("0000:82:00.4"), nil, quarantined, testVGPUProfile, pickFirst) + require.ErrorContains(t, err, "no available VF") +} + +func TestSelectVendorVFIOVFAvoidsGPUWithQuarantinedVF(t *testing.T) { + // Both GPUs are idle; GPU 82 sorts first by name but carries a + // quarantined VF, so the clean card wins. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:e3:00.4", ParentGPU: "0000:e3:00.0"}, + } + quarantined := map[string]struct{}{"0000:82:00.4": {}} + + vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5", "0000:e3:00.4"), nil, quarantined, testVGPUProfile, pickFirst) + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", vf) +} + +func TestSelectVendorVFIOVFPicksAmongEquivalentFreeVFs(t *testing.T) { + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + } + var offered int + pick := func(n int) int { + offered = n + return n - 1 + } + + vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5"), nil, nil, testVGPUProfile, pick) + require.NoError(t, err) + assert.Equal(t, 2, offered) + assert.Equal(t, "0000:82:00.5", vf) +} + +func TestSelectVendorVFIOVFRandomizesOnlyAmongCleanVFs(t *testing.T) { + // The dirty VF is still a candidate of last resort but must never be + // offered to the tiebreak while a clean sibling exists. + vfs := []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4", ParentGPU: "0000:82:00.0", Allocated: true, ProfileType: testVFProfileType}, + {PCIAddress: "0000:82:00.5", ParentGPU: "0000:82:00.0"}, + {PCIAddress: "0000:82:00.6", ParentGPU: "0000:82:00.0"}, + } + var offered int + pick := func(n int) int { + offered = n + return n - 1 + } + + vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.5", "0000:82:00.6"), nil, nil, testVGPUProfile, pick) + require.NoError(t, err) + assert.Equal(t, 2, offered) + assert.Equal(t, "0000:82:00.6", vf) + + vf, _, err = selectVendorVFIOVF(vfs[:1], testVFProfiles("0000:82:00.4"), nil, nil, testVGPUProfile, pickLast) + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", vf) +} + func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { // GPU 82 carries a claim whose profile is no longer creatable anywhere, // so its load is unknown; GPU e3 has a known 2 GB claim. Known load wins @@ -146,13 +229,13 @@ func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, } - vf, profileType, err := selectVendorVFIOVF(vfs, profiles, claims, testVGPUProfile) + vf, profileType, err := selectVendorVFIOVF(vfs, profiles, claims, nil, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:e3:00.5", vf) assert.Equal(t, testVFProfileType, profileType) // With no alternative, the GPU with unknown load is still used. - vf, _, err = selectVendorVFIOVF(vfs[:2], profiles, claims[:1], testVGPUProfile) + vf, _, err = selectVendorVFIOVF(vfs[:2], profiles, claims[:1], nil, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) } From 2e3f62a82e6fcae03322a0e7f95a3ba18f592420 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 06:35:56 +0000 Subject: [PATCH 08/21] Initialize VF health store explicitly and define the assignment key format Load the VF health store from main instead of as a side effect of devices.NewManager, so the lifecycle is visible next to the threshold config and tests can point the store at a path directly. Add FormatVFAssignedAt so failure and success reports agree on the AssignedAt key. Drop the two exported getters only tests used, log a single warning when the store is unavailable in /resources, and document why the address pattern and parent-dir sync look the way they do. --- cmd/api/main.go | 3 ++ lib/devices/manager.go | 4 --- lib/devices/vendor_vfio_linux_test.go | 2 +- lib/devices/vf_health.go | 46 ++++++++++++++------------- lib/devices/vf_health_test.go | 42 +++++++++++++----------- lib/resources/gpu.go | 31 ++++++++++-------- lib/resources/gpu_test.go | 11 ++++--- 7 files changed, 75 insertions(+), 64 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 3b0ec54e1..13d73e305 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -205,6 +205,9 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) + if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState()); err != nil { + slog.Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 6b9f6340a..30763c04d 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "log/slog" "os" "runtime" "strings" @@ -86,9 +85,6 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { - if err := initVFHealth(p.VFHealthState()); err != nil { - slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) - } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 647002bf9..411625afb 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -195,7 +195,7 @@ func TestVendorVFIOConfigureRefusesQuarantinedVF(t *testing.T) { func TestVendorVFIOConfigureFailsClosedWhenVFHealthUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, initVFHealth(path)) + require.Error(t, InitVFHealth(path)) sysfs := newTestVendorVFIOSysfs(t) const vfAddress = "0000:82:00.4" diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index a84367a32..f79566610 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -35,6 +35,11 @@ type vfHealthFile struct { } // VFInitFailureReport describes one guest-reported driver init failure. +// +// InstanceID and AssignedAt together identify the assignment. AssignedAt is +// the instance's stored GPUClaimedAt rendered with FormatVFAssignedAt; a +// success report only clears a failure whose AssignedAt string matches +// exactly, so every reporter must use that formatting. type VFInitFailureReport struct { VFAddress string InstanceID string @@ -42,12 +47,19 @@ type VFInitFailureReport struct { } // VFInitSuccessReport identifies the assignment that successfully initialized. +// AssignedAt follows the same format as VFInitFailureReport.AssignedAt. type VFInitSuccessReport struct { VFAddress string InstanceID string AssignedAt string } +// FormatVFAssignedAt renders a claim time as the AssignedAt key used in VF +// health reports. +func FormatVFAssignedAt(claimedAt time.Time) string { + return claimedAt.UTC().Format(time.RFC3339Nano) +} + // VFReportOutcome describes how a failure report changed a VF's health state. type VFReportOutcome int @@ -84,6 +96,10 @@ type vfHealthStore struct { syncDirFunc func(string) error } +// vfHealthAddressPattern is stricter than ValidatePCIAddress on purpose: +// addresses are map keys compared against sysfs entry names, which are +// lowercase with a 0-7 function digit, and this file also builds on macOS +// where ValidatePCIAddress always returns false. var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) var ( @@ -98,7 +114,11 @@ var ( vendorVFIOMu sync.Mutex ) -func initVFHealth(path string) error { +// InitVFHealth loads persisted VF health state from path. Call it after +// SetVFQuarantineThreshold so loaded tallies are evaluated against the +// configured threshold. A load error leaves the store unavailable, which +// fails vGPU placement closed until a later load succeeds. +func InitVFHealth(path string) error { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.path = path @@ -276,27 +296,6 @@ func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { return vfHealth.reportSuccess(report) } -// VFHealthStoreUnavailable reports whether persisted state failed to load or -// the last write failed. -func VFHealthStoreUnavailable() bool { - vfHealth.mu.Lock() - defer vfHealth.mu.Unlock() - return vfHealth.loadErr != nil || vfHealth.persistErr != nil -} - -// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. -func TotalQuarantinedVFs() int { - vfHealth.mu.Lock() - defer vfHealth.mu.Unlock() - count := 0 - for _, record := range vfHealth.records { - if record.QuarantinedAt != nil { - count++ - } - } - return count -} - func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { records := make([]vfHealthRecord, 0, len(s.records)) for _, record := range s.records { @@ -446,6 +445,9 @@ func (s *vfHealthStore) writeStateLocked() (bool, error) { if err := os.MkdirAll(dirPath, 0755); err != nil { return false, fmt.Errorf("create VF health state dir: %w", err) } + // The first write creates the state directory; syncing its parent makes + // that creation durable. Doing it on every write keeps the path + // stateless and cheap relative to how rarely the store is written. if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { return false, fmt.Errorf("sync VF health state parent dir: %w", err) } diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 00f14dbf7..555131b94 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,7 +15,7 @@ import ( func resetVFHealthStore(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "vf-health.json") - require.NoError(t, initVFHealth(path)) + require.NoError(t, InitVFHealth(path)) t.Cleanup(func() { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() @@ -28,6 +29,12 @@ func resetVFHealthStore(t *testing.T) string { return path } +func vfHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil || vfHealth.persistErr != nil +} + func quarantinedVFs() []vfHealthRecord { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() @@ -76,7 +83,7 @@ func TestVGPUAvailability(t *testing.T) { func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) - require.Error(t, initVFHealth(path)) + require.Error(t, InitVFHealth(path)) _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) require.ErrorContains(t, err, "VF health state unavailable") @@ -104,7 +111,7 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.Error(t, err) - assert.True(t, VFHealthStoreUnavailable()) + assert.True(t, vfHealthStoreUnavailable()) _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) require.ErrorContains(t, err, "last write failed") @@ -112,7 +119,7 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.NoError(t, err) assert.Equal(t, VFReportQuarantined, result.Outcome) - assert.False(t, VFHealthStoreUnavailable()) + assert.False(t, vfHealthStoreUnavailable()) available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) require.NoError(t, err) @@ -158,7 +165,7 @@ func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { vfHealth.records = make(map[string]vfHealthRecord) vfHealth.threshold = 2 vfHealth.mu.Unlock() - require.NoError(t, initVFHealth(path)) + require.NoError(t, InitVFHealth(path)) records := quarantinedVFs() require.Len(t, records, 1) @@ -171,7 +178,7 @@ func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { result, err := ReportVFInitFailure(VFInitFailureReport{ VFAddress: "0000:e3:00.4", InstanceID: "instance-1", - AssignedAt: "2026-08-20T15:00:00Z", + AssignedAt: FormatVFAssignedAt(time.Date(2026, 8, 20, 15, 0, 0, 0, time.UTC)), }) require.NoError(t, err) assert.Equal(t, VFReportRecorded, result.Outcome) @@ -190,7 +197,6 @@ func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { records := quarantinedVFs() require.Len(t, records, 1) - assert.Equal(t, 1, TotalQuarantinedVFs()) assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) require.NotNil(t, records[0].QuarantinedAt) require.Len(t, records[0].Failures, 2) @@ -200,7 +206,7 @@ func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { require.NoError(t, err) assert.Equal(t, VFReportUnchanged, result.Outcome) - require.NoError(t, initVFHealth(path)) + require.NoError(t, InitVFHealth(path)) reloaded := quarantinedVFs() require.Len(t, reloaded, 1) assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) @@ -251,7 +257,7 @@ func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { require.NoError(t, err) assert.Zero(t, successResult.Cleared) - require.NoError(t, initVFHealth(path)) + require.NoError(t, InitVFHealth(path)) result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) require.NoError(t, err) assert.Equal(t, VFReportRecorded, result.Outcome) @@ -369,14 +375,14 @@ func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} _, err := ReportVFInitFailure(report) require.ErrorContains(t, err, "sync VF health state parent dir") - assert.True(t, VFHealthStoreUnavailable()) + assert.True(t, vfHealthStoreUnavailable()) result, err := ReportVFInitFailure(report) require.NoError(t, err) assert.Equal(t, VFReportRecorded, result.Outcome) assert.Equal(t, 3, parentSyncs) assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") - assert.False(t, VFHealthStoreUnavailable()) + assert.False(t, vfHealthStoreUnavailable()) } func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { @@ -401,13 +407,13 @@ func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { require.Len(t, state.Records, 1) assert.NotNil(t, state.Records[0].QuarantinedAt) require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") - assert.True(t, VFHealthStoreUnavailable()) + assert.True(t, vfHealthStoreUnavailable()) vfHealth.syncDirFunc = syncDir result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) require.NoError(t, err) assert.Equal(t, VFReportRecorded, result.Outcome) - assert.False(t, VFHealthStoreUnavailable()) + assert.False(t, vfHealthStoreUnavailable()) data, err = os.ReadFile(path) require.NoError(t, err) @@ -441,13 +447,13 @@ func TestReportRetriesFailedThresholdPersistence(t *testing.T) { require.NoError(t, os.WriteFile(blocker, nil, 0644)) vfHealth.path = filepath.Join(blocker, "vf-health.json") SetVFQuarantineThreshold(2) - assert.True(t, VFHealthStoreUnavailable()) + assert.True(t, vfHealthStoreUnavailable()) vfHealth.path = path result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) require.NoError(t, err) assert.Equal(t, VFReportUnchanged, result.Outcome) - assert.False(t, VFHealthStoreUnavailable()) + assert.False(t, vfHealthStoreUnavailable()) data, err := os.ReadFile(path) require.NoError(t, err) @@ -525,8 +531,8 @@ func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { t.Run(tt.name, func(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) - require.ErrorContains(t, initVFHealth(path), tt.wantErr) - assert.True(t, VFHealthStoreUnavailable()) + require.ErrorContains(t, InitVFHealth(path), tt.wantErr) + assert.True(t, vfHealthStoreUnavailable()) assert.Empty(t, quarantinedVFs()) _, err := vfHealth.checkedAddresses() @@ -540,7 +546,7 @@ func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { quarantineVF(t, "0000:e3:00.4") require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, initVFHealth(path)) + require.Error(t, InitVFHealth(path)) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) require.Error(t, err) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 2c7843532..6a355cbd8 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -48,26 +48,29 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d } } + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + } + // Profile listing fails on the same unavailable health store, so check + // availability first and report the cause once. + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none and no profiles", "framework", framework, "error", err) + return status, err + } + status.AllocatableSlots = allocatableSlots + status.QuarantinedSlots = quarantinedSlots + // Get available profiles (reuse VFs to avoid redundant discovery) profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) - status := &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - AllocatableSlots: allocatableSlots, - QuarantinedSlots: quarantinedSlots, - Profiles: profiles, - } - if err != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) - status.AllocatableSlots = 0 - } - return status, err + status.Profiles = profiles + return status, nil } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go index 825a20260..69afc18af 100644 --- a/lib/resources/gpu_test.go +++ b/lib/resources/gpu_test.go @@ -16,15 +16,16 @@ import ( func initVFHealthForTest(t *testing.T, state []byte) { t.Helper() - dataDir := t.TempDir() + path := paths.New(t.TempDir()).VFHealthState() if state != nil { - path := paths.New(dataDir).VFHealthState() require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) require.NoError(t, os.WriteFile(path, state, 0o644)) } - devices.NewManager(paths.New(dataDir)) - resetDir := t.TempDir() - t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) + err := devices.InitVFHealth(path) + if state == nil { + require.NoError(t, err) + } + t.Cleanup(func() { require.NoError(t, devices.InitVFHealth(paths.New(t.TempDir()).VFHealthState())) }) } func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { From 25d56c3e84315a7f0739ec2a67f1bdbaddd7c35c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 06:52:15 +0000 Subject: [PATCH 09/21] Retry failed VF health persist on read A failed write latched the store closed, and only a guest report could retry it. Closed placement produces no new reports, so a transient disk error wedged vGPU placement until restart. Retry the write when placement or /resources reads the quarantine set. --- lib/devices/GPU.md | 6 ++++-- lib/devices/vf_health.go | 9 ++++++--- lib/devices/vf_health_test.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 44432898a..97760526f 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -301,8 +301,10 @@ placement. A reported init success clears failures only when that exact assignment has a recorded failure, removing the match and older tallies; if that assignment crossed the threshold, its later success also rescinds the quarantine. If the state file exists but cannot be loaded, or the last write -to it failed, placement and advertised availability fail closed until a load -or write succeeds. Recorded tallies are re-evaluated against the configured +to it failed, placement and advertised availability fail closed; the load or +write is retried on the next placement or `/resources` read, so the store +recovers on its own once the file is repaired or the disk is writable again. +Recorded tallies are re-evaluated against the configured threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs whose persisted failures already meet the new value. diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index f79566610..5d7901c35 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -232,8 +232,11 @@ func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { if err := s.ensureLoadedLocked(); err != nil { return nil, fmt.Errorf("VF health state unavailable: %w", err) } - if s.persistErr != nil { - return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + // A failed write closes placement, which stops the guest reports that + // would otherwise retry it. Retrying here lets the store recover on the + // next placement or /resources read once the disk is writable again. + if err := s.retryPersistLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", err) } addresses := make(map[string]struct{}, len(s.records)) for address, record := range s.records { @@ -422,7 +425,7 @@ func (s *vfHealthStore) retryPersistLocked() error { } // persistLocked writes the current records to disk. A failure is latched and -// fails placement closed until a later write succeeds. The returned boolean +// fails placement closed until a retry succeeds. The returned boolean // reports whether the rename made the new state visible. func (s *vfHealthStore) persistLocked() (bool, error) { if s.path == "" { diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 555131b94..f895b3591 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -127,6 +127,35 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { assert.Equal(t, 1, quarantined) } +func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "instance-2"}) + require.Error(t, err) + assert.True(t, vfHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, nil) + require.ErrorContains(t, err, "last write failed") + + // No report arrives while placement is closed; a read alone must clear the latch. + vfHealth.syncDirFunc = syncDir + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}, {PCIAddress: "0000:e3:00.5"}}) + require.NoError(t, err) + assert.False(t, vfHealthStoreUnavailable()) + assert.Equal(t, 1, available) + assert.Equal(t, 1, quarantined) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1, "the retried write must persist the rolled-back in-memory state") + assert.Equal(t, "0000:e3:00.4", state.Records[0].VFAddress) +} + func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { path := resetVFHealthStore(t) SetVFQuarantineThreshold(3) From eb544806fed5c6283c08fd2bca2ca2e82d885010 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:57:10 +0000 Subject: [PATCH 10/21] Read VF health once per status and surface load-time persist failures GetVGPUAvailability now returns a snapshot that ListGPUProfilesWithVFs consumes, so a /resources read touches the VF health store once instead of twice. SetVFQuarantineThreshold and the load path return the persist error from threshold re-evaluation instead of logging it, so InitVFHealth reports the failure and startup logs it. Document why re-evaluation keeps the in-memory quarantine on a failed write, align the rescind rule comments with the code, and note why the claim time is set outside setStoredVGPUDevice. --- cmd/api/main.go | 6 +- lib/devices/GPU.md | 11 ++-- lib/devices/mdev_darwin.go | 2 +- lib/devices/vendor_vfio_linux.go | 15 ++--- lib/devices/vendor_vfio_linux_test.go | 6 +- lib/devices/vf_health.go | 63 +++++++++++++------ lib/devices/vf_health_test.go | 91 +++++++++++++++++---------- lib/devices/vgpu_linux.go | 11 +++- lib/instances/vgpu.go | 3 + lib/resources/gpu.go | 12 ++-- 10 files changed, 140 insertions(+), 80 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 13d73e305..11650aaa0 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,9 +204,11 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) - devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) + if err := devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold); err != nil { + return fmt.Errorf("configure VF quarantine threshold: %w", err) + } if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState()); err != nil { - slog.Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + slog.Error("failed to initialize VF health state; vGPU placement is disabled until the state file is repaired or the next write succeeds", "error", err) } // Initialize OpenTelemetry (before wire initialization) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 97760526f..9665a03c6 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -299,11 +299,12 @@ overflow-only — deprioritized for new placements. Selection among a card's equivalent free VFs is randomized so a wedged VF cannot capture every placement. A reported init success clears failures only when that exact assignment has a recorded failure, removing the match and older tallies; if -that assignment crossed the threshold, its later success also rescinds the -quarantine. If the state file exists but cannot be loaded, or the last write -to it failed, placement and advertised availability fail closed; the load or -write is retried on the next placement or `/resources` read, so the store -recovers on its own once the file is repaired or the disk is writable again. +that assignment is the most recent failure recorded (the one that crossed +the threshold), its later success also rescinds the quarantine. If the state +file exists but cannot be loaded, or the last write to it failed, placement +and advertised availability fail closed; the load or write is retried on the +next placement or `/resources` read, so the store recovers on its own once +the file is repaired or the disk is writable again. Recorded tallies are re-evaluated against the configured threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs whose persisted failures already meet the new value. diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index cc7b0e78c..956d7cba2 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -23,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, availability VGPUAvailability) ([]GPUProfile, error) { return []GPUProfile{}, nil } diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a871ecacf..2fb552095 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -90,15 +90,12 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free VF advertising a type as one creatable -// instance, matching the driver-reported units that mdev sums through -// available_instances. This is a best-effort snapshot because creating on one -// VF may revoke the type from siblings that share its GPU framebuffer. -func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { - quarantined, err := vfHealth.checkedAddresses() - if err != nil { - return nil, err - } +// listProfiles counts each free, non-quarantined VF advertising a type as +// one creatable instance, matching the driver-reported units that mdev sums +// through available_instances. This is a best-effort snapshot because +// creating on one VF may revoke the type from siblings that share its GPU +// framebuffer. +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { profilesByType := make(map[string]VGPUProfileType) creatableVFs := make(map[string]int) profilesByVF, err := s.profileTypes(vfs) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 411625afb..5464c4f32 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -49,7 +49,7 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { vfs, err := sysfs.discoverVFs() require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) + profiles, err := sysfs.listProfiles(vfs, nil) require.NoError(t, err) assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) } @@ -216,7 +216,9 @@ func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) vfs, err := sysfs.discoverVFs() require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs, availability.quarantined) require.NoError(t, err) assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) } diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 5d7901c35..6625972a2 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -3,7 +3,6 @@ package devices import ( "encoding/json" "fmt" - "log/slog" "os" "path/filepath" "regexp" @@ -48,6 +47,11 @@ type VFInitFailureReport struct { // VFInitSuccessReport identifies the assignment that successfully initialized. // AssignedAt follows the same format as VFInitFailureReport.AssignedAt. +// +// A success clears the matched failure and every older tally. It rescinds a +// quarantine only when the matched failure is the most recent one recorded: +// the report that crossed the threshold, or the newest tally when a lowered +// threshold quarantined the VF at load. type VFInitSuccessReport struct { VFAddress string InstanceID string @@ -116,8 +120,8 @@ var ( // InitVFHealth loads persisted VF health state from path. Call it after // SetVFQuarantineThreshold so loaded tallies are evaluated against the -// configured threshold. A load error leaves the store unavailable, which -// fails vGPU placement closed until a later load succeeds. +// configured threshold. An error leaves the store unavailable, which fails +// vGPU placement closed until a later load or write succeeds. func InitVFHealth(path string) error { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() @@ -127,17 +131,24 @@ func InitVFHealth(path string) error { // SetVFQuarantineThreshold configures the number of failed assignments // required to quarantine a VF. Already-recorded tallies are re-evaluated so a -// lowered threshold applies to failures persisted before the change. -func SetVFQuarantineThreshold(n int) { +// lowered threshold applies to failures persisted before the change. An +// error means the re-evaluated quarantines could not be persisted; they stay +// in effect and placement fails closed until a write succeeds. +func SetVFQuarantineThreshold(n int) error { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.threshold = n - vfHealth.requarantineLocked() + return vfHealth.requarantineLocked() } // requarantineLocked quarantines records whose failure tallies meet the // current threshold, so threshold changes and loaded state agree. -func (s *vfHealthStore) requarantineLocked() { +// +// Unlike reports, a failed persist here does not roll memory back: the +// tallies already meet the threshold, so dropping the quarantine would +// readmit a VF the store has judged unhealthy. The latched error closes +// placement until a later read or report re-persists the in-memory state. +func (s *vfHealthStore) requarantineLocked() error { changed := false for address, record := range s.records { if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { @@ -148,11 +159,11 @@ func (s *vfHealthStore) requarantineLocked() { s.records[address] = record changed = true } - if changed { - if _, err := s.persistLocked(); err != nil { - slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) - } + if !changed { + return nil } + _, err := s.persistLocked() + return err } func (s *vfHealthStore) loadLocked() error { @@ -215,8 +226,7 @@ func (s *vfHealthStore) loadLocked() error { loaded[record.VFAddress] = record } s.records = loaded - s.requarantineLocked() - return nil + return s.requarantineLocked() } func (s *vfHealthStore) ensureLoadedLocked() error { @@ -253,21 +263,35 @@ func QuarantinedVFAddresses() (map[string]struct{}, error) { return vfHealth.checkedAddresses() } -// VGPUAvailability returns free allocatable and quarantined VF counts. -func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { +// VGPUAvailability is one VF health snapshot applied to discovered VFs. +// Pass it to ListGPUProfilesWithVFs so profile availability is computed from +// the same snapshot without reading the store again. +type VGPUAvailability struct { + AllocatableSlots int // free VFs eligible for placement + QuarantinedSlots int + quarantined map[string]struct{} +} + +// GetVGPUAvailability counts free allocatable and quarantined VFs. It fails +// while the health store is unavailable so callers fail closed. +func GetVGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (VGPUAvailability, error) { if framework != VGPUFrameworkVendorVFIO { - return countFreeVFs(vfs, nil), 0, nil + return VGPUAvailability{AllocatableSlots: countFreeVFs(vfs, nil)}, nil } addresses, err := vfHealth.checkedAddresses() if err != nil { - return 0, 0, err + return VGPUAvailability{}, err + } + availability := VGPUAvailability{ + AllocatableSlots: countFreeVFs(vfs, addresses), + quarantined: addresses, } for _, vf := range vfs { if _, ok := addresses[vf.PCIAddress]; ok { - quarantined++ + availability.QuarantinedSlots++ } } - return countFreeVFs(vfs, addresses), quarantined, nil + return availability, nil } func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { @@ -390,6 +414,7 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu break } } + // Only the newest failure can rescind a quarantine; see VFInitSuccessReport. if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { return VFSuccessResult{}, nil } diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index f895b3591..f1e2f4476 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -50,11 +50,11 @@ func quarantinedVFs() []vfHealthRecord { func quarantineVF(t *testing.T, address string) { t.Helper() - SetVFQuarantineThreshold(1) + require.NoError(t, SetVFQuarantineThreshold(1)) result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) require.NoError(t, err) require.Equal(t, VFReportQuarantined, result.Outcome) - SetVFQuarantineThreshold(defaultVFQuarantineThreshold) + require.NoError(t, SetVFQuarantineThreshold(defaultVFQuarantineThreshold)) } func TestVGPUAvailability(t *testing.T) { @@ -69,15 +69,15 @@ func TestVGPUAvailability(t *testing.T) { {PCIAddress: "0000:82:00.6"}, } - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) require.NoError(t, err) - assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") - assert.Equal(t, 1, quarantined) + assert.Equal(t, 1, availability.AllocatableSlots, "a below-threshold failure tally must not remove the VF from placement") + assert.Equal(t, 1, availability.QuarantinedSlots) - available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + availability, err = GetVGPUAvailability(VGPUFrameworkMdev, vfs) require.NoError(t, err) - assert.Equal(t, 2, available) - assert.Zero(t, quarantined) + assert.Equal(t, 2, availability.AllocatableSlots) + assert.Zero(t, availability.QuarantinedSlots) } func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { @@ -85,25 +85,25 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) require.Error(t, InitVFHealth(path)) - _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + _, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) require.ErrorContains(t, err, "VF health state unavailable") - available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + availability, err := GetVGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) require.NoError(t, err) - assert.Equal(t, 1, available) - assert.Zero(t, quarantined) + assert.Equal(t, 1, availability.AllocatableSlots) + assert.Zero(t, availability.QuarantinedSlots) restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) - available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + availability, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) require.NoError(t, err, "a repaired state file must re-enable placement without a new report") - assert.Zero(t, available) - assert.Equal(t, 1, quarantined) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) } func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { resetVFHealthStore(t) - SetVFQuarantineThreshold(1) + require.NoError(t, SetVFQuarantineThreshold(1)) blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0o644)) goodPath := vfHealth.path @@ -112,7 +112,7 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.Error(t, err) assert.True(t, vfHealthStoreUnavailable()) - _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) require.ErrorContains(t, err, "last write failed") vfHealth.path = goodPath @@ -121,15 +121,15 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { assert.Equal(t, VFReportQuarantined, result.Outcome) assert.False(t, vfHealthStoreUnavailable()) - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) require.NoError(t, err) - assert.Zero(t, available) - assert.Equal(t, 1, quarantined) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) } func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { path := resetVFHealthStore(t) - SetVFQuarantineThreshold(1) + require.NoError(t, SetVFQuarantineThreshold(1)) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.NoError(t, err) @@ -137,16 +137,16 @@ func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "instance-2"}) require.Error(t, err) assert.True(t, vfHealthStoreUnavailable()) - _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, nil) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, nil) require.ErrorContains(t, err, "last write failed") // No report arrives while placement is closed; a read alone must clear the latch. vfHealth.syncDirFunc = syncDir - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}, {PCIAddress: "0000:e3:00.5"}}) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}, {PCIAddress: "0000:e3:00.5"}}) require.NoError(t, err) assert.False(t, vfHealthStoreUnavailable()) - assert.Equal(t, 1, available) - assert.Equal(t, 1, quarantined) + assert.Equal(t, 1, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) data, err := os.ReadFile(path) require.NoError(t, err) @@ -158,14 +158,14 @@ func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { path := resetVFHealthStore(t) - SetVFQuarantineThreshold(3) + require.NoError(t, SetVFQuarantineThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) require.Equal(t, VFReportRecorded, result.Outcome) } - SetVFQuarantineThreshold(2) + require.NoError(t, SetVFQuarantineThreshold(2)) records := quarantinedVFs() require.Len(t, records, 1) @@ -181,7 +181,7 @@ func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { path := resetVFHealthStore(t) - SetVFQuarantineThreshold(3) + require.NoError(t, SetVFQuarantineThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) @@ -457,16 +457,16 @@ func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { } require.True(t, found) - available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) require.NoError(t, err) - assert.Zero(t, available) - assert.Equal(t, 1, quarantined) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) } func TestReportRetriesFailedThresholdPersistence(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" - SetVFQuarantineThreshold(3) + require.NoError(t, SetVFQuarantineThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) require.NoError(t, err) @@ -475,7 +475,7 @@ func TestReportRetriesFailedThresholdPersistence(t *testing.T) { blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0644)) vfHealth.path = filepath.Join(blocker, "vf-health.json") - SetVFQuarantineThreshold(2) + require.Error(t, SetVFQuarantineThreshold(2)) assert.True(t, vfHealthStoreUnavailable()) vfHealth.path = path @@ -592,3 +592,28 @@ func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { require.Len(t, records, 2, "reload must recover the previously persisted quarantine") assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) } + +func TestInitVFHealthFailsWhenReevaluatedQuarantineCannotPersist(t *testing.T) { + path := resetVFHealthStore(t) + // Two persisted tallies meet the default threshold, so loading them + // quarantines the VF and must write that back. + state := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[` + + `{"instance_id":"instance-1","reported_at":"2026-08-20T00:00:00Z"},` + + `{"instance_id":"instance-2","reported_at":"2026-08-20T01:00:00Z"}]}]}` + require.NoError(t, os.WriteFile(path, []byte(state), 0o644)) + vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } + + err := InitVFHealth(path) + require.ErrorContains(t, err, "injected sync failure") + require.Len(t, quarantinedVFs(), 1, "the re-evaluated quarantine must stay in effect in memory") + assert.True(t, vfHealthStoreUnavailable()) + _, err = GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.syncDirFunc = syncDir + availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err, "a read must retry the failed write once the disk recovers") + assert.False(t, vfHealthStoreUnavailable()) + assert.Zero(t, availability.AllocatableSlots) + assert.Equal(t, 1, availability.QuarantinedSlots) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index f7ab42ec5..a545c5733 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -38,16 +38,21 @@ func ListGPUProfiles() ([]GPUProfile, error) { if err != nil { return nil, err } - return ListGPUProfilesWithVFs(framework, vfs) + availability, err := GetVGPUAvailability(framework, vfs) + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs, availability) } // ListGPUProfilesWithVFs returns available profiles for discovered VFs. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { +// Quarantined VFs from availability are excluded from vendor VFIO counts. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, availability VGPUAvailability) ([]GPUProfile, error) { switch framework { case VGPUFrameworkMdev: return listMdevGPUProfilesWithVFs(vfs) case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.listProfiles(vfs) + return hostVendorVFIO.listProfiles(vfs, availability.quarantined) default: return nil, nil } diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 98d4a9d13..b6998e078 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -117,6 +117,9 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str SysfsPath: filepath.Clean(devices.GetDeviceSysfsPath(vfAddress)), } setStoredVGPUDevice(&meta.StoredMetadata, device) + // Only vendor VFIO claims carry an assignment identity, so the claim time + // is set here rather than in setStoredVGPUDevice. clearStoredVGPUDevice + // resets it with the rest of the device fields. claimedAt := m.nowUTC() meta.GPUClaimedAt = &claimedAt if err := m.saveMetadata(meta); err != nil { diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6a355cbd8..2a56ff7f3 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -53,18 +53,18 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d TotalSlots: len(vfs), UsedSlots: usedSlots, } - // Profile listing fails on the same unavailable health store, so check - // availability first and report the cause once. - allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + // One VF health snapshot serves both the slot counts and the profile + // listing, so a status read touches the store once. + availability, err := devices.GetVGPUAvailability(framework, vfs) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none and no profiles", "framework", framework, "error", err) return status, err } - status.AllocatableSlots = allocatableSlots - status.QuarantinedSlots = quarantinedSlots + status.AllocatableSlots = availability.AllocatableSlots + status.QuarantinedSlots = availability.QuarantinedSlots // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil From 8d17bb952866ad8c38445975fdc9dabfa7d17420 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:29:00 +0000 Subject: [PATCH 11/21] Take the quarantine threshold in InitVFHealth Passing the threshold to InitVFHealth removes the ordering dependency between configuring it and loading persisted tallies. The load error for a duplicate failure now names both halves of the assignment key. --- cmd/api/main.go | 5 +-- lib/devices/vendor_vfio_linux_test.go | 2 +- lib/devices/vf_health.go | 35 +++++++++++---------- lib/devices/vf_health_test.go | 45 ++++++++++++--------------- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 11650aaa0..9eb676622 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,10 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) - if err := devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold); err != nil { - return fmt.Errorf("configure VF quarantine threshold: %w", err) - } - if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState()); err != nil { + if err := devices.InitVFHealth(paths.New(cfg.DataDir).VFHealthState(), cfg.GPU.VFQuarantineThreshold); err != nil { slog.Error("failed to initialize VF health state; vGPU placement is disabled until the state file is repaired or the next write succeeds", "error", err) } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 5464c4f32..a92084491 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -195,7 +195,7 @@ func TestVendorVFIOConfigureRefusesQuarantinedVF(t *testing.T) { func TestVendorVFIOConfigureFailsClosedWhenVFHealthUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, InitVFHealth(path)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) sysfs := newTestVendorVFIOSysfs(t) const vfAddress = "0000:82:00.4" diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 6625972a2..28eda1026 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -118,27 +118,27 @@ var ( vendorVFIOMu sync.Mutex ) -// InitVFHealth loads persisted VF health state from path. Call it after -// SetVFQuarantineThreshold so loaded tallies are evaluated against the -// configured threshold. An error leaves the store unavailable, which fails -// vGPU placement closed until a later load or write succeeds. -func InitVFHealth(path string) error { +// InitVFHealth loads persisted VF health state from path and evaluates the +// loaded tallies against threshold, the number of failed assignments that +// quarantine a VF. A lowered threshold therefore applies to failures +// persisted before the change. An error leaves the store unavailable, which +// fails vGPU placement closed until a later load or write succeeds. An empty +// path keeps state in memory only. +func InitVFHealth(path string, threshold int) error { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() vfHealth.path = path + vfHealth.threshold = threshold return vfHealth.loadLocked() } -// SetVFQuarantineThreshold configures the number of failed assignments -// required to quarantine a VF. Already-recorded tallies are re-evaluated so a -// lowered threshold applies to failures persisted before the change. An -// error means the re-evaluated quarantines could not be persisted; they stay -// in effect and placement fails closed until a write succeeds. -func SetVFQuarantineThreshold(n int) error { - vfHealth.mu.Lock() - defer vfHealth.mu.Unlock() - vfHealth.threshold = n - return vfHealth.requarantineLocked() +// setThreshold changes the quarantine threshold on a loaded store and +// re-evaluates recorded tallies against it. +func (s *vfHealthStore) setThreshold(n int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.threshold = n + return s.requarantineLocked() } // requarantineLocked quarantines records whose failure tallies meet the @@ -170,6 +170,9 @@ func (s *vfHealthStore) loadLocked() error { s.records = make(map[string]vfHealthRecord) s.loadErr = nil s.persistErr = nil + if s.path == "" { + return nil + } data, err := os.ReadFile(s.path) if err != nil { @@ -214,7 +217,7 @@ func (s *vfHealthStore) loadLocked() error { } key := failure.InstanceID + "\x00" + failure.AssignedAt if _, exists := assignments[key]; exists { - s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for instance %q assigned at %q", i, failure.InstanceID, failure.AssignedAt) return s.loadErr } assignments[key] = struct{}{} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index f1e2f4476..92db8586f 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -15,7 +15,7 @@ import ( func resetVFHealthStore(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "vf-health.json") - require.NoError(t, InitVFHealth(path)) + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) t.Cleanup(func() { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() @@ -50,11 +50,11 @@ func quarantinedVFs() []vfHealthRecord { func quarantineVF(t *testing.T, address string) { t.Helper() - require.NoError(t, SetVFQuarantineThreshold(1)) + require.NoError(t, vfHealth.setThreshold(1)) result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) require.NoError(t, err) require.Equal(t, VFReportQuarantined, result.Outcome) - require.NoError(t, SetVFQuarantineThreshold(defaultVFQuarantineThreshold)) + require.NoError(t, vfHealth.setThreshold(defaultVFQuarantineThreshold)) } func TestVGPUAvailability(t *testing.T) { @@ -83,7 +83,7 @@ func TestVGPUAvailability(t *testing.T) { func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) - require.Error(t, InitVFHealth(path)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) _, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) require.ErrorContains(t, err, "VF health state unavailable") @@ -103,7 +103,7 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { resetVFHealthStore(t) - require.NoError(t, SetVFQuarantineThreshold(1)) + require.NoError(t, vfHealth.setThreshold(1)) blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0o644)) goodPath := vfHealth.path @@ -129,7 +129,7 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, SetVFQuarantineThreshold(1)) + require.NoError(t, vfHealth.setThreshold(1)) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.NoError(t, err) @@ -156,16 +156,16 @@ func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { assert.Equal(t, "0000:e3:00.4", state.Records[0].VFAddress) } -func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { +func TestSetThresholdReevaluatesRecordedFailures(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, SetVFQuarantineThreshold(3)) + require.NoError(t, vfHealth.setThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) require.Equal(t, VFReportRecorded, result.Outcome) } - require.NoError(t, SetVFQuarantineThreshold(2)) + require.NoError(t, vfHealth.setThreshold(2)) records := quarantinedVFs() require.Len(t, records, 1) @@ -181,20 +181,15 @@ func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, SetVFQuarantineThreshold(3)) + require.NoError(t, vfHealth.setThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) require.Equal(t, VFReportRecorded, result.Outcome) } - // Simulate a restart where the threshold is configured lower before the - // persisted tallies are loaded. - vfHealth.mu.Lock() - vfHealth.records = make(map[string]vfHealthRecord) - vfHealth.threshold = 2 - vfHealth.mu.Unlock() - require.NoError(t, InitVFHealth(path)) + // Simulate a restart with a lower configured threshold. + require.NoError(t, InitVFHealth(path, 2)) records := quarantinedVFs() require.Len(t, records, 1) @@ -235,7 +230,7 @@ func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { require.NoError(t, err) assert.Equal(t, VFReportUnchanged, result.Outcome) - require.NoError(t, InitVFHealth(path)) + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) reloaded := quarantinedVFs() require.Len(t, reloaded, 1) assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) @@ -286,7 +281,7 @@ func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { require.NoError(t, err) assert.Zero(t, successResult.Cleared) - require.NoError(t, InitVFHealth(path)) + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) require.NoError(t, err) assert.Equal(t, VFReportRecorded, result.Outcome) @@ -466,7 +461,7 @@ func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { func TestReportRetriesFailedThresholdPersistence(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" - require.NoError(t, SetVFQuarantineThreshold(3)) + require.NoError(t, vfHealth.setThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) require.NoError(t, err) @@ -475,7 +470,7 @@ func TestReportRetriesFailedThresholdPersistence(t *testing.T) { blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0644)) vfHealth.path = filepath.Join(blocker, "vf-health.json") - require.Error(t, SetVFQuarantineThreshold(2)) + require.Error(t, vfHealth.setThreshold(2)) assert.True(t, vfHealthStoreUnavailable()) vfHealth.path = path @@ -547,7 +542,7 @@ func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { { name: "duplicate assignment", state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, - wantErr: "duplicate failure for assignment", + wantErr: `duplicate failure for instance "instance-1" assigned at "a"`, }, { name: "duplicate address", @@ -560,7 +555,7 @@ func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { t.Run(tt.name, func(t *testing.T) { path := resetVFHealthStore(t) require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) - require.ErrorContains(t, InitVFHealth(path), tt.wantErr) + require.ErrorContains(t, InitVFHealth(path, defaultVFQuarantineThreshold), tt.wantErr) assert.True(t, vfHealthStoreUnavailable()) assert.Empty(t, quarantinedVFs()) @@ -575,7 +570,7 @@ func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { quarantineVF(t, "0000:e3:00.4") require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) - require.Error(t, InitVFHealth(path)) + require.Error(t, InitVFHealth(path, defaultVFQuarantineThreshold)) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) require.Error(t, err) @@ -603,7 +598,7 @@ func TestInitVFHealthFailsWhenReevaluatedQuarantineCannotPersist(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(state), 0o644)) vfHealth.syncDirFunc = func(string) error { return errors.New("injected sync failure") } - err := InitVFHealth(path) + err := InitVFHealth(path, defaultVFQuarantineThreshold) require.ErrorContains(t, err, "injected sync failure") require.Len(t, quarantinedVFs(), 1, "the re-evaluated quarantine must stay in effect in memory") assert.True(t, vfHealthStoreUnavailable()) From 5fa4534b70af2e8153504bd49d86a2ab3760fce1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:29:18 +0000 Subject: [PATCH 12/21] Stop nvidia-persistenced in the SR-IOV recovery runbook persistenced holds the GPU open like DCGM does, and the runbook dropped it when the quiesce step was expanded. --- lib/devices/GPU.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 9665a03c6..656d41695 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -330,9 +330,10 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: -with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` -fails with `Cannot obtain unbindLock` on first contact. +requires no vGPU assignments on that GPU). Quiescing the services that hold +the GPU open is not optional: with `nv-hostengine`/`dcgm-exporter` or +`nvidia-persistenced` attached, `sriov-manage -d` fails with `Cannot obtain +unbindLock` on first contact. Any manual edit to `vf-health.json` needs an immediate hypeman restart: the store loads only at startup, and a failure report landing first re-persists @@ -352,14 +353,14 @@ GPU; once none remain, run the cycle below. ```bash # 1. Quiesce the services holding the GPU (required for the unbind lock). -systemctl stop nvidia-dcgm-exporter nvidia-dcgm +systemctl stop nvidia-dcgm-exporter nvidia-dcgm nvidia-persistenced # 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e # 3. Restart the quiesced services. -systemctl start nvidia-dcgm nvidia-dcgm-exporter +systemctl start nvidia-persistenced nvidia-dcgm nvidia-dcgm-exporter ``` After the cycle, remove the card's entries from `vf-health.json`, restart, From 721563b6ec4d38f28dde84454d6c6e5c66d90aa6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:30:05 +0000 Subject: [PATCH 13/21] Report why vGPU placement is disabled and export quarantine metrics GetFullStatus dropped the GPU status error, so a broken VF health state was indistinguishable from a full host in /resources. The response now carries placement_disabled_reason when allocatable_slots is 0 for that reason, and the hypeman_resources_gpu_slots gauge exports allocatable and quarantined counts alongside total and used. --- cmd/api/api/resources.go | 3 + lib/devices/GPU.md | 7 +- lib/oapi/oapi.go | 499 ++++++++++++++++--------------- lib/resources/gpu.go | 5 + lib/resources/gpu_test.go | 16 +- lib/resources/monitoring.go | 4 +- lib/resources/monitoring_test.go | 10 +- lib/resources/resource.go | 7 +- openapi.yaml | 4 + 9 files changed, 300 insertions(+), 255 deletions(-) diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index dec9f35eb..db38fd0fc 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -93,6 +93,9 @@ func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceS AllocatableSlots: gs.AllocatableSlots, QuarantinedSlots: gs.QuarantinedSlots, } + if gs.PlacementDisabledReason != "" { + result.PlacementDisabledReason = &gs.PlacementDisabledReason + } // Convert profiles (vGPU mode) if len(gs.Profiles) > 0 { diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 656d41695..9be00b241 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -310,7 +310,12 @@ threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs whose persisted failures already meet the new value. `used_slots` includes quarantined VFs still held by running instances, so it -can overlap `quarantined_slots`; use `allocatable_slots` for admission. +can overlap `quarantined_slots`; use `allocatable_slots` for admission. While +the store is unavailable, `allocatable_slots` is 0 and +`placement_disabled_reason` carries the load or write error, so a broken +state file is distinguishable from a full host. The +`hypeman_resources_gpu_slots` gauge exports the same counts under +`kind=allocatable` and `kind=quarantined`. Quarantine only removes capacity — it never touches a running instance. diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index afd5e03de..b1d2067b8 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1028,6 +1028,9 @@ type GPUResourceStatus struct { // Mode GPU mode (vgpu for SR-IOV/mdev, passthrough for whole GPU) Mode GPUResourceStatusMode `json:"mode"` + // PlacementDisabledReason Present when allocatable_slots is 0 because the VF health state could not be read or written rather than because the host is full. vGPU placement is refused until the state file is repaired or the next write succeeds. + PlacementDisabledReason *string `json:"placement_disabled_reason,omitempty"` + // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` @@ -19378,253 +19381,255 @@ var swaggerSpec = []string{ "vT1KA8a9n/RzZF4oYA9OfqzK01pub6u1n1aoAdT2CY4om2623u6AQbA2jXW4gq9O3721ZYOaEJf1Uhal", "hQzYch/9WhTW0kstS1SlfsBSWKMnE0CtXxjJhIeiUX4ShCB4Vhb01GcPPNgGo7eIGsAxYDRx5soKog2I", "8oqJRL/nWGCmKCMxev+T/B4NrMv8/U/IZNFYBkSlH9hY1RSfhba0MRvudLaQNMKJWRZT2oMy30oJR7q1", - "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLjLU", - "LduyHW+tCr7XepE8LhdYHY8Kmgju/U9VYjEXj0lVteZayqgC6C3AW94oBmkQovvoBC+sVp8BBzddeQio", - "eDIxAJQFZxMkIVhCNWqJ3v/UXxsmqLjCSdMczvVDe2o29IT0nuphdlFWIUo4SR7yk9/t0yADK+cTcHpA", - "h3ULeh8ds/AhNLEogN1mYJikpFOmNQ1pw1EizIqVXNq7KicPGinqWHed6sJVptMNsKMQxYR4p8mtWxa6", - "l6tj8svqoeGX6ysymkaa+z106X81L4vVbBx/gyxBFxeDzKfGjWJtdZJkWGBFkkUtQLeKsE+WQxzINYlu", - "kH/4Ur/+yVRVyQUZqZkgcsaTamDMbne5Mq+EIPQ5scXIzJw8T5DiKMXiEk6Z0+xQzswKVHMYdtcBDs2U", - "ym4wqZ/Pz0+NuUcRMcdJPQtGLoV8HJEEL9CYqCtCmJsKlgj7QdD1LGLZUBBKqFFGBOXVNezsBvo9M4Hx", - "aCpwRJD5ytX1LtiaPnptl9L2EoCwjCIiZcP+bq/aX/vpJE/a7XFoWNtry+BHN9ng88NTV9ioKBztlnln", - "eZVPieiZI+cqSK/e2h25usSW64oZZOIlCXJMoOaWTT3yM4NdwBvUFNOfV7JxPeYg/fxR2w+cArNUXXPO", - "P7RSN+rHPRTZkWIWhwpwm4wTg9UwBQA3CHsXOegCNDbRcOZO9u5nmzgjCI4pI1LWEt2jXCSdbqc3sbPa", - "39rS/D4BDMu93e3nW6vjilcGlNv4uVFMVxkcXJSdicNyadUGABAmXSWJLZxlLUyyZh3X3A/AnpYDWKHo", - "tr7bPJHfZTQMBks5+9c4Uq5MINhoKz547B9bQJquCjK6wdRU/tG61wv/fA6CQREZVrMq+W8t0T4ESkGQ", - "q6YRY/+qraMh8o9B6ZSLUC04LpRNAR0TFwVb3IcuxtRhKle8tYPn/iyfPnmy+2QdHwJmUzvm9twFpmre", - "ria5E2+57fm1DUC8XlXmcEd6dTlbvS5raEpzxCWSWr2gPCPsRuv5ZG9352br2XYixy5OsMaXQhhBhydH", - "RibSmiWmjAiUEoVjrHCVyYBxU3MZKDaESQp5YpPvV7OWhoAaH/TntpXSvlQ4RkPRxLcOIzzFjE5ASTJv", - "+j3LGd558nTflHaNyWTvydN+v39TKJSXJfZJq63YMlGbHipKX84+bx/uAPGkzVz+6JwenP+sGVkuhbm0", - "tuSYsn3v38U/ywfwh/nnmLIwUkqbasB0slQFuBqgmFsoahLvo7Lgu5N72gSMNXgnIFwd4JmCuIOVsN27", - "AxgsaJxW60jcIFl8RfK0FlfesGTRuDa3LuxblrxXXkFf367Qorgv/bg64MLZP+Ed26exdBR1j5dDLW5V", - "uVquLO65VPstI6wo55kk5q+IM0B6DtX2rFyR7lmL0nBwjdgacEWX/o9F796Ph/5AvN9daTnvJ1vk88MN", - "Y6RWCqR/W5ZD13MhJ46uOcxhY3RxK7QtqGyBCYPJAQ98F94mjrDa+5vpf/3+f+Tps79v//76/fv/nr/6", - "r6Nf6X+/T07ffBbozWpMygcFlvxiWJIQPFcBlGxLSidYRQEbnVb/GlbYPjEWBxXNoAQsGpP9Ieuh11QR", - "YQoK1rJhhx20QUBTgq+0uAvVkkwi4qb++NS4uPXHfzgx+FO9jdiiFAi7IQX4jMzHMU8xZZtDNmS2LeQm", - "IkEv0H/FKMKZqQpIGdL67wKNBZRwtD7HsvMu+gNn2afNIQOrLLk28OoZhip4kyINkLmIATsqExdtXycF", - "DolJUR2y4rYuQBmN47lfVnCgJKknkTUsymr9zWpOzwch+EpIcNIbCVWKQAUpKFuTUZF5hZ4PNpf1uTU6", - "RkFDK8jPRmaYDNiDPGQubsqaPSIxjYCvuMTRmU0tLnJ2DaVZI14m+PUC9uatyWaMEc7VTPOiyCItRJxf", - "UtKFLe2CfxRCgeBLE+Ax41lvvOjNeFagbmBhwp+wCZGoKtn/p2cn2ntPBJ3YnoLgCZpEAkInHBk7M5OD", - "WlgXliZ2bgq8MC36zIl93VQKkqY4jIntV7lgrn4JgSqmAO9SUB8JyeTfoyihYHWSM54nMZoBAqPSzYRA", - "FDuDIocKj6OYTOr/rsa47Dx5Chqs+/fuTusUZrN0q6gsTwI6bepYXwuObdgkDMCIByNnCF8TlaZvQOvY", - "BzuF4vDfM+QaKk9cwUiMp88kVUpbDiORXjrlZjDvzR4DiyEzwvY0tbmPlk5hJY2uRQsmXAU+S1qg17w0", - "6bvnr8+QIiJ1gAobkd4dOCUGqqRHpcxtZbaDw5OXm/1OEHmr4tKCrVqZZlcddAB8w4avNEXllDYanJIu", - "Oj6C9Gl7rZS6GOS7/MQFSsytWF5G+4DeUjX3YFPT8fjICqDJooyBMWLLsLPpWszq19s+eluogLgYSpEI", - "W9KWa7K8TKBZGxFpknGWWq/lTYN/zKp/9j6G1BsoeGl4MaCHNt5f7W2ODotMX1Q1C9mNLyQ/LKnR/uXt", - "/ZfG2f7yMvruzWR064UeZTMsQ9Q9872a8NLSvvuO7Cq7F81BZJV+R5IGz9bfXNkf7xpSRN9zlc9DwLJP", - "etvb59t7Nzff3RQiuYqN5uEmFijJ7eGN7wImOAD6S9WoMdsA6cc2t8DZRd6foBmW7DsFD2vWke3dZ22M", - "EtBr2zh9P0KfT8yQCi7lgNaK+HIDOXdJk8QIMJJOGU7QC7Rxdvzql+PXrzdRD715c1LfilVfBPfnFmjJ", - "cAvAOpqUwwDWVgUjAhXJpOfnr+FwJQTycYwcfnl7DOW1psUWmMpucK9O34HjH8uRi+RtTl7FZQI4uaZS", - "yWWYvVYB8Z+D4Ww+LQ1jbSZp2rAhfmuBoH+uIBUHcRM37wDB2WUzLC3nA4AbP2TW6NcHrLwSCvlz8Yyt", - "neGO4Iwbr7QQFHANIuNJ0+12e2DiOxlOBWEoxLZ8Ccel9N8aCbjboYF05gMbx4eOT8vSX6UzwjVfm9OL", - "nf720+dQvXZ70Iaxpzha0ffJwWH7zgc75pbZx+P9KN4Hhf22PitL2EYFwckVXkD1R7O0w465MD3t1ju2", - "VpFsFV+zDLh8O3zluhjXgKAM4qwLXJKjdGXxmRb5qnUUvTS3GKEpTRIqScRZLKsy8gxLJDMDjWuKsBQS", - "/JDBALuoqIUNUgrCUSTy0vRopWsr7+eZpXsoBJtxpnUAqATxC1lIlFJwghbdQ+ijREVaVDxkG8Kl0BW5", - "clADNtY/QEJK16Y6xF2IGoZiM/qDIZOzXGkmttlHh5zJPCXCWmXRmILHaBPJ3Ki0MF5YjYVmmJLGRAyZ", - "fi0AvvtHoZ7sPx0MBoNup9DkdvW/ByFqulPnZ9+CS5skcIB/ZBZmGnAlRc5QzmIiioLwxJBDPUTuho7T", - "z0SVdp+3E6/s56VcFT6Y60Co26FLfy6kLwy1QT+H6NBbKOdPbi+it0pCc/KrTT+zX41uEsFAUMTzJNYa", - "31jfdsYgR2JrhpREGe5cZIK8M8VYq1O3oceKo99zIhbo/clJJexBkInmAe0mDlyiYR94dqNt2FljI1k7", - "mpu4lz0A5PsAPa5LKp6E+MUhjn0Po8uKNxRaMWxVFOdV9jWtVQYzcigz+6SJZsUEa/V4DYZIGRjptyoX", - "cmIBP60o7XIqLAB5gQdSOvXlQm6Nc7mVRXTL5jJtAVjLcwBr2Qtm08dkPsrzkGqkHzl0nnfvjo/QBvwC", - "YMMmQabSPcZPt58Pnr/oPR9vP+3txYPtHt7efdrbeYIHk93o2e72zu6KpKIW6ZW3z5gMasyBOOYian3k", - "oudDQc1NuQs12cTGY19RFvOryvUXDJD1e7fBt+u6Xw6tbz2EYEpQgqUy5osGTnYClzyJdNsmIN2m8Ba1", - "t8KGzqfng+3Ptf7A4BruiHORM+NWNaAThQsh9Qbsb1Z1nLdj+TAgl/iybrX8ztsv2mD/yYv9J5+7aC55", - "Y90Y6+R0j5vbFBHmQLlr2SEuZdWzIzkDZcfKRMaqb5NJOt1Oke8Cf4MwUIulLh63SuJqOrDdMBtZda00", - "ZNMfV/QViFQxoIzxvpZUnD4CJT4KzAYtAh0mPI+RZ4szGHXghzv2dBfdDLjFrInOpNGaZAzIjKTS5vNR", - "phkx+B91Izb1fh+9gnfhEU6NWmcHYQra+K43HC9MvIw+X65ro2StHvKZ1a/gG61sIf0vmLZeBmuyXd2E", - "kc720a8cvim0Pcbrtl/zOqhZy6/X7cQbFr/dQalAZ1bU3Ec/FeJlIaBagXRDEvvnyDKsEsFos4IjYXe8", - "o6ml3DkPE6HbMSva6XbcQgF2wjKKwruS6pfOn0+KoUAyghM4y2XSeK5oYnHbYSZUKhpJmzyiN7dJ7LG5", - "mSQeGeWpKSbVpL5aBav4yElV70/QBkBz/gVZw7b+12YRv1q563Ze7L14+mznxdNWAFzlANeLxoeQJ788", - "uLVycpTlI2sbaZr64ek7Y/uIjFWhiH15f+LjnWSCa9ajZ+4a9Dt/0X/h447FPB8nnmPRghQamGPYsCC0", - "XsGLGuIgf6fJnE4m7PeP0eXO3wVNt6+fyp3xdgOesukobHY79oMLlmzUZNwzdbPC0FBAUEI2oqe9JRJm", - "gM6IQkA/PYQjUG+KfGpLcg5jza54kLD2dnd3nz97stOKruzovIMzAiNc4FK2I/COGLyJNt6enaEtj+BM", - "mw5kBCDvmVV9w+cM2aLXg6pA2t8e7IaopOHiLqnGtj1PG5f8vVUf7aTsokPydqFaLp3y4Grv7g6e7T15", - "/qTdMbbm4ZG4Xs1hXMqSWR5bmcHf+Q2QJs8PThEkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCeP3v6ZG93", - "Z7sdDGAo6MQCXFYObJV3BQ5dgCgCuxFYimXW2226LULilCGwtyRKME0PIpdiUbt9DOr/SJjXyk1oczFY", - "DXzp4mrxbSvjVmGyMgk6RjTgAuWsqDXTX++S/SKe1Wauba6H9Vw9lJbD9OpZvCpTU+8WS5kJMqc8l1+g", - "Ia5Mzuwk4Vzc6NsmheUtkXmijM2GSvT+5DvgKZrWkFQkq+pQlhpXoHrdcnI3Os8VEgkTedNitdqNNlu/", - "asLdhlPbXQWoUeEGjVh6seZcOVsf/HmIkyiH6kq42E89KwCFA0iALEsWJrY/SThnKJphBk4S4UFgoRlP", - "4n4wElY/GU2CURX8CiXcoIBfEpLZwkNmEPozLcLQOUEbfsk9Q0q1QrhPUsNkbGmZKjU+ScMVPbEMJasV", - "qfB6PbHiHkC1+aRiCU34VIJSqCBroV+vi5BhYZIRMDOFtOap0SUDAdeBIdaYeehGNTcpn1gF14ockGhu", - "VhJHgksPnOr9SS1/eUXOW5HFvD6gszrYFqRrHJqBq8zAo7Wutxe6HwP5PJ9zQwINQ87gilBJZ5xMMcuh", - "FJFHyNYQ328dDjnjUo0KkK8bDlaqEdQXyQUp4ROLrPvCHuTeCd6LjrXdZrls3PGtvl6iqnBTTQNs5qnB", - "FQ2vVregwRAZL6OcrQRWK+Hm6mBWN0EvLAtSUAmtUg/HDm1AzovHljxMws02QTJhlVX3s6St2nKxr/cG", - "Z21x/lbD+p1iNTtmEx7A/riB59RZom20akaEg9qLCaMkdrpk4UK1pi5IGE8kQXFO7MoZ+VRgu+DYHG/w", - "WTFnI6NsWuP19Q7bmIfNGFaXH4F+7Yttwp1kOKH2XOSwViZeUSJcpta2CgKlchR2Zy03LMg0T7BAFqGz", - "zZDlIk0ou2zTulykY57QCOkP6n7xCU8SfjXSj+QPMJfNVrPTH4yaalWdmcHZvECzIbV+yyn8oGe5WctK", - "BkvMlvl+CxyjbaLHgpHiP9GEWKTEd4xee4Rexeff2xk0Zcs3NFrJk1+GCr0p57YkGzzxuQzkFq6UclyV", - "LRLboglG7MlyaQr+tLiVHCqvcwHezqNTTRz5PGiSQ8Ova8AkaEwg78dNbZlrtGCLbaYSrDWSyxn6Ox9X", - "DaJtw34DFew2WAmRIcgkGN8PO7rSIG3eWFoTb3dvgkEBbFVPFD66IbTDulp/ZXxVEz95u1T2bkbsklE3", - "R1MCr0VJFxf/UcAX2F7b4xjUCxQG4pUBpUaqBZT6hfpKC6/qpkRjLgRAkmsJhzM3G4Bd0TKPXmuHe4XO", - "Z2SBBEkxZUNGWWEkBTA1ghiZE+FlyXKhlawpifvob56KByDuaaYWtjoAGM+/k4hfsWKMQ+YPUjeeS93O", - "ATOWRZFnqlI/VDcLWp8mFMhaBieYElBQk6oZmggiZ/7cQ0VUtYx3xUXcWJ1qgdwrUPQIfKxI8UvCfFZW", - "NBNUDU1DI/PVchSfqYAMT63+iSpFiVG96PDq/nJJRFhILKZUvNIqdMU7Kp5yYkBgABEFik3avwyLL1BQ", - "WmCelM3/1TVZ/nRaNF79rfaah2vicKcPjNk2aIKNTBpPLdin6klbG6oCaXCr0GyWfQlow4VQu4o9VUnA", - "q5zT6p5sl4lXTxZwo9mSJKr2vvf8ybOnLUsXfZazzqB3fWnX3Dxd4ZJr2KmTNn6f50+ev3ixu/fkxc6N", - "PCwur6Rhf5pyS/z9QRvkWunDmvzrH/98f1Lz+jyBGOzBjQZlMkvCQ2rILqkO6P3Jv/7xTzeqWw8oxGiW", - "IeMb/PaNUTqJv5MuUKDqwmvnJFuh3x9UjAS4YDNogwAUN52TkVm3XjmYGgxIOykYZziiahFg5PjKRLsX", - "r9Swttu4g6qDDYm8pm2Liqo5l8zHZdLphusc/afxDddo4XnrCmgyHzf5od/UezVe6NJr4cc4tAgxkEXx", - "/WUDdzGfKywrAd367wjyLlyG2XK2jXljNepuPRUColhsoT8vFDCEfF+TJ+1H/vbXttPzW1bMOvUV/7Di", - "HDYfwRtZfQM3csDoG61Pra3xB3sB3u6r0divTbiy+GOlkGF569683xbZw8uFM4ob7Ob9eQmTN/mwjgkM", - "9GjHYJe8bLtbIYkGavJyYQIGNJ6Qnle8wBR+krnxCOozb2HmAxmc0SWfTKpYt0+asdEB9geSvVwvWCmt", - "mXQRuXY2izqwtsH4GXaeyGFHqwDDznY67NTcVsH0yRRfj2wHVWyXwSqw8jL9vTZI6WYwTnh0acruQTX3", - "PhqglGAmUc7g8Ne8atuD1d6hbifz9qaABicmxGmJbcGYxmSG5xSqe1ifyrQSiEmuqZIQMArt7KOYG7Sn", - "Ss1hO0P9mklu3C8nDZcOZgvbsG5Qv8eZi2gt3wUD3wQqHbOPRPCuBSvQHPvNm5OuCWCA0EMzsEp8o5uo", - "GYFmkEUXtfIK5e/h+OFxQkYw7jpcf7q8jn5OOnhWBZFESYvfXZJDjQhQxHOm6jj+aTtFrppWtnwl5QyC", - "/Wz4B+Cy2d4NgaCYRHAi5fJZrBL6LYi7ljdgVzqUOLAbImE4FOBLCvuK31qHcH0AxtjglQs37fhx3cZL", - "OJKK2/pyxakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKKSt30b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", - "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtfUa", - "76COoYk3v1UlQxuq/hDFDO1bd1LAcGl3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", - "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmKZljI2tgZnc5UsqgG4OwFsJQ+", - "q8qnIIowZyhss/PlbroPl6Pd7Hb6rYeE4zMPGmippIkVSUercLMPS2+btc5neAFWnEYn4bPdvcFgd2dw", - "K+BsN6wbLNdh+YmtiVltpymlzvvOOvorUap+C0WS9XKh5StBIVe7WCapBMHpPiTeZDgiKCETAMkrElrX", - "exbrXa8evBWobBZtQf9uo+y+OR98tWRO0ZXFHHfT6DjnYhWDyH++xiHawGaiJUi9QM7dbm/w9Hx7d//J", - "0/3t7bsAuy4WqSnb49nH7atnyQ6e7CXPF89+3549m+6ku0E97JKaykBtaPUX/W5jlE15SVaxjCosDW3Y", - "OWRE1Cto1yvPS5JQRnqyyJBan6a4ghcY//va838zO7+ZwUrZ4aw6SV+EwKpcnAplPQz+lp3MSt9FfTbH", - "R6tncasMpPpAwvRWHwqQV7vBQIWK7c5nIjPkrOU19M57sfVFtDIrbt1VFPKww0kP7nLDiofIuwbM4M16", - "1QW+fMkFbKdTLqiapatvi+K1AkYc4qY/ShVX8Z766HjKoHy+/3MRJucrUfrjTreTfNyrnhn7e3vkL4tA", - "XBCg3WpfKmgRRpaQOUlWrwK8UioewkSya11dj/mH7d72C4hDSD7u/TDovahGHHTNavnLt+3ervw6aLOG", - "fglAVzpq+8WNIq7deq6ioF9oqIBdeS9bbGJL42Wxcnd1uITbygaXj5f2uIbk0yiAfq6kZy+3kS80xSTB", - "ixA2vWeolTXt0ScyNCZTymQbu+3uoDDcPkmHnT46sADhoMsqXvTjN69pxacTmqYkplrGNKp/cwbDTktb", - "XF2XuFltEvdVQFrrh8W1F+shEtYlXK27JvufkY/7WdpvO413FXoH2NWcigoYYvBiF9EJwqxWoJSyOU5o", - "bBPpITES4tX2HVBbSbKWB8hSDnR2ki6acoXKFPqW9racNdsFi/GTa7C3rsDMMASx80UAUQoAMbqKfR0f", - "oUzwOI/K/NEEBl0ifoi8BtG2QshfH5J7l/YNSMyecIHW2zeaDBrt7JNN+12zTWqCbd7q7cH6rb4To0i3", - "k2fxeh5mXmrHwW6E3L4mBTFgoqkue00S9CbzoQVHf+uv4LLOa2zJkRaJ8sw5WDRNLVNSwN0CLoZQXO8R", - "SYi+ppYbQTyJyywJKksuup6lbj99PmtycYJHankgvxCSaV0F8I+gvxSzRXBgruxocZdsDBzatzQOr54p", - "V2RXqzq4Z2slscat8k24TSUUDJev2bwNXsqlZ/4uML590WwZAcUx/IqQ9ra5BID90oW9NdqP78Is95BC", - "2hvreqjBtjpQ4QId3fVfxgJrsa5KvHsh93yILM7xFGrlrfGjNJJKqK4kODkwUtgWt4ICm/qfCKqISgfY", - "4gQE+Nxl9Npig4gqhKe4XjmiqSyl8USvjyFqLvRybk0ETZi89XRY3/x+0PsfY25Ho/7+1g9/+b97H/4z", - "aHavGRAkEb2YTCDi6pIseqYKk8LT6hX6hykBobWKqT0zBKdgTAO0d8uV/PE+GRTcc/ErTpemAKFqXgml", - "7bUT+st/NAd6ecv4Di6MtWf3syuk3EUlWcXdvbyREjF1QfUuo26zP2SQpHdJFhJ5hdmsbOdO7Hey+MQL", - "xUcXhtz7hM0v0JhCpUs5ZFq9x1FEMq1W2Vo/1JRr58CGBcGJ344tEOfOi/XMmsAKgt6fLMEZv3l3/uOb", - "d78ejd6cvvz14Hj0y8v/hliXq57pIe5p2tt78tQWafdXcjtYKOTm9S766MTmK9iYh0kOmj0AlkmU5iqH", - "6BhyHSW5pHPnKVXJ7StbLGct375SxGdCISuVhMIzLGR3QicEAhzgXrXRRVQ6YqQSqttbKw9laFl0MYQz", - "7MCVoriTJIJ1RfRWhFe73Njqoj+dtbtLDEhs4LBD6q8+bAEr6msqAbjDBcF4L6MNSKFxJXhdBvHmzUBr", - "D4oGgyGYX7jS0uDFl6iG+m5l+dM5T3paz2soGRE0q5u1CKYQQFMmNaPT5H2ZjgPKjLVxT+kUBxwuIcfK", - "F6la6ga0NnVsaf8by7eFEzqO6vU0zLE0S1Wr/1CzlkjVa873SLV434BADBDLJomXekGG1YzdlKktW104", - "BBwSc0B1X5W2XZ4yBxPZg4/WZyOvVDC9mXkjad6bE6dH1TS9FQt0qpfmakYE8TYCPijrFNxwyWyCUgs4", - "GlOdMSOiDN512U1aPAe/u0QbhQnMLUGRdr3sF1hdh+IEXxc9gE8JyyVHLMyjrIO1/epHqBnw1tX+pBPX", - "BAyjpuWGEfKrVLRqTRxVLW+GT1XL8zbvBw+e5VUruF/T2aoRZ9lHhTRD9Pg3TNVPXIBe3Az+cudA+3D5", - "x0QAGF4dRr8VBj1NSTziuVp9/vVrWno0V35RH7asL+xsABiIOKrkNTfxAgdPUo5heaX1cpAoF1QtzvR6", - "2ah2yAd1RX1hIaEj+LnsGAqpfvoE1vNJIHPmFWFE0AjK1OrzmGIGGhN6f+JVKzSFK5eAa0EEenN4bO0u", - "DvsY9GiqgPRcAOrB6XGn25kTYWwPnUF/tz+Aw5wRhjPa2e/s9rf7gw5oVTOY4tY4p0lsE8mtRl1o8Mex", - "lYR+dC/pLwVOiYIvfgtAIkAApn0dVBA89ZTIDFNhtcgsAagGQzBUfw11F9yFum9u5a5Z9tbGY8i3hjQg", - "kr2xm/sBBGU4OzDNncHAIrwre/1CEpPJnNj6uw2jLfttJdXZJQqUIVhS85xsWSz9p25nb7B9ozGtGgqc", - "3VDH7xi22cwEtPMnN1yIW3V6zEx+os02t3Fh/okDQvLP2m8f9J7JPE2xWLgF81cr47JJMCYSYfeu0eOU", - "RJFmFVAsqY/eMGKeI6wQNiHcImdQY9p9qCm0egpM226TC7SmH3m8+GJLWOnD2Sg+VdmZPi6fluj5y9FO", - "QcbLG2kfOahxQ7X3QEA/4qJA+oOdlL3Bi7vv9JCzSUIjhXoFAdvAbCoh9ikB4HQHwsQF+j3nCqMir+ER", - "HWkrs44LcuuWV9HWHzT+ZI53QkL+gFMiUsxMloh5Z82hXzrOxjdTHueVt5oj/OOjjr2pHBqRuahAkKse", - "Uf/aqguDy9fRXgCKwvZpphc/IOHv3cMJt5MtauQ+5JGDyqQol+QxHSfraxyXQkhQlntF1NdC84P7vLJs", - "NYU/4Sl6LAT8ihQSXrlbS5fCViZyZhTgoAT4tszctN99VxX+zssnXrgQ+DV001DXQxkHM44XfeTW1Cj9", - "agFYU4LAPOPla+VUD+9rOWE793HCYMaFp+jbNfXtmlp1yg21uCnAwfROeQsbxI0sEH8++8ONrQ/fbA/t", - "bQ+tLA+MXFnrwt/5uI9saG7EY4LkjOdJjMYEGeAnF4SjsOhPPyIsohmdE0D3g2p1eaJohgWE2KQoxgob", - "H3qjYWKlWaJobks313MBmeUC1wE9JBlBCMyoCYizDMWkjJEY6U9s1EyJq7hUV92c/aCBvWiwvBrR1YxL", - "UgAbMuXd5pDnLY12DM32h+zcIt7qBYSocsdrJEkAt3eF/YczhIfMfvC9YyEuIk7itORcWAB4IjUQnWZb", - "lnP89EhHMuIh0KFzwjBTPZmRiE5oZKd1SRY2sDXYYKsCVHrAbpzvT4rMFbSzGQaug9ClMErxUfEMWUqq", - "+m8YRINHSR6XTi6HpYTFGCdJsELJNOFjnIzM+lySgE/wFbxhF6V0uJTeJMZjYmrpZws148z8nY9zpnLz", - "91jwK0nEsLPZHzLISLFrTeJuKSCiK6hol2ZcnzPBU9Pnlhni1h+XZPGpP2QHcUqZowj4BCeSI3IN30Hc", - "GICHGO7VQA/mNIX94Ie5VDz1IWAd3Zlh8lxlubKpNZKobgj+dMgUR384kMtPW3+UPX4CZzHBsaYT7xUz", - "JZCtm0YtR1jPfgSvBtztBBZg2NEXqQnzmArMlMEvLVA60dTf0o2iTASUjq2vcIQZynhmSmwAUc2wJrlK", - "GwBagZMEKThK7lstuMNONszHYhCm40YAQoMYVztGlKGTH73DNNh7Hj5PkkSChCJK/uvsza8IbmW9B+a1", - "MlzL5LYwLTCgOAfXqeNpL3E0Q8ZRBVUVhx0aDzuFOzfehLHm0obL9HrgU/xBD+0H002Xxj/0+7op467c", - "R7/9YVrZ12cpSw0g6rDzqYu8B1OqZvm4ePYhvKBNOG5nFUaANsw1twmcBFOA3PFufHNFYhYjbm+BZIEw", - "KjmQH7gypgyLxaqMysDS2xXkExPJ6C3GH0OIXBx29ocudnHY6Q47hM3hNxvgOOx8Cq+A9Vo2l/CD+6xw", - "bhZE9HQw2FwPCW7XN+CzbOEY+MI6YKNWVNQf1Tto8Wj/XP6Bf2v9s3D9YKY7LzGajOLvjO+P0AHhSey+", - "JhpwQdTEbswikjixe72h5/6dB3qzIpIk902gD0WehXusKFnwqMgRNqs8RivN9w9McYP7ulQqZvuHod9H", - "Zz8PWM+t7ZzMXahzuGALgPFYVRqZlxGW6AzG1DvTyvdL+LVv/+t0PwCXvEj49GLfqO4o4VOUUGbzAbxA", - "ZS0e2LWEjwweT/Gdhedx1fI2jCTxr3/8EwZF2fRf//inBbn/1z/+Ccd9y+DMQbHtixnBQo0JVhf76BdC", - "sh5O6Jy4yUA5XDInYoF2B9bmD4+QV/PfSmlyyIbsLVG5YF7ehClcJ22D1lWg50NZTqTFM4I0oYmtqmNi", - "GwN2G3eWzVLe64nuBnAhYQbeBPSt6GgAQPWoqThuNdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkM", - "LHHo3MEDO2m0cXb2crOPQNsyVAGVk0B3KJuxakT/G09az5MMR6kyFFhlw5sinOExTagzOTaUfTFHMMXR", - "jDJSxhcXoOuuiX03Us1jDk6PkQ2E7MKrQ/bmbAtMrIpEKhekazmBsFCrZV04bvNcoAfgX1RBdFjPvjtk", - "E4IhT+j4yDABD428SIwsGmaAaAIxrlRVStB1h8xA6loIZ33wUh6TBD6C/qdYkSu86KKi6K8rE5NgpRVi", - "2dUvD5lJNbRr0APMFuQNsw/8zAyp5yJ5bc6WIJNEq8YQgW/qn0PfGxMukI1w7pZ5pa47k21qhqUXLcXR", - "mzM9vylogtzYA6GlN2duNza7SHIUJRSoIcJsyKYQCORQjDmr7GqRUDbDIu5FXF8CPqrVJeNXCYmnTTz2", - "0CeyO5RkKv0EjtPPdXJ9bMLFbHkC+hAbpL7Vnrsj+047151t8c/ku7MVMW/gvDMWXGL4jVndb468Fo68", - "8Lo5p17Is3bkoCjvLuLXdPFAAb+O9pbX3DzxluwhLHpow2H8gFeEC3R6eIxwHAsi5ea/t71Pz9RQaSn/", - "6ftRs+KHCD2xY+HCoh9ae0uVQB4LO3hrR42wm1e90LB/v21VqhA13nRFQaLyyrv726PW6U2ukVLoLWnt", - "202yNtiWyohDvcWSWnogGiWkEF+Kc+pT0TqrsgnjLa6cleKSZc/HR+5A3p992Xads/rdcA9M8ajGEB+Q", - "EVZTrf3y4Y+Jmt8Vu+hgt1eYn78u0hzcnxR036boEJk/JnUxri2b5oIG6KTxAn1FlIE3uUs93fYQmPgZ", - "Ee5Um4EuzKyLaZlPkcFpgQmBJWa17ntsXmmn+pr2/kyaLyzPTSQWu+TfRJQWym65VqsU3GNbC/vu9NsK", - "kts9h61YAgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0B8wuva5nO6rWU", - "oUlCpzPrBIjpBGL1lF/IHEa5cw+jLAqGC6yIDVF8jHm/p3qRrRd4ToSH1OhfqVt/QNDqelXJMa+Vt+u7", - "t697hEU8LpwnzTKpffKFFSZD/5Vc3vs/dY8wn5U68aBJYPyM/TfB5KiA7vxfOz9Z8M7/tfMTTjLKyP/a", - "PUiwIlJt3hmxDO7rprtvBeYRE5/WX2h10ZZY05bCUz8PtUqZDr72JpRZgZ11xu4ikeaLUd+XF/LqWL2t", - "JLx7o3utSd03YsmxK0tgtpQLh15VgyT+9xb6zioEfe/2ukr3VMIIQLB7hFHKXDjUVWxhq6kjbsub2BRw", - "vdcYI4q3Wtoj3Pt/KpOEmfSNjBLFun6zS7SxS/jLtdI0YbfiTo0Tpo8H8r4XxBZabXj0DW7rHhw6liI9", - "uK2Kh7sE3JpxqeDR48NesFcJLSjOvzZaeibLA7ny+nCke3zUhYWEsv9QdsimNt+Tn9KN494Vb9vv/Qs9", - "B+mYTnOeSz9rOsUqmhFpEQUSUmXAj80kUF7PjUaBr5hKB/d5ddy7zv+N7u/IGlHfUMO8TbDBOpnfvdVW", - "5rfva5nfoC1b1AVbG6nr6uZtNiSBOLzltmRcgaVeTk4JjSuki6B3WlEp1QUEGsT+kP1vrX/8pghOP/zg", - "0rvzwWDnKfxO2PzDDy7Dm504UiFMCWrLnB78egQRHlMI4odKqCWYRH0cKM2lMqTnSp782ylIZZBLew3J", - "UeE3DamVhuQt12oNye7F3apI1bJJ964jOXoLLbitd/Dn1JL+5K7bigYn88mERpQwKD4Fxj+5FKtsNLlv", - "XttbgiUwGyvhBTpWJJHWamTBtdZI6GXh/y8ZSdhtrEHBEVaKpJlCU4EjMskTU7UFyVmuYn7FXEkKmKCr", - "bkbL+YSud9fUyDUSTpANl2hvq+kWZRnvW9W1HT/SDFWe2QrjVrksRZtm7fJhifdudcoWV+39a5WPmcSM", - "+ra8dJnWEAIl1oybJ81NOm/xZYnO2Efn569d6q5WT4Qr2Ke4q9LnKjkPmV+lr49eluUPzQuuBa0+kNim", - "+kNCs617FxMcJ5QRyHUgMpRlW62t+aDH4stLwOHCofft4m5zLG1N7IeTgB+MFdyLrFn49JV1yBZH06sp", - "WpwWJ2/CqXlU/MoyoADjCcl6WzhXvGfBALZm3CBEhkFyTxMcAUaufs3AN1r8FYPX6jcFoCqCJwkRBpYz", - "y5UTt4asGBxliheFp61kdqGbH+VM0eSia0INAVtJIswWFptuyCqdWZkPMBIA/wNGKEhmRlyroqsHTXku", - "4S2AM/C7RDi5wgs5ZBZVwXwOpdcFiQyCbZL00c8cAG1MNWmP8ZpSrt/JIbugcUJGFo/mAlGJ5IwLRRiJ", - "UcrnRFb7JVgklAiYxCHWKydRihcADGkwcs368IwY8MUK6g3X/8YsplAUVPdcTHl/yDDaGQxQSjCTFsNC", - "4glcOLYNBIOoDOh7hNHe4IX9qrZvAF7uln9DnyYhyJxHeJwsENFUbIptb8IGprZIr6n6rrdvQoU0+1XY", - "N231xcrGUulqzsZdlLMSpQNs/TkrQDX0dqlcMJin9QISKopr0AITjUmE9XoyXu0HIGF5FOUidEHqrfaq", - "Rf87Co7e9M5gqcIYGAmYDCISw54zrmZwpjkcpc3vG6iqJKo/x0UTPCRcIIw8ui4tGiTKgTVuAITqRVn6", - "lLlS5heb37uzo4+vZQTu+BsQ08dyPwER8cmkcgDXX03mAK/KPVsm4T/rOT10Na99FhdTPGVcKho5ZugQ", - "qn2l+ZtC2EohXL2yQWqecHHZHHD8ExeXbTUwFxj5uBQxf4ZfoSNCDw9A8B/eHwHWcKOsaKK5dyWtTl/F", - "KQWhiypZhASjhLOpPkWlVf7e3Qa+VrdhAC31ZSqMs7uAH9NKyMj+aMpme2Hd4GKIbKsPzYt07/fgjPqV", - "K0TTLCEpgbLaPUNserNLqLrxwkLpFYBtN+OV+lT5uApGF5Qm/qDrxCGgK7dhGyC9L29XkKkmfLoeELXo", - "3KF/BhBRh+ydNKUKLozr6QIVPFgLtKb8CLqa0WgG6Kigt+r2DXgqzrKLAhh+cx+9goPs4+ND5xum6Iim", - "NckTYkBP52l6sb9cOPr9yQl8ZIBRTYnoi33kikUX94fUb/lop3oWCZYK/WoxXDcKZRx29EJhrW8W89u0", - "OKglcP+QhTBRGbmyDdIJuvDgUS8asPscv33Np/KrcRWV5VbMXBRHVnUE2iQs7jQFedAk7PjZHgxCVQBa", - "orSaYdwxSOvSYF7zaVHqpULKOMvakq8dJlDxPE1X0DDa8CAfpYp5rv4iVUyEgI8tdTcRN9rAkS3zhy81", - "oVqAT3ewN4H8gqFMpvZCcKk0U+10O4TlaWf/N/uveZp2uh07Hv3dlcpSr3bDDYT8Nai39QaXQ2/0DnnQ", - "tt/E85uA1laZvodaW7tBrFrdLJm/NS/86b2Gznb3gGQIckLNmPs1iaLeeKuGH8YL9F0Y2Yv7GBlA9KIo", - "4ZJUHD2PB+DPGrxqsmOzwcitcU8PL85dRbQ2kSxn9tMz9+VXoIOvixlxY0ZuuvcePLI8gscMViCXZjPh", - "oo4Kty6q5KsnpC+3JUtTbUMh32jz5tbGVoSp9YVlFmE/iE2FTJwrnmJFI6jOFs04lx7ZFxDupo6iNSIX", - "lAkmFqPt2kyCC02qF9YcfWHViX1rOkPYf2T76MPnNv8g/IV7VH7xk2cdKDh+16kAUMFEIozGgpIJynAu", - "iZbq8pSgaBFprmjK8REczVCEM5ULApVGCUopo2me+tj8esfmGHCELrbTiy4a5wolWExBOzMPXdBNxNOU", - "sJiAnW7IZgTPqVYtBUqwIixa9CSBCuVzgq64uEw4jsHUkMUYPD5Q4VQQTYFQ6CAlCsdYYRB0LvSJH5lk", - "pouiaLlR7xm5LqkhHjKRs+9N1RXd7IUb6AUiUFaAyllR3DbCMWFREG7/7OtmY1/eJn1GVH2iDxQhdCte", - "+pAhQ77t1Q3n64gmerTQEC3Y/AqhVzarsNUsEEdG/55H2szVzfGBHE3FEq86xV+Hh6kguq/Gy/TwbiQu", - "UJyb7rxTCWT+Z/UNFQzFD7qCDFOzjbd1EBVVPItlvhHP2/rD/Xl8C1veV8IJu42KfVO9uHLSXwPLtat6", - "K577QEZMa0vybXIPx4JdZNeDiU9ceFzusRhbKwhtBd/2uZMSGLQvzr6x7TrbtoEPt2Xbzja75Nr3GDll", - "PYgVDXNwa8ZtZNXWdPBvmpVSm53HMh+cRZaei3uHW3SsMcOLhOP4zxAsvMJ/FHEhDAwGAGs8Johoz2ro", - "pwmAba4sRNl1WZvvT042m7iEUCt5hFCPmEN4qTn6szReNuC+mRMhaOxgMA9PjmzYLpVI5KyP3qRUIcXR", - "JSFZmdkC2YV9PT8HCFIbdh35o9shTIlFxilTa0dRvno3gyl/4BYX5SsUJW3NgW/u8NbucLDsPz52BlwG", - "cjfMBFZrpgqrtbWQKZtwkRq5DI95rlvXPEgvk95Pg1gwoQmRC6lIaqITJ3kCxw3q19ga5fY7s8tdiM3V", - "J8ekzWVEpFRKypkcMpszkhGh+9af6/a9QKugQ0Dhgr+eGib5dQTx6cGYuDWsmlYNoJug9nFnv7OFs2wr", - "xgo3BIrZ4X3GkH6CqDwkF+mYJzRCCWWXEm0k9NKoJ2guUaL/2FwZ1jeC7750Bfbbnyy90sdswoP1LQ3N", - "FsT8p8rusmzNOSYfHVt7RfzD4vgPbHSYra2v8S4ITnpQM90B+KBc0YR+NKxON0KlopFJPcLF2r0/KZhq", - "f8hOiBL6HQwpbklikA1Au9zKBI+2hvlgsBtlFFDgdgkMDhhe8+MUejw8fWfSUUnKxaI7ZPof0PD5wanx", - "7k6wtSZ4A7XF3dHx1ps1gc5nsEz/xhGCZoIrUQyCG/7NJXhzrJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", - "2hUA7KmYzUYB8OVQucI2hDlP8lT/w/xxvA7fTOFo9h5e/WqkXTOctd24CT6KQ2nnFBNTf/dBnB5mwR5r", - "zKpeODcFEGIq0YDBW+BA/Rmp+8ub7/11/ArdnXZFXW3rr+Zs3ffNZ8fgkDb89Xgsx9xQmpuJ4qutT1eY", - "Nluffkx4dCktJItvNtR6G+Cs6x9LXGzrIgQxATJEkYUyMoBZRHaHrGaANMg/EmGkiEgpw8kWzNk0Agjf", - "zoqF55xConYEeSo9SWPATkoAxhtg8PRswFDlGvA8utJW//Pf8Z2RiqMxiXhKHOr5Zkh1+xum6icuqhDm", - "XwtfPPfWH6ABMQV7+xrU9uYePwvF/QRfQ6h0nFuHshvRxite/mhMQV0EezPs7A7ksNNFw85OOuzoHTjE", - "YELFCj1BKWW5IrKPjox9C1Jxnw6QJBFnsXTg686CtzuQTYm5hiwbsjyfwnf3KfZYqoKlfGs7CbEH/R7S", - "30PSDtrwD5w9k3EXDl2MeK6Mud+eK/tWTBSYRzbv3VfrnZFvun0bTv43e3wrPAp2WbNLb+sNZ89yOSPN", - "JrfXpqBRrsYA6u0KIMsZ+jsfyy5i5MpYw4VU/SW+p78+NR3cR8EB3dVNig3YuX+rNNCi0kC5VmHQRhNg", - "qa9kRx0GuZFcZ1woQHO0OfeGhkCTAAQJKFP45vB4yCLNigzEoCApB+5kcdHNLXzwtzP08vBtFx1BMV70", - "cz7e7KM3LFnYGvbWRzNkRhIzzCvCDI0N1ZI4dD2bsQP13GWwuO7ggarbm5MR8Ky4vXJB4t3OjOAYJJI/", - "Oq+56SyAPvz2tT5AAABsviy2vbNS+Oi8JUosegcTRcRysyc2T4oV2Bn2knZQdFZwMwCYukPpENjKPo1s", - "YCAydnc6AcSMT9+KP9x9Eef78ZKZOBFTdm+cq0dbvxUOYsEcQyzQv66L8glNWcKWl61UMKDLpsjvr8jk", - "vpJ3VTDm/11PF8z00Tqasso+aSIuyq6s9fS65OCZgUW2jqoIZziiatFFOEnsHWVvgiIipVeIv2NB8GXM", - "r1h/yN4WBV9sQi86PH3XdY5aFFN5aVqwvtg+ejMnQubjYnAIDprxGsOak3jIFEcRTqI80eIGmUxIBLm4", - "UMdFNvhyi6F07vDslJ0Ei854Ue35o6t1F6YJ2L2SLOoUt2W2ekuQKME0bQYht4IaBBxCqMFYN8oZomyS", - "2JCqSHApkW2qRxI6pePEBgjJPjqfESRxSoYsSzBjRKBcmqh4PfReJoiUuUnw1g0AWK+hqC4qAQYzwZUN", - "TUg4F9JEE2gKf3+CpCLZCjJ7a1o+gTnfkWxrGrc9PZCRujaGZlOIfQXpDTGUYhZc01GeuADGew1FNwN6", - "aCnxsRz8c0GnUyL0qcCGyZpwPHOs3XKaQ1/JWG6se3lWvNWu7mXRqpeV6GXsrQSIG5WY23HnZlF/gc4v", - "aSOGoH10syziX/RHLfuuZquGB2EffeYsQyU8/x2rZZ55SYJtDVglhT82c5I38spRrSTarofVap1Ze5eZ", - "rq3xsx4MNusxo2XhSvpsk8L79RHC4H5RHu672Nrjpq0K2lVFN21I+V+Pqv9VUODdwOk/MMrJLeD0v6q8", - "e8A7fzj8k+BBfag8+orv2RXd/dMj4t9V+ryBxQc4tqb0ecP1bPDqSkXpvX2nnZpkW/wzSfA23vEG8rtb", - "9m9afwuVwVusdS5oTfAkzdTCBbRZX2UZdCbpR9JvcAQXcat35wq+RUjnlyMPR6eNAZ1/zhr5DxIzaksI", - "UomOjwLF5x8ZxqB/5ioXy5a+dXpYRDM6J81G9+oJtkuUCdLLeAbOldgsmF0Pd5cpLPrTj8g2bzFX7b+g", - "BiVA9ZMYxVSQSCULUw9UcwTTx3cSCa41AXjOxaI5SsQckZ8ETw/sbNbch/ZMWWNYGWeYLnoxVrg3d9xm", - "hQntM6I7XTylZniIMvTqR7RBrpUwlS7QRGs+iE6KJSXXESGxBJrc9Ae8PWiwbNKPZDQdtxnlipolb2xN", - "GBTlUvHU7f3xEdqAGmhTwvReaFF/ApJsJvicxiSujLEz54lZ1e2GBb2p3VULFUUBO6dcmME9iAzT5kKa", - "fqRZlS0UITFjyjAMbm1VkOqZMkn8uj9MmQvAsXvkRvHtCrOa34ZTdjQlQj1Ou4iKcwPxvPntmnvM15yf", - "DOXutMpt58JzVhuv2+VHtUxbuovCD0Xu3P2ard9/PSk9VD7KbB5rOp8XCmmT2fzrIsHB/d0P920uf/+I", - "U0BfEad8e6ZyaEC3GCKY1xDTHZM5SXiWQl10eLfT7eQi6ex3Zkpl+1tbEPs941Lt7714ttv59OHT/x8A", - "AP//G3o07F/4AQA=", + "mnFafmjtuQFlI1xUzrEPtDGfZjks4Nnb3vGb91tpTObdypggGnHGE6LHvenx2LmDZChT9yqsdd5kLnJb", + "VdQ4HJV1FesHAdCgrR2zTh16bwZoTCLsAEXqmxcVZYjHEHsBVq8rQZXSkiu2RmeInikbMaB5Ek3yJOlb", + "BuiGbMzME+DDHoKpuao0+cDzDIMmVYhR1wo6JUjmUURIXAtfqY/ao7d9lLMUCznDSX1y+0WodzTDAkf6", + "Pvzu+rtOuI6QHpxsy+g96ixumtZk6d0rAXr0zl3TEX//U/V4mqveJAdbAzllVAHYGSBcbxSDNJjcfXSC", + "F9aOksGdaeml3DE8mRjIz+IuESQhWEL9b4ne/9RfG5ipuMJJ0xzO9UPLpzb0hPQp0sPsoqzCBoB3eVhb", + "frdPg1dGOZ+Amwk6rPss+uiYhdmeif4BtDwDfCUlnTKt20kbABRhVqzk0t5V786gWaiOLtipLlxlOt3A", + "BRCimNBtZbIZl9Wc5Xqk/LLKpvjl+hqYppHmfg9dwmXNr2V1SXduIS/TRSIh86nlGMY6KkmGBVYkWdRC", + "oqs1DchyUAm5JtENMj5f6tc/mTo2uSAjNRNEznhSDUXa7S7XQpYQ9j8ntvybmZPne1McpVhcwilzujTK", + "mVmBatbI7jqIp5lS2Q0m9fP5+akxsCki5jip5x3JpSCbI5LgBRoTdUUIc1PBEmE/7Lyety0bSnAJNcqI", + "oLy6hp3dQL9nJhUBTQWOCDJfuUrqBVvTR6/tUtpeAqChUUSkbNjf7VX7az+d5Em7PQ4Na3vdDqvoJht8", + "fnjqSkkVpbrdMu8sr/IpET1z5FzN7tVbuyNXFzVzXTGDBb0ks48JVDmzyV5+LrYLMYQqbvrzSv6zxxyk", + "n7Fr+4FTYJaqa875h1YKXv24h2JpUsziUMlzk+Nj0DGmAJkHiQYiB+2LxkZSMneydz/bVCUtaVFGpKxB", + "C0S5SDrdTm9iZ7W/taX5fQKooXu728+3VkdyrwzhtxGLo5iuMvG4uEYT+eYS2Q3kIky6ShJbOMtaGMHN", + "Oq65H4A9LYcMQ5lzfbd5SpbLIRkMllASrnGkXGFGsIpXoh6wf2wB27sqyOgGU1NrSWu7L/zzOQiGoWRY", + "zarkv7VE+xCaBmHFmkaMxbG2jobIPwalUy5C1fe4UDbpdkxc3HFxH7qoXodiXfGPD577s3z65Mnuk3V8", + "CJhN7ZjbcxeYqnm7CitAvOW259c2ABGSVZnDHenVBYT1uqyhKc0Rl0hq9YLyjLAbreeTvd2dm61n24kc", + "u8jMGl8KoTIdnhwZmUjr8pgyIlBKFI6xwlUmA+ZkzWWgvBMmKWTmTb5fzVoaQph8mKXb1qb7UgEwDWUq", + "3zpU9hQzOgElybzp9yxneOfJ031TTDcmk70nT/v9/k3BZ16WaDOttmLLxMl6ODR9Ofu8fbgDjJk2c/mj", + "c3pw/rNmZLkU5tLakmPK9r1/F/8sH8Af5p9jysLYNG3qL9PJUt3lakhobsG/SbyPyhL7Tu5pE6LX4A+C", + "BAEAxAoiPVYCpe8O0rGgcVqt3HGD9PwV6epaXHnDkkXj2ty6lDIrbKHKK6Hs2xValFOmH1eHuDiLM7xj", + "+zSWjqLS9HJwy61qhcuV5VSXqu1lhBUFVJPE/BVxBtjaoWqqlSvSPWtRjA+uEVt1r+jS/7Ho3fvx0B+I", + "97sr5uf9ZMuqfrhhVNpKgfRvy3Loei7kxNE1hzls/i9uhbYlrC0UZDAd44HvwttEblZ7fzP9r9//jzx9", + "9vft31+/f//f81f/dfQr/e/3yembz4IZWo0C+qBQnl8MvRPCFSsQnm1J6QSrKGCj0+pfwwrbJ8bioKIZ", + "FN1FY7I/ZD30mioiTAnHWv7xsIM2CGhK8JUWd6E+lUn93NQfn5qgAv3xH04M/lRvI7a4EMJuSAH3I/Nx", + "zFNM2eaQDZltC7mJSNAL9F8xinBm6jBShrT+u0BjAUUzrZe37LyL/sBZ9mlzyMAqS64NoH2Goe7gpPBN", + "MBejYUdlItHt66RAfjFJwUNW3NYFDKZx9ffLmhmUJPW0vYZFWa2/Wc3p+SAEGAopZXojoS4UqCAFZWsy", + "KnLd0PPB5rI+t0bHKGhoBfnZWBiTc3yQh8zFTXnKRySmEfAVl6o7s8ncRZa0oTRrxMsEv17A3rw1+aMx", + "wrmaaV4UWWyLiPNLSrqwpV3wSEPwFXxpQmpmPOuNF70ZzwqcEyxMwBk2QSlVJfv/9OxEe++JoBPbUxCu", + "QpNIQOiEI2NnZrJ+C+vC0sTOTUkdpkWfObGvm9pM0pTjMdkUKhfMVYwhUDcWAHUK6iMhmfx7FCUUrE5y", + "Bj68GWBeKt1MCLayMyiy1vA4ismk/u9qVNHOk6egwbp/7+60Tho3S7eKyvIkoNOmjvW14NiGTcIAjHgw", + "cobwNXGA+ga0oRRgp1Ac/nuGXEPliSsYifH0mTRWaQuQJNJLYN0MZhraY2BRe0bYnqY299HSKawkLrZo", + "wQQIwWdJC7yglyZh+vz1GVJEpA7CYiPSuwOnxIDD9KiUua2Fd3B48nKz3wlinVVcWrBVKxMbq4MOwJ3Y", + "gKGmOKjSRoNT0kXHR5Cwbq+VUheDDKOfuECJuRXLy2gf8HKq5h5sqmgeH1kBNFmUUUdGbBl2Nl2LWf16", + "20dvCxUQF0MpUo9L2nJNlpcJNGtjUE3601LrtUx18I9Z9c/ex5DsBCVGDS8GvNbG+6u9zdGhv+mLqmYh", + "u/GF5AeCNdq/vL3/0sjmX15G372ZjG690KNshmWIume+VxNeWtp335FdZfeiOWyv0u9I0uDZ+psrtORd", + "Q4roe67yeQjK90lve/t8e+/m5rubglJX0eg8pMoCl7o9oPRdADMHYJapGjXmdyD92GZzOLvI+xM0w5J9", + "p+BhzTqyvfusjVECem2bGeHnRPCJGVLBpRy0XRHRb0D+LmmSGAFG0inDCXqBNs6OX/1y/Pr1JuqhN29O", + "6lux6ovg/twCnxpuAVhHExwVQDeroHKgIn33/Pw1HK6EQAaUkcMvb49avda02ALF2g3u1ek7cPxjOXKx", + "083pwrhMuSfXVCq5DGzYKgXhc1CzzaelYazNJE0bNqhyLfT2zxVs6CBS5eYdYGa7/JGl5XwAOOmHzNP9", + "+qCsV4JPfy6CtLUz3BGAdOOVFgJfroGSPGm63W4PBX0nw6lgOoXYli/hOBCFW2Mvdzs0kEB+YOP40PFp", + "WWytdEa45mtzerHT3376HOoFbw/aMPYURyv6Pjk4bN/5YMfcMvt4vB/F+6Cw39ZnZQnbqCA4ucILqLdp", + "lnbYMRemp916x9Yqkq3ia5Yhrm+HaF0X4xowq0GcdYFLcpSuLPfTIkO4jluY5haVNaVJQiWJOItlVUae", + "YYlkZsCITdmbQoIfMhhgFxXVx0FKQTiKRF6aHq10beX9PLN0D6V3M860DgC1N34hC4lSCk7QonsIfZSo", + "SESLh2xDuKTFIjsRqu7G+gdIAera5JK4C1HDUN5HfzBkcpYrzcQ2++iQM5mnRFirLBpT8BhtIpkblRbG", + "C6ux0AxT0piIIdOvBeCO/yjUk/2ng8Fg0O0Umtyu/vcgRE136vzsWzhvk3YPgJvMAntDtLvIGcpZTERR", + "gp8YcqiHyN3QcfqZON7u83bilf28lKvCB3Md7Hc7PO/PBVGGoTbo5xAdegvl/MntRfRWaX9OfrUJf/ar", + "0U0iGGxKhtb4xvq2MwY5ElszpCQ216PIvXlnyt9Wp25DjxVHv+dELND7k5NK2IPN0Gg3ceASDfvAsxtt", + "w84aG8na0dzEvexBTt8HzHRdUvEkxC8OKu17GB0OgaHQimGrojivsq9prTKYA0WZ2SdNNCsmWKuAbFBb", + "ysBIv1W5kBMLsWpFaZdTYSHfCwSW0qkvF3JrnMutLKJbNntsC+BxngM8zl4QvyAm81Geh1Qj/cjhIb17", + "d3yENuAXgHc2CTKV7jF+uv188PxF7/l4+2lvLx5s9/D27tPezhM8mOxGz3a3d3ZXJBW1SGi9fY5qUGMO", + "xDEXUesjFz0fCmpuyl2oySY2HvuKsphfVa6/YICs37sNvl3X/XJofeshBFOCEiyVMV80cLITuORJpNs2", + "Aek2abqodhY2dD49H2x/rvUHBtdwR5yLnBm3qoH5KFwIqTdgf7Oq47wdy4cBucSXdavld95+0Qb7T17s", + "P/ncRXPJG+vGWCene9zcpogwB4Neyw5xScKeHckZKDtWJjJWfZtM0ul2inwX+BuEgVosdfG4VRJX04Ht", + "htnIqmulAb/guKKvQKSKgcGM97Wk4vQRKKpSoGRoEegw4XmMPFucQQUEP9yxp7voZsAtZk10JnHZJGNA", + "ZiSVNp+PMs2Iwf+oG7FgB/voFbwLj3Bq1Do7CFNCyHe94Xhh4mX0+XJdGyVr9ZDPrH4F32hlC+l/wbT1", + "MliT7eomjHS2j37l8E2h7TFet/2a10HNWn69bifesIj5DrwGOrOi5j76qRAvCwHVCqQbktg/R5ZhlZhR", + "mxXkDrvjHU0t5c55KBTdjlnRTrfjFgrQKpZxK96VVL90/nxSDAWSEZzAWS7T9HNFE4uUDzOhUtFI2uQR", + "vblNYo/NzSTxyChPTTGpJvXVKljFR06qen+CNgAM9S/IGrb1vzaL+NXKXbfzYu/F02c7L562gjwrB7he", + "ND4EZILlwa2Vk6MsH1nbSNPUD0/fGdtHZKwKRezL+xMfYSYTXLMePXPXoN/5i/4LH+kt5vk48RyLFhbS", + "AEvDhgXBDAte1BAH+TtN5nQyYb9/jC53/i5oun39VO6MtxsQrE1HYbPbsR9csGSjJuOeqVQWBuMCghKy", + "Ea/uLZEwA3RGFAL66SEcgXpT5FNbknOodnbFg4S1t7u7+/zZk51WdGVH5x2cERjhApeyHYF3xOBNtPH2", + "7AxteQRn2nSwLlkA5qB2zpAtMz6oCqT97cFuiEoaLu6Samzb87Rxyd9b9dFOyi46JG8XquXSKQ+u9u7u", + "4Nnek+dP2h1jax4eievVHMalLJnlsbUw/J3fAGny/OAUQULwBEdV246LELvRqNSNRgV1XEz9hRsM7Pmz", + "p0/2dne22wEvhoJOLKRo5cBWeVfg0AWIIrAbgaVYZr3dptsiJE4ZAntLogTT9CByKRa128fUWRgJ81q5", + "CW0uBquBL11cLb5tZdwqTFYmQceIBlygnBXVffrrXbJfxLPazLXN9bCeq4fScphePYsQZqoY3mIpM0Hm", + "lOfyCzTElcmZnSScixt926SwvCUyT5Sx2VCJ3p98BzxF0xqSimRVHcpS4woctVtO7kbnuUIiYSJvWqxW", + "u9Fm61dNuNtwarurADUq3KARvTDWnCtn64M/D3ES5VDPChf7qWcFMHwACZBlycLE9icJ5wxFM8zASSI8", + "0DE040ncD0bC6iejSTCqgl+hhBvc9UtCMlvqyQxCf6ZFGDonaMMvcmhIqVZ6+ElqmIwt5lOlxidpuIZq", + "GFOpSIXX64kV9yDBzScVS2jCpxKUQgVZC/16JYoMC5OMgJkpXTZPjS4ZCLgODLHGzEM3qrlJ+cQquFbk", + "gERzs5I4Elx6cGDvT2r5yyty3oos5vUBndXBtiBd49AMXGUGkK51hcPQ/RjI5/mcGxJoGHIGV4RKOuNk", + "ilkOxZ88QraG+H7rcMgZl2pUwFzdcLBSjaCiSy5ICVhZZN0X9iD3TvBedKztNstl445v9fUSVYWbahpg", + "M08Nrmh4tboFDYbIeBlXbiWUXQnwVwezugleZFkChEpolXrIgWgDcl48tuShQG62CZIJq6y6nyVt1Rbo", + "fb03OGuLrLgaSPEUq9kxm/AA9scNPKfOEm2jVTMiHLhhTBglsdMlCxeqNXVBwngiCYpzYlfOyKc+/B2E", + "OGA1A3skfEjZtMbr6x22MQ+bMawu+AL92hfbhDvJcELtuchhrUy8okS4TK1tFQRK5SjszlpuWJBpnmCB", + "LCZqmyHLRZpQdtmmdblIxzyhEdIf1P3iE54k/GqkH8kfYC6brWanPxg1VQc7M4OzeYFmQ2r9llP4Qc9y", + "s5aVDJaYLfP9FjhG20SPBSPFf6IJsdiU7xi99gi9WhFhb2fQlC3f0GglT34ZnPWmnNuSbPDE5zKQW7hS", + "ynF1zUhsy1QYsSfLpSmx1OJWcjjIzgV4O49ONXHk86BJDg2/rgGToDGBvB83tWWu0YIttplKsLpLLmfo", + "73xcNYi2DfsN1AzcYCVEhiCTYHw/7OhKg7R5Y2lNvN29CQYFsFU9UfjohtAO66orlvFVTfzk7VKhwRmx", + "S0bdHE3RwRZFdFz8RwFfYHttj2NQLwkZiFcGlBqpFlBcGSpaLbw6pxKNuRAAAq8lHM7cbAB2Rcs8eq0d", + "7hU6n5EFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHsDmp5la2HoMYDz/TiJ+xYoxDpk/SN14", + "LnU7B8xYFkWeqUrFVt0saH2aUCBrGZxgSkAJU6pmaCKInPlzD5Wt1TLeFRdxYz2wBXKvQJkp8LEixS8J", + "81lZ0UxQNTQNjcxXy1F8puY0PLX6J6qUgUb1Ms+r+8slEWEhsZhS8Uqr0BXvqHjKiQGBAUQUKO9p/zIs", + "vkBBaYF5Ujb/V9dk+dNp0Xj1t9prHq6JQ/o+MGbboAk2Mmk8tWCfqidtbagKpMGtQrNZ9iWgDRdC7Wok", + "VSUBr1ZRq3uyXSZePVnAjWZLkqja+97zJ8+etiwW9VnOOoPe9aVdc/N0hUuuYadO2vh9nj95/uLF7t6T", + "Fzs38rC4vJKG/WnKLfH3B22Qa6UPa/Kvf/zz/UnN6/MEYrAHNxqUySwJD6khu6Q6oPcn//rHP92obj2g", + "EKNZBulv8Ns3Rukk/k66QIGqC6+dk2yFfn9QMRLggs2gDQJQ3HRORmbdeuVgajAg7aRgnOGIqkWAkeMr", + "E+1evFLD2m7jDqoONiTymrYtKqrmXDIfl0mnG65z9J/GN1yjheeta87JfNzkh35T79V4oUuvhR/j0CLE", + "wFBE2MBdzOcKy0pAt/47grwLl2G2nG1j3liNultPhYAoFlta0QsFDNUaqMmT9iN/+2vb6fktK2ad+op/", + "WHEOm4/gjay+gRs5YPSN1qfW1viDvQBv99Vo7FeDXFlus1I6srx1b95vi+zh5VIlxQ128/68hMmbfFjH", + "BAZ6tGOwS1623a2QRAM1ebkwAQMaT0jPK15gSm3J3HgE9Zm3MPOBDM7okk8mVazbJ83Y6AD7A8lerhes", + "lNZMuohcO5tFHVjbYPwMO0/ksKNVgGFnOx12am6rYPpkiq9HtoMqtstgFVh5mf5eG6R0MxgnPLo0hQ6h", + "fn4fDVBKMJMoZ3D4a1617cFq71C3k3l7U0CDExPitMS2YExjMsNzCvVUrE9lWgnEJNdUSQgYhXb2UcwN", + "2lOlyrOdoX7NJDful5OGSwezhW1YN6jf48xFtJbvgoFvArWl2UcieNeCFWiO/ebNSdcEMEDooRlYJb7R", + "TdSMQDPIootaeYXy93D88DghIxh3Ha4/XV5HPycdPKuCSKKkxe8uyaFGBCjiOVN1HP+0nSJXTStbvpJy", + "BsF+NvwDcNls74ZAUEwiOJFy+SxWCf0WxF3LG7ArHUoc2A2RMByKFfV33lqHcH0AxtjgFWg37fhx3cZL", + "OJKK24p+xakekeuIkLgO+Bl+pW2svP0yGCv/GluMoKJ2un0b4p2XZ9e/uwQvGGvTavsx/YyzHqCTuC21", + "SCIGGtBi1VQJrQI97kFajELwqqEX2mRck+vVa/0ruVaAjx7niQG9C5OuZVX2Mlq34rfObGw60FyQtRUy", + "76BypIk3v1XtSBuq/hDlI+1bd1Iycml3zohy755ZMmrcoWqhl4pLywX8u1eqMTaGlLrIXvBoO92skeDe", + "LGwVsaC8LXM0GU7JKBNkQq9XEI95wSjGVViT8iAVGQwGX3Qjxddo7xmU/pK1sTM6nalkUQ3A2QtgKX1W", + "XVVBFGHOUNhm58vddB8uR7vZ7fRbDwnHZx400FJJEyuSjlbhZh+W3jZrnc/wAqw4jU7CZ7t7g8HuzuBW", + "wNluWDdYrsPyE1uFtNpOU0qd95119FeiVP0WiiTr5dLWplyd55SUShCc7kPiTYYjghIyAZC8IqF1vWex", + "3vXqwVuBymbRFvTvNsrum/PBV0vmFF1ZzHE3jY5zLlYxiPznaxyiDWwmWoLUC+Tc7fYGT8+3d/efPN3f", + "3r4LsOtikZqyPZ593L56luzgyV7yfPHs9+3Zs+lOuhvUwy6pqQzUhlZ/0e82RtmUl2QVy6jC0tCGnUNG", + "RL1meb3WvyQJZaQniwyp9WmKK3iB8b+vPf83s/ObGayUHc6qk/RFCKzKxalQ1sPgb9nJrPRd1GdzfLR6", + "FrfKQKoPJExv9aEAebUbDFSo2O58JjJDzlpeQ++8F1tfRCuz4tZdRSEPO5z04C43rHiIvGvADN6sV13g", + "y5dcwHY65YKqWbr6tiheK2DEIW76o1RxFe+pj46njGvtyf+5CJPzlSj9cafbST7uVc+M/b098pdFIC4I", + "0G61LxW0CCNLyJwkq1cBXikVD2Ei2bWursf8w3Zv+wXEISQf934Y9F5UIw66ZrX85dt2b1d+HbRZQ78E", + "oCsdtf3iRhHXbj1XUdAvNFTArryXLTaxpfGyPLy7OlzCbWWDy8dLe1xD8mkUQD9X0rOX28gXmmKS4EUI", + "m94z1Mqa9ugTGRqTKWWyjd12d1AYbp+kw04fHViAcNBlFS/68ZvXtOLTCU1TElMtYxrVvzmDYaelLa6u", + "S9ysNon7KiCt9cPi2ov1EAnrEq7WXZP9z8jH/Sztt53Guwq9A+xqTkUFDDF4sYvoBGFWK1DqilUbpQMS", + "IyFebd8BtZUka3mALOVAZyfpoilXqEyhb2lvy1mzXbAYP7kGe+sKzAxDEDtfBBClABCjq9jX8RHKBI/z", + "qMwfTWDQJeKHyGsQbSuE/PUhuXdp34DE7AkXaL19o8mg0c4+2bTfNdukJtjmrd4erN/qOzGKdDt5Fq/n", + "YealdhzsRsjta1IQAyaa6rLXJEFvMh9acPS3/gou67zGlhxpkSjPnINF09QyJQXcLeBiCMX1HpGE6Gtq", + "uRHEk7jMkqCy5KLrWer20+ezJhcneKSWB/ILIZnWVQD/CPpLMVsEB+bKjhZ3ycbAoX1L4/DqmXJFdrWq", + "g3u2VhJr3CrfhNtUQsFw+ZrN2+ClXHrm7wLj2xfNlhFQHMOvCGlvm0sA2C9d2Fuj/fguzHIPKaS9sa6H", + "GmyrAxUu0NFd/2UssBbrqsS7F3LPh8jiHE+hVt4aP0ojqYTqSoKTAyOFbXErKLCp/4mgiqh0gC1OQIDP", + "XUavLTaIqEJ4iuuVI5rKUhpP9PoYouZCL+fWRNCEyVtPh/XN7we9/zHmdjTq72/98Jf/u/fhP4Nm95oB", + "QRLRi8kEIq4uyaJnqjApPK1eoX+YEhBaq5jaM0NwCsY0QHu3XMkf75NBwT0Xv+J0aQoQquaVUNpeO6G/", + "/EdzoJe3jO/gwlh7dj+7QspdVJJV3N3LGykRUxdU7zLqNvtDBkl6l2QhkVeYzcp27sR+J4tPvFB8dGHI", + "vU/Y/AKNKVS6lEOm1XscRSTTapWt9UNNuXYObFgQnPjt2AJx7rxYz6wJrCDo/ckSnPGbd+c/vnn369Ho", + "zenLXw+OR7+8/G+IdbnqmR7inqa9vSdPbZF2fyW3g4VCbl7voo9ObL6CjXmY5KDZA2CZRGmucoiOIddR", + "kks6d55Sldy+ssVy1vLtK0V8JhSyUkkoPMNCdid0QiDAAe5VG11EpSNGKqG6vbXyUIaWRRdDOMMOXCmK", + "O0kiWFdEb0V4tcuNrS7601m7u8SAxAYOO6T+6sMWsKK+phKAO1wQjPcy2oAUGleC12UQb94MtPagaDAY", + "gvmFKy0NXnyJaqjvVpY/nfOkp/W8hpIRQbO6WYtgCgE0ZVIzOk3el+k4oMxYG/eUTnHA4RJyrHyRqqVu", + "QGtTx5b2v7F8Wzih46heT8McS7NUtfoPNWuJVL3mfI9Ui/cNCMQAsWySeKkXZFjN2E2Z2rLVhUPAITEH", + "VPdVadvlKXMwkT34aH028koF05uZN5LmvTlxelRN01uxQKd6aa5mRBBvI+CDsk7BDZfMJii1gKMx1Rkz", + "IsrgXZfdpMVz8LtLtFGYwNwSFGnXy36B1XUoTvB10QP4lLBccsTCPMo6WNuvfoSaAW9d7U86cU3AMGpa", + "bhghv0pFq9bEUdXyZvhUtTxv837w4FletYL7NZ2tGnGWfVRIM0SPf8NU/cQF6MXN4C93DrQPl39MBIDh", + "1WH0W2HQ05TEI56r1edfv6alR3PlF/Vhy/rCzgaAgYijSl5zEy9w8CTlGJZXWi8HiXJB1eJMr5eNaod8", + "UFfUFxYSOoKfy46hkOqnT2A9nwQyZ14RRgSNoEytPo8pZqAxofcnXrVCU7hyCbgWRKA3h8fW7uKwj0GP", + "pgpIzwWgHpwed7qdORHG9tAZ9Hf7AzjMGWE4o539zm5/uz/ogFY1gylujXOaxDaR3GrUhQZ/HFtJ6Ef3", + "kv5S4JQo+OK3ACQCBGDa10EFwVNPicwwFVaLzBKAajAEQ/XXUHfBXaj75lbummVvbTyGfGtIAyLZG7u5", + "H0BQhrMD09wZDCzCu7LXLyQxmcyJrb/bMNqy31ZSnV2iQBmCJTXPyZbF0n/qdvYG2zca06qhwNkNdfyO", + "YZvNTEA7f3LDhbhVp8fM5CfabHMbF+afOCAk/6z99kHvmczTFIuFWzB/tTIumwRjIhF27xo9TkkUaVYB", + "xZL66A0j5jnCCmETwi1yBjWm3YeaQqunwLTtNrlAa/qRx4svtoSVPpyN4lOVnenj8mmJnr8c7RRkvLyR", + "9pGDGjdUew8E9CMuCqQ/2EnZG7y4+04POZskNFKoVxCwDcymEmKfEgBOdyBMXKDfc64wKvIaHtGRtjLr", + "uCC3bnkVbf1B40/meCck5A84JSLFzGSJmHfWHPql42x8M+VxXnmrOcI/PurYm8qhEZmLCgS56hH1r626", + "MLh8He0FoChsn2Z68QMS/t49nHA72aJG7kMeOahMinJJHtNxsr7GcSmEBGW5V0R9LTQ/uM8ry1ZT+BOe", + "osdCwK9IIeGVu7V0KWxlImdGAQ5KgG/LzE373XdV4e+8fOKFC4FfQzcNdT2UcTDjeNFHbk2N0q8WgDUl", + "CMwzXr5WTvXwvpYTtnMfJwxmXHiKvl1T366pVafcUIubAhxM75S3sEHcyALx57M/3Nj68M320N720Mry", + "wMiVtS78nY/7yIbmRjwmSM54nsRoTJABfnJBOAqL/vQjwiKa0TkBdD+oVpcnimZYQIhNimKssPGhNxom", + "Vpoliua2dHM9F5BZLnAd0EOSEYTAjJqAOMtQTMoYiZH+xEbNlLiKS3XVzdkPGtiLBsurEV3NuCQFsCFT", + "3m0Oed7SaMfQbH/Izi3irV5AiCp3vEaSBHB7V9h/OEN4yOwH3zsW4iLiJE5LzoUFgCdSA9FptmU5x0+P", + "dCQjHgIdOicMM9WTGYnohEZ2WpdkYQNbgw22KkClB+zG+f6kyFxBO5th4DoIXQqjFB8Vz5ClpKr/hkE0", + "eJTkcenkclhKWIxxkgQrlEwTPsbJyKzPJQn4BF/BG3ZRSodL6U1iPCamln62UDPOzN/5OGcqN3+PBb+S", + "RAw7m/0hg4wUu9Yk7pYCIrqCinZpxvU5Ezw1fW6ZIW79cUkWn/pDdhCnlDmKgE9wIjki1/AdxI0BeIjh", + "Xg30YE5T2A9+mEvFUx8C1tGdGSbPVZYrm1ojieqG4E+HTHH0hwO5/LT1R9njJ3AWExxrOvFeMVMC2bpp", + "1HKE9exH8GrA3U5gAYYdfZGaMI+pwEwZ/NICpRNN/S3dKMpEQOnY+gpHmKGMZ6bEBhDVDGuSq7QBoBU4", + "SZCCo+S+1YI77GTDfCwGYTpuBCA0iHG1Y0QZOvnRO0yDvefh8yRJJEgoouS/zt78iuBW1ntgXivDtUxu", + "C9MCA4pzcJ06nvYSRzNkHFVQVXHYofGwU7hz400Yay5tuEyvBz7FH/TQfjDddGn8Q7+vmzLuyn302x+m", + "lX19lrLUAKIOO5+6yHswpWqWj4tnH8IL2oTjdlZhBGjDXHObwEkwBcgd78Y3VyRmMeL2FkgWCKOSA/mB", + "K2PKsFisyqgMLL1dQT4xkYzeYvwxhMjFYWd/6GIXh53usEPYHH6zAY7DzqfwClivZXMJP7jPCudmQURP", + "B4PN9ZDgdn0DPssWjoEvrAM2akVF/VG9gxaP9s/lH/i31j8L1w9muvMSo8ko/s74/ggdEJ7E7muiARdE", + "TezGLCKJE7vXG3ru33mgNysiSXLfBPpQ5Fm4x4qSBY+KHGGzymO00nz/wBQ3uK9LpWK2fxj6fXT284D1", + "3NrOydyFOocLtgAYj1WlkXkZYYnOYEy9M618v4Rf+/a/TvcDcMmLhE8v9o3qjhI+RQllNh/AC1TW4oFd", + "S/jI4PEU31l4Hlctb8NIEv/6xz9hUJRN//WPf1qQ+3/9459w3LcMzhwU276YESzUmGB1sY9+ISTr4YTO", + "iZsMlMMlcyIWaHdgbf7wCHk1/62UJodsyN4SlQvm5U2YwnXSNmhdBXo+lOVEWjwjSBOa2Ko6JrYxYLdx", + "Z9ks5b2e6G4AFxJm4E1A34qOBgBUj5qK41YT7YRNpmbOFaNpPUxzKVhvPX9R5FoZ6u2ZAd6QwcASh84d", + "PLCTRhtnZy83+wi0LUMVUDkJdIeyGatG9L/xpPU8yXCUKkOBVTa8KcIZHtOEOpNjQ9kXcwRTHM0oI2V8", + "cQG67prYdyPVPObg9BjZQMguvDpkb862wMSqSKRyQbqWEwgLtVrWheM2zwV6AP5FFUSH9ey7QzYhGPKE", + "jo8ME/DQyIvEyKJhBogmEONKVaUEXXfIDKSuhXDWBy/lMUngI+h/ihW5wosuKor+ujIxCVZaIZZd/fKQ", + "mVRDuwY9wGxB3jD7wM/MkHouktfmbAkySbRqDBH4pv459L0x4QLZCOdumVfqujPZpmZYetFSHL050/Ob", + "gibIjT0QWnpz5nZjs4skR1FCgRoizIZsCoFADsWYs8quFgllMyziXsT1JeCjWl0yfpWQeNrEYw99IrtD", + "SabST+A4/Vwn18cmXMyWJ6APsUHqW+25O7LvtHPd2Rb/TL47WxHzBs47Y8Elht+Y1f3myGvhyAuvm3Pq", + "hTxrRw6K8u4ifk0XDxTw62hvec3NE2/JHsKihzYcxg94RbhAp4fHCMexIFJu/nvb+/RMDZWW8p++HzUr", + "fojQEzsWLiz6obW3VAnksbCDt3bUCLt51QsN+/fbVqUKUeNNVxQkKq+8u789ap3e5Bophd6S1r7dJGuD", + "bamMONRbLKmlB6JRQgrxpTinPhWtsyqbMN7iylkpLln2fHzkDuT92Zdt1zmr3w33wBSPagzxARlhNdXa", + "Lx/+mKj5XbGLDnZ7hfn56yLNwf1JQfdtig6R+WNSF+PasmkuaIBOGi/QV0QZeJO71NNtD4GJnxHhTrUZ", + "6MLMupiW+RQZnBaYEFhiVuu+x+aVdqqvae/PpPnC8txEYrFL/k1EaaHslmu1SsE9trWw706/rSC53XPY", + "iiWwwCKDFXXs3E5gWd3AcsGizW+RK1+cok1cY6nECjdvEheWbIOmVOhZ9yXXHTC/8LqW6axeSxmaJHQ6", + "s06AmE4gVk/5hcxhlDv3MMqiYLjAitgQxceY93uqF9l6gedEeEiN/pW69QcEra5XlRzzWnm7vnv7ukdY", + "xOPCedIsk9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOCujO/7XzkwXv/F87P+Eko4z8r92DBCsi", + "1eadEcvgvm66+1ZgHjHxaf2FVhdtiTVtKTz181CrlOnga29CmRXYWWfsLhJpvhj1fXkhr47V20rCuze6", + "15rUfSOWHLuyBGZLuXDoVTVI4n9voe+sQtD3bq+rdE8ljAAEu0cYpcyFQ13FFraaOuK2vIlNAdd7jTGi", + "eKulPcK9/6cySZhJ38goUazrN7tEG7uEv1wrTRN2K+7UOGH6eCDve0FsodWGR9/gtu7BoWMp0oPbqni4", + "S8CtGZcKHj0+7AV7ldCC4vxro6VnsjyQK68PR7rHR11YSCj7D2WHbGrzPfkp3TjuXfG2/d6/0HOQjuk0", + "57n0s6ZTrKIZkRZRICFVBvzYTALl9dxoFPiKqXRwn1fHvev83+j+jqwR9Q01zNsEG6yT+d1bbWV++76W", + "+Q3askVdsLWRuq5u3mZDEojDW25LxhVY6uXklNC4QroIeqcVlVJdQKBB7A/Z/9b6x2+K4PTDDy69Ox8M", + "dp7C74TNP/zgMrzZiSMVwpSgtszpwa9HEOExhSB+qIRagknUx4HSXCpDeq7kyb+dglQGubTXkBwVftOQ", + "WmlI3nKt1pDsXtytilQtm3TvOpKjt9CC23oHf04t6U/uuq1ocDKfTGhECYPiU2D8k0uxykaT++a1vSVY", + "ArOxEl6gY0USaa1GFlxrjYReFv7/kpGE3cYaFBxhpUiaKTQVOCKTPDFVW5Cc5SrmV8yVpIAJuupmtJxP", + "6Hp3TY1cI+EE2XCJ9raablGW8b5VXdvxI81Q5ZmtMG6Vy1K0adYuH5Z471anbHHV3r9W+ZhJzKhvy0uX", + "aQ0hUGLNuHnS3KTzFl+W6Ix9dH7+2qXuavVEuIJ9irsqfa6S85D5Vfr66GVZ/tC84FrQ6gOJbao/JDTb", + "uncxwXFCGYFcByJDWbbV2poPeiy+vAQcLhx63y7uNsfS1sR+OAn4wVjBvciahU9fWYdscTS9mqLFaXHy", + "JpyaR8WvLAMKMJ6QrLeFc8V7Fgxga8YNQmQYJPc0wRFg5OrXDHyjxV8xeK1+UwCqIniSEGFgObNcOXFr", + "yIrBUaZ4UXjaSmYXuvlRzhRNLrom1BCwlSTCbGGx6Yas0pmV+QAjAfA/YISCZGbEtSq6etCU5xLeAjgD", + "v0uEkyu8kENmURXM51B6XZDIINgmSR/9zAHQxlST9hivKeX6nRyyCxonZGTxaC4QlUjOuFCEkRilfE5k", + "tV+CRUKJgEkcYr1yEqV4AcCQBiPXrA/PiAFfrKDecP1vzGIKRUF1z8WU94cMo53BAKUEM2kxLCSewIVj", + "20AwiMqAvkcY7Q1e2K9q+wbg5W75N/RpEoLMeYTHyQIRTcWm2PYmbGBqi/Saqu96+yZUSLNfhX3TVl+s", + "bCyVruZs3EU5K1E6wNafswJUQ2+XygWDeVovIKGiuAYtMNGYRFivJ+PVfgASlkdRLkIXpN5qr1r0v6Pg", + "6E3vDJYqjIGRgMkgIjHsOeNqBmeaw1Ha/L6Bqkqi+nNcNMFDwgXCyKPr0qJBohxY4wZAqF6UpU+ZK2V+", + "sfm9Ozv6+FpG4I6/ATF9LPcTEBGfTCoHcP3VZA7wqtyzZRL+s57TQ1fz2mdxMcVTxqWikWOGDqHaV5q/", + "KYStFMLVKxuk5gkXl80Bxz9xcdlWA3OBkY9LEfNn+BU6IvTwAAT/4f0RYA03yoommntX0ur0VZxSELqo", + "kkVIMEo4m+pTVFrl791t4Gt1GwbQUl+mwji7C/gxrYSM7I+mbLYX1g0uhsi2+tC8SPd+D86oX7lCNM0S", + "khIoq90zxKY3u4SqGy8slF4B2HYzXqlPlY+rYHRBaeIPuk4cArpyG7YB0vvydgWZasKn6wFRi84d+mcA", + "EXXI3klTquDCuJ4uUMGDtUBryo+gqxmNZoCOCnqrbt+Ap+IsuyiA4Tf30Ss4yD4+PnS+YYqOaFqTPCEG", + "9HSephf7y4Wj35+cwEcGGNWUiL7YR65YdHF/SP2Wj3aqZ5FgqdCvFsN1o1DGYUcvFNb6ZjG/TYuDWgL3", + "D1kIE5WRK9sgnaALDx71ogG7z/Hb13wqvxpXUVluxcxFcWRVR6BNwuJOU5AHTcKOn+3BIFQFoCVKqxnG", + "HYO0Lg3mNZ8WpV4qpIyzrC352mECFc/TdAUNow0P8lGqmOfqL1LFRAj42FJ3E3GjDRzZMn/4UhOqBfh0", + "B3sTyC8YymRqLwSXSjPVTrdDWJ529n+z/5qnaafbsePR312pLPVqN9xAyF+DeltvcDn0Ru+QB237TTy/", + "CWhtlel7qLW1G8Sq1c2S+Vvzwp/ea+hsdw9IhiAn1Iy5X5Mo6o23avhhvEDfhZG9uI+RAUQvihIuScXR", + "83gA/qzBqyY7NhuM3Br39PDi3FVEaxPJcmY/PXNffgU6+LqYETdm5KZ778EjyyN4zGAFcmk2Ey7qqHDr", + "okq+ekL6cluyNNU2FPKNNm9ubWxFmFpfWGYR9oPYVMjEueIpVjSC6mzRjHPpkX0B4W7qKFojckGZYGIx", + "2q7NJLjQpHphzdEXVp3Yt6YzhP1Hto8+fG7zD8JfuEflFz951oGC43edCgAVTCTCaCwomaAM55JoqS5P", + "CYoWkeaKphwfwdEMRThTuSBQaZSglDKa5qmPza93bI4BR+hiO73oonGuUILFFLQz89AF3UQ8TQmLCdjp", + "hmxG8Jxq1VKgBCvCokVPEqhQPifoiovLhOMYTA1ZjMHjAxVOBdEUCIUOUqJwjBUGQedCn/iRSWa6KIqW", + "G/WekeuSGuIhEzn73lRd0c1euIFeIAJlBaicFcVtIxwTFgXh9s++bjb25W3SZ0TVJ/pAEUK34qUPGTLk", + "217dcL6OaKJHCw3Rgs2vEHplswpbzQJxZPTveaTNXN0cH8jRVCzxqlP8dXiYCqL7arxMD+9G4gLFuenO", + "O5VA5n9W31DBUPygK8gwNdt4WwdRUcWzWOYb8bytP9yfx7ew5X0lnLDbqNg31YsrJ/01sFy7qrfiuQ9k", + "xLS2JN8m93As2EV2PZj4xIXH5R6LsbWC0FbwbZ87KYFB++LsG9uus20b+HBbtu1ss0uufY+RU9aDWNEw", + "B7dm3EZWbU0H/6ZZKbXZeSzzwVlk6bm4d7hFxxozvEg4jv8MwcIr/EcRF8LAYACwxmOCiPashn6aANjm", + "ykKUXZe1+f7kZLOJSwi1kkcI9Yg5hJeaoz9L42UD7ps5EYLGDgbz8OTIhu1SiUTO+uhNShVSHF0SkpWZ", + "LZBd2Nfzc4AgtWHXkT+6HcKUWGScMrV2FOWrdzOY8gducVG+QlHS1hz45g5v7Q4Hy/7jY2fAZSB3w0xg", + "tWaqsFpbC5myCRepkcvwmOe6dc2D9DLp/TSIBROaELmQiqQmOnGSJ3DcoH6NrVFuvzO73IXYXH1yTNpc", + "RkRKpaScySGzOSMZEbpv/blu3wu0CjoEFC7466lhkl9HEJ8ejIlbw6pp1QC6CWofd/Y7WzjLtmKscEOg", + "mB3eZwzpJ4jKQ3KRjnlCI5RQdinRRkIvjXqC5hIl+o/NlWF9I/juS1dgv/3J0it9zCY8WN/S0GxBzH+q", + "7C7L1pxj8tGxtVfEPyyO/8BGh9na+hrvguCkBzXTHYAPyhVN6EfD6nQjVCoamdQjXKzd+5OCqfaH7IQo", + "od/BkOKWJAbZALTLrUzwaGuYDwa7UUYBBW6XwOCA4TU/TqHHw9N3Jh2VpFwsukOm/wENnx+cGu/uBFtr", + "gjdQW9wdHW+9WRPofAbL9G8cIWgmuBLFILjh31yCN8caaTxDsuGI8myVqsSzP30Iq5XgvtkVHqddAcCe", + "itlsFABfDpUrbEOY8yRP9T/MH8fr8M0Ujmbv4dWvRto1w1nbjZvgoziUdk4xMfV3H8TpYRbsscas6oVz", + "UwAhphINGLwFDtSfkbq/vPneX8ev0N1pV9TVtv5qztZ933x2DA5pw1+Px3LMDaW5mSi+2vp0hWmz9enH", + "hEeX0kKy+GZDrbcBzrr+scTFti5CEBMgQxRZKCMDmEVkd8hqBkiD/CMRRoqIlDKcbMGcTSOA8O2sWHjO", + "KSRqR5Cn0pM0BuykBGC8AQZPzwYMVa4Bz6MrbfU//x3fGak4GpOIp8Shnm+GVLe/Yap+4qIKYf618MVz", + "b/0BGhBTsLevQW1v7vGzUNxP8DWESse5dSi7EW284uWPxhTURbA3w87uQA47XTTs7KTDjt6BQwwmVKzQ", + "E5RSlisi++jI2LcgFffpAEkScRZLB77uLHi7A9mUmGvIsiHL8yl8d59ij6UqWMq3tpMQe9DvIf09JO2g", + "Df/A2TMZd+HQxYjnypj77bmyb8VEgXlk8959td4Z+abbt+Hkf7PHt8KjYJc1u/S23nD2LJcz0mxye20K", + "GuVqDKDergCynKG/87HsIkaujDVcSNVf4nv661PTwX0UHNBd3aTYgJ37t0oDLSoNlGsVBm00AZb6SnbU", + "YZAbyXXGhQI0R5tzb2gINAlAkIAyhW8Oj4cs0qzIQAwKknLgThYX3dzCB387Qy8P33bRERTjRT/n480+", + "esOSha1hb300Q2YkMcO8IszQ2FAtiUPXsxk7UM9dBovrDh6our05GQHPitsrFyTe7cwIjkEi+aPzmpvO", + "AujDb1/rAwQAwObLYts7K4WPzluixKJ3MFFELDd7YvOkWIGdYS9pB0VnBTcDgKk7lA6BrezTyAYGImN3", + "pxNAzPj0rfjD3Rdxvh8vmYkTMWX3xrl6tPVb4SAWzDHEAv3ruiif0JQlbHnZSgUDumyK/P6KTO4reVcF", + "Y/7f9XTBTB+toymr7JMm4qLsylpPr0sOnhlYZOuoinCGI6oWXYSTxN5R9iYoIlJ6hfg7FgRfxvyK9Yfs", + "bVHwxSb0osPTd13nqEUxlZemBeuL7aM3cyJkPi4Gh+CgGa8xrDmJh0xxFOEkyhMtbpDJhESQiwt1XGSD", + "L7cYSucOz07ZSbDojBfVnj+6WndhmoDdK8miTnFbZqu3BIkSTNNmEHIrqEHAIYQajHWjnCHKJokNqYoE", + "lxLZpnokoVM6TmyAkOyj8xlBEqdkyLIEM0YEyqWJitdD72WCSJmbBG/dAID1GorqohJgMBNc2dCEhHMh", + "TTSBpvD3J0gqkq0gs7em5ROY8x3JtqZx29MDGalrY2g2hdhXkN4QQylmwTUd5YkLYLzXUHQzoIeWEh/L", + "wT8XdDolQp8KbJisCcczx9otpzn0lYzlxrqXZ8Vb7epeFq16WYlext5KgLhRibkdd24W9Rfo/JI2Ygja", + "RzfLIv5Ff9Sy72q2angQ9tFnzjJUwvPfsVrmmZck2NaAVVL4YzMneSOvHNVKou16WK3WmbV3menaGj/r", + "wWCzHjNaFq6kzzYpvF8fIQzuF+XhvoutPW7aqqBdVXTThpT/9aj6XwUF3g2c/gOjnNwCTv+ryrsHvPOH", + "wz8JHtSHyqOv+J5d0d0/PSL+XaXPG1h8gGNrSp83XM8Gr65UlN7bd9qpSbbFP5MEb+MdbyC/u2X/pvW3", + "UBm8xVrngtYET9JMLVxAm/VVlkFnkn4k/QZHcBG3eneu4FuEdH458nB02hjQ+eeskf8gMaO2hCCV6Pgo", + "UHz+kWEM+meucrFs6Vunh0U0o3PSbHSvnmC7RJkgvYxn4FyJzYLZ9XB3mcKiP/2IbPMWc9X+C2pQAlQ/", + "iVFMBYlUsjD1QDVHMH18J5HgWhOA51wsmqNEzBH5SfD0wM5mzX1oz5Q1hpVxhumiF2OFe3PHbVaY0D4j", + "utPFU2qGhyhDr35EG+RaCVPpAk205oPopFhSch0REkugyU1/wNuDBssm/UhG03GbUa6oWfLG1oRBUS4V", + "T93eHx+hDaiBNiVM74UW9ScgyWaCz2lM4soYO3OemFXdbljQm9pdtVBRFLBzyoUZ3IPIMG0upOlHmlXZ", + "QhESM6YMw+DWVgWpnimTxK/7w5S5ABy7R24U364wq/ltOGVHUyLU47SLqDg3EM+b3665x3zN+clQ7k6r", + "3HYuPGe18bpdflTLtKW7KPxQ5M7dr9n6/deT0kPlo8zmsabzeaGQNpnNvy4SHNzf/XDf5vL3jzgF9BVx", + "yrdnKocGdIshgnkNMd0xmZOEZynURYd3O91OLpLOfmemVLa/tQWx3zMu1f7ei2e7nU8fPv3/AQAA//8A", + "qZIc0fkBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 2a56ff7f3..38aaf716c 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -17,6 +17,11 @@ type GPUResourceStatus struct { QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + + // PlacementDisabledReason is set when AllocatableSlots is 0 because the + // VF health state could not be read or written, not because the host is + // full. + PlacementDisabledReason string `json:"placement_disabled_reason,omitempty"` } // GetGPUStatus returns the current GPU resource status and any error that diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go index 69afc18af..5900cd93b 100644 --- a/lib/resources/gpu_test.go +++ b/lib/resources/gpu_test.go @@ -14,6 +14,10 @@ import ( "github.com/stretchr/testify/require" ) +const testVFQuarantineThreshold = 2 + +// initVFHealthForTest points the VF health store at a state file for one test +// and detaches it from disk again afterwards. func initVFHealthForTest(t *testing.T, state []byte) { t.Helper() path := paths.New(t.TempDir()).VFHealthState() @@ -21,11 +25,11 @@ func initVFHealthForTest(t *testing.T, state []byte) { require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) require.NoError(t, os.WriteFile(path, state, 0o644)) } - err := devices.InitVFHealth(path) + err := devices.InitVFHealth(path, testVFQuarantineThreshold) if state == nil { require.NoError(t, err) } - t.Cleanup(func() { require.NoError(t, devices.InitVFHealth(paths.New(t.TempDir()).VFHealthState())) }) + t.Cleanup(func() { require.NoError(t, devices.InitVFHealth("", testVFQuarantineThreshold)) }) } func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { @@ -77,7 +81,15 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { }) err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + statusMgr, _, _ := monitoringTestManager(t) + full, err := statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + require.NotNil(t, full.GPU) + assert.Equal(t, "VF health state unavailable: read failed", full.GPU.PlacementDisabledReason) setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + full, err = statusMgr.GetFullStatus(ctx) + require.NoError(t, err) + assert.Empty(t, full.GPU.PlacementDisabledReason) status.AllocatableSlots = 1 require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index d69b1e2a4..7c4d5dc3d 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -191,7 +191,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_slots", - metric.WithDescription("Total and used GPU slots"), + metric.WithDescription("Total, used, allocatable, and quarantined GPU slots"), ) if err != nil { return err @@ -242,6 +242,8 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { if snapshot.status.GPU != nil { o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.UsedSlots), metric.WithAttributes(attribute.String("kind", "used"))) o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.TotalSlots), metric.WithAttributes(attribute.String("kind", "total"))) + o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.AllocatableSlots), metric.WithAttributes(attribute.String("kind", "allocatable"))) + o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.QuarantinedSlots), metric.WithAttributes(attribute.String("kind", "quarantined"))) for _, profile := range snapshot.status.GPU.Profiles { o.ObserveInt64(gpuProfileSlots, int64(profile.Available), metric.WithAttributes( diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index 39166856e..77df034d2 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -200,9 +200,11 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { originalProvider := currentGPUStatusProvider() setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ - Mode: "vgpu", - TotalSlots: 8, - UsedSlots: 3, + Mode: "vgpu", + TotalSlots: 8, + UsedSlots: 3, + AllocatableSlots: 4, + QuarantinedSlots: 1, Profiles: []devices.GPUProfile{ {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, @@ -225,6 +227,8 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { rm := collectMonitoringMetrics(t, reader) require.Equal(t, int64(3), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "used"})) require.Equal(t, int64(8), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "total"})) + require.Equal(t, int64(4), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "allocatable"})) + require.Equal(t, int64(1), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "quarantined"})) require.Equal(t, int64(5), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-1Q", "kind": "available"})) require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) } diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 9c28fda0f..4784acf76 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -427,7 +427,12 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus, _ := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) + if gpuStatus != nil && gpuStatusErr != nil { + withReason := *gpuStatus + withReason.PlacementDisabledReason = gpuStatusErr.Error() + gpuStatus = &withReason + } return &FullResourceStatus{ CPU: *cpuStatus, diff --git a/openapi.yaml b/openapi.yaml index ffa6d7e51..3621c3e8c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1832,6 +1832,10 @@ components: type: integer description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. example: 2 + placement_disabled_reason: + type: string + description: Present when allocatable_slots is 0 because the VF health state could not be read or written rather than because the host is full. vGPU placement is refused until the state file is repaired or the next write succeeds. + example: "VF health state unavailable: unmarshal VF health state: invalid character 'x'" profiles: type: array description: Available vGPU profiles (only in vGPU mode) From 891915734f9a54b0810130c549a953ef8294e3e5 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:40:22 +0000 Subject: [PATCH 14/21] Read GPU status for admission before taking the resource lock The GPU status provider walks sysfs and may retry a failed VF health write, which fsyncs. Fetch it before the manager lock so a slow disk cannot stall CPU or memory admission. --- lib/resources/resource.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 4784acf76..96c77df2a 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -625,7 +625,23 @@ func (m *Manager) admissionStatusLocked(rt ResourceType, visibleAllocated int64, return status, nil } -func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string, req pendingAllocation) error { +// gpuAdmission is the GPU status an admission check consumes. It is read +// before the manager lock is taken: the provider walks sysfs and may retry a +// failed VF health write, and neither should stall CPU or memory admission. +type gpuAdmission struct { + status *GPUResourceStatus + err error +} + +func (m *Manager) gpuAdmissionFor(ctx context.Context, req pendingAllocation) gpuAdmission { + if req.GPUSlots == 0 { + return gpuAdmission{} + } + status, err := currentGPUStatusProvider()(ctx) + return gpuAdmission{status: status, err: err} +} + +func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string, req pendingAllocation, gpu gpuAdmission) error { usage, err := m.collectAdmissionUsageLocked(ctx) if err != nil { return err @@ -696,7 +712,7 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := gpu.status, gpu.err if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } @@ -721,20 +737,22 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Returns nil if allocation is allowed, or a detailed error describing // which resource is insufficient and the current capacity/usage. func (m *Manager) ValidateAllocation(ctx context.Context, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, diskBytes int64, needsGPU bool) error { + req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) + gpu := m.gpuAdmissionFor(ctx, req) + m.mu.RLock() defer m.mu.RUnlock() - - req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) - return m.validateAllocationLocked(ctx, "", req) + return m.validateAllocationLocked(ctx, "", req, gpu) } // ReserveAllocation tentatively reserves resources for an in-flight operation. func (m *Manager) ReserveAllocation(ctx context.Context, instanceID string, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, diskBytes int64, needsGPU bool) error { + req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) + gpu := m.gpuAdmissionFor(ctx, req) + m.mu.Lock() defer m.mu.Unlock() - - req := newPendingAllocation(vcpus, memoryBytes, networkDownloadBps, networkUploadBps, diskIOBps, diskBytes, needsGPU) - if err := m.validateAllocationLocked(ctx, instanceID, req); err != nil { + if err := m.validateAllocationLocked(ctx, instanceID, req, gpu); err != nil { return err } if existing, ok := m.pending[instanceID]; ok { From 0ee1ad876da092f035fe54de7ef7dfd9186f0e24 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:41:14 +0000 Subject: [PATCH 15/21] Set placement_disabled_reason where the GPU status is built getVGPUStatus already knows why allocatable capacity could not be determined, so record the reason on the status there instead of copying the struct in GetFullStatus. The error still reaches admission. Drop the warning on every status read; the reason is visible in /resources and the admission error, and a broken store would otherwise log on every metrics tick. --- lib/resources/gpu.go | 7 ++++--- lib/resources/gpu_test.go | 5 ++++- lib/resources/resource.go | 10 +++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 38aaf716c..ddece4f22 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -25,8 +25,9 @@ type GPUResourceStatus struct { } // GetGPUStatus returns the current GPU resource status and any error that -// prevents determining allocatable vGPU capacity. It returns nil if no GPU is -// available or the mode is "none". +// prevents determining allocatable vGPU capacity. The status is still +// returned alongside such an error, with PlacementDisabledReason set. It +// returns nil if no GPU is available or the mode is "none". func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { @@ -62,7 +63,7 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d // listing, so a status read touches the store once. availability, err := devices.GetVGPUAvailability(framework, vfs) if err != nil { - logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none and no profiles", "framework", framework, "error", err) + status.PlacementDisabledReason = err.Error() return status, err } status.AllocatableSlots = availability.AllocatableSlots diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go index 5900cd93b..e15def07f 100644 --- a/lib/resources/gpu_test.go +++ b/lib/resources/gpu_test.go @@ -39,6 +39,7 @@ func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { assert.Zero(t, status.AllocatableSlots) assert.Zero(t, status.QuarantinedSlots) require.ErrorContains(t, err, "VF health state unavailable") + assert.Equal(t, err.Error(), status.PlacementDisabledReason) } func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { @@ -76,8 +77,10 @@ func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) require.ErrorContains(t, err, "no allocatable vgpu slots") + disabled := *status + disabled.PlacementDisabledReason = "VF health state unavailable: read failed" setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { - return status, errors.New("VF health state unavailable: read failed") + return &disabled, errors.New(disabled.PlacementDisabledReason) }) err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 96c77df2a..d674c5d01 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -426,13 +426,9 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } } - // Get GPU status - gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) - if gpuStatus != nil && gpuStatusErr != nil { - withReason := *gpuStatus - withReason.PlacementDisabledReason = gpuStatusErr.Error() - gpuStatus = &withReason - } + // A GPU status error only means vGPU placement is disabled. The status + // carries the reason, so it is reported rather than failing the read. + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, From 5f6fe50cf5603d5e13f1150824060f7fe42b4e42 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:42:02 +0000 Subject: [PATCH 16/21] Export a gauge for disabled vGPU placement kind=allocatable reads 0 both when the host is full and when the VF health store is unavailable. hypeman_resources_gpu_placement_disabled separates the two so the runbook condition is alertable from metrics alone. --- lib/devices/GPU.md | 4 +++- lib/resources/monitoring.go | 15 ++++++++++++++- lib/resources/monitoring_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 9be00b241..d5539b35f 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -315,7 +315,9 @@ the store is unavailable, `allocatable_slots` is 0 and `placement_disabled_reason` carries the load or write error, so a broken state file is distinguishable from a full host. The `hypeman_resources_gpu_slots` gauge exports the same counts under -`kind=allocatable` and `kind=quarantined`. +`kind=allocatable` and `kind=quarantined`, and +`hypeman_resources_gpu_placement_disabled` is 1 while the store is +unavailable, so the condition is alertable without scraping `/resources`. Quarantine only removes capacity — it never touches a running instance. diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 7c4d5dc3d..59851658d 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -205,6 +205,14 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { return err } + gpuPlacementDisabled, err := meter.Int64ObservableGauge( + "hypeman_resources_gpu_placement_disabled", + metric.WithDescription("1 while vGPU placement is refused because the VF health state is unavailable, otherwise 0"), + ) + if err != nil { + return err + } + if _, err := meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error { snapshot, ok := mgr.currentMonitoringSnapshot() if !ok { @@ -244,6 +252,11 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.TotalSlots), metric.WithAttributes(attribute.String("kind", "total"))) o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.AllocatableSlots), metric.WithAttributes(attribute.String("kind", "allocatable"))) o.ObserveInt64(gpuSlots, int64(snapshot.status.GPU.QuarantinedSlots), metric.WithAttributes(attribute.String("kind", "quarantined"))) + var placementDisabled int64 + if snapshot.status.GPU.PlacementDisabledReason != "" { + placementDisabled = 1 + } + o.ObserveInt64(gpuPlacementDisabled, placementDisabled) for _, profile := range snapshot.status.GPU.Profiles { o.ObserveInt64(gpuProfileSlots, int64(profile.Available), metric.WithAttributes( @@ -255,7 +268,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { } return nil - }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots); err != nil { + }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots, gpuPlacementDisabled); err != nil { return err } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index 77df034d2..357e1c79c 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -3,6 +3,7 @@ package resources import ( "bytes" "context" + "errors" "os" "path/filepath" "sync" @@ -231,6 +232,37 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { require.Equal(t, int64(1), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "quarantined"})) require.Equal(t, int64(5), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-1Q", "kind": "available"})) require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) +} + +func TestStartMonitoringPublishesGPUPlacementDisabled(t *testing.T) { + mgr, _, _ := monitoringTestManager(t) + + originalProvider := currentGPUStatusProvider() + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return &GPUResourceStatus{ + Mode: "vgpu", + TotalSlots: 8, + UsedSlots: 3, + PlacementDisabledReason: "VF health state unavailable: read failed", + }, errors.New("VF health state unavailable: read failed") + }) + defer func() { + setGPUStatusProvider(originalProvider) + }() + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, mgr.StartMonitoring(ctx, provider.Meter("test"), time.Hour)) + waitForMonitoringSnapshot(t, mgr) + + rm := collectMonitoringMetrics(t, reader) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_resources_gpu_slots", map[string]string{"kind": "allocatable"})) + require.Equal(t, int64(1), int64GaugeValue(t, rm, "hypeman_resources_gpu_placement_disabled", nil)) } func TestStartMonitoringPublishesDiskUtilizationFromCachedSnapshot(t *testing.T) { From c60588d0f96e950ae8c9d7a75f7716db483347cc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:42:33 +0000 Subject: [PATCH 17/21] Pass the quarantine set to ListGPUProfilesWithVFs directly Callers no longer hand back a VGPUAvailability just so the function can read an unexported field from it. The set is exported on the snapshot and the profile listing takes it explicitly. --- lib/devices/mdev_darwin.go | 2 +- lib/devices/vendor_vfio_linux_test.go | 2 +- lib/devices/vf_health.go | 8 ++++---- lib/devices/vgpu_linux.go | 8 ++++---- lib/resources/gpu.go | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 956d7cba2..f0f009d34 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -23,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, availability VGPUAvailability) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { return []GPUProfile{}, nil } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index a92084491..3c7438532 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -218,7 +218,7 @@ func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) require.NoError(t, err) availability, err := GetVGPUAvailability(VGPUFrameworkVendorVFIO, vfs) require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs, availability.quarantined) + profiles, err := sysfs.listProfiles(vfs, availability.Quarantined) require.NoError(t, err) assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) } diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 28eda1026..a75e0b701 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -267,12 +267,12 @@ func QuarantinedVFAddresses() (map[string]struct{}, error) { } // VGPUAvailability is one VF health snapshot applied to discovered VFs. -// Pass it to ListGPUProfilesWithVFs so profile availability is computed from -// the same snapshot without reading the store again. +// Pass Quarantined to ListGPUProfilesWithVFs so profile availability is +// computed from the same snapshot without reading the store again. type VGPUAvailability struct { AllocatableSlots int // free VFs eligible for placement QuarantinedSlots int - quarantined map[string]struct{} + Quarantined map[string]struct{} // PCI addresses of quarantined VFs } // GetVGPUAvailability counts free allocatable and quarantined VFs. It fails @@ -287,7 +287,7 @@ func GetVGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (VGPUAv } availability := VGPUAvailability{ AllocatableSlots: countFreeVFs(vfs, addresses), - quarantined: addresses, + Quarantined: addresses, } for _, vf := range vfs { if _, ok := addresses[vf.PCIAddress]; ok { diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index a545c5733..d53659efd 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -42,17 +42,17 @@ func ListGPUProfiles() ([]GPUProfile, error) { if err != nil { return nil, err } - return ListGPUProfilesWithVFs(framework, vfs, availability) + return ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) } // ListGPUProfilesWithVFs returns available profiles for discovered VFs. -// Quarantined VFs from availability are excluded from vendor VFIO counts. -func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, availability VGPUAvailability) ([]GPUProfile, error) { +// Quarantined VFs are excluded from vendor VFIO counts. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction, quarantined map[string]struct{}) ([]GPUProfile, error) { switch framework { case VGPUFrameworkMdev: return listMdevGPUProfilesWithVFs(vfs) case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.listProfiles(vfs, availability.quarantined) + return hostVendorVFIO.listProfiles(vfs, quarantined) default: return nil, nil } diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index ddece4f22..6dfd537ae 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -70,7 +70,7 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d status.QuarantinedSlots = availability.QuarantinedSlots // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs, availability.Quarantined) if err != nil { logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil From 408e53b3625f2fb848ddcdf473d08a2402baaeac Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:44:40 +0000 Subject: [PATCH 18/21] Move the VF health threshold setter into the tests Production only sets the threshold through InitVFHealth. The tests keep a helper that changes it on the loaded store and re-evaluates tallies. --- lib/devices/vf_health.go | 9 --------- lib/devices/vf_health_test.go | 28 +++++++++++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index a75e0b701..51a1947a2 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -132,15 +132,6 @@ func InitVFHealth(path string, threshold int) error { return vfHealth.loadLocked() } -// setThreshold changes the quarantine threshold on a loaded store and -// re-evaluates recorded tallies against it. -func (s *vfHealthStore) setThreshold(n int) error { - s.mu.Lock() - defer s.mu.Unlock() - s.threshold = n - return s.requarantineLocked() -} - // requarantineLocked quarantines records whose failure tallies meet the // current threshold, so threshold changes and loaded state agree. // diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 92db8586f..6e60d0eb1 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -29,6 +29,16 @@ func resetVFHealthStore(t *testing.T) string { return path } +// setVFHealthThreshold changes the quarantine threshold on the loaded store +// and re-evaluates recorded tallies, as a restart with a new +// gpu.vf_quarantine_threshold would. +func setVFHealthThreshold(n int) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n + return vfHealth.requarantineLocked() +} + func vfHealthStoreUnavailable() bool { vfHealth.mu.Lock() defer vfHealth.mu.Unlock() @@ -50,11 +60,11 @@ func quarantinedVFs() []vfHealthRecord { func quarantineVF(t *testing.T, address string) { t.Helper() - require.NoError(t, vfHealth.setThreshold(1)) + require.NoError(t, setVFHealthThreshold(1)) result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) require.NoError(t, err) require.Equal(t, VFReportQuarantined, result.Outcome) - require.NoError(t, vfHealth.setThreshold(defaultVFQuarantineThreshold)) + require.NoError(t, setVFHealthThreshold(defaultVFQuarantineThreshold)) } func TestVGPUAvailability(t *testing.T) { @@ -103,7 +113,7 @@ func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { resetVFHealthStore(t) - require.NoError(t, vfHealth.setThreshold(1)) + require.NoError(t, setVFHealthThreshold(1)) blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0o644)) goodPath := vfHealth.path @@ -129,7 +139,7 @@ func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, vfHealth.setThreshold(1)) + require.NoError(t, setVFHealthThreshold(1)) _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) require.NoError(t, err) @@ -158,14 +168,14 @@ func TestCheckedAddressesRetriesFailedPersist(t *testing.T) { func TestSetThresholdReevaluatesRecordedFailures(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, vfHealth.setThreshold(3)) + require.NoError(t, setVFHealthThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) require.Equal(t, VFReportRecorded, result.Outcome) } - require.NoError(t, vfHealth.setThreshold(2)) + require.NoError(t, setVFHealthThreshold(2)) records := quarantinedVFs() require.Len(t, records, 1) @@ -181,7 +191,7 @@ func TestSetThresholdReevaluatesRecordedFailures(t *testing.T) { func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { path := resetVFHealthStore(t) - require.NoError(t, vfHealth.setThreshold(3)) + require.NoError(t, setVFHealthThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) require.NoError(t, err) @@ -461,7 +471,7 @@ func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { func TestReportRetriesFailedThresholdPersistence(t *testing.T) { path := resetVFHealthStore(t) vf := "0000:e3:00.4" - require.NoError(t, vfHealth.setThreshold(3)) + require.NoError(t, setVFHealthThreshold(3)) for _, instance := range []string{"instance-1", "instance-2"} { _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) require.NoError(t, err) @@ -470,7 +480,7 @@ func TestReportRetriesFailedThresholdPersistence(t *testing.T) { blocker := filepath.Join(t.TempDir(), "blocker") require.NoError(t, os.WriteFile(blocker, nil, 0644)) vfHealth.path = filepath.Join(blocker, "vf-health.json") - require.Error(t, vfHealth.setThreshold(2)) + require.Error(t, setVFHealthThreshold(2)) assert.True(t, vfHealthStoreUnavailable()) vfHealth.path = path From a1ccc88996351f9886838ebe3ba11fea8a8d4f9f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:44:40 +0000 Subject: [PATCH 19/21] Group the vendor VFIO selector inputs in a struct selectVendorVFIOVF took six positional arguments, two of them nilable, and claimVGPU repeated the full list at each of its three call sites. The host state it chooses from now travels as one value that the retry paths update in place. --- lib/instances/vgpu.go | 44 +++++++++++++++++++++++--------------- lib/instances/vgpu_test.go | 24 +++++++++++---------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index b6998e078..0296ef34d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -64,7 +64,8 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if err != nil { return nil, err } - vfAddress, profileType, err := selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex) + candidates := vendorVFIOCandidates{vfs: vfs, profilesByVF: profilesByVF, claims: allMetadata, quarantined: quarantined} + vfAddress, profileType, err := selectVendorVFIOVF(candidates, profileName, m.pickVFIndex) if err != nil { // A dirty unclaimed VF consumes framebuffer, which can make the // requested profile vanish from every creatable list before the @@ -75,10 +76,11 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str if _, vfs, err = m.discoverVGPUDevices(); err != nil { return nil, err } - if profilesByVF, err = listProfiles(vfs); err != nil { + if candidates.profilesByVF, err = listProfiles(vfs); err != nil { return nil, err } - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex); err != nil { + candidates.vfs = vfs + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, err } } @@ -90,7 +92,7 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str // clean siblings. log := logger.FromContext(ctx) for { - vf, ok := vfByAddress(vfs, vfAddress) + vf, ok := vfByAddress(candidates.vfs, vfAddress) if !ok || !vf.Allocated { break } @@ -104,8 +106,8 @@ func (m *manager) claimVGPU(ctx context.Context, meta *metadata, profileName str break } log.WarnContext(ctx, "dirty vGPU VF refused reset; trying another VF", "vf", vf.PCIAddress, "error", repairErr) - vfs = withoutVF(vfs, vf.PCIAddress) - if vfAddress, profileType, err = selectVendorVFIOVF(vfs, profilesByVF, allMetadata, quarantined, profileName, m.pickVFIndex); err != nil { + candidates.vfs = withoutVF(candidates.vfs, vf.PCIAddress) + if vfAddress, profileType, err = selectVendorVFIOVF(candidates, profileName, m.pickVFIndex); err != nil { return nil, fmt.Errorf("repair dirty VF %s before claim: %w", vf.PCIAddress, repairErr) } } @@ -171,15 +173,23 @@ func (m *manager) quarantinedVFAddresses() (map[string]struct{}, error) { return quarantined() } +// vendorVFIOCandidates is the host state a vendor VFIO placement chooses from. +type vendorVFIOCandidates struct { + vfs []devices.VirtualFunction + profilesByVF map[string][]devices.VGPUProfileType + claims []StoredMetadata // every instance; those with a device path hold a VF + quarantined map[string]struct{} +} + // selectVendorVFIOVF picks the VF to claim for profileName. Quarantined VFs // are never candidates and count against their parent GPU, so placement // drifts away from cards carrying a wedged VF. Among equally ranked // candidates on the chosen GPU, pick selects the index (nil is uniform // random), so a single VF cannot capture every placement on an idle host. -func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][]devices.VGPUProfileType, allMetadata []StoredMetadata, quarantined map[string]struct{}, profileName string, pick func(n int) int) (string, string, error) { +func selectVendorVFIOVF(c vendorVFIOCandidates, profileName string, pick func(n int) int) (string, string, error) { profilesByName := make(map[string]devices.VGPUProfileType) - advertises := make(map[string]map[string]struct{}, len(profilesByVF)) - for vfAddress, profiles := range profilesByVF { + advertises := make(map[string]map[string]struct{}, len(c.profilesByVF)) + for vfAddress, profiles := range c.profilesByVF { advertises[vfAddress] = make(map[string]struct{}, len(profiles)) for _, profile := range profiles { profilesByName[profile.Name] = profile @@ -188,21 +198,21 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ } requested, found := profilesByName[profileName] if !found { - if len(profilesByName) == 0 && len(vfs) > 0 { + if len(profilesByName) == 0 && len(c.vfs) > 0 { return "", "", fmt.Errorf("no creatable vGPU profiles on any VF (GPUs at capacity or dirty VFs consuming framebuffer): profile %q", profileName) } return "", "", fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) } - vfsByAddress := make(map[string]devices.VirtualFunction, len(vfs)) - for _, vf := range vfs { + vfsByAddress := make(map[string]devices.VirtualFunction, len(c.vfs)) + for _, vf := range c.vfs { vfsByAddress[vf.PCIAddress] = vf } claimed := make(map[string]struct{}) usageByGPU := make(map[string]int) unknownUsageByGPU := make(map[string]bool) - for i := range allMetadata { - stored := &allMetadata[i] + for i := range c.claims { + stored := &c.claims[i] if stored.GPUDevicePath == "" { continue } @@ -221,15 +231,15 @@ func selectVendorVFIOVF(vfs []devices.VirtualFunction, profilesByVF map[string][ } parentAdvertises := make(map[string]bool) - for _, vf := range vfs { + for _, vf := range c.vfs { if _, ok := advertises[vf.PCIAddress][requested.TypeName]; ok { parentAdvertises[vf.ParentGPU] = true } } quarantinedByGPU := make(map[string]int) freeByGPU := make(map[string][]devices.VirtualFunction) - for _, vf := range vfs { - if _, bad := quarantined[vf.PCIAddress]; bad { + for _, vf := range c.vfs { + if _, bad := c.quarantined[vf.PCIAddress]; bad { quarantinedByGPU[vf.ParentGPU]++ continue } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index ea35b15fa..be346dee0 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -120,9 +120,11 @@ func TestSelectVendorVFIOVFFailsClosedOnClaimedVF(t *testing.T) { profiles := map[string][]devices.VGPUProfileType{ "0000:82:00.4": {{TypeName: testVFProfileType, Name: testVGPUProfile, FramebufferMB: 2048}}, } - _, _, err := selectVendorVFIOVF(vfs, profiles, []StoredMetadata{{ - GPUDevicePath: testVFDevicePath, - }}, nil, testVGPUProfile, nil) + _, _, err := selectVendorVFIOVF(vendorVFIOCandidates{ + vfs: vfs, + profilesByVF: profiles, + claims: []StoredMetadata{{GPUDevicePath: testVFDevicePath}}, + }, testVGPUProfile, nil) require.ErrorContains(t, err, "no available VF") } @@ -145,11 +147,11 @@ func TestSelectVendorVFIOVFSkipsQuarantinedVF(t *testing.T) { } quarantined := map[string]struct{}{"0000:82:00.4": {}} - vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5"), nil, quarantined, testVGPUProfile, pickFirst) + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5"), quarantined: quarantined}, testVGPUProfile, pickFirst) require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) - _, _, err = selectVendorVFIOVF(vfs[:1], testVFProfiles("0000:82:00.4"), nil, quarantined, testVGPUProfile, pickFirst) + _, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:1], profilesByVF: testVFProfiles("0000:82:00.4"), quarantined: quarantined}, testVGPUProfile, pickFirst) require.ErrorContains(t, err, "no available VF") } @@ -163,7 +165,7 @@ func TestSelectVendorVFIOVFAvoidsGPUWithQuarantinedVF(t *testing.T) { } quarantined := map[string]struct{}{"0000:82:00.4": {}} - vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5", "0000:e3:00.4"), nil, quarantined, testVGPUProfile, pickFirst) + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5", "0000:e3:00.4"), quarantined: quarantined}, testVGPUProfile, pickFirst) require.NoError(t, err) assert.Equal(t, "0000:e3:00.4", vf) } @@ -179,7 +181,7 @@ func TestSelectVendorVFIOVFPicksAmongEquivalentFreeVFs(t *testing.T) { return n - 1 } - vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.4", "0000:82:00.5"), nil, nil, testVGPUProfile, pick) + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.4", "0000:82:00.5")}, testVGPUProfile, pick) require.NoError(t, err) assert.Equal(t, 2, offered) assert.Equal(t, "0000:82:00.5", vf) @@ -199,12 +201,12 @@ func TestSelectVendorVFIOVFRandomizesOnlyAmongCleanVFs(t *testing.T) { return n - 1 } - vf, _, err := selectVendorVFIOVF(vfs, testVFProfiles("0000:82:00.5", "0000:82:00.6"), nil, nil, testVGPUProfile, pick) + vf, _, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: testVFProfiles("0000:82:00.5", "0000:82:00.6")}, testVGPUProfile, pick) require.NoError(t, err) assert.Equal(t, 2, offered) assert.Equal(t, "0000:82:00.6", vf) - vf, _, err = selectVendorVFIOVF(vfs[:1], testVFProfiles("0000:82:00.4"), nil, nil, testVGPUProfile, pickLast) + vf, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:1], profilesByVF: testVFProfiles("0000:82:00.4")}, testVGPUProfile, pickLast) require.NoError(t, err) assert.Equal(t, "0000:82:00.4", vf) } @@ -229,13 +231,13 @@ func TestSelectVendorVFIOVFPrefersGPUWithKnownLoad(t *testing.T) { {GPUProfile: testVGPUProfile, GPUDevicePath: "/sys/bus/pci/devices/0000:e3:00.4"}, } - vf, profileType, err := selectVendorVFIOVF(vfs, profiles, claims, nil, testVGPUProfile, nil) + vf, profileType, err := selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs, profilesByVF: profiles, claims: claims}, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:e3:00.5", vf) assert.Equal(t, testVFProfileType, profileType) // With no alternative, the GPU with unknown load is still used. - vf, _, err = selectVendorVFIOVF(vfs[:2], profiles, claims[:1], nil, testVGPUProfile, nil) + vf, _, err = selectVendorVFIOVF(vendorVFIOCandidates{vfs: vfs[:2], profilesByVF: profiles, claims: claims[:1]}, testVGPUProfile, nil) require.NoError(t, err) assert.Equal(t, "0000:82:00.5", vf) } From 9aea9c48dac3aaf8f23b257cabfe503a494b20ee Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:19:03 +0000 Subject: [PATCH 20/21] Deep-copy GPUClaimedAt in cloneStoredMetadata --- lib/instances/fork.go | 4 ++++ lib/instances/fork_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index ec0c7c9d2..4c0046c7e 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -638,6 +638,10 @@ func cloneStoredMetadata(src StoredMetadata) StoredMetadata { guestAgentReadyAt := *src.GuestAgentReadyAt dst.GuestAgentReadyAt = &guestAgentReadyAt } + if src.GPUClaimedAt != nil { + gpuClaimedAt := *src.GPUClaimedAt + dst.GPUClaimedAt = &gpuClaimedAt + } if src.ExitCode != nil { exitCode := *src.ExitCode dst.ExitCode = &exitCode diff --git a/lib/instances/fork_test.go b/lib/instances/fork_test.go index 26bc6cfcb..98479175a 100644 --- a/lib/instances/fork_test.go +++ b/lib/instances/fork_test.go @@ -711,6 +711,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { t.Parallel() startedAt := time.Now().Add(-2 * time.Minute) stoppedAt := time.Now().Add(-1 * time.Minute) + gpuClaimedAt := time.Now().Add(-3 * time.Minute) expiresAt := time.Now().Add(time.Hour) notBefore := time.Now().Add(5 * time.Minute) pid := 1234 @@ -731,6 +732,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { ExpiresAt: &expiresAt, StartedAt: &startedAt, StoppedAt: &stoppedAt, + GPUClaimedAt: &gpuClaimedAt, HypervisorProcessIdentity: HypervisorProcessIdentity{HypervisorPID: &pid}, ExitCode: &exitCode, AutoStandby: &autostandby.Policy{ @@ -786,6 +788,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { *cloned.ExpiresAt = now *cloned.StartedAt = now *cloned.StoppedAt = now + *cloned.GPUClaimedAt = now require.Equal(t, "1", src.Env["A"]) require.Equal(t, "x", src.Tags["m"]) @@ -806,6 +809,7 @@ func TestCloneStoredMetadataForFork_DeepCopiesReferenceFields(t *testing.T) { require.Equal(t, expiresAt, *src.ExpiresAt) require.Equal(t, startedAt, *src.StartedAt) require.Equal(t, stoppedAt, *src.StoppedAt) + require.Equal(t, gpuClaimedAt, *src.GPUClaimedAt) } func TestCloneStoredMetadataWithoutPendingStandbyCompression_ClearsPendingPlan(t *testing.T) { From 7cc1a45a5f62858ba69a47561e0c2da20884d573 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:03:58 +0000 Subject: [PATCH 21/21] Clear older tallies on success while a VF stays quarantined A success matching an older assignment on a quarantined VF returned without changing the record, which contradicts the documented semantics: the match and every older tally are cleared, and only the rescind is withheld unless the match is the newest failure. --- lib/devices/vf_health.go | 6 ++--- lib/devices/vf_health_test.go | 45 +++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go index 51a1947a2..8de8e23a0 100644 --- a/lib/devices/vf_health.go +++ b/lib/devices/vf_health.go @@ -408,15 +408,15 @@ func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResu break } } - // Only the newest failure can rescind a quarantine; see VFInitSuccessReport. - if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + if match < 0 { return VFSuccessResult{}, nil } + // Only the newest failure can rescind a quarantine; see VFInitSuccessReport. remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) result := VFSuccessResult{ Cleared: len(previous.Failures) - len(remaining), - Rescinded: previous.QuarantinedAt != nil, + Rescinded: previous.QuarantinedAt != nil && len(remaining) == 0, } if len(remaining) == 0 { delete(s.records, report.VFAddress) diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go index 6e60d0eb1..7c2e93d15 100644 --- a/lib/devices/vf_health_test.go +++ b/lib/devices/vf_health_test.go @@ -327,6 +327,51 @@ func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T assert.Empty(t, quarantinedVFs()) } +func TestReportVFInitSuccessForOlderAssignmentKeepsQuarantine(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + older := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + } + _, err := ReportVFInitFailure(older) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: older.VFAddress, + InstanceID: older.InstanceID, + AssignedAt: older.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 1, success.Cleared) + assert.False(t, success.Rescinded) + + require.NoError(t, InitVFHealth(path, defaultVFQuarantineThreshold)) + quarantined := quarantinedVFs() + require.Len(t, quarantined, 1) + require.Len(t, quarantined[0].Failures, 1) + assert.Equal(t, trigger.InstanceID, quarantined[0].Failures[0].InstanceID) + + success, err = ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 1, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { resetVFHealthStore(t) _, err := ReportVFInitFailure(VFInitFailureReport{