diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index ebb6ad951..dec9f35eb 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -87,9 +87,11 @@ func convertResourceStatus(rs resources.ResourceStatus) oapi.ResourceStatus { func convertGPUResourceStatus(gs *resources.GPUResourceStatus) oapi.GPUResourceStatus { result := oapi.GPUResourceStatus{ - Mode: oapi.GPUResourceStatusMode(gs.Mode), - TotalSlots: gs.TotalSlots, - UsedSlots: gs.UsedSlots, + Mode: oapi.GPUResourceStatusMode(gs.Mode), + TotalSlots: gs.TotalSlots, + UsedSlots: gs.UsedSlots, + AllocatableSlots: gs.AllocatableSlots, + QuarantinedSlots: gs.QuarantinedSlots, } // Convert profiles (vGPU mode) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index b46f4f2e5..c250a3621 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -269,7 +269,8 @@ type SnapshotConfig struct { // GPUConfig holds GPU-related settings. type GPUConfig struct { - ProfileCacheTTL string `koanf:"profile_cache_ttl"` + ProfileCacheTTL string `koanf:"profile_cache_ttl"` + VFQuarantineThreshold int `koanf:"vf_quarantine_threshold"` } // Config is the top-level Hypeman server configuration. @@ -494,7 +495,8 @@ func defaultConfig() *Config { }, GPU: GPUConfig{ - ProfileCacheTTL: "30m", + ProfileCacheTTL: "30m", + VFQuarantineThreshold: 2, }, } } @@ -647,6 +649,9 @@ func (c *Config) Validate() error { if c.Build.MaxConcurrentSourceBuilds <= 0 { return fmt.Errorf("build.max_concurrent_source_builds must be positive, got %d", c.Build.MaxConcurrentSourceBuilds) } + if c.GPU.VFQuarantineThreshold < 1 { + return fmt.Errorf("gpu.vf_quarantine_threshold must be >= 1, got %d", c.GPU.VFQuarantineThreshold) + } if c.Limits.MaxConcurrentPushes <= 0 { return fmt.Errorf("limits.max_concurrent_pushes must be positive, got %d", c.Limits.MaxConcurrentPushes) } diff --git a/cmd/api/config/config_test.go b/cmd/api/config/config_test.go index 5660d878e..efd52f208 100644 --- a/cmd/api/config/config_test.go +++ b/cmd/api/config/config_test.go @@ -250,6 +250,18 @@ func TestValidateRejectsInvalidMetricsPort(t *testing.T) { } } +func TestValidateRejectsInvalidVFQuarantineThreshold(t *testing.T) { + for _, threshold := range []int{0, -1} { + cfg := defaultConfig() + cfg.GPU.VFQuarantineThreshold = threshold + + err := cfg.Validate() + if err == nil { + t.Fatalf("expected validation error for vf_quarantine_threshold %d", threshold) + } + } +} + func TestValidateRejectsInvalidMetricExportInterval(t *testing.T) { cfg := defaultConfig() cfg.Otel.MetricExportInterval = "not-a-duration" diff --git a/cmd/api/main.go b/cmd/api/main.go index faa9ac15a..d6fba1b43 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -204,6 +204,7 @@ func run() error { // Configure GPU profile cache TTL devices.SetGPUProfileCacheTTL(cfg.GPU.ProfileCacheTTL) + devices.SetVFQuarantineThreshold(cfg.GPU.VFQuarantineThreshold) // Initialize OpenTelemetry (before wire initialization) otelCfg := otel.Config{ diff --git a/config.example.yaml b/config.example.yaml index ebef41257..70d55fa68 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -170,6 +170,12 @@ data_dir: /var/lib/hypeman # idle_ttl: "" # delete builders idle this long (e.g. "24h"); # # destructive, empty = disabled +# gpu: +# profile_cache_ttl: 30m # vGPU profile metadata cache TTL +# vf_quarantine_threshold: 2 # distinct instance assignments that must report +# # a guest driver init failure before the VF is +# # quarantined (must be >= 1) + # ============================================================================= # Resource Limits # ============================================================================= diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index d04dcf599..678871562 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -49,8 +49,10 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59}, + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57}, {"name": "L40S-2Q", "framebuffer_mb": 2048, "available": 30}, {"name": "L40S-4Q", "framebuffer_mb": 4096, "available": 16} ] @@ -121,6 +123,8 @@ curl -s http://localhost:4973/resources | jq .gpu "mode": "passthrough", "total_slots": 4, "used_slots": 2, + "allocatable_slots": 2, + "quarantined_slots": 0, "devices": [ {"name": "NVIDIA L40S", "available": true}, {"name": "NVIDIA L40S", "available": false} @@ -185,8 +189,10 @@ Returns GPU status along with other resources: "mode": "vgpu", "total_slots": 64, "used_slots": 5, + "allocatable_slots": 57, + "quarantined_slots": 2, "profiles": [ - {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 59} + {"name": "L40S-1Q", "framebuffer_mb": 1024, "available": 57} ] } } @@ -282,10 +288,28 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) ``` (0x65 = timeout; the guest's init requests are never answered, and -`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because -placement is deterministic least-loaded, an idle host re-picks the same VF for -every request, so one wedged VF presents as all vGPU instances failing while -`/resources` reports full capacity. +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). + +Hypeman tracks these failures in `/gpu/vf-health.json` (it survives +restarts): each reported init failure is tallied per instance assignment, and +once failures accumulate from `gpu.vf_quarantine_threshold` distinct +assignments (default 2), the VF is quarantined: excluded from placement and +from advertised profile availability, and its parent GPU becomes +overflow-only — deprioritized for new placements. Selection among a card's +equivalent free VFs is randomized so a wedged VF cannot capture every +placement. A reported init success clears failures only when that exact +assignment has a recorded failure, removing the match and older tallies; if +that assignment crossed the threshold, its later success also rescinds the +quarantine. If the state file exists but cannot be loaded, or the last write +to it failed, placement and advertised availability fail closed until a load +or write succeeds. Recorded tallies are re-evaluated against the configured +threshold at load, so lowering `gpu.vf_quarantine_threshold` quarantines VFs +whose persisted failures already meet the new value. + +`used_slots` includes quarantined VFs still held by running instances, so it +can overlap `quarantined_slots`; use `allocatable_slots` for admission. + +Quarantine only removes capacity — it never touches a running instance. The wedge itself leaves no host-side log: no kernel error, no XID, no plugin crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is @@ -303,18 +327,44 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it. Confirm by assigning the same profile on a different VF: if that guest initializes, the VF is wedged, not the driver stack. Remediate by cycling SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it -requires no vGPU assignments on that GPU): +requires no vGPU assignments on that GPU). The DCGM quiesce is not optional: +with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d` +fails with `Cannot obtain unbindLock` on first contact. + +Any manual edit to `vf-health.json` needs an immediate hypeman restart: the +store loads only at startup, and a failure report landing first re-persists +the in-memory set over your edit. The restart does not disturb running VMs — +startup reconciliation protects live VFs. + +**Draining the parent GPU.** Overflow-only is a preference, not a cordon: +under capacity pressure new placements still land on the card's healthy VFs +and refill it. To drain the card, quarantine all of its VFs by hand — add +records to the versioned `vf-health.json` (`{"version": 1, "records": +[{"vf_address": "...", "quarantined_at": "..."}]}`) and restart. Running +instances are untouched and +drain through their normal lifecycle: standby is blocked for vGPU instances, +so only a running VM pins a VF, and each stop or delete frees one for good. +Monitor by listing instances whose `gpu.device_path` sits under the parent +GPU; once none remain, run the cycle below. ```bash +# 1. Quiesce the services holding the GPU (required for the unbind lock). +systemctl stop nvidia-dcgm-exporter nvidia-dcgm + +# 2. Cycle SR-IOV on the parent GPU. /usr/lib/nvidia/sriov-manage -d /usr/lib/nvidia/sriov-manage -e + +# 3. Restart the quiesced services. +systemctl start nvidia-dcgm nvidia-dcgm-exporter ``` +After the cycle, remove the card's entries from `vf-health.json`, restart, +and boot a GPU instance to verify recovery. + Do not unbind/rebind the VF from the nvidia driver — it breaks the nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) and the VF stops accepting assignments entirely until the SR-IOV cycle. -Services holding the GPU (DCGM, persistenced) must be stopped for the cycle -to obtain the unbind lock. ### vGPU assignment fails diff --git a/lib/devices/manager.go b/lib/devices/manager.go index 30763c04d..6b9f6340a 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "os" "runtime" "strings" @@ -85,6 +86,9 @@ type manager struct { // NewManager creates a new device manager. // Use SetLivenessChecker after construction to enable accurate orphan detection. func NewManager(p *paths.Paths) Manager { + if err := initVFHealth(p.VFHealthState()); err != nil { + slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err) + } return &manager{ paths: p, vfioBinder: NewVFIOBinder(), diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index a1378a3c7..0d8a27383 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -8,12 +8,12 @@ import ( "fmt" "log/slog" "maps" + "math/rand/v2" "os" "path/filepath" "sort" "strconv" "strings" - "sync" "syscall" "time" @@ -30,6 +30,26 @@ type vendorVFIOOwner struct { assignedAt time.Time } +type vendorVFIOGPUPlacement struct { + usage int + unknownUsage bool + quarantined int + freeVFs []VirtualFunction +} + +func (p *vendorVFIOGPUPlacement) preferredTo(other *vendorVFIOGPUPlacement, gpu, otherGPU string) bool { + if p.quarantined != other.quarantined { + return p.quarantined < other.quarantined + } + if p.unknownUsage != other.unknownUsage { + return !p.unknownUsage + } + if p.usage != other.usage { + return p.usage < other.usage + } + return gpu < otherGPU +} + type vendorVFIOSysfs struct { pciDevicesPath string procPath string @@ -37,18 +57,16 @@ type vendorVFIOSysfs struct { owners map[string]vendorVFIOOwner framebufferByType map[string]int openVFIOPathsFunc func() (map[string]struct{}, error) + pickVFIndex func(n int) int } -var ( - hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]vendorVFIOOwner), - framebufferByType: make(map[string]int), - } - vendorVFIOMu sync.Mutex -) +var hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]vendorVFIOOwner), + framebufferByType: make(map[string]int), +} func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(s.pciDevicesPath) @@ -110,6 +128,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { // available_instances. This is a best-effort snapshot because creating on one // VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return nil, err + } profilesByType := make(map[string]profileMetadata) creatableVFs := make(map[string]int) for _, vf := range vfs { @@ -121,9 +143,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) continue } + _, bad := quarantined[vf.PCIAddress] for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { + if !vf.Allocated && !bad { creatableVFs[profile.TypeName]++ } } @@ -314,56 +337,75 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } -func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { - usageByGPU := make(map[string]int) - unknownUsageByGPU := make(map[string]bool) - freeByGPU := make(map[string][]VirtualFunction) - for _, vf := range vfs { - if vf.Allocated { - // framebufferByType only covers currently creatable profiles, so - // after a restart an allocated type can be missing when its - // capacity is exhausted. Prefer GPUs whose load is fully known - // instead of rejecting placement outright; the kernel driver - // still enforces real capacity through creatable_vgpu_types. - framebuffer, ok := s.framebufferByType[vf.ProfileType] - if !ok { - unknownUsageByGPU[vf.ParentGPU] = true - continue - } - usageByGPU[vf.ParentGPU] += framebuffer - continue +func (s vendorVFIOSysfs) addVFToPlacement(placement *vendorVFIOGPUPlacement, vf VirtualFunction, profileType string, quarantined bool) { + if quarantined { + placement.quarantined++ + if !vf.Allocated { + return } - profiles, err := s.readCreatableProfiles(vf.PCIAddress) - if err != nil { - // An unreadable free VF is just not a placement candidate. - slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) - continue + } + if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + placement.unknownUsage = true + return } - for _, profile := range profiles { - if profile.TypeName == profileType { - freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) - break - } + placement.usage += framebuffer + return + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + return + } + for _, profile := range profiles { + if profile.TypeName == profileType { + placement.freeVFs = append(placement.freeVFs, vf) + return } } +} - gpus := make([]string, 0, len(freeByGPU)) - for gpu := range freeByGPU { - gpus = append(gpus, gpu) +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + quarantined, err := vfHealth.checkedAddresses() + if err != nil { + return "", err } - sort.Slice(gpus, func(i, j int) bool { - if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { - return !unknownUsageByGPU[gpus[i]] + placementByGPU := make(map[string]*vendorVFIOGPUPlacement) + for _, vf := range vfs { + placement := placementByGPU[vf.ParentGPU] + if placement == nil { + placement = &vendorVFIOGPUPlacement{} + placementByGPU[vf.ParentGPU] = placement } - if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { - return gpus[i] < gpus[j] + _, bad := quarantined[vf.PCIAddress] + s.addVFToPlacement(placement, vf, profileType, bad) + } + + gpus := make([]string, 0, len(placementByGPU)) + for gpu, placement := range placementByGPU { + if len(placement.freeVFs) > 0 { + gpus = append(gpus, gpu) } - return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + } + sort.Slice(gpus, func(i, j int) bool { + return placementByGPU[gpus[i]].preferredTo(placementByGPU[gpus[j]], gpus[i], gpus[j]) }) if len(gpus) == 0 { return "", nil } - return freeByGPU[gpus[0]][0].PCIAddress, nil + candidates := placementByGPU[gpus[0]].freeVFs + pick := s.pickVFIndex + if pick == nil { + pick = rand.IntN + } + return candidates[pick(len(candidates))].PCIAddress, nil } func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 098d016e3..a65fd0a88 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -621,3 +621,69 @@ func assertFileValue(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(value)) } + +func TestVendorVFIOSkipsQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIONoVFWhenAllQuarantined(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.ErrorContains(t, err, "no available VF") +} + +func TestVendorVFIOCardBiasAvoidsGPUWithQuarantinedVF(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(int) int { return 0 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectUsesTiebreakAmongFreeVFs(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.pickVFIndex = func(n int) int { return n - 1 } + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOListProfilesExcludesQuarantinedFromAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q")) +} diff --git a/lib/devices/vf_health.go b/lib/devices/vf_health.go new file mode 100644 index 000000000..3ae8d02c0 --- /dev/null +++ b/lib/devices/vf_health.go @@ -0,0 +1,482 @@ +package devices + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "sort" + "sync" + "time" +) + +const ( + vfHealthFileVersion = 1 + defaultVFQuarantineThreshold = 2 +) + +type vfInitFailure struct { + InstanceID string `json:"instance_id,omitempty"` + AssignedAt string `json:"assigned_at,omitempty"` + ReportedAt time.Time `json:"reported_at"` +} + +type vfHealthRecord struct { + VFAddress string `json:"vf_address"` + Failures []vfInitFailure `json:"failures,omitempty"` + QuarantinedAt *time.Time `json:"quarantined_at,omitempty"` +} + +type vfHealthFile struct { + Version int `json:"version"` + Records []vfHealthRecord `json:"records"` +} + +// VFInitFailureReport describes one guest-reported driver init failure. +type VFInitFailureReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFInitSuccessReport identifies the assignment that successfully initialized. +type VFInitSuccessReport struct { + VFAddress string + InstanceID string + AssignedAt string +} + +// VFReportOutcome describes how a failure report changed a VF's health state. +type VFReportOutcome int + +const ( + // VFReportUnchanged means the VF was already quarantined or this + // assignment was already recorded. + VFReportUnchanged VFReportOutcome = iota + // VFReportRecorded means the failure was tallied below the quarantine threshold. + VFReportRecorded + // VFReportQuarantined means this report crossed the threshold and quarantined the VF. + VFReportQuarantined +) + +// VFReportResult is the outcome of recording a driver init failure. +type VFReportResult struct { + Outcome VFReportOutcome + Failures int + Threshold int +} + +// VFSuccessResult describes how a successful init changed a VF's health state. +type VFSuccessResult struct { + Cleared int + Rescinded bool +} + +type vfHealthStore struct { + mu sync.Mutex + path string + records map[string]vfHealthRecord + threshold int + loadErr error + persistErr error + syncDirFunc func(string) error +} + +var vfHealthAddressPattern = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +var ( + vfHealth = &vfHealthStore{ + records: make(map[string]vfHealthRecord), + threshold: defaultVFQuarantineThreshold, + syncDirFunc: syncDir, + } + // vendorVFIOMu is acquired before vfHealth.mu. It serializes quarantine + // mutations with vendor-VFIO create, destroy, and reconciliation so + // placement cannot select a VF while it is being quarantined. + vendorVFIOMu sync.Mutex +) + +func initVFHealth(path string) error { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = path + return vfHealth.loadLocked() +} + +// SetVFQuarantineThreshold configures the number of failed assignments +// required to quarantine a VF. Already-recorded tallies are re-evaluated so a +// lowered threshold applies to failures persisted before the change. +func SetVFQuarantineThreshold(n int) { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.threshold = n + vfHealth.requarantineLocked() +} + +// requarantineLocked quarantines records whose failure tallies meet the +// current threshold, so threshold changes and loaded state agree. +func (s *vfHealthStore) requarantineLocked() { + changed := false + for address, record := range s.records { + if record.QuarantinedAt != nil || len(record.Failures) < s.threshold { + continue + } + now := time.Now().UTC() + record.QuarantinedAt = &now + s.records[address] = record + changed = true + } + if changed { + if _, err := s.persistLocked(); err != nil { + slog.Default().Error("failed to persist re-evaluated VF quarantines; vGPU placement is disabled until a write succeeds", "error", err) + } + } +} + +func (s *vfHealthStore) loadLocked() error { + s.records = make(map[string]vfHealthRecord) + s.loadErr = nil + s.persistErr = nil + + data, err := os.ReadFile(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + s.loadErr = fmt.Errorf("read VF health state: %w", err) + return s.loadErr + } + var state vfHealthFile + if err := json.Unmarshal(data, &state); err != nil { + s.loadErr = fmt.Errorf("unmarshal VF health state: %w", err) + return s.loadErr + } + if state.Version != vfHealthFileVersion { + s.loadErr = fmt.Errorf("validate VF health state: unsupported version %d", state.Version) + return s.loadErr + } + if state.Records == nil { + s.loadErr = fmt.Errorf("validate VF health state: expected a records array") + return s.loadErr + } + loaded := make(map[string]vfHealthRecord, len(state.Records)) + for i, record := range state.Records { + if !vfHealthAddressPattern.MatchString(record.VFAddress) { + s.loadErr = fmt.Errorf("validate VF health state record %d: invalid VF address %q", i, record.VFAddress) + return s.loadErr + } + if record.QuarantinedAt != nil && record.QuarantinedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d: missing quarantine timestamp", i) + return s.loadErr + } + if record.QuarantinedAt == nil && len(record.Failures) == 0 { + s.loadErr = fmt.Errorf("validate VF health state record %d: neither quarantined nor any recorded failures", i) + return s.loadErr + } + assignments := make(map[string]struct{}, len(record.Failures)) + for j, failure := range record.Failures { + if failure.ReportedAt.IsZero() { + s.loadErr = fmt.Errorf("validate VF health state record %d failure %d: missing report timestamp", i, j) + return s.loadErr + } + key := failure.InstanceID + "\x00" + failure.AssignedAt + if _, exists := assignments[key]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate failure for assignment %q", i, failure.InstanceID) + return s.loadErr + } + assignments[key] = struct{}{} + } + if _, exists := loaded[record.VFAddress]; exists { + s.loadErr = fmt.Errorf("validate VF health state record %d: duplicate VF address %q", i, record.VFAddress) + return s.loadErr + } + loaded[record.VFAddress] = record + } + s.records = loaded + s.requarantineLocked() + return nil +} + +func (s *vfHealthStore) ensureLoadedLocked() error { + if s.loadErr == nil { + return nil + } + return s.loadLocked() +} + +func (s *vfHealthStore) checkedAddresses() (map[string]struct{}, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return nil, fmt.Errorf("VF health state unavailable: %w", err) + } + if s.persistErr != nil { + return nil, fmt.Errorf("VF health state unavailable: last write failed: %w", s.persistErr) + } + addresses := make(map[string]struct{}, len(s.records)) + for address, record := range s.records { + if record.QuarantinedAt != nil { + addresses[address] = struct{}{} + } + } + return addresses, nil +} + +// VGPUAvailability returns free allocatable and quarantined VF counts. +func VGPUAvailability(framework VGPUFramework, vfs []VirtualFunction) (allocatable, quarantined int, err error) { + if framework != VGPUFrameworkVendorVFIO { + return countFreeVFs(vfs, nil), 0, nil + } + addresses, err := vfHealth.checkedAddresses() + if err != nil { + return 0, 0, err + } + for _, vf := range vfs { + if _, ok := addresses[vf.PCIAddress]; ok { + quarantined++ + } + } + return countFreeVFs(vfs, addresses), quarantined, nil +} + +func countFreeVFs(vfs []VirtualFunction, quarantined map[string]struct{}) int { + available := 0 + for _, vf := range vfs { + if vf.Allocated { + continue + } + if _, ok := quarantined[vf.PCIAddress]; !ok { + available++ + } + } + return available +} + +// ReportVFInitFailure records a guest-reported driver init failure and +// quarantines the VF once failures from enough distinct assignments accumulate. +func ReportVFInitFailure(report VFInitFailureReport) (VFReportResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportFailure(report) +} + +// ReportVFInitSuccess clears failures through an exactly matched successful +// assignment. A quarantine is rescinded only when that assignment triggered it. +func ReportVFInitSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + return vfHealth.reportSuccess(report) +} + +// VFHealthStoreUnavailable reports whether persisted state failed to load or +// the last write failed. +func VFHealthStoreUnavailable() bool { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + return vfHealth.loadErr != nil || vfHealth.persistErr != nil +} + +// TotalQuarantinedVFs returns the number of quarantined VFs in persisted state. +func TotalQuarantinedVFs() int { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + count := 0 + for _, record := range vfHealth.records { + if record.QuarantinedAt != nil { + count++ + } + } + return count +} + +func (s *vfHealthStore) sortedRecordsLocked() []vfHealthRecord { + records := make([]vfHealthRecord, 0, len(s.records)) + for _, record := range s.records { + records = append(records, record) + } + sort.Slice(records, func(i, j int) bool { return records[i].VFAddress < records[j].VFAddress }) + return records +} + +func (s *vfHealthStore) reportFailure(report VFInitFailureReport) (VFReportResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFReportResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFReportResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFReportResult{}, err + } + + previous, existed := s.records[report.VFAddress] + result := VFReportResult{Failures: len(previous.Failures), Threshold: s.threshold} + if previous.QuarantinedAt != nil { + return result, nil + } + for _, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + return result, nil + } + } + + record := vfHealthRecord{ + VFAddress: report.VFAddress, + Failures: append(append([]vfInitFailure(nil), previous.Failures...), vfInitFailure{ + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + ReportedAt: time.Now().UTC(), + }), + } + result.Failures = len(record.Failures) + result.Outcome = VFReportRecorded + if result.Failures >= s.threshold { + now := time.Now().UTC() + record.QuarantinedAt = &now + result.Outcome = VFReportQuarantined + } + s.records[report.VFAddress] = record + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + if existed { + s.records[report.VFAddress] = previous + } else { + delete(s.records, report.VFAddress) + } + } + return VFReportResult{}, err + } + return result, nil +} + +func sameVFAssignment(failure vfInitFailure, instanceID, assignedAt string) bool { + return failure.InstanceID == instanceID && failure.AssignedAt == assignedAt +} + +func (s *vfHealthStore) reportSuccess(report VFInitSuccessReport) (VFSuccessResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.ensureLoadedLocked(); err != nil { + return VFSuccessResult{}, err + } + if !vfHealthAddressPattern.MatchString(report.VFAddress) { + return VFSuccessResult{}, fmt.Errorf("invalid VF address %q", report.VFAddress) + } + if err := s.retryPersistLocked(); err != nil { + return VFSuccessResult{}, err + } + previous, ok := s.records[report.VFAddress] + if !ok || len(previous.Failures) == 0 { + return VFSuccessResult{}, nil + } + + match := -1 + for i, failure := range previous.Failures { + if sameVFAssignment(failure, report.InstanceID, report.AssignedAt) { + match = i + break + } + } + if match < 0 || (previous.QuarantinedAt != nil && match != len(previous.Failures)-1) { + return VFSuccessResult{}, nil + } + + remaining := append([]vfInitFailure(nil), previous.Failures[match+1:]...) + result := VFSuccessResult{ + Cleared: len(previous.Failures) - len(remaining), + Rescinded: previous.QuarantinedAt != nil, + } + if len(remaining) == 0 { + delete(s.records, report.VFAddress) + } else { + record := previous + record.Failures = remaining + s.records[report.VFAddress] = record + } + renamed, err := s.persistLocked() + if err != nil { + if !renamed { + s.records[report.VFAddress] = previous + } + return VFSuccessResult{}, err + } + return result, nil +} + +func (s *vfHealthStore) retryPersistLocked() error { + if s.persistErr == nil { + return nil + } + _, err := s.persistLocked() + return err +} + +// persistLocked writes the current records to disk. A failure is latched and +// fails placement closed until a later write succeeds. The returned boolean +// reports whether the rename made the new state visible. +func (s *vfHealthStore) persistLocked() (bool, error) { + if s.path == "" { + return false, nil + } + renamed, err := s.writeStateLocked() + s.persistErr = err + return renamed, err +} + +func (s *vfHealthStore) writeStateLocked() (bool, error) { + data, err := json.MarshalIndent(vfHealthFile{ + Version: vfHealthFileVersion, + Records: s.sortedRecordsLocked(), + }, "", " ") + if err != nil { + return false, fmt.Errorf("marshal VF health state: %w", err) + } + dirPath := filepath.Dir(s.path) + if err := os.MkdirAll(dirPath, 0755); err != nil { + return false, fmt.Errorf("create VF health state dir: %w", err) + } + if err := s.syncDirFunc(filepath.Dir(dirPath)); err != nil { + return false, fmt.Errorf("sync VF health state parent dir: %w", err) + } + tmp := s.path + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return false, fmt.Errorf("create VF health state: %w", err) + } + if _, err := f.Write(data); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("write VF health state: %w", err) + } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return false, fmt.Errorf("sync VF health state: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("close VF health state: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + os.Remove(tmp) + return false, fmt.Errorf("rename VF health state: %w", err) + } + if err := s.syncDirFunc(dirPath); err != nil { + return true, fmt.Errorf("sync VF health state dir: %w", err) + } + return true, nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/lib/devices/vf_health_test.go b/lib/devices/vf_health_test.go new file mode 100644 index 000000000..00f14dbf7 --- /dev/null +++ b/lib/devices/vf_health_test.go @@ -0,0 +1,559 @@ +package devices + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func resetVFHealthStore(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "vf-health.json") + require.NoError(t, initVFHealth(path)) + t.Cleanup(func() { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + vfHealth.path = "" + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = defaultVFQuarantineThreshold + vfHealth.loadErr = nil + vfHealth.persistErr = nil + vfHealth.syncDirFunc = syncDir + }) + return path +} + +func quarantinedVFs() []vfHealthRecord { + vfHealth.mu.Lock() + defer vfHealth.mu.Unlock() + records := vfHealth.sortedRecordsLocked() + result := records[:0] + for _, record := range records { + if record.QuarantinedAt != nil { + result = append(result, record) + } + } + return result +} + +func quarantineVF(t *testing.T, address string) { + t.Helper() + SetVFQuarantineThreshold(1) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: address, InstanceID: "quarantine-helper"}) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + SetVFQuarantineThreshold(defaultVFQuarantineThreshold) +} + +func TestVGPUAvailability(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:82:00.4") + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:82:00.6", InstanceID: "instance-1"}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + vfs := []VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + } + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, vfs) + require.NoError(t, err) + assert.Equal(t, 1, available, "a below-threshold failure tally must not remove the VF from placement") + assert.Equal(t, 1, quarantined) + + available, quarantined, err = VGPUAvailability(VGPUFrameworkMdev, vfs) + require.NoError(t, err) + assert.Equal(t, 2, available) + assert.Zero(t, quarantined) +} + +func TestVGPUAvailabilityFailsWhenStoreUnavailable(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte("not json"), 0o644)) + require.Error(t, initVFHealth(path)) + + _, _, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.ErrorContains(t, err, "VF health state unavailable") + + available, quarantined, err := VGPUAvailability(VGPUFrameworkMdev, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err) + assert.Equal(t, 1, available) + assert.Zero(t, quarantined) + + restored := `{"version":1,"records":[{"vf_address":"0000:82:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0o644)) + available, quarantined, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + require.NoError(t, err, "a repaired state file must re-enable placement without a new report") + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestVGPUAvailabilityFailsClosedAfterPersistFailure(t *testing.T) { + resetVFHealthStore(t) + SetVFQuarantineThreshold(1) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0o644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + assert.True(t, VFHealthStoreUnavailable()) + _, _, err = VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.ErrorContains(t, err, "last write failed") + + vfHealth.path = goodPath + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: "0000:e3:00.4"}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestSetVFQuarantineThresholdReevaluatesRecordedFailures(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + SetVFQuarantineThreshold(2) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt, "the re-evaluated quarantine must be persisted") +} + +func TestLoadReevaluatesTalliesAgainstConfiguredThreshold(t *testing.T) { + path := resetVFHealthStore(t) + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: instance}) + require.NoError(t, err) + require.Equal(t, VFReportRecorded, result.Outcome) + } + + // Simulate a restart where the threshold is configured lower before the + // persisted tallies are loaded. + vfHealth.mu.Lock() + vfHealth.records = make(map[string]vfHealthRecord) + vfHealth.threshold = 2 + vfHealth.mu.Unlock() + require.NoError(t, initVFHealth(path)) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} + +func TestReportVFInitFailureQuarantinesAtThreshold(t *testing.T) { + path := resetVFHealthStore(t) + + result, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Equal(t, defaultVFQuarantineThreshold, result.Threshold) + assert.Empty(t, quarantinedVFs(), "one failure must not quarantine at the default threshold") + + result, err = ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Equal(t, VFReportQuarantined, result.Outcome) + assert.Equal(t, 2, result.Failures) + + records := quarantinedVFs() + require.Len(t, records, 1) + assert.Equal(t, 1, TotalQuarantinedVFs()) + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) + require.NotNil(t, records[0].QuarantinedAt) + require.Len(t, records[0].Failures, 2) + assert.Equal(t, "instance-1", records[0].Failures[0].InstanceID) + + result, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + + require.NoError(t, initVFHealth(path)) + reloaded := quarantinedVFs() + require.Len(t, reloaded, 1) + assert.Equal(t, "0000:e3:00.4", reloaded[0].VFAddress) + require.Len(t, reloaded[0].Failures, 2) +} + +func TestReportVFInitFailureDeduplicatesAssignments(t *testing.T) { + resetVFHealthStore(t) + + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + + result, err = ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.Equal(t, 1, result.Failures) + assert.Empty(t, quarantinedVFs(), "a rescanned assignment must not count toward the threshold twice") +} + +func TestReportVFInitSuccessClearsFailureTally(t *testing.T) { + path := resetVFHealthStore(t) + report := VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + } + + _, err := ReportVFInitFailure(report) + require.NoError(t, err) + + success := VFInitSuccessReport{ + VFAddress: report.VFAddress, + InstanceID: report.InstanceID, + AssignedAt: report.AssignedAt, + } + successResult, err := ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Equal(t, 1, successResult.Cleared) + assert.False(t, successResult.Rescinded) + + successResult, err = ReportVFInitSuccess(success) + require.NoError(t, err) + assert.Zero(t, successResult.Cleared) + + require.NoError(t, initVFHealth(path)) + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: report.VFAddress, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 1, result.Failures) +} + +func TestReportVFInitSuccessRescindsQuarantineTriggeredByAssignment(t *testing.T) { + resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-1", + AssignedAt: "2026-08-20T14:00:00Z", + }) + require.NoError(t, err) + trigger := VFInitFailureReport{ + VFAddress: vf, + InstanceID: "instance-2", + AssignedAt: "2026-08-20T15:00:00Z", + } + result, err := ReportVFInitFailure(trigger) + require.NoError(t, err) + require.Equal(t, VFReportQuarantined, result.Outcome) + + success, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: trigger.VFAddress, + InstanceID: trigger.InstanceID, + AssignedAt: trigger.AssignedAt, + }) + require.NoError(t, err) + assert.Equal(t, 2, success.Cleared) + assert.True(t, success.Rescinded) + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitSuccessWithoutMatchingFailureClearsNothing(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-1", + AssignedAt: "2026-08-20T15:00:00Z", + }) + require.NoError(t, err) + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "instance-2", + AssignedAt: "2026-08-20T16:00:00Z", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) +} + +func TestReportVFInitSuccessNeverClearsAnotherAssignmentsQuarantine(t *testing.T) { + resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + result, err := ReportVFInitSuccess(VFInitSuccessReport{ + VFAddress: "0000:e3:00.4", + InstanceID: "another-instance", + }) + require.NoError(t, err) + assert.Zero(t, result.Cleared) + assert.False(t, result.Rescinded) + require.Len(t, quarantinedVFs(), 1) +} + +func TestReportVFInitFailureRejectsInvalidAddress(t *testing.T) { + resetVFHealthStore(t) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "not-a-pci-address"}) + require.ErrorContains(t, err, "invalid VF address") + assert.Empty(t, quarantinedVFs()) +} + +func TestReportVFInitFailureRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + + vfHealth.mu.Lock() + _, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + assert.False(t, exists, "a failure whose persist failed must be retried by the next report") +} + +func TestReportVFInitFailureRetriesParentSyncAfterFailure(t *testing.T) { + resetVFHealthStore(t) + parentDir := t.TempDir() + vfHealth.path = filepath.Join(parentDir, "gpu", "vf-health.json") + + parentSyncs := 0 + retrySawPersistErr := false + vfHealth.syncDirFunc = func(path string) error { + if path != parentDir { + return syncDir(path) + } + parentSyncs++ + if parentSyncs == 1 { + return errors.New("injected parent sync failure") + } + if parentSyncs == 2 { + retrySawPersistErr = vfHealth.persistErr != nil + } + return syncDir(path) + } + + report := VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"} + _, err := ReportVFInitFailure(report) + require.ErrorContains(t, err, "sync VF health state parent dir") + assert.True(t, VFHealthStoreUnavailable()) + + result, err := ReportVFInitFailure(report) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.Equal(t, 3, parentSyncs) + assert.True(t, retrySawPersistErr, "retry must sync the parent before clearing the write failure") + assert.False(t, VFHealthStoreUnavailable()) +} + +func TestReportVFInitFailureRetainsRenamedStateAfterSyncFailure(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-1"}) + require.NoError(t, err) + + vfHealth.syncDirFunc = func(path string) error { + if path == filepath.Dir(vfHealth.path) { + return errors.New("injected sync failure") + } + return syncDir(path) + } + _, err = ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-2"}) + require.ErrorContains(t, err, "sync VF health state dir") + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) + require.Len(t, quarantinedVFs(), 1, "memory must retain state already renamed into place") + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.syncDirFunc = syncDir + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5", InstanceID: "other-instance"}) + require.NoError(t, err) + assert.Equal(t, VFReportRecorded, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err = os.ReadFile(path) + require.NoError(t, err) + state = vfHealthFile{} + require.NoError(t, json.Unmarshal(data, &state)) + found := false + for _, record := range state.Records { + if record.VFAddress == vf { + found = true + assert.NotNil(t, record.QuarantinedAt, "a later write must not erase the renamed quarantine") + } + } + require.True(t, found) + + available, quarantined, err := VGPUAvailability(VGPUFrameworkVendorVFIO, []VirtualFunction{{PCIAddress: vf}}) + require.NoError(t, err) + assert.Zero(t, available) + assert.Equal(t, 1, quarantined) +} + +func TestReportRetriesFailedThresholdPersistence(t *testing.T) { + path := resetVFHealthStore(t) + vf := "0000:e3:00.4" + SetVFQuarantineThreshold(3) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: instance}) + require.NoError(t, err) + } + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + vfHealth.path = filepath.Join(blocker, "vf-health.json") + SetVFQuarantineThreshold(2) + assert.True(t, VFHealthStoreUnavailable()) + + vfHealth.path = path + result, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: vf, InstanceID: "instance-3"}) + require.NoError(t, err) + assert.Equal(t, VFReportUnchanged, result.Outcome) + assert.False(t, VFHealthStoreUnavailable()) + + data, err := os.ReadFile(path) + require.NoError(t, err) + var state vfHealthFile + require.NoError(t, json.Unmarshal(data, &state)) + require.Len(t, state.Records, 1) + assert.NotNil(t, state.Records[0].QuarantinedAt) +} + +func TestReportVFInitSuccessRollsBackOnPersistFailure(t *testing.T) { + resetVFHealthStore(t) + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.NoError(t, err) + + blocker := filepath.Join(t.TempDir(), "blocker") + require.NoError(t, os.WriteFile(blocker, nil, 0644)) + goodPath := vfHealth.path + vfHealth.path = filepath.Join(blocker, "vf-health.json") + + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.4", InstanceID: "instance-1"}) + require.Error(t, err) + vfHealth.path = goodPath + + vfHealth.mu.Lock() + record, exists := vfHealth.records["0000:e3:00.4"] + vfHealth.mu.Unlock() + require.True(t, exists, "a clear whose persist failed must be restored in memory") + assert.Len(t, record.Failures, 1) +} + +func TestCheckedAddressesFailsClosedOnInvalidRecord(t *testing.T) { + tests := []struct { + name string + state string + wantErr string + }{ + { + name: "unsupported version", + state: `{"version":2,"records":[]}`, + wantErr: "unsupported version 2", + }, + { + name: "missing records", + state: `{"version":1}`, + wantErr: "expected a records array", + }, + { + name: "invalid address", + state: `{"version":1,"records":[{"vf_address":"not-a-pci-address","quarantined_at":"2026-08-20T00:00:00Z"}]}`, + wantErr: "invalid VF address", + }, + { + name: "neither quarantined nor failed", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4"}]}`, + wantErr: "neither quarantined nor any recorded failures", + }, + { + name: "failure missing report timestamp", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1"}]}]}`, + wantErr: "missing report timestamp", + }, + { + name: "duplicate assignment", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","failures":[{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-20T00:00:00Z"},{"instance_id":"instance-1","assigned_at":"a","reported_at":"2026-08-21T00:00:00Z"}]}]}`, + wantErr: "duplicate failure for assignment", + }, + { + name: "duplicate address", + state: `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"},{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-21T00:00:00Z"}]}`, + wantErr: "duplicate VF address", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := resetVFHealthStore(t) + require.NoError(t, os.WriteFile(path, []byte(tt.state), 0644)) + require.ErrorContains(t, initVFHealth(path), tt.wantErr) + assert.True(t, VFHealthStoreUnavailable()) + assert.Empty(t, quarantinedVFs()) + + _, err := vfHealth.checkedAddresses() + require.Error(t, err) + }) + } +} + +func TestReportVFInitFailureRefusesToClobberUnloadedState(t *testing.T) { + path := resetVFHealthStore(t) + quarantineVF(t, "0000:e3:00.4") + + require.NoError(t, os.WriteFile(path, []byte("not json"), 0644)) + require.Error(t, initVFHealth(path)) + + _, err := ReportVFInitFailure(VFInitFailureReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + _, err = ReportVFInitSuccess(VFInitSuccessReport{VFAddress: "0000:e3:00.5"}) + require.Error(t, err) + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, "not json", string(data), "a failed load must not be overwritten by later reports") + + restored := `{"version":1,"records":[{"vf_address":"0000:e3:00.4","quarantined_at":"2026-08-20T00:00:00Z"}]}` + require.NoError(t, os.WriteFile(path, []byte(restored), 0644)) + quarantineVF(t, "0000:e3:00.5") + records := quarantinedVFs() + require.Len(t, records, 2, "reload must recover the previously persisted quarantine") + assert.Equal(t, "0000:e3:00.4", records[0].VFAddress) +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index ba5df2db0..f2948267f 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1018,6 +1018,9 @@ type GPUProfile struct { // GPUResourceStatus GPU resource status. Null if no GPUs available. type GPUResourceStatus struct { + // AllocatableSlots Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + AllocatableSlots int `json:"allocatable_slots"` + // Devices Physical GPUs (only in passthrough mode) Devices *[]PassthroughDevice `json:"devices,omitempty"` @@ -1027,10 +1030,13 @@ type GPUResourceStatus struct { // Profiles Available vGPU profiles (only in vGPU mode) Profiles *[]GPUProfile `json:"profiles,omitempty"` + // QuarantinedSlots VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + QuarantinedSlots int `json:"quarantined_slots"` + // TotalSlots Total slots (VFs for vGPU, physical GPUs for passthrough) TotalSlots int `json:"total_slots"` - // UsedSlots Slots currently in use + // UsedSlots Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. UsedSlots int `json:"used_slots"` } @@ -18991,249 +18997,252 @@ var swaggerSpec = []string{ "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJhhK", - "kRmVgZF+q3IhJxbf0YrS2MlDBm+6gH8onfpyIbfGudzKIrpl82+2AJvjGWBz7AWTp2MyH+V5SDXSjxwY", - "y7t3J8doA34BbFlIoawSMMZPtp8Nnj3vPRtvP+ntxYPtHt7efdLb2ceDyW70dHd7Z3dFIkyLbLrbJ8gF", - "NeZAHHMRtT5y0fOhoOam3IWabGLjsa8oi/lV5foLBsj6vdvg23XdL4fWtx5CMCEnwVIZ80UDJzuFS55E", - "um0TkG4zNotSS2FD55O3g+3Ptf7A4BruiLciZ8atajAGChdC6g3Y36zqOG/H8mFALvFl3Wr5nbdftMHB", - "/vOD/c9dNJe8sW6MdXK6x81tighzGMy17BCXoejZkZyBsmNlImPVt8kknW6nyHeBv0EYqMVSF49bJXE1", - "HdhumI2sulYakqdPKvoKRKoYDL74QEsqTh+Big5Fir4WgY4SnsfIs8UZSDLww514uotuBtxi1kRnIEZN", - "MobWcQDTGipHUKYZMfgfdSM20/oAvYR34RFOjVpnB2Hql/iuNxwvTLyMPl+ua6NkrR7yudWv4ButbCH9", - "L5i2XgZrsl3dhJHODtCvHL4ptD3G67Zf8zqoWcuv1+3EGxau2yFnQGdW1DxAPxXiZSGgWoF0QxL758gy", - "rBKwZrMCG2B3vKOppdw5LwW+2zEr2ul23EJBqvxy0vy7kuqXzp9PiqFAMoITOMtljnCuaGJhumEmVCoa", - "SZs8oje3SeyxpZVIPDLKU1NMqkk8tQpW8ZGTqt6fog1AYvwLsoZt/a/NIn61ctftPN97/uTpzvMnrfCW", - "ygGuF42PIC16eXBr5eQoy0fWNtI09aOzd8b2ERmrQhH78v7Uh7fIBNesR8/cNeh3/rz/3IeZink+TjzH", - "osWkM6i2sGFBJLWCFzXEQf5OkzmdTNjvH6PLnb8Lmm5fP5E74+0G+FzTUdjsduIHFyzZqMm4Z8okhZGA", - "gKCEbATLekMkzACdE4WAfnoIR6DeFNnMluQcpJZd8SBh7e3u7j57ur/Tiq7s6LyDMwIjXOBStiPwjhi8", - "iTbenJ+jLY/gTJsOUwIQzplVfcPnDNkax4OqQNrfHuyGqKTh4i6pxrY9TxuX/L1VH+2k7KJDUnahWi6d", - "8uBq7+4Onu7tP9tvd4yteXgkrldzGJeyZJbHAvH7O78B0uTbwzMECcETHFVtOy5C7EajUjcaFRSRMODv", - "NxjYs6dP9vd2d7bbob6Fgk4snmHlwFZ5V+DQBYgisBuBpVhmvd2m2yIkThkCe0OiBNP0MHIpFrXbx4C8", - "j4R5rdyENheD1cCXLq4W37YybhUmK5OgY0QDLlDOitIi/fUu2S/iWW3m2uZ6WM/VQ2k5TK+ehScyJdRu", - "sZSZIHPKc/kFGuLK5MxOEs7Fjb5tUljeEJknythsqETvT78DnqJpDUlFsqoOZalxBYjTLSd3o/NcIZEw", - "kTctVqvdaLP1qybcbTi13VWAGhVu0AidFmvOlbP1wZ9HOIlyKKaDi/3UswIMMIAEyLJkYWL7k4RzhqIZ", - "ZuAkER7iEZrxJO4HI2H1k9EkGFXBr1DCDejzJSGZrTNjBqE/0yIMnRO04VdYM6RUq3u6nxomYyuJVKlx", - "Pw0XcMQylKxWpMLr9cSKe3jE5pOKJTThUwlKoYKshX4dBj/DwiQjYGbqJs1To0sGAq4DQ6wx89CNam5S", - "PrEKrhU5INHcrCSOBJcSkYROoUbP+9Na/vKKnLcii3l9QGd1sC1I1zg0A1eZQcNqXV4tdD8G8nk+54YE", - "GoacwRWhks44mWKWQ+UZj5CtIb7fOhxyxqUaFbhUNxysVCMoJ5ELUqLlFVn3hT3IvRO8Fx1ru81y2bjj", - "W329RFXhppoG2MxTgysaXq1uQYMhMl5G5loJBlaii9WhpG4CVlfWH6ASWqUebBnagJwXjy15EHSbbYJk", - "wiqr7mdJW7XVQV/tDc7bwrqtRnE7w2p2wiY8gP1xA8+ps0TbaNWMiJRCQRUUE0ZJ7HTJwoVqTV2QMJ5I", - "guKc2JUz8qnAdsGxOd7gs2LORkbZtMbr6x22MQ+bMayuNgH92hfbhDvJcELtW5HDWpl4RYlwmVrbKgiU", - "ylHYnbXcsCDTPMECWUDGNkOWizSh7LJN63KRjnlCI6Q/qPvFJzxJ+NVIP5I/wFw2W81OfzBqKk10bgZn", - "8wLNhtT6Lafwg57lZi0rGSwxW+b7LXCMtokeC0aK/0QTYtH93jF67RF6FY59b2fQlC3f0GglT34ZGfKm", - "nNuSbPDE5zKQW7hSynFFlUhsMfKN2JPl0tR3aXErORBW5wK8nUenmjjyedAkR4Zf14BJ0JhA3o+b2jLX", - "aMEW20wlWFoilzP0dz6uGkTbhv0GCpZtsBIiQ5BJML4fdnSlQdq8sbQm3u7eBIMC2KqeKHx0Q2iHdaXd", - "yviqJn7yZqnK2YzYJaNujqbiWYsKHi7+o4AvsL22xzGo16MLxCsDSo1UC6jsCuV0Fl6RRYnGXAhAoNYS", - "DmduNgC7omUevdYO9wq9nZEFEiTFlA0ZZYWRFMDUCGJkToSXJcuFVrKmJO6jv3kqHmB2p5laWDB4MJ5/", - "JxG/YsUYh8wfpG48l7qdQ2YsiyLPVKVcpG4WtD5NKJC1DE4wJaB+IlUzNBFEzvy5h2pmahnviou4sRjR", - "ArlXoMYN+FiR4peE+aysaCaoGpqGRuar5Sg+U/AWnlr9E1Vq0KJ6jdnV/eWSiLCQWEypeKVV6Ip3VDzl", - "xIDAACIK1Ba0fxkWX6CgtMA8KZv/q2uy/OmsaLz6W+01D9fEwQwfGrNt0AQbmTSeWrBP1ZO2NlQF0uBW", - "odks+xLQhguhdgVaqpKAVyil1T3ZLhOvnizgRrMlSVTtfe/Z/tMnLSvVfJazzqB3fWnX3Dxd4ZJr2KnT", - "Nn6fZ/vPnj/f3dt/vnMjD4vLK2nYn6bcEn9/0Aa5VvqwJv/6xz/fn9a8PvsQgz240aBMZkl4SA3ZJdUB", - "vT/91z/+6UZ16wGFGM0yQniD374xSifxd9IFClRdeO2cZCv0+8OKkQAXbAZtkMmEgBl0ZNatVw6mBgPS", - "TgrGGY6oWgQYOb4y0e7FKzWk6zbuoOpgQyKvaduiomrOJfNxmXS64TpH/2l8wzVaeNa64JXMx01+6Nf1", - "Xo0XuvRa+DEOLUIMZFFrfdnAXcznCstKQLf+O4K8C5dhtpxtY95YjbpbT4WAKBZb180LBQyhtdfkSfuR", - "v/217fT8lhWzTn3FP6w4h81H8EZW38CNHDD6RutTa2v8wV6At/tqNPZL0a2s9VepW1feujfvt0X28HKd", - "hOIGu3l/XsLkTT6sYwIDPdox2CUv2+5WSKKBmrxcmIABjSekVwTq2UQZJHPjEdRn3sLMBzI4o0s+mVSx", - "bvebsdEB9geSvVwvWCmtmXQRuXY2izqwtsH4GXb25bCjVYBhZzsddmpuq2D6ZIqvR7aDKrbLYBVYeZn+", - "XhukdDMYJzy6NFXWoHh3Hw1QSjCTKGdw+Gtete3Bau9Qt5N5e1NAgxMT4rTEtmBMYzLDcwoVKaxPZVoJ", - "xCTXVEkIGIV2DlDMDdpTpcSsnaF+zSQ3HpSThksHs4VtWDeo3+PMRbSW74KBbwKFbdlHInjXghVojv36", - "9WnXBDBA6KEZWCW+0U3UjEAzyKKLWnmF8vdw/PA4ISMYdx2uP11eRz8nHTyrgkiipMXvLsmhRgQo4jlT", - "dRz/tJ0iV00rW76ScgbBfjb8A3DZbO+GQFBMIjiRcvksVgn9FsRdyxuwKx1KHNgNkTAcCvAlhX3Fb6xD", - "uD4AY2zwqkObdvy4buMlHEnFbTmx4lSPyHVESFwH/Ay/0jZW3n4ZjJV/hS1GUFG42b4N8c7Ls+vfXYIX", - "jLVptf2YfsZZD9BJ3JZaJBEDDWixaqqEVoEe9yAtRiF41dALbTKuyfXqtf6VXCvAR4/zxIDehUnXsip7", - "Ga1b8VtnNjYdaC7I2vJ8d1C2zsSb36pwnQ1Vf4jadfatO6lXt7Q750S5d88tGTXuULXQS8Wl5QL+3SvV", - "GBtDSl1kL3i0nW7WSHBvFraKWFDeljmaDKdklAkyodcriMe8YBTjKqxJeZCKDAaDL7qR4mu09xRFMyxk", - "beyMTmcqWVQDcPYCWEqfVdRREEWYMxS22flyN92Hy9Fudjv91kPC8bkHDbRU0sSKpKNVuNlHpbfNWucz", - "vAArTqOT8Onu3mCwuzO4FXC2G9YNluuo/MSWQKy205RS531nHf2VKFW/hSLJermu7pWgkKtdLJNUguD0", - "ABJvMhwRlJAJgOQVCa3rPYv1rlcP3gpUNou2oH+3UXbfnA++WjKn6MpijrtpdJxzsYpB5D9f4xBtYDPR", - "EqReIOdutzd48nZ792D/ycH29l2AXReL1JTt8fTj9tXTZAdP9pJni6e/b8+eTnfS3aAedklNZaA2tPqL", - "frcxyqa8JKtYRhWWhjbsHDIi6gWT64XGJUkoIz1ZZEitT1NcwQuM/33t+b+Znd/MYKXscF6dpC9CYFUu", - "ToWyHgZ/y05mpe+iPpuT49WzuFUGUn0gYXqrDwXIq91goELFduczkRly1vIaeue92PoiWpkVt+4qCnnY", - "4aQHd7lhxUPkXQNm8Ga96gJfvuQCttMpF1TN0tW3RfFaASMOcdMfpYqreE99dDJlUC3d/7kIk/OVKP1x", - "p9tJPu5Vz4z9vT3yl0UgLgjQbrUvFbQII4Ni/KtXAV4pFQ9hItm1rq7H/MN2b/s5xCEkH/d+GPSeVyMO", - "uma1/OXbdm9Xfh20WUO/BKArHbX9/EYR1249V1HQLzRUwK68ly02saXxsja1uzpcwm1lg8vHS3tcQ/Jp", - "FEA/V9Kzl9vIF5pikuBFCJveM9TKmvboExkakyllso3ddndQGG7302Gnjw4tQDjosooX/fjNQw16j05o", - "mpKYahnTqP7NGQw7LW1xdV3iZrVJ3FcBaa0fFteer4dIWJdwte6a7H9GPu5nab/tNN5V6B1gV3MqKmCI", - "wYtdRCcIs1qBUsrmOKGxTaSHxEiIVztwQG0lyVoeIEs50NlJumjKFSpT6Fva23LWbBcsxk+uwd66AjPD", - "EMTOFwFEKQDE6Cr2dXKMMsHjPCrzRxMYdIn4IfIaRNsKIX99SO5d2jcgMXvCBVpv32gyaLSzTzbtd802", - "qQm2eau3B+u3+k6MIt1OnsXreZh5qR0HuxFy+5oUxICJprrsNUnQm8yHFhz9jb+CyzqvsSVHWiTKM+dg", - "0TS1TEkBdwu4GEJxvcckIfqaWm4E8SQusySoLLnoepa6/eTZrMnFCR6p5YH8QkimdRXAP4L+UswWwYG5", - "sqPFXbIxcGjf0ji8eqZckV2t6uCerpXEGrfKN+E2lVAwXL5m8zZ4KZee+bvA+PZFs2UEFMfwK0Lam+YS", - "APZLF/bWaD++C7PcQwppr63roQbb6kCFC3R0138ZC6zFuirx7oXc8yGyeGs14yYo2noWqG91Puz9j7Ey", - "o1H/YOuHv/zfvQ//GbQ21/RmSUQvJhMINLoki54pPqR19H4ViBUqH2hhempJheAUbEgAcm4Poz/e/UHB", - "NBa/4nRpChCh5VUO2l47ob/8R3N8k7eM74BPriXZzy4MchcFVBV319FGSsTUxZK7RLLN/pBBbtolWUjk", - "1SOzIo0j1O9k8YkXgY4ujBjYJ2x+gcYUCjzKIdNaLY4ikmltwpa4oaZKOQfuIwhO/HZsXTSX+G0dkiae", - "gKD3p0sovq/fvf3x9btfj0evz178engy+uXFf0OIx1XP9BD3NO3t7T+xtcn9ldwO1se4eZmHPjq1YfrW", - "1T/JQaEFnC6J0lzlEBRCrqMkl3TuHIQquX1Bh+Vk3dsXSPhMBGClklBUgkWqTuiEgF8frhMbVEOlI0Yq", - "oai7NW5QhpZvbEM4ww5wUq8mf6icht6K8GqXG1td9CezdizUYKMGDjtkvEL1/YD2QiXgVbjYD+9ltAGZ", - "I67yrEuc3bwZVuth0WAw8vALFxgaPP8SRUDfraz6OedJT6s3DZUSgtZksxbByHloymQkdJqcDtNxQIa3", - "pt0pneKAnyHkT/gixTrdgNZmTC3tf2PVsnAew3G9jIQ5lmapamUPakYCqXrNaQ6plmobgHcBWdjkrlIv", - "tq6aqJoytWWL6obwMmIOYOarspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzNiCDeRsAH", - "JTz/DZfM5uW0QGExRQkzIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1eUXTvF10QO4UrBc8j/CPMryT9sv", - "fwSo/Deu5CWduCZgGDXlLgwMX6WiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLHv2GqfuIC", - "1MFmzJM7x5eHyz8mAjDg6ujxraDXaUriEc/V6vNvK+rbK78oi1qW1XWqLwYijirpvE28wKFylGNYXmm9", - "HCTKBVWLc71eNpgb0iBdLVtYSOgIfi47hvqhnz6B0XgSSBh5SRgRNILqrPo8ppiBxoTen3pF+ky9xiW8", - "VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT3IKK+jZ/", - "2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfQ7kBd6EemFu5a5a9", - "tc0U0owh+4Vkr+3mfgBBGc4OTHNnMLDA5spev5C7YxIGtv5uo0fLfltJdXaJAuj7S2qeky2Lpf/U7ewN", - "tm80plVDgbMb6vgdwzaJl4B2vn/DhbhVpyfMpOXZJGsbDuWfOCAk/6z99kHvmczTFIuFWzB/tTIumwRj", - "IhF27xo9TkkUaVYBNYL66DUj5jnCCmETuSxyBqWV3YeaQqunwLTtNrkAKfqRx4svtoSVPpyN4lOVnenj", - "8mmJnr8c7RRkvLyR9pFD2DZUew8E9CMu6oI/2EnZGzy/+06POJskNFKoVxCwjUemEkJ+EsALd9hDXKDf", - "c64wKsL5H9GRtjLruCC3bnkVbf1B40/meCckZAY/IyLFzCRHmHfWHPql42xcEuVxXnmrOcI/Oe7Ym8qB", - "8JiLCgS56hH1r626MLh8He0FEBhsn2Z68QMS/t49nHA72aI07EMeOSjIiXJJHtNxsi62cSmEBGW5l0R9", - "LTQ/uM8ryxYR+BOeosdCwC9JIeGVu7V0KWxlImdGAQ5KgG/KhEX73XdV4e9t+cSLkgG/hm4aylko41fF", - "8aKP3JoapV8tAGJJEJhnvHytnOnhfS0nbOc+ThjMuPAUfbumvl1Tq065oRY3BTiY3ilvYYO4kQXiz2d/", - "uLH14Zvtob3toZXlgZEra134Ox/3kY1IjXhMkJzxPInRmCCDd+RiTxQW/elHhEU0o3MCoHZQpC1PFM2w", - "gMiSFMVYYeNDbzRMrDRLFM1t6eZ6Lg6xXOA6joUkI8DhGzXhT5YRiJQxEiP9iYXuK+EEl8qJm7MfNLAX", - "DZZXI7qacUkKPD+mvNsc0pul0Y6h2f6QvbVAr3oBIZja8RpJEoCrXWH/4QzhIbMffO9YiAsEkzgtORcW", - "gBlIDTKl2Zbl1DY90pGMeAhr5y1hmKmezEhEJzSy07okCxvPGWywVd0lPWA3zvenRcIG2tkM47UBPGMY", - "nPe4eIYsJVX9NwyCoKMkj0snl4MQwmKMkyRYmGOa8DFORmZ9LknAJ/gS3rCLUjpcSm8S4zExJeSzhZpx", - "Zv7OxzlTufl7LPiVJGLY2ewPGSRi2LUmcbcUENEVFHJLM67PmeCp6XPLDHHrj0uy+NQfssM4pcxRBHyC", - "E8kRuYbvoL4VYGYY7tVAD+Y0hf3gR7lUPPWRTx3dmWHyXGW5shklkqhuCPVzyBRHfzhsx09bf5Q9fgJn", - "McGxphPvFTMlkK2bRi1HWM9+BK8G3O0EFmDY0RepCfOYCsyUge0swCnR1N/SjaI6AlRMra9whBnKeGYq", - "SwBRzbAmuUobgNWAkwQpOEruWy24w042zMdC76XjRtw9A5RWO0aUodMfvcM02HsWPk+SRIKEIkr+6/z1", - "rwhuZb0H5rUyXMukdDAtMKA4B9ep42kvcDRDxlEFxQSHHRoPO4U7N96EsebShsv0euBT/EEP7QfTTZfG", - "P/T7uinjrjxAv/1hWjnQZylLDQ7osPOpi7wHU6pm+bh49iG8oE3wZecVRoA2zDW3CZwEU0Ca8W58c0Vi", - "FiNub4FkgTAqOZAfuDKmDIvFqkTCwNLbFeQTE8noLcYfQ4hcHHYOhi52cdjpDjuEzeE3G+A47HwKr4D1", - "WjZXroP7rHBuFkT0ZDDYXI+Ebdc34LNs4Rj4wjpgo1ZUlN3UO2hhWP9c/oF/a/2zcP1gpjsvoYmM4u+M", - "74/QAeFJ7L4mGnBB1MRuzCKSOLF7vaHn/p0HerMikiT3TaAPRZ6Fe6xA6n9U5AibVR6jleb7B6a4wX1d", - "KhWz/cPQ76Oznwes59Z2TuYu1DlcpwQwaKwqjczLCEt0DmPqnWvl+wX82rf/dbofYCpeJHx6cWBUd5Tw", - "KUoos/kAXqCyFg/sWsJHBoam+M6i0rgicRtGkvjXP/4Jg6Js+q9//NNiu//rH/+E475l4NWgxvTFjGCh", - "xgSriwP0CyFZDyd0TtxkoAosmROxQLsDa/OHR8grdW+lNDlkQ/aGqFwwL2/C1GuTtkHrKtDzoSwn0sL4", - "6BfpxBaTMbGNAbuNO8tmKe/1RHcDcIgwA28C+lZ0NABYctQU2raaaCdsMjVzrhhN62GaS8F66/mLItfK", - "UG/PDPCGDAaWOHTu4IGdNNo4P3+x2UegbRmqgIJBoDuUzVg1ov+NJ63nSYajVBkKrLLhTRHO8Jgm1Jkc", - "G6qdmCOY4mhGGSnjiwuscdfEgRup5jGHZyfIBkJ24dUhe32+BSZWRSKVC9K1nEBYhNGyHBq3eS7QA/Av", - "qiA6rGffHbIJwZAndHJsmIAHwl3kAxYNMwDygBhXqiqV17pDZpBkLXKxPngpj0kCH0H/U6zIFV50UVHr", - "1lVHSbDSCrHs6peHzGC92jXoAVQJ8obZB35mhtRzkbw2Z0uQSaJVY4jAN2W/oe+NCRfIRjh7Vf5ddybJ", - "0gxLL1qKo9fnen5T0AS5sQdCS6/P3W5sdpHkKEooUEOE2ZBNIRDIgfdyVtnVIqFshkXci7i+BHwwp0vG", - "rxIST5t47JFPZHcoyVT6CRynn+vk+tiEi9nyBPQhNgB1qz13x/addq472+KfyXdnC0HewHlnLLjE8Buz", - "ut8ceS0ceeF1c069kGft2CEw3l3Er+nigQJ+He0tr7l54i3ZQ1j00IaDtgGvCBfo7OgE4TgWRMrNf297", - "n56podJS/tP3o2bFDxF6YsfChQX9s/aWKoE8Fnbwxo4aYTeven1d/37bqhTfabzpijo85ZV397dHrdOb", - "XCOl0FvS2rebZG2wLZURhzKDJbX0QDRKSCG+FOfUp6J1VmUTxltcOSvFJcueT47dgbw/+7LtOmf1u+Ee", - "mOJxjSE+ICOsplr7VbMfEzW/K3bRoU2vMD9/XaQ5uD8p6L5N0SEyf0zqYlxbNs0FDdBJ4wX6kigDb3KX", - "errtITDxcyLcqTYDXZhZF9MynyKD0wITAkvMat33xLzSTvU17f2ZNF9YnptILHbJv4koLZTdcq1WKbgn", - "tgT03em30MON1NsvF7ZiCSywyGBFHTu3E1hWN7BcsGjzW+TKF6doE9dYKrHCzZvEhSXboCkVetZ9yXWH", - "zK83rmU6q9dShiYJnc6sEyCmE4jVU379bhjlzj2MsqiTLbAiNkTxMeb9nulFtl7gOREKvT46MevvX6lb", - "f0DQ6npVyTGvlbfruzeveoRFPC6cJ80yqX3yhRUmQ/+VXN77P3WPMJ+VOvGgSWD8jP03weTIxL/3Kf9f", - "Oz8ldCywWPyvnZ9wklFG/tfuYYIVkWrzzohlcF833X0rMI+Y+LT+QquLBqyJTQEydo3AX7zVUuZ37/+p", - "xH4z6RsJ/sW6fpP928j+/nKtFP/tVtypAmD6eCAPV0FsodWGR98gbe7BaGop0oO0qXiRSlCbGZcKHj2+", - "/GYbVE4LivOvjZbW//JArrw+HOmeHHdhIaGiNFS0sOmD9+QLcOO4d+HW9nv/joDDdEynOc+ln5mYYhXN", - "iLRZuwmpMuDHJnaX13Oj4P0VU+ngPq+Oe5erv9H9HUn89Q01zNs49NbJ/O6ttjK/fV/L/AbR1GY227Ib", - "XVeSabMh0NphmrYl4wr063IAeGhcIV0EvdOKSqkuINAgDobsf2v94zdFcPrhB5dCmQ8GO0/gd8LmH35w", - "WZTs1JEKYUpQW0Hv8Ndj8KJOIVAWiuyVCdv1cZia3UB6rqzAv52CVDqS22tIjgq/aUitNCRvuVZrSHYv", - "7lZFqpYmuXcdydFbaMEtpvifU0v6k7tHKhqczCcTGlHCoMALJKbLpXhAo8l984zcMiGZWX+kF0xUkURa", - "q5EF11ojoZc1pe89kOykLKJ139qjK1/9OBOreGbrwVp9rZQWmhW2r40eBvd7e92/ovaYScxoRMtLl2mh", - "O1AZyNSESnOThVZ8WYKK9dHbt69cxpmW+IWrM6W4Ky7l6m4OmV9cqo9elFW7zAuuBS2Rk9hmqEIeni3X", - "FBMcJ5QRCNElMpQcVi0J96DH4ssLleF6d62Eyns+lraC6cMJlQ/GCu5FfDupFIbmpY7vl8IrTosT4eDU", - "PCp+ZRlQgPGExKctnCveszmsWzNugM3C2I5nCY4A2lG/ZlDHLGyAgRn0mwIsAMGThAiDJpflytVAHLJi", - "cJR5Nd5tjZsL3fwoZ4omF10TIQOQIBJhtrCQSkNW6QwrRdJMQWovpK3DCAXJzIhrxR/1oCnPJbwFWbh+", - "lwgnV3ghh8wmA5vPoVCuIJEBXkySPvqZAw4DwlNMmcd4TQXC7+SQXdA4ISMLo3CBqERyxoUijMQo5XMi", - "q/0SLBJKBEziCOuVkyjFC8AzM9COZn14RgxmWAWsget/YxZTqGWney6mfDBkGO0MBiglmEmbei3xBC4c", - "2waCQVQG9D3CaG/w3H5V2zfA3HXLv6FPkxBkziM8ThaIaCoG8Ae1CRuY2tqSpkav3r4JFdLsV2EytEXD", - "KhtLpSuVGHdRzsrkcjCf56zIBdfbpXLBYJ7WsUaoKK5Bi6cxJhHW68l4tR9AMuRRlIvQBam32ity+u8o", - "OHrTO4elCqduJ6CFRySGPWdczeBMczhKm983UFVJVH+OiyZ4SLhAGHl0XRoJSJQDa9wA5L+LsmIfcxV4", - "Lza/d2dHH1/LCNzxN9h7j+V+AiLik0nlAK6/mswBXpUysUzCf9ZzeuRKtfosLqZ4yrhUNHLMsF7Z/ZtC", - "2FohXL2yQWqecHHpy1ZV+v2Ji8u2GpjFE6WPSxHzZ/gV2vb18AC7+eFN/GBgNsqKJpp7V9Lq9FWcUhC6", - "qJIudpijhLOpPkWlofveLfG+VrdhcNj0ZSqM/7hAzdFKyMj+aKq96snYWppgtY9sqw/Ni3Tv9+Df+ZUr", - "RNMsISmBarA9Q2x6s0uEJaicT6WHM3QzXqlPlZ8ObHRBaVz6XScOAV25DdsA6X15u4JMNeHT9Th+RecO", - "tC4A5DdkprI/QRfGm3OBCh6sBVqDmo+uZjSaAagf6K26fYP5h7PsosAz3jxAL+Eg+7DO0PmGwcrXtCZ5", - "QgxW3zxNLw6W652+Pz2Fjwyen6lsenGAXI3T4v6Q+i0fpE/PIsFSoV8t9OBGoYzDjl4orPXNYn6bFr6v", - "xJseshCUHyNXtkE6QRceqt9FA+SU47ev+PTBhLFuc5UAMxfFkVUdgTYJiztNcRM0CQP6bQ8GIfDqluCC", - "Zhh3jC24NJhXfFpUKKiQMs6ytuRrhwlUPE/TFTSMNjykMqlinqu/SBUTIeBjS91NxI02cGSrU+FLTagW", - "l84d7E0gv2B0kIEMDy6VZqqdboewPO0c/Gb/NU/TTrdjx+NBjd9AuF8D0lhvcDmKRe+Mh8T4TSy/CcZi", - "ldl7IIu1m8Oq080S+Rvzwp/eW+hsdg9IhiAf1Iy4X5MI6o23avBhvACLhJE9v4+RAaIkihIuScXB83jw", - "qKyhqyYzNhuK3Br39PDi3BXwaRMUcm4/PXdffgW697pYETdm5KZ770EjyyN4zLm1cmk2Ey7qIEbrokm+", - "ekL6cluyNNU2FPKNNm9uZWxFmFpPWGYR9oPYFHTDueIpVjSCYkLRjHPpkX2BOGzKflnjcUGZYFoxWq4N", - "yr/QpHphzdAXVo04sCYzhP1Hto8+fG5D+cNfuEflFz95VoGC43ed6A+A+1DtXFAyQRnOJdFSXZ4SFC0i", - "zRVN9SiCoxmKcKZyQaAwHkEpZTTNUx9KWu/YHAPsxcV2etFF41yhBIspaGXmoQu2iXiaEhYTsM8N2Yzg", - "OdUqpUAJVoRFi54kUFB3TtAVF5cJxzGYGLIYg6cHCvIJoikQcLlTonCMFQZB50Kf+JHJC7ooauwatZ6R", - "65Ia4iETOfveFAnQzV64gV4gAijYVM6KWowRjgmLgujQ5183G/vytuhzouoTfaDIoFvx0ocMFfJtrm44", - "X0cU0SMLb+bCbmMbNr9C6JXNKmw1ocKR0b/nkTZzdXN8IAdTscSrTvHX4VkqiO6r8S49vPuICxTnpjvv", - "VAKZ/1l9QgVD8YOtIFnTbONtHUNF0blimW/E87b+cH+e3MKW95Vwwm6jYt9U3qic9NfAcu2q3ornPpAR", - "09qSfJvcw7FgF9H1YOITFx6XeyzGVsuwzdEs+LbPnZTAoH1x9o1t19m2DXi4Ldt2ttkll77HyCnrQYxo", - "mINbM24jq7amg3/TbJTa7DyW+eAssvRc3BtbPCkYoWGNGV4kHMd/hiDhFf6jiAthECUAo+IxIZp6VkM/", - "PQBsc2XdtK7L1nx/errZxCWEWskjhHrEHMJLydGfpfGyAff1nAhBYwv8iY5Oj224LpVI5KyPXqdUIcXR", - "JSFZmdECWYV9PT+HrbFco70CotHtEKbEIuOUqbWjKF+9m8F8ulVl93vmkxYi+5s7vLU7HCz7j4+dAZeB", - "nA0zgdWaqcJqbelOyiZcpEYuw2Oe69Y1D9LLpPfTlMCc0ITIhVQkNVGJkzyB4wblFmxJXfud2eUuxOTq", - "k2PS5TIiUiol5UwOmc0VyYjQfevPdftegFXQIaBwwV/PDJP8OoL39GBMvBpWTasGKEhQqrNz0NnCWbYV", - "Y4UbAsTs8D5jSD9BNB6Si3TMExqhhLJLiTYSemnUEzSXKNF/bK4M5xvBd1+6YPDtT5Ze6RM24cFybIZm", - "C2L+U2V1WbbmHJOPjq29JP5hcfwHNjrM1taXJBYEJz0o8euwcFCuaEI/GlanG6FS0cikHOFi7d6fFky1", - "P2SnRAn9DobUtiQxiAagXW5lgkdbw3ww2I0yCoBquwQGBwyv+XEKPR6dvTNpqCTlYtEdMv0PaPjt4Znx", - "7k6wtSZ4A7W1iNHJ1us1Ac7nsEz/xhGCZoIr0QuCG/7NJXhzjJHGMyQbjijPVqlKPPvTh7BaCe6bXeFx", - "2hUA5KmYzcZU4AiEYjnLVcyvWNiGMOdJnup/mD9O1kGFKRzN3sOrX420a4azths3wUdxKO2cYmLKRT6I", - "08Ms2GONWdUL56YAQkwlGjB4CxyqPyN1f3nzvb+OX6G7066oK8X61Zyt+7757Bgcwoa/Ho/lmBtKczNR", - "fLX16QrTZuvTjwmPLqWFYvHNhlpvA8hy/WMJMW1dhCAmQGYoshBGBiiLyO6Q1QyQBvFHIowUESllONmC", - "OZtGACzbWbHwnFNI0I4gT6UnaQyYSQkgYgP8nZ4NGKpcA55HV9piVf47vjNScTQmEU+JAxDfDKluf8NU", - "/cRFFQ38a+GLb731B0hATMHevgYAvbnHzwJEP8XXECod59ah7Ea08ZKXPxpTUBfB3gw7uwM57HTRsLOT", - "Djt6B44wmFCxQvsopSxXRPbRsbFvQQrukwGSJOIslg7H3FnwdgeyKSHXkGVDducT+O4+xR5LVbCUb2wn", - "Ifag30P6e0jaQRv+gbNnMu7CoYsRz5Ux99tzZd+KiQLzyOa9+2q9M/JNt2/Dyf9mj2+FR8Eua3bpbb3h", - "7FkuZ6TZ5PbK1AbK1RjwsV29TjlDf+dj2UWMXBlruJCqv8T39NdnpoP7wO7XXd0Et9/O/RtofwvQ/nKt", - "wmCNJsBSX8mOOgxiI7nOuFCA4mhz7Q0NgSYByBE8wgl6fXQyZJFmRQZaUJCUA3eyEOPmFj782zl6cfSm", - "i46hdiT6OR9v9tFrlixcBW/joxkyI4kZ5hVhhsaGakkcup7N2IF67jJYXHfwQMWYzckIeFbcXrkg8W5n", - "RnAMEskfnVfcdBZAHX7zSh8gAP41Xxbb3lkpfHTeECUWvcOJImK52VObJ8UKzAx7STsIOiu4GeBL3aF0", - "yGtln0Y2MNAYuzudAFLGp291FO6+5uj9eMlMnIipYDfOAWmUQZIBjhePK5ZJzlDBHEMs0L+ui0oETVnC", - "lpetVDCgy6bI76/I5L6Sd1Ww5f9dTxfM9NE6mrLKPmkiLiqYrPX0uuTgmYFDto6qCGc4omrRRThJ7B1l", - "b4IiIqVXiL9jQfBlzK9Yf8jeFLVTbEIvOjp713WOWhRTeWlasL7YPno9J0Lm42JwCA6a8RrDmpN4yBRH", - "EU6iPNHiBplMSAS5uFASRTb4couhdO7w7JSdBOu3eFHt+aMrGxemCdi9kizqFLdltnpLkCjBNG0GH7eC", - "GgQcQqjBWDfKGaJsktiQqkhwKZFtqkcSOqXjxAYIyT56OyNI4pQMWZZgxohAuTRR8XrovUwQKXOT4K0b", - "AJBeQ1FdVAILZoIrG5qQcC6kiSbQFP7+FElFshVk9sa0fApzviPZ1jRue3ogI3VtDM2mEPsK0htiKMUs", - "uKajPHEBjPcaim4G9NBS4mM5+G8FnU6J0KcCGyZrwvHMsXbLaQ59JWO5sYTkefFWuxKSRateVqKXsbcS", - "GG5UYm3HnZtF/QU6v6SN2IH20c2yiH/RH7Xsu5qtGh6EffSZs/yzVOY/95IE2xqwSgp/bOYkb+SVo1pJ", - "tF0Pq9U6s/YuM11b42c9GGzWY0bLwpX02SaF9+sjhMH9ojzcd5G1x01bFbSrim7akPK/Hk3/q6DAu4HR", - "f2CUk1vA6H9VefeAc/5w+CfBg/pQefQV37OrX/unR8K/q/R5A4cPcGxN6fOG69ng1ZWK0nv7Tjs1ybb4", - "Z5LgbbzjDeR3t+zftP4WKoO3WOtc0JrgSZqphQtos77KMuhM0o+k3+AILuJW784VfIuQzi9HHo5OGwM6", - "/5zl5h8kZtSWDqQSnRwH6rg/MoxB/8xVLpYtfev0sIhmdE6aje7VE2yXKBOkl/EMnCuxWTC7Hu4uU1j0", - "px+Rbd5irtp/Qe1JgOonMYqpIJFKFqYOqOYIpo/vJBJcawLwnItFc5SIOSI/CZ4e2tmsuQ/tmbLGsDLO", - "MF30Yqxwb+64zQoT2mdEd7p4Ss3wEGXo5Y9og1wrYSpcoInWfBCdFEtqCvhLoMlNf8DbgwbLJv1IRtNx", - "m1GuqFXy2taCQVEuFU/d3p8cow2ofTYlTO+FFvUnIMlmgs9pTOLKGDtznphV3W5Y0JvaXbVQURSuc8qF", - "GdyDyDBtLqTpR5pV2UIREjOmDMPg1lYFqZ4pk8Sv+8OUuQAcu0duFN+uMKv5bThlR1Mi1OG0i6g4NxDP", - "m9+uucd8zfnJUO5Oq9x2LjxntfG6XX5Uy7Sluyj8UOTO3a/Z+v3Xk9JD5aPM5rGm83mhkDaZzb8uEhzc", - "3/1w3+by9484BfQlccq3ZyqHBnSLIYJ5BTHdMZmThGcp1EOHdzvdTi6SzkFnplR2sLUFsd8zLtXB3vOn", - "u51PHz79/wEAAP//zu89IfTvAQA=", + "LWTAlvvo16Kwll5qWaIq9QOWwho9mQBq/cJIJjwUjfKTIATBs7Kgpz574ME2GL1F1ACOAaOJM1dWEG1A", + "lFdMJPo9xwIzRRmJ0fuf5PdoYF3m739CJovGMiAq/cDGqqb4NLSljdlwZ7OFpBFOzLKY0h6U+VZKONKt", + "1Yyz8kNrzw0oG+Gico59oI35NMthAc/f9E5ev99KYzLvVsYE0YgznhA97k2Px84dJEOZuldhrfMmc5Gh", + "btmW7XhrVfC91ovkcbnA6nhU0ERw73+qEou5eEyqqjXXUkYVQG8B3vJGMUiDEN1Hp3hhtfoMOLjpykNA", + "xZOJAaAsOJsgCcESqlFL9P6n/towQcUVTprm8FY/tKdmQ09I76keZhdlFaKEk+QhP/ndPgkysHI+AacH", + "dFi3oPfRCQsfQhOLAthtBoZJSjplWtOQNhwlwqxYyaW9q3LyoJGijnXXqS5cZTrdADsKUUyId5rcumWh", + "e7k6Jr+sHhp+ub4io2mkud8jl/5X87JYzcbxN8gSdHExyHxq3CjWVidJhgVWJFnUAnSrCPtkOcSBXJPo", + "BvmHL/Trn0xVlVyQkZoJImc8qQbG7HaXK/NKCEKfE1uMzMzJ8wQpjlIsLuGUOc0O5cysQDWHYXcd4NBM", + "qewGk/r57dszY+5RRMxxUs+CkUshH8ckwQs0JuqKEOamgiXCfhB0PYtYNhSEEmqUEUF5dQ07u4F+z01g", + "PJoKHBFkvnJ1vQu2po9e26W0vQQgLKOISNmwv9ur9td+OsmTdnscGtb22jL40U02+O3RmStsVBSOdsu8", + "s7zKZ0T0zJFzFaRXb+2OXF1iy3XFDDLxkgQ5JlBzy6Ye+ZnBLuANaorpzyvZuB5zkH7+qO0HToFZqq45", + "5x9aqRv14x6K7Egxi0MFuE3GicFqmAKAG4S9ixx0ARqbaDhzJ3v3s02cEQTHlBEpa4nuUS6STrfTm9hZ", + "HWxtaX6fAIbl3u72s63VccUrA8pt/NwopqsMDi7KzsRhubRqAwAIk66SxBbOshYmWbOOa+4HYE/LAaxQ", + "dFvfbZ7I7zIaBoOlnP1rHClXJhBstBUfPPaPLSBNVwUZ3WBqKv9o3eu5fz4HwaCIDKtZlfy3lmgfAqUg", + "yFXTiLF/1dbREPnHoHTKRagWHBfKpoCOiYuCLe5DF2PqMJUr3trBM3+WT/b3d/fX8SFgNrVjbs9dYKrm", + "7WqSO/GW255f2wDE61VlDnekV5ez1euyhqY0R1wiqdULyjPCbrSe+3u7Ozdbz7YTOXFxgjW+FMIIOjo9", + "NjKR1iwxZUSglCgcY4WrTAaMm5rLQLEhTFLIE5t8v5q1NATU+KA/t62U9qXCMRqKJr5xGOEpZnQCSpJ5", + "0+9ZzvDO/pMDU9o1JpO9/Sf9fv+mUCgvSuyTVluxZaI2PVSUvpx93j7cAeJJm7n80Tk7fPuzZmS5FObS", + "2pJjyg68fxf/LB/AH+afY8rCSCltqgHTyVIV4GqAYm6hqEl8gMqC707uaRMw1uCdgHB1gGcK4g5Wwnbv", + "DmCwoHFarSNxg2TxFcnTWlx5zZJF49rcurBvWfJeeQV9fbtCi+K+9OPqgAtn/4R3bJ/G0lHUPV4OtbhV", + "5Wq5srjnUu23jLCinGeSmL8izgDpOVTbs3JFumctSsPBNWJrwBVd+j8WvXs/HvkD8X53peW8n2yRzw83", + "jJFaKZD+bVkOXc+FnDi65jCHjdHFrdC2oLIFJgwmBzzwXXibOMJq76+n//X7/5FnT/++/fur9+//e/7y", + "v45/pf/9Pjl7/VmgN6sxKR8UWPKLYUlC8FwFULItKZ1iFQVsdFr9a1hh+8RYHFQ0gxKwaEwOhqyHXlFF", + "hCkoWMuGHXbQBgFNCb7S4i5USzKJiJv64zPj4tYf/+HE4E/1NmKLUiDshhTgMzIfxzzFlG0O2ZDZtpCb", + "iAS9QP8VowhnpiogZUjrvws0FlDC0focy8676A+cZZ82hwyssuTawKtnGKrgTYo0QOYiBuyoTFy0fZ0U", + "OCQmRXXIitu6AGU0jud+WcGBkqSeRNawKKv1N6s5PRuE4CshwUlvJFQpAhWkoGxNRkXmFXo22FzW59bo", + "GAUNrSA/G5lhMmAP85C5uClr9pjENAK+4hJHZza1uMjZNZRmjXiZ4NcL2Js3JpsxRjhXM82LIou0EHF+", + "SUkXtrQL/lEIBYIvTYDHjGe98aI341mBuoGFCX/CJkSiqmT/n56daO89EXRiewqCJ2gSCQidcGTszEwO", + "amFdWJrYW1PghWnRZ07s66ZSkDTFYUxsv8oFc/VLCFQxBXiXgvpISCb/HkUJBauTnPE8idEMEBiVbiYE", + "otgZFDlUeBzFZFL/dzXGZWf/CWiw7t+7O61TmM3SraKyPAnotKljfS04tmGTMAAjHoycIXxNVJq+Aa1j", + "H+wUisN/z5FrqDxxBSMxnj6TVCltOYxEeumUm8G8N3sMLIbMCNvT1OY+WjqFlTS6Fi2YcBX4LGmBXvPC", + "pO++fXWOFBGpA1TYiPTuwCkxUCU9KmVuK7MdHp2+2Ox3gshbFZcWbNXKNLvqoAPgGzZ8pSkqp7TR4JR0", + "0ckxpE/ba6XUxSDf5ScuUGJuxfIyOgD0lqq5B5uajifHVgBNFmUMjBFbhp1N12JWv94O0JtCBcTFUIpE", + "2JK2XJPlZQLN2ohIk4yz1Hotbxr8Y1b9s/cxpN5AwUvDiwE9tPH+am9zdFhk+qKqWchufCH5YUmN9i9v", + "7780zvaXl9F3byajWy/0KJthGaLume/VhJeW9t13ZFfZvWgOIqv0O5I0eLb+5sr+eNeQIvqeq3weApbd", + "721vv93eu7n57qYQyVVsNA83sUBJbg9vfBcwwQHQX6pGjdkGSD+2uQXOLvL+FM2wZN8peFizjmzvPm1j", + "lIBe28bp+xH6fGKGVHApB7RWxJcbyLlLmiRGgJF0ynCCnqON85OXv5y8erWJeuj169P6Vqz6Irg/t0BL", + "hlsA1tGkHAawtioYEahIJn379hUcroRAPo6Rwy9vj6G81rTYAlPZDe7l2Ttw/GM5cpG8zcmruEwAJ9dU", + "KrkMs9cqIP5zMJzNp6VhrM0kTRs2xG8tEPTPFaTiIG7i5h0gOLtshqXlfABw44fMGv36gJVXQiF/Lp6x", + "tTPcEZxx45UWggKuQWTsN91utwcmvpPhVBCGQmzLl3BcSv+tkYC7HRpIZz60cXzo5Kws/VU6I1zztTk9", + "3+lvP3kG1Wu3B20Ye4qjFX2fHh6173ywY26ZAzw+iOIDUNhv67OyhG1UEJxc4QVUfzRLO+yYC9PTbr1j", + "axXJVvE1y4DLt8NXrotxDQjKIM66wCU5SlcWn2mRr1pH0UtzixGa0iShkkScxbIqI8+wRDIz0LimCEsh", + "wQ8ZDLCLilrYIKUgHEUiL02PVrq28n6eWbqHQrAZZ1oHgEoQv5CFRCkFJ2jRPYQ+SlSkRcVDtiFcCl2R", + "Kwc1YGP9AySkdG2qQ9yFqGEoNqM/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF1VhohilpTMSQ", + "6dcC4Lt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtuLRJAgf4R2ZhpgFXUuQM5SwmoigITww51EPkbug4", + "/UxUafd5O/HKfl7KVeGDuQ6Euh269OdC+sJQG/RziA69hXK+f3sRvVUSmpNfbfqZ/Wp0kwgGgiKeJ7HW", + "+Mb6tjMGORJbM6QkynDnIhPknSnGWp26DT1WHP2eE7FA709PK2EPgkw0D2g3ceASDfvAsxttw84aG8na", + "0dzEvewBIN8H6HFdUvEkxC8Ocex7GF1WvKHQimGrojivsq9prTKYkUOZ2SdNNCsmWKvHazBEysBIv1W5", + "kBML+GlFaZdTYQHICzyQ0qkvF3JrnMutLKJbNpdpC8BangFYy14wmz4m81Geh1Qj/cih87x7d3KMNuAX", + "ABs2CTKV7jF+sv1s8Ox579l4+0lvLx5s9/D27pPezj4eTHajp7vbO7srkopapFfePmMyqDEH4piLqPWR", + "i54PBTU35S7UZBMbj31FWcyvKtdfMEDW790G367rfjm0vvUQgilBCZbKmC8aONkpXPIk0m2bgHSbwlvU", + "3gobOp+8HWx/rvUHBtdwR7wVOTNuVQM6UbgQUm/A/mZVx3k7lg8Dcokv61bL77z9og0O9p8f7H/uornk", + "jXVjrJPTPW5uU0SYA+WuZYe4lFXPjuQMlB0rExmrvk0m6XQ7Rb4L/A3CQC2WunjcKomr6cB2w2xk1bXS", + "kE1/UtFXIFLFgDLGB1pScfoIlPgoMBu0CHSU8DxGni3OYNSBH+7E0110M+AWsyY6k0ZrkjEgM5JKm89H", + "mWbE4H/UjdjU+wP0Et6FRzg1ap0dhClo47vecLww8TL6fLmujZK1esjnVr+Cb7SyhfS/YNp6GazJdnUT", + "Rjo7QL9y+KbQ9hiv237N66BmLb9etxNvWPx2B6UCnVlR8wD9VIiXhYBqBdINSeyfI8uwSgSjzQqOhN3x", + "jqaWcuc8TIRux6xop9txCwXYCcsoCu9Kql86fz4phgLJCE7gLJdJ47miicVth5lQqWgkbfKI3twmscfm", + "ZpJ4ZJSnpphUk/pqFaziIydVvT9FGwDN+RdkDdv6X5tF/Grlrtt5vvf8ydOd509aAXCVA1wvGh9Bnvzy", + "4NbKyVGWj6xtpGnqR2fvjO0jMlaFIvbl/amPd5IJrlmPnrlr0O/8ef+5jzsW83yceI5FC1JoYI5hw4LQ", + "egUvaoiD/J0mczqZsN8/Rpc7fxc03b5+InfG2w14yqajsNntxA8uWLJRk3HP1M0KQ0MBQQnZiJ72hkiY", + "ATonCgH99BCOQL0p8qktyTmMNbviQcLa293dffZ0f6cVXdnReQdnBEa4wKVsR+AdMXgTbbw5P0dbHsGZ", + "Nh3ICEDeM6v6hs8ZskWvB1WBtL892A1RScPFXVKNbXueNi75e6s+2knZRYfk7UK1XDrlwdXe3R083dt/", + "tt/uGFvz8Ehcr+YwLmXJLI+tzODv/AZIk28PzxAkBE9wVLXtuAixG41K3WhUUFXEVAO4wcCePX2yv7e7", + "s90OBjAUdGIBLisHtsq7AocuQBSB3QgsxTLr7TbdFiFxyhDYGxIlmKaHkUuxqN0+BvV/JMxr5Sa0uRis", + "Br50cbX4tpVxqzBZmQQdIxpwgXJW1Jrpr3fJfhHPajPXNtfDeq4eSsthevUsXpWpqXeLpcwEmVOeyy/Q", + "EFcmZ3aScC5u9G2TwvKGyDxRxmZDJXp/+h3wFE1rSCqSVXUoS40rUL1uObkbnecKiYSJvGmxWu1Gm61f", + "NeFuw6ntrgLUqHCDRiy9WHOunK0P/jzCSZRDdSVc7KeeFYDCASRAliULE9ufJJwzFM0wAyeJ8CCw0Iwn", + "cT8YCaufjCbBqAp+hRJuUMAvCcls4SEzCP2ZFmHonKANv+SeIaVaIdz91DAZW1qmSo37abiiJ5ahZLUi", + "FV6vJ1bcA6g2n1QsoQmfSlAKFWQt9Ot1ETIsTDICZqaQ1jw1umQg4DowxBozD92o5iblE6vgWpEDEs3N", + "SuJIcOmBU70/reUvr8h5K7KY1wd0VgfbgnSNQzNwlRl4tNb19kL3YyCf53NuSKBhyBlcESrpjJMpZjmU", + "IvII2Rri+63DIWdcqlEB8nXDwUo1gvoiuSAlfGKRdV/Yg9w7wXvRsbbbLJeNO77V10tUFW6qaYDNPDW4", + "ouHV6hY0GCLjZZSzlcBqJdxcHczqJuiFZUEKKqFV6uHYoQ3IefHYkodJuNkmSCassup+lrRVWy721d7g", + "vC3O32pYvzOsZidswgPYHzfwnDpLtI1WzYhwUHsxYZTETpcsXKjW1AUJ44kkKM6JXTkjnwpsFxyb4w0+", + "K+ZsZJRNa7y+3mEb87AZw+ryI9CvfbFNuJMMJ9S+FTmslYlXlAiXqbWtgkCpHIXdWcsNCzLNEyyQRehs", + "M2S5SBPKLtu0LhfpmCc0QvqDul98wpOEX430I/kDzGWz1ez0B6OmWlXnZnA2L9BsSK3fcgo/6Flu1rKS", + "wRKzZb7fAsdom+ixYKT4TzQhFinxHaPXHqFX8fn3dgZN2fINjVby5JehQm/KuS3JBk98LgO5hSulHFdl", + "i8S2aIIRe7JcmoI/LW4lh8rrXIC38+hUE0c+D5rkyPDrGjAJGhPI+3FTW+YaLdhim6kEa43kcob+zsdV", + "g2jbsN9ABbsNVkJkCDIJxvfDjq40SJs3ltbE292bYFAAW9UThY9uCO2wrtZfGV/VxE/eLJW9mxG7ZNTN", + "0ZTAa1HSxcV/FPAFttf2OAb1AoWBeGVAqZFqAaV+ob7Swqu6KdGYCwGQ5FrC4czNBmBXtMyj19rhXqG3", + "M7JAgqSYsiGjrDCSApgaQYzMifCyZLnQStaUxH30N0/FAxD3NFMLWx0AjOffScSvWDHGIfMHqRvPpW7n", + "kBnLosgzVakfqpsFrU8TCmQtgxNMCSioSdUMTQSRM3/uoSKqWsa74iJurE61QO4VKHoEPlak+CVhPisr", + "mgmqhqahkflqOYrPVECGp1b/RJWixKhedHh1f7kkIiwkFlMqXmkVuuIdFU85MSAwgIgCxSbtX4bFFygo", + "LTBPyub/6posfzorGq/+VnvNwzVxuNOHxmwbNMFGJo2nFuxT9aStDVWBNLhVaDbLvgS04UKoXcWeqiTg", + "Vc5pdU+2y8SrJwu40WxJElV733u2//RJy9JFn+WsM+hdX9o1N09XuOQaduq0jd/n2f6z58939/af79zI", + "w+LyShr2pym3xN8ftEGulT6syb/+8c/3pzWvzz7EYA9uNCiTWRIeUkN2SXVA70//9Y9/ulHdekAhRrMM", + "Gd/gt2+M0kn8nXSBAlUXXjsn2Qr9/rBiJMAFm0EbBKC46ZyMzLr1ysHUYEDaScE4wxFViwAjx1cm2r14", + "pYa13cYdVB1sSOQ1bVtUVM25ZD4uk043XOfoP41vuEYLz1pXQJP5uMkP/breq/FCl14LP8ahRYiBLIrv", + "Lxu4i/lcYVkJ6NZ/R5B34TLMlrNtzBurUXfrqRAQxWIL/XmhgCHk+5o8aT/yt7+2nZ7fsmLWqa/4hxXn", + "sPkI3sjqG7iRA0bfaH1qbY0/2Avwdl+Nxn5twpXFHyuFDMtb9+b9tsgeXi6cUdxgN+/PS5i8yYd1TGCg", + "RzsGu+Rl290KSTRQk5cLEzCg8YT0vOIFpvCTzI1HUJ95CzMfyOCMLvlkUsW63W/GRgfYH0j2cr1gpbRm", + "0kXk2tks6sDaBuNn2NmXw45WAYad7XTYqbmtgumTKb4e2Q6q2C6DVWDlZfp7bZDSzWCc8OjSlN2Dau59", + "NEApwUyinMHhr3nVtgervUPdTubtTQENTkyI0xLbgjGNyQzPKVT3sD6VaSUQk1xTJSFgFNo5QDE3aE+V", + "msN2hvo1k9x4UE4aLh3MFrZh3aB+jzMX0Vq+Cwa+CVQ6Zh+J4F0LVqA59uvXp10TwAChh2ZglfhGN1Ez", + "As0giy5q5RXK38Pxw+OEjGDcdbj+dHkd/Zx08KwKIomSFr+7JIcaEaCI50zVcfzTdopcNa1s+UrKGQT7", + "2fAPwGWzvRsCQTGJ4ETK5bNYJfRbEHctb8CudChxYDdEwnAowJcU9hW/sQ7h+gCMscErF27a8eO6jZdw", + "JBW39eWKUz0i1xEhcR3wM/xK21h5+2UwVv4VthhBRSVv+zbEOy/Prn93CV4w1qbV9mP6GWc9QCdxW2qR", + "RAw0oMWqqRJaBXrcg7QYheBVQy+0ybgm16vX+ldyrQAfPc4TA3oXJl3LquxltG7Fb53Z2HSguSBr6zXe", + "QR1DE29+q0qGNlT9IYoZ2rfupIDh0u6cE+XePbdk1LhD1UIvFZeWC/h3r1RjbAwpdZG94NF2ulkjwb1Z", + "2CpiQXlb5mgynJJRJsiEXq8gHvOCUYyrsCblQSoyGAy+6EaKr9HeUxTNsJC1sTM6nalkUQ3A2QtgKX1W", + "lU9BFGHOUNhm58vddB8uR7vZ7fRbDwnH5x400FJJEyuSjlbhZh+V3jZrnc/wAqw4jU7Cp7t7g8HuzuBW", + "wNluWDdYrqPyE1sTs9pOU0qd95119FeiVP0WiiTr5ULLV4JCrnaxTFIJgtMDSLzJcERQQiYAklcktK73", + "LNa7Xj14K1DZLNqC/t1G2X1zPvhqyZyiK4s57qbRcc7FKgaR/3yNQ7SBzURLkHqBnLvd3uDJ2+3dg/0n", + "B9vbdwF2XSxSU7bH04/bV0+THTzZS54tnv6+PXs63Ul3g3rYJTWVgdrQ6i/63cYom/KSrGIZVVga2rBz", + "yIioV9CuV56XJKGM9GSRIbU+TXEFLzD+97Xn/2Z2fjODlbLDeXWSvgiBVbk4Fcp6GPwtO5mVvov6bE6O", + "V8/iVhlI9YGE6a0+FCCvdoOBChXbnc9EZshZy2vonfdi64toZVbcuqso5GGHkx7c5YYVD5F3DZjBm/Wq", + "C3z5kgvYTqdcUDVLV98WxWsFjDjETX+UKq7iPfXRyZRB+Xz/5yJMzlei9Medbif5uFc9M/b39shfFoG4", + "IEC71b5U0CKMLCFzkqxeBXilVDyEiWTXuroe8w/bve3nEIeQfNz7YdB7Xo046JrV8pdv271d+XXQZg39", + "EoCudNT28xtFXLv1XEVBv9BQAbvyXrbYxJbGy2Ll7upwCbeVDS4fL+1xDcmnUQD9XEnPXm4jX2iKSYIX", + "IWx6z1Ara9qjT2RoTKaUyTZ2291BYbjdT4edPjq0AOGgyype9OM3r2nFpxOapiSmWsY0qn9zBsNOS1tc", + "XZe4WW0S91VAWuuHxbXn6yES1iVcrbsm+5+Rj/tZ2m87jXcVegfY1ZyKChhi8GIX0QnCrFaglLI5Tmhs", + "E+khMRLi1Q4cUFtJspYHyFIOdHaSLppyhcoU+pb2tpw12wWL8ZNrsLeuwMwwBLHzRQBRCgAxuop9nRyj", + "TPA4j8r80QQGXSJ+iLwG0bZCyF8fknuX9g1IzJ5wgdbbN5oMGu3sk037XbNNaoJt3urtwfqtvhOjSLeT", + "Z/F6HmZeasfBboTcviYFMWCiqS57TRL0JvOhBUd/46/gss5rbMmRFonyzDlYNE0tU1LA3QIuhlBc7zFJ", + "iL6mlhtBPInLLAkqSy66nqVuP3k2a3JxgkdqeSC/EJJpXQXwj6C/FLNFcGCu7Ghxl2wMHNq3NA6vnilX", + "ZFerOrinayWxxq3yTbhNJRQMl6/ZvA1eyqVn/i4wvn3RbBkBxTH8ipD2prkEgP3Shb012o/vwiz3kELa", + "a+t6qMG2OlDhAh3d9V/GAmuxrkq8eyH3fIgs3lrNuAmKtp4F6ludD3v/Y6zMaNQ/2PrhL/9378N/Bq3N", + "Nb1ZEtGLyQQCjS7JomeKD2kdvV8FYoXKB1qYnlpSITgFGxKAnNvD6I93f1AwjcWvOF2aAkRoeZWDttdO", + "6C//0Rzf5C3jO+CTa0n2swuD3EUBVcXddbSREjF1seQukWyzP2SQm3ZJFhJ59cisSOMI9TtZfOJFoKML", + "Iwb2CZtfoDGFAo9yyLRWi6OIZFqbsCVuqKlSzoH7CIITvx1bF80lfluHpIknIOj96RKK7+t3b398/e7X", + "49Hrsxe/Hp6Mfnnx3xDicdUzPcQ9TXt7+09sbXJ/JbeD9TFuXuahj05tmL519U9yUGgBp0uiNFc5BIWQ", + "6yjJJZ07B6FKbl/QYTlZ9/YFEj4TAVipJBSVYJGqEzoh4NeH68QG1VDpiJFKKOpujRuUoeUb2xDOsAOc", + "1KvJHyqnobcivNrlxlYX/cmsHQs12KiBww4Zr1B9P6C9UAl4FS72w3sZbUDmiKs86xJnN2+G1XpYNBiM", + "PPzCBYYGz79EEdB3K6t+znnS0+pNQ6WEoDXZrEUwch6aMhkJnSanw3QckOGtaXdKpzjgZwj5E75IsU43", + "oLUZU0v731i1LJzHcFwvI2GOpVmqWtmDmpFAql5zmkOqpdoG4F1AFja5q9SLrasmqqZMbdmiuiG8jJgD", + "mPmqbOXylDl0xB58tD4Jd6Ve5c3MG0nz3pw69aGm4KxYoDO9NFczIoi3EfBBCc9/wyWzeTktUFhMUcKM", + "iDJm1SX1aKkU3M0SbRSWH7cERbbxsjl8dfmFU3xd9ACuFCyX/I8wj7L80/bLHwEq/40reUknrgkYRk25", + "CwPDV6lo1Zo4qlreDJ+qludt3g8ePMurVnC/prNVI86yjwpphujxb5iqn7gAdbAZ8+TO8eXh8o+JAAy4", + "Onp8K+h1mpJ4xHO1+vzbivr2yi/KopZldZ3qi4GIo0o6bxMvcKgc5RiWV1ovB4lyQdXiXK+XDeaGNEhX", + "yxYWEjqCn8uOoX7op09gNJ4EEkZeEkYEjaA6qz6PKWagMaH3p16RPlOvcQmvFUSg10cn1tzgIH9BfaQK", + "SM/FXR6enXS6nTkRRuXuDPq7/QEc5owwnNHOQWe3v90fdECrmsEUt6Civs2ftvnGheJ6EltJ6Ef3kv5S", + "4JQo+OK3ABIAxB3a10EFwVNPicwwFVaLzBJAKDAEQ/XXUG7AXagH5lbummVvbTOFNGPIfiHZa7u5H0BQ", + "hrMD09wZDCywubLXL+TumISBrb/b6NGy31ZSnV2iAPr+kprnZMti6T91O3uD7RuNadVQ4OyGOn7HsE3i", + "JaCd799wIW7V6QkzaXk2ydqGQ/knDgjJP2u/fdB7JvM0xWLhFsxfrYzLJsGYSITdu0aPUxJFmlVAjaA+", + "es2IeY6wQthELoucQWll96Gm0OopMG27TS5Ain7k8eKLLWGlD2ej+FRlZ/q4fFqi5y9HOwUZL2+kfeQQ", + "tg3V3gMB/YiLuuAPdlL2Bs/vvtMjziYJjRTqFQRs45GphJCfBPDCHfYQF+j3nCuMinD+R3Skrcw6Lsit", + "W15FW3/Q+JM53gkJmcHPiEgxM8kR5p01h37pOBuXRHmcV95qjvBPjjv2pnIgPOaiAkGuekT9a6suDC5f", + "R3sBBAbbp5le/ICEv3cPJ9xOtigN+5BHDgpyolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ", + "8EtSSHjlbi1dCluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZ", + "L18rZ3p4X8sJ27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTK", + "Whf+zsd9ZCNSIx4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80S", + "RXNburmei0MsF7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpXLi5uwHDexFg+XViK5mXJICz48p7zaH", + "9GZptGNotj9kby3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQj", + "HsLaeUsYZqonMxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ", + "49LJ5SCEsBjjJAkW5pgmfIyTkVmfSxLwCb6EN+yilA6X0pvEeExMCflsoWacmb/zcc5Ubv4eC34liRh2", + "NvtDBokYdq1J3C0FRHQFhdzSjOtzJnhq+twyQ9z645IsPvWH7DBOKXMUAZ/gRHJEruE7qG8FmBmGezXQ", + "gzlNYT/4US4VT33kU0d3Zpg8V1mubEaJJKobQv0cMsXRHw7b8dPWH2WPn8BZTHCs6cR7xUwJZOumUcsR", + "1rMfwasBdzuBBRh29EVqwjymAjNlYDsLcEo09bd0o6iOABVT6yscYYYynpnKEkBUM6xJrtIGYDXgJEEK", + "jpL7VgvusJMN87HQe+m4EXfPAKXVjhFl6PRH7zAN9p6Fz5MkkSChiJL/On/9K4JbWe+Bea0M1zIpHUwL", + "DCjOwXXqeNoLHM2QcVRBMcFhh8bDTuHOjTdhrLm04TK9HvgUf9BD+8F006XxD/2+bsq4Kw/Qb3+YVg70", + "WcpSgwM67HzqIu/BlKpZPi6efQgvaBN82XmFEaANc81tAifBFJBmvBvfXJGYxYjbWyBZIIxKDuQHrowp", + "w2KxKpEwsPR2BfnERDJ6i/HHECIXh52DoYtdHHa6ww5hc/jNBjgOO5/CK2C9ls2V6+A+K5ybBRE9GQw2", + "1yNh2/UN+CxbOAa+sA7YqBUVZTf1DloY1j+Xf+DfWv8sXD+Y6c5LaCKj+Dvj+yN0QHgSu6+JBlwQNbEb", + "s4gkTuxeb+i5f+eB3qyIJMl9E+hDkWfhHiuQ+h8VOcJmlcdopfn+gSlucF+XSsVs/zD0++js5wHrubWd", + "k7kLdQ7XKQEMGqtKI/MywhKdw5h651r5fgG/9u1/ne4HmIoXCZ9eHBjVHSV8ihLKbD6AF6isxQO7lvCR", + "gaEpvrOoNK5I3IaRJP71j3/CoCib/usf/7TY7v/6xz/huG8ZeDWoMX0xI1ioMcHq4gD9QkjWwwmdEzcZ", + "qAJL5kQs0O7A2vzhEfJK3VspTQ7ZkL0hKhfMy5sw9dqkbdC6CvR8KMuJtDA++kU6scVkTGxjwG7jzrJZ", + "yns90d0AHCLMwJuAvhUdDQCWHDWFtq0m2gmbTM2cK0bTepjmUrDeev6iyLUy1NszA7whg4ElDp07eGAn", + "jTbOz19s9hFoW4YqoGAQ6A5lM1aN6H/jSet5kuEoVYYCq2x4U4QzPKYJdSbHhmon5gimOJpRRsr44gJr", + "3DVx4Eaqeczh2QmygZBdeHXIXp9vgYlVkUjlgnQtJxAWYbQsh8Ztngv0APyLKogO69l3h2xCMOQJnRwb", + "JuCBcBf5gEXDDIA8IMaVqkrlte6QGSRZi1ysD17KY5LAR9D/FCtyhRddVNS6ddVREqy0Qiy7+uUhM1iv", + "dg16AFWCvGH2gZ+ZIfVcJK/N2RJkkmjVGCLwTdlv6HtjwgWyEc5elX/XnUmyNMPSi5bi6PW5nt8UNEFu", + "7IHQ0utztxubXSQ5ihIK1BBhNmRTCARy4L2cVXa1SCibYRH3Iq4vAR/M6ZLxq4TE0yYee+QT2R1KMpV+", + "Asfp5zq5PjbhYrY8AX2IDUDdas/dsX2nnevOtvhn8t3ZQpA3cN4ZCy4x/Mas7jdHXgtHXnjdnFMv5Fk7", + "dgiMdxfxa7p4oIBfR3vLa26eeEv2EBY9tOGgbcArwgU6OzpBOI4FkXLz39vep2dqqLSU//T9qFnxQ4Se", + "2LFwYUH/rL2lSiCPhR28saNG2M2rXl/Xv9+2KsV3Gm+6og5PeeXd/e1R6/Qm10gp9Ja09u0mWRtsS2XE", + "ocxgSS09EI0SUogvxTn1qWidVdmE8RZXzkpxybLnk2N3IO/Pvmy7zln9brgHpnhcY4gPyAirqdZ+1ezH", + "RM3vil10aNMrzM9fF2kO7k8Kum9TdIjMH5O6GNeWTXNBA3TSeIG+JMrAm9ylnm57CEz8nAh3qs1AF2bW", + "xbTMp8jgtMCEwBKzWvc9Ma+0U31Ne38mzReW5yYSi13ybyJKC2W3XKtVCu6JLQF9d/ot9HAj9fbLha1Y", + "AgssMlhRx87tBJbVDSwXLNr8FrnyxSnaxDWWSqxw8yZxYck2aEqFnnVfct0h8+uNa5nO6rWUoUlCpzPr", + "BIjpBGL1lF+/G0a5cw+jLOpkC6yIDVF8jHm/Z3qRrRd4ToRCr49OzPr7V+rWHxC0ul5Vcsxr5e367s2r", + "HmERjwvnSbNMap98YYXJ0H8ll/f+T90jzGelTjxoEhg/Y/9NMDky8e99yv/Xzk8JHQssFv9r5yecZJSR", + "/7V7mGBFpNq8M2IZ3NdNd98KzCMmPq2/0OqiAWtiU4CMXSPwF2+1lPnd+38qsd9M+kaCf7Gu32T/NrK/", + "v1wrxX+7FXeqAJg+HsjDVRBbaLXh0TdIm3swmlqK9CBtKl6kEtRmxqWCR48vv9kGldOC4vxro6X1vzyQ", + "K68PR7onx11YSKgoDRUtbPrgPfkC3DjuXbi1/d6/I+AwHdNpznPpZyamWEUzIm3WbkKqDPixid3l9dwo", + "eH/FVDq4z6vj3uXqb3R/RxJ/fUMN8zYOvXUyv3urrcxv39cyv0E0tZnNtuxG15Vk2mwItHaYpm3JuAL9", + "uhwAHhpXSBdB77SiUqoLCDSIgyH731r/+E0RnH74waVQ5oPBzhP4nbD5hx9cFiU7daRCmBLUVtA7/PUY", + "vKhTCJSFIntlwnZ9HKZmN5CeKyvwb6cglY7k9hqSo8JvGlIrDclbrtUakt2Lu1WRqqVJ7l1HcvQWWnCL", + "Kf7n1JL+5O6RigYn88mERpQwKPACielyKR7QaHLfPCO3TEhm1h/pBRNVJJHWamTBtdZI6GVN6XsPJDsp", + "i2jdt/boylc/zsQqntl6sFZfK6WFZoXta6OHwf3eXvevqD1mEjMa0fLSZVroDlQGMjWh0txkoRVflqBi", + "ffT27SuXcaYlfuHqTCnuiku5uptD5heX6qMXZdUu84JrQUvkJLYZqpCHZ8s1xQTHCWUEQnSJDCWHVUvC", + "Peix+PJCZbjeXSuh8p6Ppa1g+nBC5YOxgnsR304qhaF5qeP7pfCK0+JEODg1j4pfWQYUYDwh8WkL54r3", + "bA7r1owbYLMwtuNZgiOAdtSvGdQxCxtgYAb9pgALQPAkIcKgyWW5cjUQh6wYHGVejXdb4+ZCNz/KmaLJ", + "RddEyAAkiESYLSyk0pBVOsNKkTRTkNoLaeswQkEyM+Ja8Uc9aMpzCW9BFq7fJcLJFV7IIbPJwOZzKJQr", + "SGSAF5Okj37mgMOA8BRT5jFeU4HwOzlkFzROyMjCKFwgKpGccaEIIzFK+ZzIar8Ei4QSAZM4wnrlJErx", + "AvDMDLSjWR+eEYMZVgFr4PrfmMUUatnpnospHwwZRjuDAUoJZtKmXks8gQvHtoFgEJUBfY8w2hs8t1/V", + "9g0wd93yb+jTJASZ8wiPkwUimooB/EFtwgamtrakqdGrt29ChTT7VZgMbdGwysZS6Uolxl2UszK5HMzn", + "OStywfV2qVwwmKd1rBEqimvQ4mmMSYT1ejJe7QeQDHkU5SJ0Qeqt9oqc/jsKjt70zmGpwqnbCWjhEYlh", + "zxlXMzjTHI7S5vcNVFUS1Z/jogkeEi4QRh5dl0YCEuXAGjcA+e+irNjHXAXei83v3dnRx9cyAnf8Dfbe", + "Y7mfgIj4ZFI5gOuvJnOAV6VMLJPwn/WcHrlSrT6LiymeMi4VjRwzrFd2/6YQtlYIV69skJonXFz6slWV", + "fn/i4rKtBmbxROnjUsT8GX6Ftn09PMBufngTPxiYjbKiiebelbQ6fRWnFIQuqqSLHeYo4WyqT1Fp6L53", + "S7yv1W0YHDZ9mQrjPy5Qc7QSMrI/mmqvejK2liZY7SPb6kPzIt37Pfh3fuUK0TRLSEqgGmzPEJve7BJh", + "CSrnU+nhDN2MV+pT5acDG11QGpd+14lDQFduwzZAel/eriBTTfh0PY5f0bkDrQsA+Q2ZqexP0IXx5lyg", + "ggdrgdag5qOrGY1mAOoHeqtu32D+4Sy7KPCMNw/QSzjIPqwzdL5hsPI1rUmeEIPVN0/Ti4PleqfvT0/h", + "I4PnZyqbXhwgV+O0uD+kfssH6dOzSLBU6FcLPbhRKOOwoxcKa32zmN+mhe8r8aaHLATlx8iVbZBO0IWH", + "6nfRADnl+O0rPn0wYazbXCXAzEVxZFVHoE3C4k5T3ARNwoB+24NBCLy6JbigGcYdYwsuDeYVnxYVCiqk", + "jLOsLfnaYQIVz9N0BQ2jDQ+pTKqY5+ovUsVECPjYUncTcaMNHNnqVPhSE6rFpXMHexPILxgdZCDDg0ul", + "mWqn2yEsTzsHv9l/zdO00+3Y8XhQ4zcQ7teANNYbXI5i0TvjITF+E8tvgrFYZfYeyGLt5rDqdLNE/sa8", + "8Kf3Fjqb3QOSIcgHNSPu1ySCeuOtGnwYL8AiYWTP72NkgCiJooRLUnHwPB48KmvoqsmMzYYit8Y9Pbw4", + "dwV82gSFnNtPz92XX4HuvS5WxI0Zuenee9DI8ggec26tXJrNhIs6iNG6aJKvnpC+3JYsTbUNhXyjzZtb", + "GVsRptYTllmE/SA2Bd1wrniKFY2gmFA041x6ZF8gDpuyX9Z4XFAmmFaMlmuD8i80qV5YM/SFVSMOrMkM", + "Yf+R7aMPn9tQ/vAX7lH5xU+eVaDg+F0n+gPgPlQ7F5RMUIZzSbRUl6cERYtIc0VTPYrgaIYinKlcECiM", + "R1BKGU3z1IeS1js2xwB7cbGdXnTROFcowWIKWpl56IJtIp6mhMUE7HNDNiN4TrVKKVCCFWHRoicJFNSd", + "E3TFxWXCcQwmhizG4OmBgnyCaAoEXO6UKBxjhUHQudAnfmTygi6KGrtGrWfkuqSGeMhEzr43RQJ0sxdu", + "oBeIAAo2lbOiFmOEY8KiIDr0+dfNxr68LfqcqPpEHygy6Fa89CFDhXybqxvO1xFF9MjCm7mw29iGza8Q", + "emWzCltNqHBk9O95pM1c3RwfyMFULPGqU/x1eJYKovtqvEsP7z7iAsW56c47lUDmf1afUMFQ/GArSNY0", + "23hbx1BRdK5Y5hvxvK0/3J8nt7DlfSWcsNuo2DeVNyon/TWwXLuqt+K5D2TEtLYk3yb3cCzYRXQ9mPjE", + "hcflHoux1TJsczQLvu1zJyUwaF+cfWPbdbZtAx5uy7adbXbJpe8xcsp6ECMa5uDWjNvIqq3p4N80G6U2", + "O49lPjiLLD0X98YWTwpGaFhjhhcJx/GfIUh4hf8o4kIYRAnAqHhMiKae1dBPDwDbXFk3reuyNd+fnm42", + "cQmhVvIIoR4xh/BScvRnabxswH09J0LQ2AJ/oqPTYxuuSyUSOeuj1ylVSHF0SUhWZrRAVmFfz89hayzX", + "aK+AaHQ7hCmxyDhlau0oylfvZjCfblXZ/Z75pIXI/uYOb+0OB8v+42NnwGUgZ8NMYLVmqrBaW7qTsgkX", + "qZHL8JjnunXNg/Qy6f00JTAnNCFyIRVJTVTiJE/guEG5BVtS135ndrkLMbn65Jh0uYyIlEpJOZNDZnNF", + "MiJ03/pz3b4XYBV0CChc8NczwyS/juA9PRgTr4ZV06oBChKU6uwcdLZwlm3FWOGGADE7vM8Y0k8QjYfk", + "Ih3zhEYooexSoo2EXhr1BM0lSvQfmyvD+Ubw3ZcuGHz7k6VX+oRNeLAcm6HZgpj/VFldlq05x+SjY2sv", + "iX9YHP+BjQ6ztfUliQXBSQ9K/DosHJQrmtCPhtXpRqhUNDIpR7hYu/enBVPtD9kpUUK/gyG1LUkMogFo", + "l1uZ4NHWMB8MdqOMAqDaLoHBAcNrfpxCj0dn70waKkm5WHSHTP8DGn57eGa8uxNsrQneQG0tYnSy9XpN", + "gPM5LNO/cYSgmeBK9ILghn9zCd4cY6TxDMmGI8qzVaoSz/70IaxWgvtmV3icdgUAeSpmszEVOAKhWM5y", + "FfMrFrYhzHmSp/of5o+TdVBhCkez9/DqVyPtmuGs7cZN8FEcSjunmJhykQ/i9DAL9lhjVvXCuSmAEFOJ", + "BgzeAofqz0jdX95876/jV+jutCvqSrF+NWfrvm8+OwaHsOGvx2M55obS3EwUX219usK02fr0Y8KjS2mh", + "WHyzodbbALJc/1hCTFsXIYgJkBmKLISRAcoisjtkNQOkQfyRCCNFREoZTrZgzqYRAMt2Viw85xQStCPI", + "U+lJGgNmUgKI2AB/p2cDhirXgOfRlbZYlf+O74xUHI1JxFPiAMQ3Q6rb3zBVP3FRRQP/WvjiW2/9ARIQ", + "U7C3rwFAb+7xswDRT/E1hErHuXUouxFtvOTlj8YU1EWwN8PO7kAOO1007Oykw47egSMMJlSs0D5KKcsV", + "kX10bOxbkIL7ZIAkiTiLpcMxdxa83YFsSsg1ZNmQ3fkEvrtPscdSFSzlG9tJiD3o95D+HpJ20IZ/4OyZ", + "jLtw6GLEc2XM/fZc2bdiosA8snnvvlrvjHzT7dtw8r/Z41vhUbDLml16W284e5bLGWk2ub0ytYFyNQZ8", + "bFevU87Q3/lYdhEjV8YaLqTqL/E9/fWZ6eA+sPt1VzfB7bdz/wba3wK0v1yrMFijCbDUV7KjDoPYSK4z", + "LhSgONpce0NDoEkAcgSPcIJeH50MWaRZkYEWFCTlwJ0sxLi5hQ//do5eHL3pomOoHYl+zsebffSaJQtX", + "wdv4aIbMSGKGeUWYobGhWhKHrmczdqCeuwwW1x08UDFmczICnhW3Vy5IvNuZERyDRPJH5xU3nQVQh9+8", + "0gcIgH/Nl8W2d1YKH503RIlF73CiiFhu9tTmSbECM8Ne0g6CzgpuBvhSdygd8lrZp5ENDDTG7k4ngJTx", + "6VsdhbuvOXo/XjITJ2Iq2I1zQBplkGSA48XjimWSM1QwxxAL9K/rohJBU5aw5WUrFQzosiny+ysyua/k", + "XRVs+X/X0wUzfbSOpqyyT5qIiwomaz29Ljl4ZuCQraMqwhmOqFp0EU4Se0fZm6CISOkV4u9YEHwZ8yvW", + "H7I3Re0Um9CLjs7edZ2jFsVUXpoWrC+2j17PiZD5uBgcgoNmvMaw5iQeMsVRhJMoT7S4QSYTEkEuLpRE", + "kQ2+3GIonTs8O2UnwfotXlR7/ujKxoVpAnavJIs6xW2Zrd4SJEowTZvBx62gBgGHEGow1o1yhiibJDak", + "KhJcSmSb6pGETuk4sQFCso/ezgiSOCVDliWYMSJQLk1UvB56LxNEytwkeOsGAKTXUFQXlcCCmeDKhiYk", + "nAtpogk0hb8/RVKRbAWZvTEtn8Kc70i2NY3bnh7ISF0bQ7MpxL6C9IYYSjELrukoT1wA472GopsBPbSU", + "+FgO/ltBp1Mi9KnAhsmacDxzrN1ymkNfyVhuLCF5XrzVroRk0aqXlehl7K0EhhuVWNtx52ZRf4HOL2kj", + "dqB9dLMs4l/0Ry37rmarhgdhH33mLP8slfnPvSTBtgasksIfmznJG3nlqFYSbdfDarXOrL3LTNfW+FkP", + "Bpv1mNGycCV9tknh/foIYXC/KA/3XWTtcdNWBe2qops2pPyvR9P/KijwbmD0Hxjl5BYw+l9V3j3gnD8c", + "/knwoD5UHn3F9+zq1/7pkfDvKn3ewOEDHFtT+rzhejZ4daWi9N6+005Nsi3+mSR4G+94A/ndLfs3rb+F", + "yuAt1joXtCZ4kmZq4QLarK+yDDqT9CPpNziCi7jVu3MF3yKk88uRh6PTxoDOP2e5+QeJGbWlA6lEJ8eB", + "Ou6PDGPQP3OVi2VL3zo9LKIZnZNmo3v1BNslygTpZTwD50psFsyuh7vLFBb96Udkm7eYq/ZfUHsSoPpJ", + "jGIqSKSShakDqjmC6eM7iQTXmgA852LRHCVijshPgqeHdjZr7kN7pqwxrIwzTBe9GCvcmztus8KE9hnR", + "nS6eUjM8RBl6+SPaINdKmAoXaKI1H0QnxZKaAv4SaHLTH/D2oMGyST+S0XTcZpQrapW8trVgUJRLxVO3", + "9yfHaANqn00J03uhRf0JSLKZ4HMak7gyxs6cJ2ZVtxsW9KZ2Vy1UFIXrnHJhBvcgMkybC2n6kWZVtlCE", + "xIwpwzC4tVVBqmfKJPHr/jBlLgDH7pEbxbcrzGp+G07Z0ZQIdTjtIirODcTz5rdr7jFfc34ylLvTKred", + "C89Zbbxulx/VMm3pLgo/FLlz92u2fv/1pPRQ+SizeazpfF4opE1m86+LBAf3dz/ct7n8/SNOAX1JnPLt", + "mcqhAd1iiGBeQUx3TOYk4VkK9dDh3U63k4ukc9CZKZUdbG1B7PeMS3Ww9/zpbufTh0//fwAAAP//5962", + "rgXyAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 814dc1432..833a7a911 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -343,6 +343,11 @@ func (p *Paths) DeviceMetadata(id string) string { return filepath.Join(p.DeviceDir(id), "metadata.json") } +// VFHealthState returns the path to the persisted vGPU VF health file. +func (p *Paths) VFHealthState() string { + return filepath.Join(p.dataDir, "gpu", "vf-health.json") +} + // Volume path methods // VolumesDir returns the root volumes directory. diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 054e3744e..2c7843532 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -10,16 +10,19 @@ import ( // GPUResourceStatus represents the GPU resource status for the API response. // Returns nil if no GPU is available on the host. type GPUResourceStatus struct { - Mode string `json:"mode"` // "vgpu" or "passthrough" - TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough - UsedSlots int `json:"used_slots"` // Slots currently in use - Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only - Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only + Mode string `json:"mode"` // "vgpu" or "passthrough" + TotalSlots int `json:"total_slots"` // VFs for vGPU, physical GPUs for passthrough + UsedSlots int `json:"used_slots"` // Slots currently in use, including assigned quarantined VFs + AllocatableSlots int `json:"allocatable_slots"` // Healthy free slots used by admission control + QuarantinedSlots int `json:"quarantined_slots"` // Quarantined VFs; may overlap UsedSlots + Profiles []devices.GPUProfile `json:"profiles,omitempty"` // vGPU mode only + Devices []devices.PassthroughDevice `json:"devices,omitempty"` // passthrough mode only } -// GetGPUStatus returns the current GPU resource status. -// Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus(ctx context.Context) *GPUResourceStatus { +// GetGPUStatus returns the current GPU resource status and any error that +// prevents determining allocatable vGPU capacity. It returns nil if no GPU is +// available or the mode is "none". +func GetGPUStatus(ctx context.Context) (*GPUResourceStatus, error) { framework, vfs, err := devices.DiscoverVGPU() if err != nil { // Only report passthrough once vGPU discovery confirms no vGPU @@ -27,16 +30,16 @@ func GetGPUStatus(ctx context.Context) *GPUResourceStatus { // expose the PFs/VFs as available passthrough slots while active vGPU // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return nil, nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) } - return getPassthroughStatus() + return getPassthroughStatus(), nil } // getVGPUStatus returns GPU status for vGPU mode (SR-IOV). -func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) (*GPUResourceStatus, error) { usedSlots := 0 // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { @@ -51,13 +54,20 @@ func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []d logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } - - return &GPUResourceStatus{ - Mode: string(devices.GPUModeVGPU), - TotalSlots: len(vfs), - UsedSlots: usedSlots, - Profiles: profiles, + allocatableSlots, quarantinedSlots, err := devices.VGPUAvailability(framework, vfs) + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: len(vfs), + UsedSlots: usedSlots, + AllocatableSlots: allocatableSlots, + QuarantinedSlots: quarantinedSlots, + Profiles: profiles, + } + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to count allocatable vGPU slots; reporting none", "framework", framework, "error", err) + status.AllocatableSlots = 0 } + return status, err } // getPassthroughStatus returns GPU status for whole-GPU passthrough mode. @@ -92,9 +102,10 @@ func getPassthroughStatus() *GPUResourceStatus { } return &GPUResourceStatus{ - Mode: string(devices.GPUModePassthrough), - TotalSlots: len(passthroughDevices), - UsedSlots: usedSlots, - Devices: passthroughDevices, + Mode: string(devices.GPUModePassthrough), + TotalSlots: len(passthroughDevices), + UsedSlots: usedSlots, + AllocatableSlots: len(passthroughDevices) - usedSlots, + Devices: passthroughDevices, } } diff --git a/lib/resources/gpu_test.go b/lib/resources/gpu_test.go new file mode 100644 index 000000000..825a20260 --- /dev/null +++ b/lib/resources/gpu_test.go @@ -0,0 +1,88 @@ +package resources + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initVFHealthForTest(t *testing.T, state []byte) { + t.Helper() + dataDir := t.TempDir() + if state != nil { + path := paths.New(dataDir).VFHealthState() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, state, 0o644)) + } + devices.NewManager(paths.New(dataDir)) + resetDir := t.TempDir() + t.Cleanup(func() { devices.NewManager(paths.New(resetDir)) }) +} + +func TestGetVGPUStatusFailsClosedWhenVFHealthIsUnavailable(t *testing.T) { + initVFHealthForTest(t, []byte("not json")) + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{{PCIAddress: "0000:82:00.4"}}) + assert.Zero(t, status.AllocatableSlots) + assert.Zero(t, status.QuarantinedSlots) + require.ErrorContains(t, err, "VF health state unavailable") +} + +func TestGetVGPUStatusReportsQuarantinedSlots(t *testing.T) { + initVFHealthForTest(t, nil) + for _, instance := range []string{"instance-1", "instance-2"} { + _, err := devices.ReportVFInitFailure(devices.VFInitFailureReport{VFAddress: "0000:82:00.4", InstanceID: instance}) + require.NoError(t, err) + } + + status, err := getVGPUStatus(context.Background(), devices.VGPUFrameworkVendorVFIO, []devices.VirtualFunction{ + {PCIAddress: "0000:82:00.4"}, + {PCIAddress: "0000:82:00.5", Allocated: true}, + {PCIAddress: "0000:82:00.6"}, + }) + require.NoError(t, err) + assert.Equal(t, 3, status.TotalSlots) + assert.Equal(t, 1, status.UsedSlots) + assert.Equal(t, 1, status.AllocatableSlots) + assert.Equal(t, 1, status.QuarantinedSlots) +} + +func TestReserveAllocationUsesAllocatableGPUSlots(t *testing.T) { + status := &GPUResourceStatus{ + Mode: string(devices.GPUModeVGPU), + TotalSlots: 4, + UsedSlots: 1, + AllocatableSlots: 0, + } + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + t.Cleanup(func() { setGPUStatusProvider(nil) }) + + mgr := NewManager(&config.Config{}, paths.New(t.TempDir())) + ctx := context.Background() + + err := mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { + return status, errors.New("VF health state unavailable: read failed") + }) + err = mgr.ValidateAllocation(ctx, 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "vGPU placement is disabled: VF health state unavailable") + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return status, nil }) + + status.AllocatableSlots = 1 + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-a", 0, 0, 0, 0, 0, 0, true)) + err = mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true) + require.ErrorContains(t, err, "no allocatable vgpu slots") + + mgr.FinishAllocation("pending-a") + require.NoError(t, mgr.ReserveAllocation(ctx, "pending-b", 0, 0, 0, 0, 0, 0, true)) +} diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index bef0740dc..39166856e 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func(context.Context) *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) (*GPUResourceStatus, error) { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, @@ -207,7 +207,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { {Name: "L40S-1Q", Available: 5}, {Name: "L40S-2Q", Available: 2}, }, - } + }, nil }) defer func() { setGPUStatusProvider(originalProvider) diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 86f644bda..9c28fda0f 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) (*GPUResourceStatus, error) { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) (*GPUResourceStatus, error)) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, _ := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,15 +691,18 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()(ctx) + gpuStatus, gpuStatusErr := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } - availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots - pending.GPUSlots + availableSlots := gpuStatus.AllocatableSlots - pending.GPUSlots if availableSlots < req.GPUSlots { + if gpuStatusErr != nil { + return fmt.Errorf("insufficient GPU: vGPU placement is disabled: %w", gpuStatusErr) + } if availableSlots <= 0 { - return fmt.Errorf("insufficient GPU: all %d %s slots are in use", - gpuStatus.TotalSlots, gpuStatus.Mode) + return fmt.Errorf("insufficient GPU: no allocatable %s slots available (%d total, %d in use)", + gpuStatus.Mode, gpuStatus.TotalSlots, gpuStatus.UsedSlots) } return fmt.Errorf("insufficient GPU: requested %d %s slot(s), but only %d available", req.GPUSlots, gpuStatus.Mode, availableSlots) diff --git a/openapi.yaml b/openapi.yaml index 002c9fcce..eb31e5863 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1800,7 +1800,7 @@ components: type: object description: GPU resource status. Null if no GPUs available. nullable: true - required: [mode, total_slots, used_slots] + required: [mode, total_slots, used_slots, allocatable_slots, quarantined_slots] properties: mode: type: string @@ -1813,8 +1813,16 @@ components: example: 64 used_slots: type: integer - description: Slots currently in use + description: Slots currently in use. Includes quarantined VFs that are still assigned, so this can overlap quarantined_slots. example: 5 + allocatable_slots: + type: integer + description: Free slots eligible for placement, matching admission control (excludes quarantined VFs; 0 while VF health state is unavailable) + example: 57 + quarantined_slots: + type: integer + description: VFs quarantined after guest driver init failures (vGPU mode only). May overlap used_slots until the affected instance releases its VF. + example: 2 profiles: type: array description: Available vGPU profiles (only in vGPU mode)