From 58bbeb822d4d9eff3bdfb4afce28ba612a66570a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:57:41 +0000 Subject: [PATCH 01/24] Support vendor VFIO vGPU devices Linux 6.8 hosts with NVIDIA R580 drop the mdev interface: vGPUs are assigned by writing a type ID to a VF's nvidia/current_vgpu_type and passed to QEMU as a plain VFIO PCI device. Add a vendor VFIO backend behind the existing framework dispatch: profile discovery from the capacity-dependent creatable catalogs, least-loaded VF placement, create/verify/rollback, and release. Because the same VF path is reused across assignments (unlike mdev UUIDs), release is guarded: an in-process owner map covers the window before QEMU opens the device, and an open-VFIO-handle scan refuses to clear a VF a running VM still holds. Reconciliation clears orphaned assignments on startup, skipping VFs protected by the caller and failing closed when the protected set is unavailable. Branch the vGPU integration test by discovered framework and extend it to cover release on stop and reacquisition on start. --- integration/vgpu_test.go | 110 ++++-- lib/devices/GPU.md | 46 +-- lib/devices/gpu_mode.go | 30 -- lib/devices/mdev_darwin.go | 13 +- lib/devices/mdev_linux.go | 32 +- lib/devices/types.go | 15 +- lib/devices/vendor_vfio_linux.go | 476 ++++++++++++++++++++++++++ lib/devices/vendor_vfio_linux_test.go | 374 ++++++++++++++++++++ lib/devices/vgpu_linux.go | 116 ++++++- lib/devices/vgpu_linux_test.go | 50 +++ lib/resources/gpu.go | 35 +- lib/resources/monitoring_test.go | 2 +- lib/resources/resource.go | 8 +- 13 files changed, 1159 insertions(+), 148 deletions(-) delete mode 100644 lib/devices/gpu_mode.go create mode 100644 lib/devices/vendor_vfio_linux.go create mode 100644 lib/devices/vendor_vfio_linux_test.go create mode 100644 lib/devices/vgpu_linux_test.go diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 12861a6a7..3f8fdfd65 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" @@ -21,21 +23,23 @@ import ( "github.com/stretchr/testify/require" ) -// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works. +// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works +// on the host's framework: mdev or NVIDIA's vendor-specific VFIO. // // This test automatically detects vGPU availability and skips if: -// - No SR-IOV VFs are found in /sys/class/mdev_bus/ +// - No vGPU framework (mdev or vendor VFIO) is discovered // - No vGPU profiles are available -// - Not running as root (required for mdev creation) +// - Not running as root (required for sysfs vGPU assignment) // - KVM is not available // // To run manually: // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies mdev creation and PCI device visibility inside the VM. -// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA -// guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -159,9 +163,18 @@ func TestVGPU(t *testing.T) { instanceID = inst.Id t.Logf("Instance created: %s", inst.Id) - // Verify mdev UUID was assigned - require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned") - t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + // Verify the assignment matches the host's framework + require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned") + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned") + t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + case devices.VGPUFrameworkVendorVFIO: + require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID") + t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath) + default: + t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework) + } // Step 5: Check GPU resources AFTER creating instance t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) { @@ -180,12 +193,9 @@ func TestVGPU(t *testing.T) { assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM") }) - // Step 6: Verify mdev was created in sysfs - t.Run("MdevCreated", func(t *testing.T) { - mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID - _, err := os.Stat(mdevPath) - assert.NoError(t, err, "mdev device should exist at %s", mdevPath) - t.Logf("mdev exists at: %s", mdevPath) + // Step 6: Verify the assignment exists in sysfs + t.Run("VGPUAssignedInSysfs", func(t *testing.T) { + assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath) }) // Step 7: Wait for guest agent to be ready @@ -225,13 +235,68 @@ func TestVGPU(t *testing.T) { require.NoError(t, err) assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match") - assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") - t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID) + assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match") + assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set") + if inst.GPUFramework == devices.VGPUFrameworkMdev { + assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") + } + t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) + }) + + t.Log("Step 10: Stopping instance to release the vGPU...") + _, err = instanceManager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "stop should succeed") + + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) + + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") + + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) }) t.Log("✅ vGPU test PASSED!") } +func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.NoError(t, err, "mdev device should exist at %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + +func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + // checkVGPUTestPrerequisites checks if vGPU test can run. // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met. func checkVGPUTestPrerequisites() (string, string) { @@ -245,10 +310,13 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires root (sudo) for mdev creation", "" } - // Check for vGPU mode (SR-IOV VFs present) - mode := devices.DetectHostGPUMode() - if mode != devices.GPUModeVGPU { - return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", "" + // Check for a vGPU framework (mdev or vendor VFIO) + framework, _, err := devices.DiscoverVGPU() + if err != nil { + return "vGPU test failed to discover vGPU framework: " + err.Error(), "" + } + if framework == devices.VGPUFrameworkNone { + return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } // Check for available profiles diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..cdd269dc8 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati | Mode | Description | Use Case | |------|-------------|----------| -| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources | +| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources | | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance | The host's GPU mode is determined by the host driver configuration: -- If `/sys/class/mdev_bus/` contains VFs → vGPU mode -- If NVIDIA GPUs are available for VFIO → passthrough mode +- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode +- If VFs expose `/sys/bus/pci/devices//nvidia/current_vgpu_type` → vendor VFIO vGPU mode +- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode ## vGPU Mode (Recommended) -vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU. +vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`. ### How It Works @@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \ }' ``` -The response includes the assigned mdev UUID: +On an mdev host, the response also includes the assigned mdev UUID: ```json { @@ -87,19 +88,16 @@ The response includes the assigned mdev UUID: } ``` -### Ephemeral mdev Lifecycle +### Ephemeral vGPU Lifecycle -mdev devices are **ephemeral**: created on instance start, destroyed on instance delete. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts. ``` -Instance Create → Create mdev → Attach to VM → Instance Running -Instance Delete → Stop VM → Destroy mdev → VF available again +Instance Create → Assign profile to VF → Attach VF to VM → Instance Running +Instance Stop/Delete → Release profile → VF available again ``` -This ensures: -- **Security**: No VRAM data leakage between instances -- **Clean state**: Fresh vGPU for each instance -- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. ## Passthrough Mode @@ -241,7 +239,8 @@ To upgrade the NVIDIA driver version: 1. Check host GPU mode detection: ```bash - ls /sys/class/mdev_bus/ # Should show VFs for vGPU mode + ls /sys/class/mdev_bus/ + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' ``` 2. Verify NVIDIA drivers are loaded on host: @@ -265,17 +264,18 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` -### mdev creation fails +### vGPU assignment fails -1. Check if VFs are available: - ```bash - ls /sys/class/mdev_bus/ - ``` +Check the files for the framework detected on the host: -2. Verify mdev types: - ```bash - cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances - ``` +```bash +# mdev +cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances + +# vendor VFIO +cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types +cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type +``` ## Performance Tuning diff --git a/lib/devices/gpu_mode.go b/lib/devices/gpu_mode.go deleted file mode 100644 index 40b3b2ba0..000000000 --- a/lib/devices/gpu_mode.go +++ /dev/null @@ -1,30 +0,0 @@ -package devices - -import ( - "os" -) - -// DetectHostGPUMode determines the host's GPU configuration mode. -// -// Returns: -// - GPUModeVGPU if /sys/class/mdev_bus has entries (SR-IOV VFs present) -// - GPUModePassthrough if NVIDIA GPUs are available for VFIO passthrough -// - GPUModeNone if no GPUs are available -// -// Note: A host is configured for either vGPU or passthrough, not both, -// because the host driver determines which mode is available. -func DetectHostGPUMode() GPUMode { - // Check for vGPU mode first (SR-IOV VFs present) - entries, err := os.ReadDir("/sys/class/mdev_bus") - if err == nil && len(entries) > 0 { - return GPUModeVGPU - } - - // Check for passthrough mode (physical GPUs available) - gpus, err := DiscoverAvailableDevices() - if err == nil && len(gpus) > 0 { - return GPUModePassthrough - } - - return GPUModeNone -} diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 1427a5095..4274063ed 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) { // No-op on macOS } -// DiscoverVFs returns an empty list on macOS. -// SR-IOV Virtual Functions are not available on macOS. -func DiscoverVFs() ([]VirtualFunction, error) { - return []VirtualFunction{}, nil +// DiscoverVGPU reports no vGPU framework on macOS. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return VGPUFrameworkNone, nil, nil } // ListGPUProfiles returns an empty list on macOS. @@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { return []GPUProfile{}, nil } @@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 1a398a418..e2891efc9 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,14 +89,13 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. -// These are discovered by scanning /sys/class/mdev_bus/ which contains -// VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { +// discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, +// discovered by scanning /sys/class/mdev_bus/. +func discoverMdevVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(mdevBusPath) if err != nil { if os.IsNotExist(err) { - return nil, nil // No mdev_bus means no vGPU support + return nil, nil // No mdev_bus means no mdev vGPU support } return nil, fmt.Errorf("read mdev_bus: %w", err) } @@ -133,20 +132,9 @@ func DiscoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// ListGPUProfiles returns available vGPU profiles with availability counts. -// Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +// listMdevGPUProfilesWithVFs returns available vGPU profiles using +// pre-discovered VFs. Uses parallel sysfs reads for fast availability counting. +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -305,7 +293,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -531,7 +519,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -697,7 +685,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/types.go b/lib/devices/types.go index 809d669fe..c76d239ed 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -63,12 +63,13 @@ type GPUMode string type VGPUFramework string const ( - VGPUFrameworkNone VGPUFramework = "" - VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio" // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" - // GPUModeVGPU indicates SR-IOV + mdev based vGPU + // GPUModeVGPU indicates vGPU mode GPUModeVGPU GPUMode = "vgpu" // GPUModeNone indicates no GPU available GPUModeNone GPUMode = "none" @@ -76,9 +77,10 @@ const ( // VirtualFunction represents an SR-IOV Virtual Function for vGPU type VirtualFunction struct { - PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" - ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" + ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + ProfileType string `json:"profile_type,omitempty"` } // VGPUAssignment identifies an existing vGPU assignment to release. @@ -86,6 +88,7 @@ type VGPUAssignment struct { Framework VGPUFramework DevicePath string MdevUUID string + InstanceID string } type VGPUDevice struct { diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go new file mode 100644 index 000000000..23ab1e89a --- /dev/null +++ b/lib/devices/vendor_vfio_linux.go @@ -0,0 +1,476 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + pciDevicesPath = "/sys/bus/pci/devices" + vfioDevicesPath = "/dev/vfio/devices" +) + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +var ( + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + } + vendorVFIOMu sync.Mutex +) + +func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read PCI devices: %w", err) + } + + vfs := make([]VirtualFunction, 0) + for _, entry := range entries { + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(filepath.Join(nvidiaPath, "creatable_vgpu_types")); err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + } + + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + } + + parentGPU := "" + if target, err := os.Readlink(filepath.Join(vfPath, "physfn")); err == nil { + parentGPU = filepath.Base(target) + } + vfs = append(vfs, VirtualFunction{ + PCIAddress: entry.Name(), + ParentGPU: parentGPU, + Allocated: currentType != "0", + ProfileType: currentType, + }) + } + + sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) + return vfs, nil +} + +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + profilesByType := make(map[string]profileMetadata) + availability := make(map[string]int) + for _, vf := range vfs { + creatable, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range creatable { + profilesByType[profile.TypeName] = profile + if !vf.Allocated { + availability[profile.TypeName]++ + } + } + } + + metadata := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + metadata = append(metadata, profile) + } + sort.Slice(metadata, func(i, j int) bool { return metadata[i].Name < metadata[j].Name }) + + profiles := make([]GPUProfile, 0, len(metadata)) + for _, profile := range metadata { + profiles = append(profiles, GPUProfile{ + Name: profile.Name, + FramebufferMB: profile.FramebufferMB, + Available: availability[profile.TypeName], + }) + } + return profiles, nil +} + +func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + vfs, err := s.discoverVFs() + if err != nil { + return nil, err + } + metadata, err := s.profileMetadata(vfs) + if err != nil { + return nil, err + } + + var requested profileMetadata + found := false + for _, profile := range metadata { + if profile.Name == profileName { + requested = profile + found = true + break + } + } + if !found { + if len(metadata) == 0 && len(vfs) > 0 { + return nil, fmt.Errorf("no creatable vGPU profiles on any VF, GPUs may be at capacity: profile %q", profileName) + } + return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } + + targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + if err != nil { + return nil, err + } + if targetVF == "" { + return nil, fmt.Errorf("no available VF for profile %q", profileName) + } + + currentTypePath := filepath.Join(s.pciDevicesPath, targetVF, "nvidia", "current_vgpu_type") + if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { + return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) + } + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + if currentType != requested.TypeName { + verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + s.owners[targetVF] = instanceID + + logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", + "profile", profileName, + "vf", targetVF, + "instance_id", instanceID, + ) + return &VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + }, nil +} + +func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { + return s.destroyWithOpenPaths(ctx, vfAddress, instanceID, nil) +} + +func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, instanceID string, openPaths map[string]struct{}) error { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + log := logger.FromContext(ctx) + currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + if os.IsNotExist(err) { + delete(s.owners, vfAddress) + return nil + } + return fmt.Errorf("read current vGPU type for VF %s: %w", vfAddress, err) + } + if currentType == "0" { + delete(s.owners, vfAddress) + return nil + } + + if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } + + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) + if err != nil { + return fmt.Errorf("check vendor VFIO vGPU usage for VF %s: %w", vfAddress, err) + } + if inUse { + return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) + } + + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) + } + delete(s.owners, vfAddress) + log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress) + return nil +} + +func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + vfs, err := s.discoverVFs() + if err != nil { + return err + } + log := logger.FromContext(ctx) + protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) + for path := range protectedDevicePaths { + protectedVFs[filepath.Base(path)] = struct{}{} + } + var openPaths map[string]struct{} + for _, vf := range vfs { + if !vf.Allocated { + continue + } + if _, ok := protectedVFs[vf.PCIAddress]; ok { + log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) + continue + } + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vf.PCIAddress, openPaths) + if err != nil { + log.WarnContext(ctx, "failed to check vendor VFIO vGPU usage", "vf", vf.PCIAddress, "error", err) + continue + } + if inUse { + continue + } + if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { + log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) + } + } + return nil +} + +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { + framebufferByType := make(map[string]int, len(metadata)) + for _, profile := range metadata { + framebufferByType[profile.TypeName] = profile.FramebufferMB + } + + usageByGPU := make(map[string]int) + freeByGPU := make(map[string][]VirtualFunction) + for _, vf := range vfs { + if vf.Allocated { + usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + continue + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return "", err + } + for _, profile := range profiles { + if profile.TypeName == profileType { + freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) + break + } + } + } + + gpus := make([]string, 0, len(freeByGPU)) + for gpu := range freeByGPU { + gpus = append(gpus, gpu) + } + sort.Slice(gpus, func(i, j int) bool { + if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { + return gpus[i] < gpus[j] + } + return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + }) + if len(gpus) == 0 { + return "", nil + } + return freeByGPU[gpus[0]][0].PCIAddress, nil +} + +func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { + profilesByType := make(map[string]profileMetadata) + for _, vf := range vfs { + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile + } + } + profiles := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + profiles = append(profiles, profile) + } + sort.Slice(profiles, func(i, j int) bool { return profiles[i].Name < profiles[j].Name }) + return profiles, nil +} + +func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { + path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read creatable vGPU types for VF %s: %w", vfAddress, err) + } + return parseCreatableVGPUTypes(string(data)) +} + +func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string]struct{}) (bool, error) { + devicePaths := make([]string, 0, 2) + probeErrs := make([]error, 0, 2) + + vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } else { + for _, device := range vfioDevices { + devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) + } + } + + target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } else { + devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) + } + + for _, path := range devicePaths { + if _, ok := openPaths[path]; ok { + return true, nil + } + } + if len(probeErrs) > 0 { + return false, errors.Join(probeErrs...) + } + return false, nil +} + +func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + processes, err := os.ReadDir(s.procPath) + if err != nil { + return nil, err + } + prefix := filepath.Dir(s.vfioDevicesPath) + string(filepath.Separator) + open := make(map[string]struct{}) + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + fdPath := filepath.Join(s.procPath, process.Name(), "fd") + fds, err := os.ReadDir(fdPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) + } + if strings.HasPrefix(target, prefix) { + open[target] = struct{}{} + } + } + } + return open, nil +} + +func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { + profiles := make([]profileMetadata, 0) + for lineNumber, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + typeID, name, found := strings.Cut(line, ":") + typeID = strings.TrimSpace(typeID) + name = strings.TrimSpace(name) + if typeID == "ID" { + continue + } + if !found || name == "" { + return nil, fmt.Errorf("parse creatable vGPU types line %d: %q", lineNumber+1, line) + } + if _, err := strconv.Atoi(typeID); err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", typeID, err) + } + profiles = append(profiles, profileMetadata{ + TypeName: typeID, + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }) + } + return profiles, nil +} + +func framebufferFromProfileName(name string) int { + series := strings.LastIndexAny(name, "ABCQ") + if series <= 0 { + return 0 + } + dash := strings.LastIndex(name[:series], "-") + if dash < 0 { + return 0 + } + gb, err := strconv.Atoi(name[dash+1 : series]) + if err != nil { + return 0 + } + return gb * 1024 +} + +func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + } + return verifyErr +} + +func readCurrentVGPUType(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if _, err := strconv.Atoi(value); err != nil { + return "", fmt.Errorf("invalid current vGPU type %q", value) + } + return value, nil +} diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go new file mode 100644 index 000000000..8e79a7750 --- /dev/null +++ b/lib/devices/vendor_vfio_linux_test.go @@ -0,0 +1,374 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCreatableTypes = `ID : vGPU Name +1147 : NVIDIA L40S-1Q +1148 : NVIDIA L40S-2Q +1159 : NVIDIA L40S-48Q +` + +func TestParseCreatableVGPUTypes(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes(testCreatableTypes) + require.NoError(t, err) + require.Len(t, profiles, 3) + assert.Equal(t, profileMetadata{TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, profiles[0]) + assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) +} + +func TestVendorVFIOCreateAndDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.False(t, vfs[0].Allocated) + assert.Equal(t, "0000:82:00.0", vfs[0].ParentGPU) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, device.Framework) + assert.Equal(t, "0000:82:00.4", device.VFAddress) + assert.Equal(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"), device.SysfsPath) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "stale-instance")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + unbind func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + }{ + { + name: "missing vfio device directory", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + }, + { + name: "missing iommu group symlink", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.unbind(t, sysfs, vfAddress) + + require.NoError(t, sysfs.destroy(context.Background(), vfAddress, "instance-1")) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + }) + } +} + +func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "1159", "") + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Empty(t, profiles) + + _, err = sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "GPUs may be at capacity") +} + +func TestVendorVFIOCreateReportsAmbiguousMissingProfile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n") + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-48Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "not creatable on any VF") + assert.ErrorContains(t, err, "unknown profile or insufficient capacity") +} + +func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOReconcile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + protected := map[string]struct{}{ + filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"): {}, + } + require.NoError(t, sysfs.reconcile(context.Background(), protected)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOReconcilePreservesLegacyGroupFD(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + legacyGroup := filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(legacyGroup, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenVFIODeviceProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + vfioDevPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev") + require.NoError(t, os.RemoveAll(vfioDevPath)) + require.NoError(t, os.WriteFile(vfioDevPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenIOMMUGroupProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + iommuGroupPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group") + require.NoError(t, os.Remove(iommuGroupPath)) + require.NoError(t, os.WriteFile(iommuGroupPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDDirectoryScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + processPath := filepath.Join(sysfs.procPath, "123") + require.NoError(t, os.MkdirAll(processPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(processPath, "fd"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptors") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDLinkScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + fdPath := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(fdPath, "5"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptor 5") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestParseCreatableVGPUTypesHeaderOnly(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes("ID : vGPU Name\n") + require.NoError(t, err) + assert.Empty(t, profiles) +} + +func TestParseCreatableVGPUTypesRejectsMalformedLine(t *testing.T) { + t.Parallel() + + _, err := parseCreatableVGPUTypes("NVIDIA") + require.Error(t, err) + + _, err = parseCreatableVGPUTypes("not-an-id : NVIDIA L40S-1Q") + require.Error(t, err) +} + +func TestRollbackVendorVFIOCreate(t *testing.T) { + t.Parallel() + + verifyErr := errors.New("verification failed") + t.Run("preserves verification error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") + require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assertFileValue(t, currentTypePath, "0") + }) + + t.Run("surfaces rollback error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + }) +} + +func profileAvailability(profiles []GPUProfile, name string) int { + for _, profile := range profiles { + if profile.Name == name { + return profile.Available + } + } + return -1 +} + +type testVendorVFIOSysfs struct { + vendorVFIOSysfs +} + +func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { + t.Helper() + root := t.TempDir() + pci := filepath.Join(root, "sys", "bus", "pci", "devices") + proc := filepath.Join(root, "proc") + vfio := filepath.Join(root, "dev", "vfio", "devices") + require.NoError(t, os.MkdirAll(pci, 0755)) + require.NoError(t, os.MkdirAll(proc, 0755)) + require.NoError(t, os.MkdirAll(vfio, 0755)) + return testVendorVFIOSysfs{vendorVFIOSysfs{ + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + }} +} + +func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { + t.Helper() + parentPath := filepath.Join(s.pciDevicesPath, parent) + vfPath := filepath.Join(s.pciDevicesPath, address) + nvidiaPath := filepath.Join(vfPath, "nvidia") + require.NoError(t, os.MkdirAll(parentPath, 0755)) + require.NoError(t, os.MkdirAll(nvidiaPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte(currentType), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte(creatableTypes), 0444)) + require.NoError(t, os.Symlink(parentPath, filepath.Join(vfPath, "physfn"))) + vfioName := "vfio" + vfioID + require.NoError(t, os.MkdirAll(filepath.Join(vfPath, "vfio-dev", vfioName), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(s.vfioDevicesPath, vfioName), nil, 0600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "kernel", "iommu_groups", vfioID), filepath.Join(vfPath, "iommu_group"))) +} + +func assertFileValue(t *testing.T, path, expected string) { + t.Helper() + value, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, expected, string(value)) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index eaf210b42..a4ccd2db6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,31 +8,115 @@ import ( "path/filepath" ) +// DiscoverVGPU returns the host's active vGPU framework and virtual functions. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return discoverVGPUWith(discoverMdevVFs, hostVendorVFIO.discoverVFs) +} + +func discoverVGPUWith(discoverMdev, discoverVendorVFIO func() ([]VirtualFunction, error)) (VGPUFramework, []VirtualFunction, error) { + vfs, err := discoverMdev() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover mdev VFs: %w", err) + } + if len(vfs) > 0 { + return VGPUFrameworkMdev, vfs, nil + } + + vfs, err = discoverVendorVFIO() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + if len(vfs) == 0 { + return VGPUFrameworkNone, nil, nil + } + return VGPUFrameworkVendorVFIO, vfs, nil +} + +// ListGPUProfiles returns available vGPU profiles with availability counts. +func ListGPUProfiles() ([]GPUProfile, error) { + framework, vfs, err := DiscoverVGPU() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs) +} + +// ListGPUProfilesWithVFs returns available profiles for discovered VFs. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { + switch framework { + case VGPUFrameworkMdev: + return listMdevGPUProfilesWithVFs(vfs) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.listProfiles(vfs) + default: + return nil, nil + } +} + func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { - mdev, err := CreateMdev(ctx, profileName, instanceID) + framework, _, err := DiscoverVGPU() if err != nil { return nil, err } - return &VGPUDevice{ - Framework: VGPUFrameworkMdev, - VFAddress: mdev.VFAddress, - ProfileType: mdev.ProfileType, - ProfileName: mdev.ProfileName, - SysfsPath: mdev.SysfsPath, - MdevUUID: mdev.UUID, - }, nil + switch framework { + case VGPUFrameworkMdev: + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.create(ctx, profileName, instanceID) + default: + return nil, fmt.Errorf("vGPU framework not available") + } } func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { - if assignment.Framework != VGPUFrameworkNone && assignment.Framework != VGPUFrameworkMdev { - return fmt.Errorf("unknown vGPU framework %q", assignment.Framework) + framework := assignment.Framework + if framework == VGPUFrameworkNone && assignment.MdevUUID != "" { + framework = VGPUFrameworkMdev + } + + switch framework { + case VGPUFrameworkMdev: + mdevUUID := assignment.MdevUUID + if mdevUUID == "" { + mdevUUID = filepath.Base(assignment.DevicePath) + } + return DestroyMdev(ctx, mdevUUID) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) + case VGPUFrameworkNone: + return nil + default: + return fmt.Errorf("unknown vGPU framework %q", framework) } - mdevUUID := assignment.MdevUUID - if mdevUUID == "" { - if assignment.DevicePath == "" { +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + framework, _, err := DiscoverVGPU() + if err != nil { + return err + } + + switch framework { + case VGPUFrameworkMdev: + return ReconcileMdevs(ctx, nil) + case VGPUFrameworkVendorVFIO: + if protectedDevicePaths == nil { return nil } - mdevUUID = filepath.Base(assignment.DevicePath) + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + default: + return nil } - return DestroyMdev(ctx, mdevUUID) } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go new file mode 100644 index 000000000..805f8da95 --- /dev/null +++ b/lib/devices/vgpu_linux_test.go @@ -0,0 +1,50 @@ +//go:build linux + +package devices + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("mdev discovery failed") + vendorCalled := false + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + func() ([]VirtualFunction, error) { + vendorCalled = true + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) + assert.False(t, vendorCalled) +} + +func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("vendor VFIO discovery failed") + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, nil + }, + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) +} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 78788412e..4069692c9 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -1,7 +1,10 @@ package resources import ( + "context" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" ) // GPUResourceStatus represents the GPU resource status for the API response. @@ -16,31 +19,22 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status. // Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus() *GPUResourceStatus { - mode := devices.DetectHostGPUMode() - if mode == devices.GPUModeNone { +func GetGPUStatus(ctx context.Context) *GPUResourceStatus { + framework, vfs, err := devices.DiscoverVGPU() + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) return nil } - - switch mode { - case devices.GPUModeVGPU: - return getVGPUStatus() - case devices.GPUModePassthrough: - return getPassthroughStatus() - default: - return nil + if framework != devices.VGPUFrameworkNone { + return getVGPUStatus(ctx, framework, vfs) } + return getPassthroughStatus() } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). -func getVGPUStatus() *GPUResourceStatus { - vfs, err := devices.DiscoverVFs() - if err != nil || len(vfs) == 0 { - return nil - } - - // Count used VFs (those with mdevs) +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV). +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { usedSlots := 0 + // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { if vf.Allocated { usedSlots++ @@ -48,8 +42,9 @@ func getVGPUStatus() *GPUResourceStatus { } // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index ab6c3e4dd..bef0740dc 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func() *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) *GPUResourceStatus { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, diff --git a/lib/resources/resource.go b/lib/resources/resource.go index caaf5ba50..86f644bda 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func() *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func() *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,7 +691,7 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } From e2cbf6c922f4945c068e40af767903b208469211 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 02/24] Account for consumed vGPU profiles --- lib/devices/vendor_vfio_linux.go | 34 ++++++++++++++------------- lib/devices/vendor_vfio_linux_test.go | 32 +++++++++++++++++++++---- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 23ab1e89a..4d056db9e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -22,18 +22,20 @@ const ( ) type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string + framebufferByType map[string]int } var ( hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex ) @@ -141,7 +143,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) } - targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + targetVF, err := s.selectLeastLoadedVF(vfs, requested.TypeName) if err != nil { return nil, err } @@ -270,17 +272,16 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } -func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { - framebufferByType := make(map[string]int, len(metadata)) - for _, profile := range metadata { - framebufferByType[profile.TypeName] = profile.FramebufferMB - } - +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { - usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + } + usageByGPU[vf.ParentGPU] += framebuffer continue } profiles, err := s.readCreatableProfiles(vf.PCIAddress) @@ -320,6 +321,7 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada } for _, profile := range profiles { profilesByType[profile.TypeName] = profile + s.framebufferByType[profile.TypeName] = profile.FramebufferMB } } profiles := make([]profileMetadata, 0, len(profilesByType)) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8e79a7750..9afd56292 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -169,6 +169,29 @@ func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + _, err = sysfs.profileMetadata(vfs) + require.NoError(t, err) + for _, vfAddress := range []string{"0000:82:00.5", "0000:e3:00.4"} { + creatableTypesPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Chmod(creatableTypesPath, 0644)) + require.NoError(t, os.WriteFile(creatableTypesPath, []byte("1147 : NVIDIA L40S-1Q\n"), 0444)) + } + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() @@ -343,10 +366,11 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { require.NoError(t, os.MkdirAll(proc, 0755)) require.NoError(t, os.MkdirAll(vfio, 0755)) return testVendorVFIOSysfs{vendorVFIOSysfs{ - pciDevicesPath: pci, - procPath: proc, - vfioDevicesPath: vfio, - owners: make(map[string]string), + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + framebufferByType: make(map[string]int), }} } From 4b8fbafd512cd20d8ab339a162875f4c799c61df Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 03/24] Reject unowned vendor VFIO releases --- lib/devices/vendor_vfio_linux.go | 19 ++++++++++++------- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 4d056db9e..f702f4591 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -203,13 +203,18 @@ func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, in return nil } - if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { - log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", - "vf", vfAddress, - "owner_instance_id", owner, - "requesting_instance_id", instanceID, - ) - return nil + if owner, ok := s.owners[vfAddress]; ok { + if instanceID == "" { + return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) + } + if owner != instanceID { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } } if openPaths == nil { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 9afd56292..10b87b081 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -72,6 +72,20 @@ func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") } +func TestVendorVFIODestroyRejectsMissingInstanceIDForOwnedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + err = sysfs.destroy(context.Background(), device.VFAddress, "") + require.ErrorContains(t, err, "without instance ID") + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") +} + func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { t.Parallel() From 8d84cbda2b1983ceb88752c7464539536501c70f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 04/24] Check all vendor VFIO device paths --- lib/devices/vendor_vfio_linux.go | 14 ++++---- lib/devices/vendor_vfio_linux_test.go | 46 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index f702f4591..6841c5294 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -351,11 +351,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] probeErrs := make([]error, 0, 2) vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } } else { for _, device := range vfioDevices { devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) @@ -363,11 +362,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] } target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } } else { devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 10b87b081..8dc6e6558 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -138,6 +138,52 @@ func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { } } +func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remove func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + open func(sysfs testVendorVFIOSysfs) string + }{ + { + name: "missing iommu group", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(sysfs.vfioDevicesPath, "vfio42") + }, + }, + { + name: "missing vfio device directory", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "42") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.remove(t, sysfs, vfAddress) + + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(tt.open(sysfs), filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") + }) + } +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() From 6fb8cdc7f0962f5484c8ae3789fd65a0d8bd1daf Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 05/24] Require usable mdev types for discovery --- lib/devices/mdev_linux.go | 27 ++++++++++++++++++++++++--- lib/devices/vgpu_linux_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index e2891efc9..61e55b599 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -92,7 +92,11 @@ func getCachedProfiles(firstVF string) []profileMetadata { // discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, // discovered by scanning /sys/class/mdev_bus/. func discoverMdevVFs() ([]VirtualFunction, error) { - entries, err := os.ReadDir(mdevBusPath) + return discoverMdevVFsWith(mdevBusPath, pciDevicesPath, ListMdevDevices) +} + +func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice, error)) ([]VirtualFunction, error) { + entries, err := os.ReadDir(busPath) if err != nil { if os.IsNotExist(err) { return nil, nil // No mdev_bus means no mdev vGPU support @@ -101,7 +105,7 @@ func discoverMdevVFs() ([]VirtualFunction, error) { } // List mdevs once and build a lookup map to avoid O(n*m) performance - mdevs, _ := ListMdevDevices() + mdevs, _ := listMdevs() mdevByVF := make(map[string]bool, len(mdevs)) for _, mdev := range mdevs { mdevByVF[mdev.VFAddress] = true @@ -110,10 +114,27 @@ func discoverMdevVFs() ([]VirtualFunction, error) { var vfs []VirtualFunction for _, entry := range entries { vfAddr := entry.Name() + types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + } + usable := false + for _, typ := range types { + if typ.IsDir() { + usable = true + break + } + } + if !usable { + continue + } // Find parent GPU by checking physfn symlink // VFs have a physfn symlink pointing to their parent Physical Function - physfnPath := filepath.Join("/sys/bus/pci/devices", vfAddr, "physfn") + physfnPath := filepath.Join(pciPath, vfAddr, "physfn") parentGPU := "" if target, err := os.Readlink(physfnPath); err == nil { parentGPU = filepath.Base(target) diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 805f8da95..40b987394 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -4,6 +4,8 @@ package devices import ( "errors" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -31,6 +33,29 @@ func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { assert.False(t, vendorCalled) } +func TestDiscoverVGPUWithFallsBackFromTypelessMdevBus(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), 0755)) + + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return discoverMdevVFsWith(busPath, filepath.Join(root, "sys", "bus", "pci", "devices"), func() ([]MdevDevice, error) { + return nil, nil + }) + }, + func() ([]VirtualFunction, error) { + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, framework) + assert.Equal(t, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, vfs) +} + func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { t.Parallel() From a924f153ea75ab37357d64b8ad4be3794160c1fa Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:10 +0000 Subject: [PATCH 06/24] Test retained vendor VFIO assignments --- integration/vgpu_test.go | 50 ++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 3f8fdfd65..ce0da7fb2 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -89,6 +89,14 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { if instanceID != "" { + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }) + require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + } t.Log("Cleanup: Deleting instance...") instanceManager.DeleteInstance(ctx, instanceID) } @@ -243,26 +251,38 @@ func TestVGPU(t *testing.T) { t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) }) - t.Log("Step 10: Stopping instance to release the vGPU...") + t.Log("Step 10: Stopping instance...") _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) + case devices.VGPUFrameworkVendorVFIO: + t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + // Release requires an instance-owned assignment, so stop retains it. + assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") + assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") + assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) + }) + } t.Log("✅ vGPU test PASSED!") } From e2d7010330c2a15af1d1280b286a1db9179d71e6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:08:51 +0000 Subject: [PATCH 07/24] Always delete the test instance during vGPU cleanup --- integration/vgpu_test.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index ce0da7fb2..059513d6e 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, release on stop, reacquisition on -// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi -// or CUDA functionality since that requires NVIDIA guest drivers pre-installed -// in the image. +// Note: This test verifies vGPU assignment, stop behavior (mdev releases and +// reacquires on start; vendor VFIO retains the assignment), and PCI device +// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality +// since that requires NVIDIA guest drivers pre-installed in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -88,18 +88,23 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { - if instanceID != "" { - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }) - require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + if instanceID == "" { + return + } + if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { + t.Logf("Cleanup: stop instance: %v", err) + } + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }); err != nil { + t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) } - t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) } + t.Log("Cleanup: Deleting instance...") + instanceManager.DeleteInstance(ctx, instanceID) }) // Step 1: Ensure system files (kernel, initrd) From f17f21544318c15c38ff1087668123d7e57dcc6d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:27:38 +0000 Subject: [PATCH 08/24] Fall back to passthrough status when vGPU discovery fails --- lib/resources/gpu.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 4069692c9..6a34de5cc 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,8 +22,9 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { + // A failed vGPU probe must not hide passthrough GPUs from status reporting. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return getPassthroughStatus() } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From 820552c65f958b0c2368123574c7f09c8a9f6c5f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:39:31 +0000 Subject: [PATCH 09/24] Keep vGPU placement available when an allocated type is unknown Sort GPUs with unaccountable load last instead of rejecting placement, and stop reporting passthrough capacity when vGPU discovery fails. --- lib/devices/vendor_vfio_linux.go | 12 +++++++++++- lib/devices/vendor_vfio_linux_test.go | 27 +++++++++++++++++++++++++++ lib/resources/gpu.go | 7 +++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6841c5294..78526a45e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -279,12 +279,19 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) + unknownUsageByGPU := make(map[string]bool) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. framebuffer, ok := s.framebufferByType[vf.ProfileType] if !ok { - return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + unknownUsageByGPU[vf.ParentGPU] = true + continue } usageByGPU[vf.ParentGPU] += framebuffer continue @@ -306,6 +313,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { + return !unknownUsageByGPU[gpus[i]] + } if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { return gpus[i] < gpus[j] } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8dc6e6558..ad2fc2a35 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -252,6 +252,33 @@ func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) { + t.Parallel() + + // Simulates a restart: type 1159 is allocated but no longer creatable + // anywhere, so its framebuffer size is unknown. + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last") +} + +func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6a34de5cc..054e3744e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,9 +22,12 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { - // A failed vGPU probe must not hide passthrough GPUs from status reporting. + // Only report passthrough once vGPU discovery confirms no vGPU + // framework. On a vGPU host a transient probe failure would otherwise + // expose the PFs/VFs as available passthrough slots while active vGPU + // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return getPassthroughStatus() + return nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From 3a331f8e209aa74fbe3dfa4fc392197ce0b017a3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:53:16 +0000 Subject: [PATCH 10/24] Report vendor VFIO availability per parent GPU and thread instance ownership through releases --- lib/devices/vendor_vfio_linux.go | 17 ++++++++++++++--- lib/devices/vendor_vfio_linux_test.go | 16 ++++++++++++++++ lib/instances/create.go | 1 + lib/instances/start.go | 1 + lib/instances/vgpu.go | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 78526a45e..6f59bed3d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,9 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } +// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the +// same GPU share its framebuffer, so counting each advertising VF overreports +// availability. The driver only guarantees that a GPU still advertising a +// type can fit one more instance of it, so report that per-GPU lower bound. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - availability := make(map[string]int) + creatableGPUs := make(map[string]map[string]struct{}) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -92,7 +96,14 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - availability[profile.TypeName]++ + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } + if creatableGPUs[profile.TypeName] == nil { + creatableGPUs[profile.TypeName] = make(map[string]struct{}) + } + creatableGPUs[profile.TypeName][gpu] = struct{}{} } } } @@ -108,7 +119,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: availability[profile.TypeName], + Available: len(creatableGPUs[profile.TypeName]), }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index ad2fc2a35..51668ed11 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,6 +184,22 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } +func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "free VFs share their parent GPU's capacity, so availability is per GPU") +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() diff --git a/lib/instances/create.go b/lib/instances/create.go index add25e4e7..b4b0952da 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -316,6 +316,7 @@ func (m *manager) createInstance( Framework: gpuDevice.Framework, DevicePath: gpuDevice.SysfsPath, MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 7e7855eac..a6c832450 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -177,6 +177,7 @@ func (m *manager) startInstance( Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index cffe2ac1d..a8ca6aceb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -27,6 +27,7 @@ func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { Framework: stored.GPUFramework, DevicePath: path, MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { return err From a19a35fce6ceab5c7da6740b30f8d67ba5436e7d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:18:53 +0000 Subject: [PATCH 11/24] Keep vendor VFIO out of the create path until lifecycle integration The instance lifecycle already routes create/start/stop/delete through CreateVGPU/DestroyVGPU, so dispatching vendor VFIO creates here would activate the backend before assignment durability and release guards exist. Reject vendor VFIO creates for now; destroy stays wired so existing assignments remain releasable. The integration test skips on vendor VFIO hosts at this layer and no longer asserts the transitional stop-retention behavior. --- integration/vgpu_test.go | 66 +++++++++++++++------------------------ lib/devices/vgpu_linux.go | 5 ++- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 059513d6e..7873aa35c 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, stop behavior (mdev releases and -// reacquires on start; vendor VFIO retains the assignment), and PCI device -// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality -// since that requires NVIDIA guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -94,17 +94,10 @@ func TestVGPU(t *testing.T) { if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { t.Logf("Cleanup: stop instance: %v", err) } - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }); err != nil { - t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) - } - } t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) + if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil { + t.Errorf("cleanup: delete instance: %v", err) + } }) // Step 1: Ensure system files (kernel, initrd) @@ -260,34 +253,22 @@ func TestVGPU(t *testing.T) { _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - switch inst.GPUFramework { - case devices.VGPUFrameworkMdev: - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) - case devices.VGPUFrameworkVendorVFIO: - t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - // Release requires an instance-owned assignment, so stop retains it. - assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") - assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") - assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) - }) - } + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) t.Log("✅ vGPU test PASSED!") } @@ -343,6 +324,11 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } + if framework == devices.VGPUFrameworkVendorVFIO { + // CreateVGPU rejects vendor VFIO until the instance lifecycle + // integration lands. + return "vGPU test requires the vendor VFIO instance lifecycle integration", "" + } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index a4ccd2db6..72fe3b944 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,7 +73,10 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.create(ctx, profileName, instanceID) + // The instance lifecycle does not yet persist vendor VFIO assignments + // durably or guard their release against live claims, so keep the + // backend out of the create path until that integration lands. + return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") default: return nil, fmt.Errorf("vGPU framework not available") } From 99467d5dd99ca42c7be4f74ef6ee73d19ae04bf8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:53:56 +0000 Subject: [PATCH 12/24] Report vendor VFIO profile availability per free VF --- lib/devices/vendor_vfio_linux.go | 21 +++++++-------------- lib/devices/vendor_vfio_linux_test.go | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6f59bed3d..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,13 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the -// same GPU share its framebuffer, so counting each advertising VF overreports -// availability. The driver only guarantees that a GPU still advertising a -// type can fit one more instance of it, so report that per-GPU lower bound. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableGPUs := make(map[string]map[string]struct{}) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -96,14 +96,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } - if creatableGPUs[profile.TypeName] == nil { - creatableGPUs[profile.TypeName] = make(map[string]struct{}) - } - creatableGPUs[profile.TypeName][gpu] = struct{}{} + creatableVFs[profile.TypeName]++ } } } @@ -119,7 +112,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: len(creatableGPUs[profile.TypeName]), + Available: creatableVFs[profile.TypeName], }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 51668ed11..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,8 @@ func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "free VFs share their parent GPU's capacity, so availability is per GPU") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { From 748b9b649664f1fceefbaec5d3cba21c108c9f71 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:18:33 +0000 Subject: [PATCH 13/24] Clarify vGPU availability semantics --- lib/oapi/oapi.go | 491 ++++++++++++++++++------------------ lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 3 files changed, 248 insertions(+), 247 deletions(-) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 2ee09469e..8c66a74d3 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of instances that can be created with this profile + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18968,250 +18968,251 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9TRrsUKK7G1Wh91VahcadB3zNF", - "J6Uo/jwk2gKKxzifTIgYpQFb20/6OTIvFCgEpz9WxVstRrdVos8qmwNa9ARHlE03W69+wD5Xm8Y6mL+X", - "Z+/e2Co+TQDIequKSj8G+7iPfi3qXKGXZ+9kCXLUDxju6kB6DXldZ7OFpBFOTIumSAVlvr0NiLO1wHxW", - "fmgtkwGxOVwezR0EtDGfZjkcw/M3vZPX77fSmMy7lTFBXN2MJ0SPe9PjFnMHLlAmoVWYxLzJ8GEIQ7Y9", - "QN5aFSe49SJ55zWwOoornIxkwkMhPG/1QwQP0cb7n0x2sB5BF2WVrdS/+8g/Pn0/CZ4YAJRv6PYcOqxb", - "UCsHPKhK1hHJOtXpVToNHRWT2bQs8izXJuSX1Y3ml+vr4ZlGmvs9cslXNRu3lSuRydEy8IkuKgGZT40R", - "21pKJMmwwIoki1p4ZBXfnCw7mMk1iW6Q/fVCv/7J1LTIBRmpmSByxpNqWMJud7kuqoQQ4DmxpaDMnDw7", - "vOIoxeISLkYnV6OcmRWoRpDvroN7mSmV3WBSP799e2aUbUXEHCf1HAS55HA/JgleoDFRV4QwNxUsEfZD", - "UOs5nLKhHI9Qo4wIyqtr2NkN9HtuwpLRVOCIIPOVq6pst0TCrdl2KW0vAQDBKCJSNuzv9qr9tZ9O8qTd", - "HoeGtb22CHl0kw1+e3TmysoUZXvdMu8sr/IZET1z5Fz93tVbuyNXFzhyXTGDC7skMIwJVDyyiR9+XqYL", - "N4KKTvrzSi6kxxykn71n+4FTYJaqa875h1bCXv24h/zqKWZxqPyxifc3mfJTgM+CoGORgyuJxiYWySTv", - "G7HcT1sQBMeUESlracZRLpJOt9Ob2FkdbG0lPMIJIAju7W4/21od1bkynNdGL41iukrdczFOJgrGJbUa", - "+DWYdJUktnCWtTCImXVccz8Ae1oOH4SSx/pu8yQ8F08+GCxlTF/jSLkibWAhq3hAsX9sAee3Mh9oMDV1", - "V7So/dw/n4OgSzrDalYl/60l2ocwFQgx1DRirA+1dTRE/jEoUXERqsTFhbIJeGPiYhCL+9BF+DlE24qv", - "bPDMn+WT/f3d/XV8CJhN7ZjbcxeYqnm7mmJMvOW259c2ANFSVZnDHenVxUT1uqyhKc0Rl0hq9YLyjLAb", - "ref+3u7Ozdaz7UROXJRWjS+FEFqOTo+NTBRxpjBlRKCUKBxjhatMBkxLmstAqRdMUsjSmXy/mrU0hDP4", - "kCu3rVP1pZzhDSXr3jiE5hQzOtEM2b7p9yxneGf/yYEprBmTyd7+k36/f1Mgihcl8kSrrdgyMXMeJkVf", - "zj5vH+4Ab6LNXP7onB2+/VkzslwKc2ltyTFlB96/i3+WD+AP888xZWGcija1WOlkqQZrNTwst0DAJD5A", - "ZbltJ/e0qtcftg1DsDCA4wRR3ypBk3cH71bQOK2i+N8gVXdF6qoWV16zZNG4Nrcuq1oWHFdeOVU/+atF", - "aVX6cbW725m74B3bp8GDLqrOLju6b1U3WK4srbhUeSsjrCimmCTmr4gzwNkNVVasXJHuWYvCXHCN2Apc", - "RZf+j0Xv3o9H/kC8311hL+8nW2Lxww0jVFYKpH9blkPXcyEnjq45zGHbY3ErtC1na2HhgqHZD3wX3iaK", - "q9r76+l//f5/5NnTv2///ur9+/+ev/yv41/pf79Pzl5/FuTIakTAB4X1+2JIfqZEuw/n15aUTrGKAjY6", - "rf41rLB9YiwOKppBAU40JgdD1kOvqCLClHOr5SIOO2iDgKYEX2lxF2rVmDSwTf3xmXEw6o//cGLwp3ob", - "sc0RF3ZDCugPmY9jnmLKNodsyGxbyE1Egl6g/4pRhDNTk40ypPXfBRoLKKBnPT5l5130B86yT5tDZovm", - "G3DrDEMNskmRhMWcv9aOykSl2tdJgQJhnC5DVtzWBSSecfv1S/x8SpJ6Ck/DoqzW36zm9GwQAg+E9BK9", - "kVAjBlSQgrI1GRV5L+jZYHNZn1ujYxQ0tIL8rF/c5B8e5iFzcVPO4jGJaQR8xaXtzWxiZ5ExaSjNGvEy", - "wa8XsDdvTC5ZjHCuZpoXRTbPPeL8kpIubGkXTQQUeOiaL417fcaz3njRm/GswDzAwgSfYOOgrirZ/6dn", - "J9p7TwSd2J6CqeuaRAJCJxwZOzOTAVhYF5Ym9taU12Ba9JkT+7qp0yJNaQ4TWa1ywVz1CAI1JAFco6A+", - "EpLJv0dRQsHqJGc8T2I0A/w7pZsJQdh1BkUGCx5HMZnU/12NMNjZfwIarPv37k7rBFKzdKuoLE8COm3q", - "WF8Ljm3YJAzAiAcjZwhfExOkb0BbyArsFIrDf8+Ra6g8cQUjMd4pk9ImbTGCRHrJbJvBrCN7DCyCxwjb", - "09TmPlo6hZUkphYtmGAB+CxpgR3ywiRPvn11jhQRqUtn34j07sApMUARPSplbutiHR6dvtjsd4K4RxVX", - "FWzVyiSn6qAD0Ac2eKApJqK00eCUdNHJMSSv2mul1MUg2+AnLlBibsXyMjoA7IyquQebinonx1YATRZl", - "BIIRW4adTddiVr/eDtCbQgXExVCKNMSStlyT5WUCzdp4NJMKsdR6LWsV/GNW/bP3MSQ+QLlBw4sBu7Hx", - "/mpvc3RIUPqiqlnIbnwh+UEhjfYvb++/NMrxl5fRd28mo1sf8CibYRmi7pnv1YSXlvbddyNX2b1oDuGp", - "9DuSNHi2/uaKrnjXkCL6nqt8HoL13O9tb7/d3ru5+e6mALVVZCoPta7AqG0PLnsXIK0ByFWqRo2x3kg/", - "tpHdzi7y/hTNsGTfKXhYs45s7z5tY5SAXttGSfvx0XxihlRwKQdzVUT3GsCvS5okRoCRdMpwgp6jjfOT", - "l7+cvHq1iXro9evT+las+iK4P7fAqoVbANbRJHwFkI4qGfqoSOV7+/YVHK6EQDaEkcMvb49gu9a02ALR", - "1g3u5dk7cPxjOXJxlM2pg7hMvyXXVCq5DHLWKhz5cxB0zaftau67SZo2ytL7q2F4f67gxAZR6zbvAD/X", - "xZIvLecDQMs+ZM7e1wdruxKI9nPRZK2d4Y7AZBuvtBAQaw2gYL/pdrs9LOydDKeC7xJiW76E4xKqb43D", - "2u3QQDLpodQXD4nRyVlZeKl0Rrjma3N6vtPffvIMaoduD9ow9hRHK/o+PTxq3/lgx9wyB3h8EMUHoLDf", - "1mdlCduoIDi5wguovWeWdtgxF6an3XrH1iqSreJrluFub4duWxfjGvBrQZx1gUtylK4s/dEiW7COYZbm", - "FqExpUlCJYk4i2VVRp5hiWRmgElNCYxCgh8yGGAXFZWIQUpBOIpEXpoerXRt5f08s3QPZTgzzrQOADj8", - "v5CFRCkFJ2jRPYQ+SlQkpcRDtiFcAlORqQQVOGP9A6QDdF3YuR4aVVDqQ38wZHKWK83ENvvoiDOZp0RY", - "qywaU/AYbSKZG5UWxgursdAMU9KYiCHTrwWgT/8o1JODJ4PBYNDtFJrcrv73IERNd+r87FtoX5OCC+B7", - "zIL8AqqfyBnKWUxEUY6bGHKoh8jd0HH6mZi+7vN24pX9vJSrwgdzHQRwO2zfzwVUhaE26OcQHXoL5Xz/", - "9iJ6qxQgJ7/a5B/71egmEQwERTxPYq3xjfVtZwxyJLZmSEmU4c7mXSrRO1MKszp1G3qsOPo9J2KB3p+e", - "VsIeBJloHtBu4sAlGvaBZzfahp01NpK1o7mJe9mDn70PyNm6pOJJiF8cYNb3MLqcZEOhFcNWRXFeZV/T", - "WmUwi4Qys0+aaFZMsGbKj8l8lOchrUQ/crAk796dHFcoBeMn288Gz573no23n/T24sF2D2/vPunt7OPB", - "ZDd6uru9s7si46RFFtntE8OCqmkgYLgIDx+5MPVQ9HBTkkBNCLCBz1eUxfyqcs8EI1H93m2U67rul2PY", - "Ww8hmPmSYKmMnaCBZZzCbUoi3baJ/LaZikWJobBF8cnbwfbnmllgcA3M+K3ImfFfmtz6wlafegP2N6s6", - "ztvxVhiQyzBZt1p+5+0XbXCw//xg/3MXzWVJrBtjnZzucXObQq8c9nAtDcOlAnoGG2cJ7Fjhw5jPbdZG", - "p9spEkvgb7h1a0HLxeNW2VJNB7YbZiOr+HdD0vBJRTGAkBCDPRcfaJGgyDcd5woVqela1jhKeB4jz+hl", - "oLjA4XXiKQm6GfA/WVuYgdY0WQ9amQAsZ6iYQJlmxODo043YDOMD9BLehUc4NfqTHYSp2+H7uHC8MIEp", - "+ny5ro02s3rI51aRgW+0VoP0v2DaehmsbXR1E0YMOkC/cvimUKsYrxtZzeugzyy/XjfIbtjUX4cYAZ1Z", - "me4A/VTIcYUkaCW/DUnsnyPLsEqgls1Kurzd8Y6mlnLnvNTvbsesaKfbcQsFKeLLyeLvSqpfOn8+KYYi", - "tghO4CyXybi5oomFp4aZUKloJG2Wht7cJvnClhQi8choKU3BnybD02oyxUdOfHl/ijYAgfAvyFqQ9b82", - "i0DRyl2383zv+ZOnO8+ftMIZKge4XgY9gvzj5cGtFUijLB9ZI0TT1I/O3hkjQ2TU9yLI5P2pD+uQCa5Z", - "j565a9Dv/Hn/uQ+vFPN8nHgePIvFZtBcYcOCCGIFL2oIOPydJnM6mbDfP0aXO38XNN2+fiJ3xtsNsLGm", - "o7B968T34i8Zg8m4Z8oDhRFwgKCEbASJekMkzACdE4WAfnoIR6BHFGnDluQclJRd8SBh7e3u7j57ur/T", - "iq7s6LyDMwJrV+BStiPwjhi8iTbenJ+jLY/gTJsOSwGQvZnVMcPnDNnavoOqQNrfHuyGqKTh4i6pxrY9", - "TxuX/L3V0+yk7KJD9nOhwy2d8uBq7+4Onu7tP9tvd4ytHXYkrldzGJcbZJbHAtD7O78B0uTbwzMEmbcT", - "HFWNKC4U60ajUjcaFRRPMKDnNxjYs6dP9vd2d7bboZ2Fojssjl/lwFZ5V+DQBYgisBuBpVhmvd2m2yIk", - "ThkCe0OiBNP0MHK5DLXbx4Cbj4R5rdyENheDNf0vXVwtvm1lRSpsQyYTxogGXKCcFSU1+ut9n1/EhdnM", - "tc31sJ6rh/JfmF49C8tjSofdYikzQeaU5/ILNMSVSU6dJJyLG33bpLC8ITJPlPEzUonen34HPEXTGpKK", - "ZFUdylLjCvCiW07uRue5QiJhIm9arFa70WbrV02423Bqu6uQKyrcoBEyLNacK2froyyPcBLlUEQGF/up", - "ZwVoP5B7n2XJwgTRJwnnDEUzzMAbIWzJKjZFGM14EveDIaf6yWgSDF/gVyjhBuz4kpDM1lcxg9CfaRGG", - "zgna8CuLGVKq1fvcTw2TsRU0qtS4n4YLF2IZygorcs71emLFPRxe80nF5JjwqQSlUEF6QL8O/55hYaL+", - "MTP1guap0SUDkc2BIdaYeehGNTcpn1gF14ockNFtVhJHgkuJSEKnUJvm/WktUXhFclmRLrw+crI62Bak", - "azyHgasMrjjZuqxY6H4MJM58zg0JNAzJeStiEp1xMsUsh4orHiFbi3e/ddzhjEs1KgCgbjhYqUZQRiEX", - "pESJK9LbC3uQeyd4LzrWdpvlsgG+t/p6iarCTTUNsJmnBlc0vFrdggZDZLwMgbUSdauE8apjNt0EpK3E", - "3acSWqUePhjagOQSjy152PGbbaJRwiqr7mdJW7VVMV/tDc7b4qethks7w2p2wiY8ALJxAxels0TbsNCM", - "iJRCIREUE0ZJ7HTJwldpTV2QmZ1IguKc2JUz8qnAdsGxOd4AlMGcjYyyaY3X1ztsYx42Y1hdZQH6tS+2", - "iSuS4czVtyKHtTKBgRLhMoe1VbQllaOwO2u5YUGmeYIFqmMErhiyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJ", - "wq9G+pH8Aeay2Wp2+oNRU0meczM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j", - "1x6hV2HI93YGTWnpDY1WEtKXIRhvyrktyQZPfC4DSXwrpRxXTIjEFhveiD1ZLk1dkxa3kgMfdS7A23l0", - "qhkan4cBcmT4dQ0BBI0JJNi4qS1zjRZssc1UgiUVcjlDf+fjqkG0bXxtoFDXBiuxKASZBAPpYUdXGqTN", - "G0tr4u3uTcAegK3qicJHN8RQWFfSrAxkauInb5aqe82IXTLq5mgqfbWoXOECLQqcANtre8CAeh22QGAw", - "wMFItYCKplBGZuEVF5RozIUA5GUt4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahc", - "aCVrSuI++pun4gFWdZqphQVBB+P5dxLxK1aMccj8QerGc6nbOWTGsihyKJJfvqSbBa1PEwqkB4MTTAnq", - "8G8ngsiZP/dQrUgt411xETcW4Vkg9wrUdgEfK1L8kjCflRXNBFVD09DIfLUcLmcKvcJTq3+iSu1VVK+t", - "urq/XBIRFhKLKRWvtApd8Y6Kp5wYtBWAHoGaevYvw+ILuJEW4CJl8391TZY/nRWNV3+rveYBiDg830Nj", - "tg2aYCOTL1ML9ql60taGqkC+2SrYmGVfAtpwscquMElVEvAKhLS6J9ulvNWj8t1otiSJqr3vPdt/+qRl", - "hZbPctYZmKwv7Zqbpytccg07ddrG7/Ns/9nz57t7+893buRhcQkcDfvTlMTh7w/aINdKH9bkX//45/vT", - "mtdnH4KdBzcalEnhCA+pIY2jOqD3p//6xz/dqG49oBCjWYbibvDbN0bpJP5OukCBqguvnZNshX5/WDES", - "4ILNoA0ymRAwg47MuvXKwdTwNtpJwTjDEVWLACPHVyasvHilBindxh1UHWxI5DVtW/hRzblkPi6zOzdc", - "5+g/jW+4RgvPWhd6kvm4yQ/9ut6r8UKXXgs/xqFFiIEsaowvG7iL+VxhWYmc1n9HkODgUrmW01rMG6vh", - "bes5BxDFYuuZeaGAIVj0mjxpP/K3v7adnt+yYtapr/iHFeew+QjeyOobuJEDRt9ofQ5rjT/YC/B2X43G", - "fgm2lTXuKvXaylv35v22SNNdLkhQ3GA378/LTLzJh3XwXaBHOwa75GXb3QpJNFCTl3QSMKDxhPSKQD2b", - "kYJkbjyC+sxbPPdAqmR0ySeTKqjsfjMIOeDrQFaV6wUrpTWTLiLXzmZRR7A2YDrDzr4cdrQKMOxsp8NO", - "zW0VzFNM8fXIdlAFURmsQgUv88xrg5RuBuOER5emuhgUre6jAUoJZhLlDA5/zau2PVjtHep2Mm9vCgxu", - "YkKcltgWjGlMZnhOofSD9alMK4GY5JoqCQGj0M4BirmBVaqUVrUz1K+ZLMKDctJw6WC2sA3rBvV7nLmI", - "1vJdMPBNoKAr+0gE71pUAM2xX78+7ZoABgg9NAOrxDe6iZoRaAZZdFGrY1D+Ho4fHidkBOOu4+Kny+vo", - "J3+DZ1UQSZS0QNklOdSIAEU8Z6oOmJ+2U+Sq+VvLV1LOINjPhn8AAJrt3dblj0kEJ1Iun8Uqod+CuGt5", - "A3alQ4kDuyEShkMBvqSwr/iNdQjXB2CMDV5VZNOOH9dtvIQjqbgto1Wc6hG5jgiJ68ia4VfaxsrbL4Ox", - "8q+wBeMpChbbtyHeeXl2/bvLpIKxNq22H9PPOOsBDIjbUgvZYTD4LChMldAqGN8edsQohGMaeqFNajO5", - "Xr3Wv5JrBUDkcZ4YdLkw6VpWZS+jdSt+6xTCpgPNxfr6/ndQrs3Em9+qYJsNVX+Imm32rTup07a0O+dE", - "uXfPLRk1V+uvVFSpuLRcwL97pRpjY0ipi+wFj7bTzRoJ7s3CVhGLftsyGZLhlIwyQSb0egXxmBeMYlzF", - "DykPUpHBYIA8N1J8jfaeomiGhayNndHpTCWLagDOXgC06LOKGQqiCHOGwjY7X+6m+3A52s1up996SDg+", - "9zB4lmqHWJF0tAqg+qj0tlnrfIYXYMVpdBI+3d0bDHZ3BrdCqHbDusFyHZWf2NJ/1XaaUuq876yjvxKl", - "6rdQZDMv15O9EhSSootlkkoQnB5A4k2GI4ISMgE0uqKO93rPYr3r1YO3ApVFfino322U3Tfng6/Wpim6", - "suDebhod51ysgv34z9c4RBvYTLSEXRfIudvtDZ683d492H9ysL19F6jSxSI1ZXs8/bh99TTZwZO95Nni", - "6e/bs6fTnXQ3qIddUlOCpw2t/qLfbYyyKS/JKmhQhaWhDTuHjIh6oeB6gW1JEspITxYZUuvTFFfwAuN/", - "X3v+b2bnNzNYKTucVyfpixBYlYtToayHAbqyk1npu6jP5uR49SxulYFUH0iY3upDAfJqNxgoBbHd+UwI", - "hJy1vIbeeS+2vohWZsWtu4pCHnY46cFdbljxEHnXEBC8Wa+6wJcvuYDtdMoFVbN09W1RvFbgdUPc9Eep", - "4iqwUh+dTBlUCfd/LsLkfCVKf9zpdpKPe9UzY39vD7FloX4LArRb7UsFLcLIoAj96lWAV0rFQ5hIdq2r", - "6zH/sN3bfg5xCMnHvR8GvefViIOuWS1/+bbd25VfB23W0K+152o0bT+/UcS1W89VFPQLDVWKK+9lCwJs", - "abysyeyuDpdwW9ng8vHSHtcgcxoF0M+V9OzlNvKFppgkeBECgfcMtbKmPfpEhsZkSplsY7fdHRSG2/10", - "2OmjQ4vEDbpsWYG/0jzUXvfohKYpiamWMY3q35zBsNPSFlfXJW5WBMR9FZDW+mFx7fl6iIR1CVfrrsn+", - "Z+Tjfpb2207jXYXeAXY1p6ICWBe82EV0gjCrVQKlbI4TGttEekiMhHi1A4eIVpKs5QGylAOdnaSLplyh", - "MoW+pb0tZ812wWL85BrsrSswMwxB7HwRQJQCqYuuYl8nxygTPM6jMn80gUGXiB8ir2GhrRDy14fk3qV9", - "AxKzJ1yg9faNJoNGO/tk037XbJOaYJu3enuwfqvvxCjS7eRZvJ6HmZfacbAbQaSvSUEMmGiqy16TBL3J", - "fGjB0d/4K7is8xpbcqRFojxzDhZNU8uUFHC3gIshFNd7TBKir6nlRhBP4jJLgsqSi65nqdtPns2aXJzg", - "kVoeyC+EZFpXAfwj6C/FbBEcmKvvWdwlGwMHqy2Nw6tn6gLZ1aoO7ulaSaxxq3wTblOtAsPlazZvg5dy", - "6Zm/CzBtXzRbRkBxDL8ipL1pxtq3X7qwt0b78V2Y5R5SSHttXQ81fFSH3lvAkLv+y1hgLdZViXcv5J4P", - "kcVbqxk3Yb7Ws0B9q/Nh73+MlRmN+gdbP/zl/+59+M+gtbmmN0siejGZQKDRJVn0TJUfraP3q4inUGJA", - "C9NTSyoEp2BDAjRxexj98e4PCqax+BWnS1OACC2vRM/22gn95T+a45u8ZXwHfHItyX52BY67qFSquLuO", - "NlIipi6W3CWSbfaHDHLTLslCIq/wlxVpHKF+J4tPvAh0dGHEwD5h8ws0plBJUQ6Z1mpxFJFMaxO2lgw1", - "5cA5cB9BcOK3YwuQucRv65A08QQEvT9dgst9/e7tj6/f/Xo8en324tfDk9EvL/4bQjyueqaHuKdpb2//", - "iS0C7q/kdrAQxc3rKfTRqQ3Tt67+SQ4KLeB0SZTmKoegEHIdJbmkc+cgVMntKycsJ+vevhLBZ0LtKpWE", - "ohIsJHRCJwT8+nCd2KAaKh0xUgnV061xgzK0fGMbwhl2gJN6xe9DdSv0VoRXu9zY6qI/mbVjoQaENHDY", - "IeMVytwHtBcqAa/CxX54L6MNyBxxJV5d4uzmzUBRD4sGg5GHX7iSz+D5l6i2+W5lec05T3pavWkoSRC0", - "Jpu1CEbOQ1MmI6HT5HSYjgMyvDXtTukUB/wMIX/CF6mK6Qa0NmNqaf8by4OF8xiO6/UazLE0S1WrL1Az", - "EkjVa05zSLVUOypL/1eDZ3Jmc1epF1tXTVRNmdqy1WtDeBkxB9TwVdnK5Slz6Ig9+Gh9Eu5KvcqbmTeS", - "5r05depDTcFZsUBnemmuZkQQbyPggxIH/4ZLZvNyWqCwmOp/GRFlzKpL6tFSKbibJdooLD9uCYps42Vz", - "+Oo6B6f4uugBXClYLvkfYR5lnaXtlz8CJv0bV1uSTlwTMIyachdGYK9S0ao1cVS1vBk+VS3P27wfPHiW", - "V63gfk1nq0acZR8V0gzR498wVT9xAepgM+bJnQO5w+UfEwEYcHWY9lYY5zQl8YjnavX5t6Xr7ZVf1B8t", - "69c61RcDEUeVdN4mXuBQOcoxLK+0Xg4S5YKqxbleLxvMDWmQrmgsLCR0BD+XHUOhzk+fwGg8CSSMvCSM", - "CBpBGVR9HlPMQGNC70+9animMOISXiuIQK+PTqy5wUH+gvpIFZCei7s8PDvpdDtzIozK3Rn0d/sDOMwZ", - "YTijnYPObn+7P+iAVjWDKW5B6XqbP23zjQvF9SS2ktCP7iX9pcApUfDFbwEkAIg7tK+DCoKnnhKZYSqs", - "FpklgFBgCIbqrwHX312oB+ZW7pplb20zhTRjyH4h2Wu7uR9AUIazA9PcGQwssLmy1y/k7piEga2/2+jR", - "st9WUp1dogDM/ZKa52TLYuk/dTt7g+0bjWnVUODshjp+x7BN4iWgne/fcCFu1ekJM2l5NsnahkP5Jw4I", - "yT9rv33QeybzNMVi4RbMX62MyybBmEiE3btGj1MSRZpVQDGePnrNiHmOsELYRC6LnEENY/ehptDqKTBt", - "u00uQIp+5PHiiy1hpQ9no/hUZWf6uHxaoucvRzsFGS9vpH3kELYN1d4DAf2IiwLcD3ZS9gbP777TI84m", - "CY0U6hUEbOORqYSQnwTwwh32EBfo95wrjIpw/kd0pK3MOi7IrVteRVt/0PiTOd4JCZnBz4hIMTPJEead", - "NYd+6Tgbl0R5nFfeao7wobQH3FQOhMdcVCDIVY+of23VhcHl62gvgMBg+zTTix+Q8Pfu4YTbyRY1WB/y", - "yEHlS5RL8piOk3WxjUshJCjLvSTqa6H5wX1eWbaIwJ/wFD0WAn5JCgmv3K2lS2ErEzkzCnBQAnxTJiza", - "776rCn9vyydelAz4NXTTUM5CGb8qjhd95NbUKP1qARBLgsA84+Vr5UwP72s5YTv3ccJgxoWn6Ns19e2a", - "WnXKDbW4KcDB9E55CxvEjSwQfz77w42tD99sD+1tD60sD4xcWevC3/m4j2xEasRjguSM50mMxgQZvCMX", - "e6Kw6E8/IiyiGZ0TALWDIm15omiGBUSWpCjGChsfeqNhYqVZomhuSzfXc3GI5QLXcSwkGQEO36gJf7KM", - "QKSMkRjpTyx0XwknuFS325z9oIG9aLC8GtHVjEtS4Pkx5d3mkN4sjXYMzfaH7K0FetULCMHUjtdIkgBc", - "7Qr7D2cID5n94HvHQlwgmMRpybmwAMxAapApzbYsp7bpkY5kxENYO28Jw0z1ZEYiOqGRndYlWdh4zmCD", - "reou6QG7cb4/LRI20M5mGK8N4BnD4LzHxTNkKanqv2EQBB0leVw6uRyEEBZjnCTBwhzThI9xMjLrc0kC", - "PsGX8IZdFL++v/MmMR4TU6s9W6gZZ+bvfJwzlZu/x4JfSSKGnc3+kEEihl1rEndLARFdQSG3NOP6nAme", - "mj63zBC3/rgki0/9ITuMU8ocRcAnOJEckWv4DupbAWaG4V4N9GBOU9gPfpRLxVMf+dTRnRkmz1WWK5tR", - "IonqhlA/h0xx9IfDdvy09UfZ4ydwFhMcazrxXjFTAtm6adRyhPXsR/BqwN1OYAGGHX2RmjCPqcBMGdjO", - "ApwSTf0t3SiqI+hDullf4QgzlPHMVJYAopphTXKVNgCrAScJUnCU3LdacIedbJiPhd5Lx424ewYorXaM", - "KEOnP3qHabD3LHyeJIkECUWU/Nf5618R3Mp6D8xrZbiWSelgWmBAcQ6uU8fTXuBohoyjCooJDjs0HnYK", - "d268CWPNpQ2X6fXAp/iDHtoPppsujX/o93VTxl15gH77w7RyoM9Slhoc0GHnUxd5D6ZUzfJx8exDeEGb", - "4MvOK4wAbZhrbhM4CaaANOPd+OaKxCxG3N4CyQJhVHIgP3BlTBkWi1WJhIGltyvIJyaS0VuMP4YQuTjs", - "HAxd7OKw0x12CJvDbzbAcdj5FF4B67VsrlwH91nh3CyI6MlgsLkeCduub8Bn2cIx8IV1wEatqCi7qXfQ", - "wrD+ufwD/9b6Z+H6wUx3XkITGcXfGd8foQPCk9h9TTTggqiJ3ZhFJHFi93pDz/07D/RmRSRJ7ptAH4o8", - "C/dYgdT/qMgRNqs8RivN9w9McYP7ulQqZvuHod9HZz8PWM+t7ZzMXahzuE4JYNBYVRqZlxGW6BzG1DvX", - "yvcL+LVv/+t0P8BUvEj49OLAqO4o4VOUUGbzAbxAZS0e2LWEjwwMTfGdRaVxReI2jCTxr3/8EwZF2fRf", - "//inxXb/1z/+Ccd9y8CrQY3pixnBQo0JVhcH6BdCsh5O6Jy4yUAVWDInYoF2B9bmD4+QV+reSmlyyIbs", - "DVG5YF7ehKnXJm2D1lWg50NZTqSF8dEv0oktJmNiGwN2G3eWzVLe64nuBuAQYQbeBPSt6GgAsOSoKbRt", - "NdFO2GRq5lwxmtbDNJeC9dbzF0WulaHenhngDRkMLHHo3MEDO2m0cX7+YrOPQNsyVAEFg0B3KJuxakT/", - "G09az5MMR6kyFFhlw5sinOExTagzOTZUOzFHMMXRjDJSxhcXWOOuiQM3Us1jDs9OkA2E7MKrQ/b6fAtM", - "rIpEKhekazmBsAijZTk0bvNcoAfgX1RBdFjPvjtkE4IhT+jk2DABD4S7yAcsGmYA5AExrlRVKq91h8wg", - "yVrkYn3wUh6TBD6C/qdYkSu86KKi1q2rjpJgpRVi2dUvD5nBerVr0AOoEuQNsw/8zAyp5yJ5bc6WIJNE", - "q8YQgW/KfkPfGxMukI1w9qr8u+5MkqUZll60FEevz/X8pqAJcmMPhJZen7vd2OwiyVGUUKCGCLMhm0Ig", - "kAPv5ayyq0VC2QyLuBdxfQn4YE6XjF8lJJ428dgjn8juUJKp9BM4Tj/XyfWxCRez5QnoQ2wA6lZ77o7t", - "O+1cd7bFP5PvzhaCvIHzzlhwieE3ZnW/OfJaOPLC6+aceiHP2rFDYLy7iF/TxQMF/DraW15z88Rbsoew", - "6KENB20DXhEu0NnRCcJxLIiUm//e9j49U0Olpfyn70fNih8i9MSOhQsL+mftLVUCeSzs4I0dNcJuXvX6", - "uv79tlUpvtN40xV1eMor7+5vj1qnN7lGSqG3pLVvN8naYFsqIw5lBktq6YFolJBCfCnOqU9F66zKJoy3", - "uHJWikuWPZ8cuwN5f/Zl23XO6nfDPTDF4xpDfEBGWE219qtmPyZqflfsokObXmF+/rpIc3B/UtB9m6JD", - "ZP6Y1MW4tmyaCxqgk8YL9CVRBt7kLvV020Ng4udEuFNtBrowsy6mZT5FBqcFJgSWmNW674l5pZ3qa9r7", - "M2m+sDw3kVjskn8TUVoou+VarVJwT2wJ6LvTb6GHG6m3Xy5sxRJYYJHBijp2biewrG5guWDR5rfIlS9O", - "0SausVRihZs3iQtLtkFTKvSs+5LrDplfb1zLdFavpQxNEjqdWSdATCcQq6f8+t0wyp17GGVRJ1tgRWyI", - "4mPM+z3Ti2y9wHMiFHp9dGLW379St/6AoNX1qpJjXitv13dvXvUIi3hcOE+aZVL75AsrTIb+K7m893/q", - "HmE+K3XiQZPA+Bn7b4LJkYl/71P+v3Z+SuhYYLH4Xzs/4SSjjPyv3cMEKyLV5p0Ry+C+brr7VmAeMfFp", - "/YVWFw1YE5sCZOwagb94q6XM797/U4n9ZtI3EvyLdf0m+7eR/f3lWin+2624UwXA9PFAHq6C2EKrDY++", - "Qdrcg9HUUqQHaVPxIpWgNjMuFTx6fPnNNqicFhTnXxstrf/lgVx5fTjSPTnuwkJCRWmoaGHTB+/JF+DG", - "ce/Cre33/h0Bh+mYTnOeSz8zMcUqmhFps3YTUmXAj03sLq/nRsH7K6bSwX1eHfcuV3+j+zuS+Osbapi3", - "ceitk/ndW21lfvu+lvkNoqnNbLZlN7quJNNmQ6C1wzRtS8YV6NflAPDQuEK6CHqnFZVSXUCgQRwM2f/W", - "+sdviuD0ww8uhTIfDHaewO+EzT/84LIo2akjFcKUoLaC3uGvx+BFnUKgLBTZKxO26+MwNbuB9FxZgX87", - "Bal0JLfXkBwVftOQWmlI3nKt1pDsXtytilQtTXLvOpKjt9CCW0zxP6eW9Cd3j1Q0OJlPJjSihEGBF0hM", - "l0vxgEaT++YZuWVCMrP+SC+YqCKJtFYjC661RkIva0rfeyDZSVlE6761R1e++nEmVvHM1oO1+lopLTQr", - "bF8bPQzu9/a6f0XtMZOY0YiWly7TQnegMpCpCZXmJgut+LIEFeujt29fuYwzLfELV2dKcVdcytXdHDK/", - "uFQfvSirdpkXXAtaIiexzVCFPDxbrikmOE4oIxCiS2QoOaxaEu5Bj8WXFyrD9e5aCZX3fCxtBdOHEyof", - "jBXci/h2UikMzUsd3y+FV5wWJ8LBqXlU/MoyoADjCYlPWzhXvGdzWLdm3ACbhbEdzxIcAbSjfs2gjlnY", - "AAMz6DcFWACCJwkRBk0uy5WrgThkxeAo82q82xo3F7r5Uc4UTS66JkIGIEEkwmxhIZWGrNIZVoqkmYLU", - "XkhbhxEKkpkR14o/6kFTnkt4C7Jw/S4RTq7wQg6ZTQY2n0OhXEEiA7yYJH30MwccBoSnmDKP8ZoKhN/J", - "IbugcUJGFkbhAlGJ5IwLRRiJUcrnRFb7JVgklAiYxBHWKydRiheAZ2agHc368IwYzLAKWAPX/8YsplDL", - "TvdcTPlgyDDaGQxQSjCTNvVa4glcOLYNBIOoDOh7hNHe4Ln9qrZvgLnrln9DnyYhyJxHeJwsENFUDOAP", - "ahM2MLW1JU2NXr19Eyqk2a/CZGiLhlU2lkpXKjHuopyVyeVgPs9ZkQuut0vlgsE8rWONUFFcgxZPY0wi", - "rNeT8Wo/gGTIoygXoQtSb7VX5PTfUXD0pncOSxVO3U5AC49IDHvOuJrBmeZwlDa/b6Cqkqj+HBdN8JBw", - "gTDy6Lo0EpAoB9a4Ach/F2XFPuYq8F5sfu/Ojj6+lhG442+w9x7L/QRExCeTygFcfzWZA7wqZWKZhP+s", - "5/TIlWr1WVxM8ZRxqWjkmGG9svs3hbC1Qrh6ZYPUPOHi0petqvT7ExeXbTUwiydKH5ci5s/wK7Tt6+EB", - "dvPDm/jBwGyUFU00966k1emrOKUgdFElXewwRwlnU32KSkP3vVvifa1uw+Cw6ctUGP9xgZqjlZCR/dFU", - "e9WTsbU0wWof2VYfmhfp3u/Bv/MrV4imWUJSAtVge4bY9GaXCEtQOZ9KD2foZrxSnyo/HdjogtK49LtO", - "HAK6chu2AdL78nYFmWrCp+tx/IrOHWhdAMhvyExlf4IujDfnAhU8WAu0BjUfXc1oNANQP9BbdfsG8w9n", - "2UWBZ7x5gF7CQfZhnaHzDYOVr2lN8oQYrL55ml4cLNc7fX96Ch8ZPD9T2fTiALkap8X9IfVbPkifnkWC", - "pUK/WujBjUIZhx29UFjrm8X8Ni18X4k3PWQhKD9GrmyDdIIuPFS/iwbIKcdvX/Hpgwlj3eYqAWYuiiOr", - "OgJtEhZ3muImaBIG9NseDELg1S3BBc0w7hhbcGkwr/i0qFBQIWWcZW3J1w4TqHiepitoGG14SGVSxTxX", - "f5EqJkLAx5a6m4gbbeDIVqfCl5pQLS6dO9ibQH7B6CADGR5cKs1UO90OYXnaOfjN/muepp1ux47Hgxq/", - "gXC/BqSx3uByFIveGQ+J8ZtYfhOMxSqz90AWazeHVaebJfI35oU/vbfQ2ewekAxBPqgZcb8mEdQbb9Xg", - "w3gBFvm4EKGsqakmtTWbatwse3p4ce5K6LQJyzi3n567L78C7XddtIYbM3LTvfewjeURPObsVrk0mwkX", - "dRihdfEcXz0hfbktWZpqGwr5Rps3t/O1IkwtqS+zCPtBbEqq4VzxFCsaQTmfaMa59Mi+wPw1hbes+bag", - "TDBuGD3ThsVfaFK9sIbgCyvIH1ijFcL+I9tHHz63wfThL9yj8oufPL284PhdJ3wD5D3UGxeUTFCGc0m0", - "XJWnBEWLSHNFU7+J4GiGIpypXBAoTUdQShlN89QHc9Y7NscAPHGxnV500ThXKMFiCnqReejCXSKepoTF", - "BCxkQzYjeE61UidQghVh0aInCZS0nRN0xcVlwnEMSn4WY/C1QEk8QTQFAjJ2ShSOscIgalzoEz8ymTkX", - "RZVbo1gzcl1SQzxkImffG5h+3eyFG+gFIoBDTeWsqIYY4ZiwKIjPfP51s7Evbw0+J6o+0QeKzbkVL33I", - "YB3f6umG83XE8TyyAGMu7Da2YfMrhF7ZrERWUxocGf17HmkzVzfHB3LxFEu86hR/Hb6dgui+Gv/Owztw", - "uEBxbrrzTiWQ+Z/VK1MwFD/cCdIlzTbe1jVTlH0rlvlGPG/rD/fnyS2saV8JJ+w2KvZNBYbKSX8NLNeu", - "6q147gOZEa0tybOKPSALdjFVDyY+ceFxucdi7rQM2xzNgm/73EkJDNoXZ9/Ydp1t25CD27JtZ5tdcqp7", - "jJyyHkRphjm4NeM2smprOvg3zQepzc5jmQ/OIkvfwb2xxZOCERrWmOFFwnH8ZwjTXeHBibgQBtMBUCIe", - "E6aoZzX0A/TBNldWLuu6fMn3p6ebTVxCqJU8QqhHzCG8pBj9WRoo5/96ToSgsYXeREenxzZglkokctZH", - "r1MKBf8vCcnKnBLI6+vr+Tl0i+Uq6RUYi26HMCUWGadMrR1F+erdDObTrWqr3zOftCDV3xzSrR3SYNl/", - "fOwMuAxkTZgJrNZMFVZri2dSNuEiNXIZHvNct655kF4mvZ+mCOWEJkQupCKpiQuc5AkcNyh4YIva2u/M", - "LnchKlafHJOwlhGRUikpZ3LIbLZGRoTuW3+u2/dCnIIOAYUL/npmmOTXET6nB2MixrBqWjXAIYJimZ2D", - "zhbOsq0YK9wQomWH9xlD+gni4ZBcpGOe0AgllF1KtJHQS6OeoLlEif5jc2VA3Qi++9Ile29/svRKn7AJ", - "DxZEMzRbEPOfKq/KsjXnmHx0bO0l8Q+L4z+w0WG2tr4osCA46UGRXYdGg3JFE/rRsDrdCJWKRibpBxdr", - "9/60YKr9ITslSuh3MCSXJYnBFADtcisTPNoa5oPBbpRRgDTbJTA4YHjNj1Po8ejsnUkEJSkXi+6Q6X9A", - "w28Pz4x3d4KtNcEbqK0GjE62Xq8JMT6HZfo3jtEzE1yJHxDc8G8uwZujfDSeIdlwRHm2SlXi2Z8+iNRK", - "cN/sCo/TrgAwS8VsNqYCRyAUy1muYn7FwjaEOU/yVP/D/HGyDqxL4Wj2Hl79aqRdM5y13bgJPopDaecU", - "E1Ow8UGcHmbBHmvMql44NwUQYirRgMFb4FD9Gan7y5vv/XX8Ct2ddkVdMdSv5mzd981nx+AwLvz1eCzH", - "3FCam4niq61PV5g2W59+THh0KS0Yim821HobgIbrH0uQZ+siBDEBcjORBREyUFVEdoesZoA0mDsSYaSI", - "SCnDyRbM2TQCcNXOioXnnEKKdJRQSFKjMaAWJYBJDQB0ejZgqHINeB5dactF+e/4zkjF0ZhEPCUOwnsz", - "pLr9DVP1ExdVPO6vhS++9dYfQPkwBXv7Ggjy5h4/C5L8FF9DqHScW4eyG9HGS17+aExBXQR7M+zsDuSw", - "00XDzk467OgdOMJgQsUK7aOUslwR2UfHxr4FSbBPBkiSiLNYOiRxZ8HbHcimlFhDlg35lU/gu/sUeyxV", - "wVK+sZ2E2IN+D+nvIWkHbfgHzp7JuAuHLkY8V8bcb8+VfSsmCswjm/fuq/XOyDfdvg0n/5s9vhUeBbus", - "2aW39YazZ7mckWaT2ytTnSdXY0CodhUz5Qz9nY9lFzFyZazhQqr+Et/TX5+ZDu4DPV93dRPkfDv3b7D5", - "LWDzy7UKwyWaAEt9JTvqMJiJ5DrjQgGOos12NzQEmgRgN/AIJ+j10cmQRZoVGXA/QVIO3MmCfJtb+PBv", - "5+jF0ZsuOobqjejnfLzZR69ZsnA1tI2PZsiMJGaYV4QZGhuqJXHoejZjB+q5y2Bx3cEDlUM2JyPgWXF7", - "5YLEu50ZwTFIJH90XnHTWQD3980rfYAAetd8WWx7Z6Xw0XlDlFj0DieKiOVmT22eFCtQK+wl7UDgrOBm", - "oCd1h9Jhn5V9GtnAgFPs7nQCWBWfvlUyuPuqn/fjJTNxIqaG3DgHrE8GSQY4XjyuWCY5QwVzDLFA/7ou", - "agE0ZQlbXrZSwYAumyK/vyKT+0reVUF3/3c9XTDTR+toyir7pIm4qCGy1tPrkoNnBpDYOqoinOGIqkUX", - "4SSxd5S9CYqIlF4h/o4FwZcxv2L9IXtTVC+xCb3o6Oxd1zlqUUzlpWnB+mL76PWcCJmPi8EhOGjGawxr", - "TuIhUxxFOInyRIsbZDIhEeTiQlES2eDLLYbSucOzU3YSrKDiRbXnj65wW5gmYPdKsqhT3JbZ6i1BogTT", - "tBn+2wpqEHAIoQZj3ShniLJJYkOqIsGlRLapHknolI4TGyAk++jtjCCJUzJkWYIZIwLl0kTF66H3MkGk", - "zE2Ct24AYHINRXVRCe2XCa5saELCuZAmmkBT+PtTJBXJVpDZG9PyKcz5jmRb07jt6YGM1LUxNJtC7CtI", - "b4ihFLPgmo7yxAUw3msouhnQQ0uJj+XgvxV0OiVCnwpsmKwJxzPH2i2nOfSVjOXGIo7nxVvtijgWrXpZ", - "iV7G3kpotlGJdh13bhb1F+j8kjai99lHN8si/kV/1LLvarZqeBD20WfO8s9SG//cSxJsa8AqKfyxmZO8", - "kVeOaiXRdj2sVuvM2rvMdG2Nn/VgsFmPGS0LV9JnmxTer48QBveL8nDfZc4eN21V0K4qumlDyv96PPuv", - "ggLvBsj+gVFObgFk/1Xl3QPS+MPhnwQP6kPl0Vd8z66C7J8ei/6u0ucNID3AsTWlzxuuZ4NXVypK7+07", - "7dQk2+KfSYK38Y43kN/dsn/T+luoDN5irXNBa4InaaYWLqDN+irLoDNJP5J+gyO4iFu9O1fwLUI6vxx5", - "ODptDOj8cxZ8f5CYUVu8j0p0chyopP7IMAb9M1e5WLb0rdPDIprROWk2uldPsF2iTJBexjNwrsRmwex6", - "uLtMYdGffkS2eYu5av8F1R8BLJ/EKKaCRCpZmEqcmiOYPr6TSHCtCcBzLhbNUSLmiPwkeHpoZ7PmPrRn", - "yhrDyjjDdNGLscK9ueM2K0xonxHd6eIpNcNDlKGXP6INcq2EqTGBJlrzQXRSLKkpoS+BJjf9AW8PGiyb", - "9CMZTcdtRrmiWshrW40FRblUPHV7f3KMNqD62JQwvRda1J+AJJsJPqcxiStj7Mx5YlZ1u2FBb2p31UJF", - "UTrOKRdmcA8iw7S5kKYfaVZlC0VIzJgyDINbW5ejeqZMEr/uD1PmAnDsHrlRfLvCrOa34ZQdTYlQCdMu", - "ouLcQDxvfrvmHvM15ydDuTutctu58JzVxut2+VEt05buovBDkTt3v2br919PSg+VjzKbx5rO54VC2mQ2", - "/7pIcHB/98N9m8vfP+IU0JfEKd+eqRwa0C2GCOYVxHTHZE4SnqVQkRze7XQ7uUg6B52ZUtnB1hbEfs+4", - "VAd7z5/udj59+PT/BwAA//+uhiQVbu4BAA==", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", + "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", + "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", + "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", + "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", + "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", + "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", + "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", + "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", + "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", + "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", + "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", + "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", + "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", + "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", + "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", + "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", + "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", + "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", + "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", + "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", + "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", + "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", + "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", + "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", + "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", + "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", + "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", + "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", + "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", + "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", + "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", + "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", + "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", + "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", + "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", + "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", + "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", + "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", + "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", + "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", + "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", + "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", + "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", + "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", + "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", + "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", + "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", + "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", + "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", + "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", + "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", + "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", + "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", + "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", + "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", + "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", + "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", + "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", + "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", + "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", + "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", + "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", + "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", + "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", + "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", + "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", + "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", + "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", + "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", + "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", + "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", + "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", + "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", + "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", + "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", + "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", + "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", + "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", + "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", + "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", + "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", + "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", + "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", + "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", + "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", + "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", + "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", + "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", + "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", + "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", + "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", + "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", + "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", + "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", + "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", + "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", + "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", + "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", + "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", + "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", + "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", + "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", + "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", + "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", + "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", + "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", + "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", + "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", + "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", + "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", + "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", + "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", + "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", + "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", + "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", + "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", + "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", + "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", + "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", + "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", + "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", + "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", + "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", + "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", + "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", + "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", + "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", + "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", + "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", + "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", + "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", + "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", + "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", + "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", + "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", + "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", + "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", + "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", + "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", + "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", + "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", + "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", + "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", + "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", + "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", + "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", + "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", + "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", + "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", + "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", + "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", + "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", + "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", + "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", + "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", + "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", + "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", + "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", + "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", + "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", + "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", + "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", + "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", + "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", + "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", + "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", + "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", + "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", + "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", + "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", + "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", + "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", + "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", + "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", + "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", + "XjBRRRJprUYWXGuNhF7WlL73QLKTsojWfWuPrnz140ys4pmtB2v1tVJaaFbYvjZ6GNzv7XX/itpjJjGj", + "ES0vXaaF7kBlIFMTKs1NFlrxZQkq1kdv375yGWda4heuzpTirriUq7s5ZH5xqT56UVbtMi+4FrRETmKb", + "oQp5eLZcU0xwnFBGIESXyFByWLUk3IMeiy8vVIbr3bUSKu/5WNoKpg8nVD4YK7gX8e2kUhialzq+Xwqv", + "OC1OhINT86j4lWVAAcYTEp+2cK54z+awbs24ATYLYzueJTgCaEf9mkEds7ABBmbQbwqwAARPEiIMmlyW", + "K1cDcciKwVHm1Xi3NW4udPOjnCmaXHRNhAxAgkiE2cJCKg1ZpTOsFEkzBam9kLYOIxQkMyOuFX/Ug6Y8", + "l/AWZOH6XSKcXOGFHDKbDGw+h0K5gkQGeDFJ+uhnDjgMCE8xZR7jNRUIv5NDdkHjhIwsjMIFohLJGReK", + "MBKjlM+JrPZLsEgoETCJI6xXTqIULwDPzEA7mvXhGTGYYRWwBq7/jVlMoZad7rmY8sGQYbQzGKCUYCZt", + "6rXEE7hwbBsIBlEZ0PcIo73Bc/tVbd8Ac9ct/4Y+TUKQOY/wOFkgoqkYwB/UJmxgamtLmhq9evsmVEiz", + "X4XJ0BYNq2wsla5UYtxFOSuTy8F8nrMiF1xvl8oFg3laxxqhorgGLZ7GmERYryfj1X4AyZBHUS5CF6Te", + "aq/I6b+j4OhN7xyWKpy6nYAWHpEY9pxxNYMzzeEobX7fQFUlUf05LprgIeECYeTRdWkkIFEOrHEDkP8u", + "yop9zFXgvdj83p0dfXwtI3DH32DvPZb7CYiITyaVA7j+ajIHeFXKxDIJ/1nP6ZEr1eqzuJjiKeNS0cgx", + "w3pl928KYWuFcPXKBql5wsWlL1tV6fcnLi7bamAWT5Q+LkXMn+FXaNvXwwPs5oc38YOB2SgrmmjuXUmr", + "01dxSkHookq62GGOEs6m+hSVhu57t8T7Wt2GwWHTl6kw/uMCNUcrISP7o6n2qidja2mC1T6yrT40L9K9", + "34N/51euEE2zhKQEqsH2DLHpzS4RlqByPpUeztDNeKU+VX46sNEFpXHpd504BHTlNmwDpPfl7Qoy1YRP", + "1+P4FZ070LoAkN+Qmcr+BF0Yb84FKniwFmgNaj66mtFoBqB+oLfq9g3mH86yiwLPePMAvYSD7MM6Q+cb", + "Bitf05rkCTFYffM0vThYrnf6/vQUPjJ4fqay6cUBcjVOi/tD6rd8kD49iwRLhX610IMbhTIOO3qhsNY3", + "i/ltWvi+Em96yEJQfoxc2QbpBF14qH4XDZBTjt++4tMHE8a6zVUCzFwUR1Z1BNokLO40xU3QJAzotz0Y", + "hMCrW4ILmmHcMbbg0mBe8WlRoaBCyjjL2pKvHSZQ8TxNV9Aw2vCQyqSKea7+IlVMhICPLXU3ETfawJGt", + "ToUvNaFaXDp3sDeB/ILRQQYyPLhUmql2uh3C8rRz8Jv91zxNO92OHY8HNX4D4X4NSGO9weUoFr0zHhLj", + "N7H8JhiLVWbvgSzWbg6rTjdL5G/MC396b6Gz2T0gGYJ8UDPifk0iqDfeqsGH8QIs8nEhQllTU01qazbV", + "uFn29PDi3JXQaROWcW4/PXdffgXa77poDTdm5KZ772EbyyN4zNmtcmk2Ey7qMELr4jm+ekL6cluyNNU2", + "FPKNNm9u52tFmFpSX2YR9oPYlFTDueIpVjSCcj7RjHPpkX2B+WsKb1nzbUGZYNwweqYNi7/QpHphDcEX", + "VpA/sEYrhP1Hto8+fG6D6cNfuEflFz95ennB8btO+AbIe6g3LiiZoAznkmi5Kk8JihaR5oqmfhPB0QxF", + "OFO5IFCajqCUMprmqQ/mrHdsjgF44mI7veiica5QgsUU9CLz0IW7RDxNCYsJWMiGbEbwnGqlTqAEK8Ki", + "RU8SKGk7J+iKi8uE4xiU/CzG4GuBkniCaAoEZOyUKBxjhUHUuNAnfmQycy6KKrdGsWbkuqSGeMhEzr43", + "MP262Qs30AtEAIeayllRDTHCMWFREJ/5/OtmY1/eGnxOVH2iDxSbcyte+pDBOr7V0w3n64jjeWQBxlzY", + "bWzD5lcIvbJZiaymNDgy+vc80maubo4P5OIplnjVKf46fDsF0X01/p2Hd+BwgeLcdOedSiDzP6tXpmAo", + "frgTpEuabbyta6Yo+1Ys84143tYf7s+TW1jTvhJO2G1U7JsKDJWT/hpYrl3VW/HcBzIjWluSZxV7QBbs", + "YqoeTHziwuNyj8XcaRm2OZoF3/a5kxIYtC/OvrHtOtu2IQe3ZdvONrvkVPcYOWU9iNIMc3Brxm1k1dZ0", + "8G+aD1KbnccyH5xFlr6De2OLJwUjNKwxw4uE4/jPEKa7woMTcSEMpgOgRDwmTFHPaugH6INtrqxc1nX5", + "ku9PTzebuIRQK3mEUI+YQ3hJMfqzNFDO//WcCEFjC72Jjk6PbcAslUjkrI9epxQK/l8SkpU5JZDX19fz", + "c+gWy1XSKzAW3Q5hSiwyTplaO4ry1bsZzKdb1Va/Zz5pQaq/OaRbO6TBsv/42BlwGciaMBNYrZkqrNYW", + "z6RswkVq5DI85rluXfMgvUx6P00RyglNiFxIRVITFzjJEzhuUPDAFrW135ld7kJUrD45JmEtIyKlUlLO", + "5JDZbI2MCN23/ly374U4BR0CChf89cwwya8jfE4PxkSMYdW0aoBDBMUyOwedLZxlWzFWuCFEyw7vM4b0", + "E8TDIblIxzyhEUoou5RoI6GXRj1Bc4kS/cfmyoC6EXz3pUv23v5k6ZU+YRMeLIhmaLYg5j9VXpVla84x", + "+ejY2kviHxbHf2Cjw2xtfVFgQXDSgyK7Do0G5Yom9KNhdboRKhWNTNIPLtbu/WnBVPtDdkqU0O9gSC5L", + "EoMpANrlViZ4tDXMB4PdKKMAabZLYHDA8Jofp9Dj0dk7kwhKUi4W3SHT/4CG3x6eGe/uBFtrgjdQWw0Y", + "nWy9XhNifA7L9G8co2cmuBI/ILjh31yCN0f5aDxDsuGI8myVqsSzP30QqZXgvtkVHqddAWCWitlsTAWO", + "QCiWs1zF/IqFbQhznuSp/of542QdWJfC0ew9vPrVSLtmOGu7cRN8FIfSzikmpmDjgzg9zII91phVvXBu", + "CiDEVKIBg7fAofozUveXN9/76/gVujvtirpiqF/N2brvm8+OwWFc+OvxWI65oTQ3E8VXW5+uMG22Pv2Y", + "8OhSWjAU32yo9TYADdc/liDP1kUIYgLkZiILImSgqojsDlnNAGkwdyTCSBGRUoaTLZizaQTgqp0VC885", + "hRTpKKGQpEZjQC1KAJMaAOj0bMBQ5RrwPLrSlovy3/GdkYqjMYl4ShyE92ZIdfsbpuonLqp43F8LX3zr", + "rT+A8mEK9vY1EOTNPX4WJPkpvoZQ6Ti3DmU3oo2XvPzRmIK6CPZm2NkdyGGni4adnXTY0TtwhMGEihXa", + "RylluSKyj46NfQuSYJ8MkCQRZ7F0SOLOgrc7kE0psYYsG/Irn8B39yn2WKqCpXxjOwmxB/0e0t9D0g7a", + "8A+cPZNxFw5djHiujLnfniv7VkwUmEc2791X652Rb7p9G07+N3t8KzwKdlmzS2/rDWfPcjkjzSa3V6Y6", + "T67GgFDtKmbKGfo7H8suYuTKWMOFVP0lvqe/PjMd3Ad6vu7qJsj5du7fYPNbwOaXaxWGSzQBlvpKdtRh", + "MBPJdcaFAhxFm+1uaAg0CcBu4BFO0OujkyGLNCsy4H6CpBy4kwX5Nrfw4d/O0YujN110DNUb0c/5eLOP", + "XrNk4WpoGx/NkBlJzDCvCDM0NlRL4tD1bMYO1HOXweK6gwcqh2xORsCz4vbKBYl3OzOCY5BI/ui84qaz", + "AO7vm1f6AAH0rvmy2PbOSuGj84YosegdThQRy82e2jwpVqBW2EvagcBZwc1AT+oOpcM+K/s0soEBp9jd", + "6QSwKj59q2Rw91U/78dLZuJETA25cQ5YnwySDHC8eFyxTHKGCuYYYoH+dV3UAmjKEra8bKWCAV02RX5/", + "RSb3lbyrgu7+73q6YKaP1tGUVfZJE3FRQ2Stp9clB88MILF1VEU4wxFViy7CSWLvKHsTFBEpvUL8HQuC", + "L2N+xfpD9qaoXmITetHR2buuc9SimMpL04L1xfbR6zkRMh8Xg0Nw0IzXGNacxEOmOIpwEuWJFjfIZEIi", + "yMWFoiSywZdbDKVzh2en7CRYQcWLas8fXeG2ME3A7pVkUae4LbPVW4JECaZpM/y3FdQg4BBCDca6Uc4Q", + "ZZPEhlRFgkuJbFM9ktApHSc2QEj20dsZQRKnZMiyBDNGBMqliYrXQ+9lgkiZmwRv3QDA5BqK6qIS2i8T", + "XNnQhIRzIU00gabw96dIKpKtILM3puVTmPMdybamcdvTAxmpa2NoNoXYV5DeEEMpZsE1HeWJC2C811B0", + "M6CHlhIfy8F/K+h0SoQ+FdgwWROOZ461W05z6CsZy41FHM+Lt9oVcSxa9bISvYy9ldBsoxLtOu7cLOov", + "0PklbUTvs49ulkX8i/6oZd/VbNXwIOyjz5zln6U2/rmXJNjWgFVS+GMzJ3kjrxzVSqLtelit1pm1d5np", + "2ho/68Fgsx4zWhaupM82KbxfHyEM7hfl4b7LnD1u2qqgXVV004aU//V49l8FBd4NkP0Do5zcAsj+q8q7", + "B6Txh8M/CR7Uh8qjr/ieXQXZPz0W/V2lzxtAeoBja0qfN1zPBq+uVJTe23faqUm2xT+TBG/jHW8gv7tl", + "/6b1t1AZvMVa54LWBE/STC1cQJv1VZZBZ5J+JP0GR3ARt3p3ruBbhHR+OfJwdNoY0PnnLPj+IDGjtngf", + "lejkOFBJ/ZFhDPpnrnKxbOlbp4dFNKNz0mx0r55gu0SZIL2MZ+Bcic2C2fVwd5nCoj/9iGzzFnPV/guq", + "PwJYPolRTAWJVLIwlTg1RzB9fCeR4FoTgOdcLJqjRMwR+Unw9NDOZs19aM+UNYaVcYbpohdjhXtzx21W", + "mNA+I7rTxVNqhocoQy9/RBvkWglTYwJNtOaD6KRYUlNCXwJNbvoD3h40WDbpRzKajtuMckW1kNe2GguK", + "cql46vb+5BhtQPWxKWF6L7SoPwFJNhN8TmMSV8bYmfPErOp2w4Le1O6qhYqidJxTLszgHkSGaXMhTT/S", + "rMoWipCYMWUYBre2Lkf1TJkkft0fpswF4Ng9cqP4doVZzW/DKTuaEqESpl1ExbmBeN78ds095mvOT4Zy", + "d1rltnPhOauN1+3yo1qmLd1F4Ycid+5+zdbvv56UHiofZTaPNZ3PC4W0yWz+dZHg4P7uh/s2l79/xCmg", + "L4lTvj1TOTSgWwwRzCuI6Y7JnCQ8S6EiObzb6XZykXQOOjOlsoOtLYj9nnGpDvaeP93tfPrw6f8PAAD/", + "//30TUjf7gEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 32f584193..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Available GPU slots by vGPU profile"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index eac70a5ee..14ffd7ffc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: Number of instances that can be created with this profile + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From a21fa198c98c19f518881c34812569c4302b1c92 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 14/24] Report conservative per-GPU vGPU profile capacity Counting every free VF advertising a type overreports concurrent capacity: sibling VFs share their parent GPU's framebuffer, so one 48Q assignment revokes the type from every other VF on that GPU. Bound each GPU's contribution by both its free VFs and how many times the profile framebuffer fits into the GPU's remaining framebuffer, using the largest still-creatable profile as a lower bound on what remains. --- lib/devices/vendor_vfio_linux.go | 51 ++- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 492 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 316 insertions(+), 259 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..d67ddbc00 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,22 +81,37 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free VF advertising a type as one creatable -// instance, matching the driver-reported units that mdev sums through -// available_instances. This is a best-effort snapshot because creating on one -// VF may revoke the type from siblings that share its GPU framebuffer. +// listProfiles reports a conservative estimate of how many instances of each +// profile are concurrently creatable. Free VFs on the same GPU share its +// framebuffer, so counting every advertising VF overreports capacity: one 48Q +// assignment can revoke the type from all sibling VFs. Each GPU instead +// contributes min(free VFs advertising the type, remaining framebuffer / +// profile framebuffer), where the largest profile still creatable on the GPU +// is a lower bound on its remaining framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableVFs := make(map[string]int) + freeVFsByGPU := make(map[string]map[string]int) + remainingFBByGPU := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { - creatableVFs[profile.TypeName]++ + if vf.Allocated { + continue + } + if freeVFsByGPU[gpu] == nil { + freeVFsByGPU[gpu] = make(map[string]int) + } + freeVFsByGPU[gpu][profile.TypeName]++ + if profile.FramebufferMB > remainingFBByGPU[gpu] { + remainingFBByGPU[gpu] = profile.FramebufferMB } } } @@ -109,15 +124,35 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { + available := 0 + for gpu, freeVFs := range freeVFsByGPU { + available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) + } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: creatableVFs[profile.TypeName], + Available: available, }) } return profiles, nil } +// gpuProfileCapacity estimates how many instances of a profile one GPU can +// still create concurrently. When a profile's framebuffer is unknown (0), the +// free VF count is the only signal available. +func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { + if freeVFs == 0 { + return 0 + } + if profileFB <= 0 || remainingFB <= 0 { + return freeVFs + } + if byFB := remainingFB / profileFB; byFB < freeVFs { + return byFB + } + return freeVFs +} + func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b68a9f912 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { +func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,30 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "each free VF advertising the type counts as one creatable instance") + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), + "small profiles stay capped by the free VF count") +} + +func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { + t.Parallel() + + // One 48G GPU with a 24Q already assigned: siblings only advertise up to + // 24Q, so at most one more 24Q fits despite two free VFs. + remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 8c66a74d3..599c8a1d4 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. + // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18968,251 +18968,251 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", - "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", - "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", - "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", - "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", - "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", - "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", - "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", - "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", - "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", - "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", - "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", - "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", - "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", - "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", - "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", - "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", - "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", - "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", - "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", - "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", - "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", - "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", - "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", - "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", - "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", - "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", - "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", - "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", - "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", - "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", - "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", - "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", - "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", - "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", - "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", - "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", - "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", - "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", - "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", - "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", - "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", - "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", - "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", - "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", - "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", - "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", - "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", - "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", - "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", - "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", - "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", - "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", - "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", - "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", - "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", - "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", - "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", - "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", - "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", - "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", - "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", - "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", - "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", - "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", - "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", - "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", - "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", - "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", - "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", - "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", - "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", - "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", - "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", - "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", - "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", - "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", - "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", - "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", - "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", - "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", - "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", - "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", - "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", - "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", - "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", - "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", - "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", - "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", - "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", - "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", - "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", - "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", - "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", - "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", - "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", - "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", - "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", - "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", - "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", - "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", - "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", - "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", - "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", - "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", - "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", - "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", - "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", - "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", - "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", - "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", - "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", - "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", - "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", - "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", - "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", - "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", - "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", - "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", - "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", - "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", - "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", - "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", - "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", - "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", - "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", - "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", - "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", - "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", - "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", - "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", - "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", - "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", - "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", - "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", - "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", - "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", - "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", - "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", - "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", - "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", - "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", - "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", - "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", - "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", - "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", - "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", - "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", - "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", - "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", - "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", - "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", - "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", - "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", - "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", - "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", - "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", - "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", - "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", - "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", - "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", - "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", - "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", - "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", - "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", - "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", - "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", - "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", - "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", - "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", - "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", - "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", - "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", - "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", - "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", - "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", - "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", - "XjBRRRJprUYWXGuNhF7WlL73QLKTsojWfWuPrnz140ys4pmtB2v1tVJaaFbYvjZ6GNzv7XX/itpjJjGj", - "ES0vXaaF7kBlIFMTKs1NFlrxZQkq1kdv375yGWda4heuzpTirriUq7s5ZH5xqT56UVbtMi+4FrRETmKb", - "oQp5eLZcU0xwnFBGIESXyFByWLUk3IMeiy8vVIbr3bUSKu/5WNoKpg8nVD4YK7gX8e2kUhialzq+Xwqv", - "OC1OhINT86j4lWVAAcYTEp+2cK54z+awbs24ATYLYzueJTgCaEf9mkEds7ABBmbQbwqwAARPEiIMmlyW", - "K1cDcciKwVHm1Xi3NW4udPOjnCmaXHRNhAxAgkiE2cJCKg1ZpTOsFEkzBam9kLYOIxQkMyOuFX/Ug6Y8", - "l/AWZOH6XSKcXOGFHDKbDGw+h0K5gkQGeDFJ+uhnDjgMCE8xZR7jNRUIv5NDdkHjhIwsjMIFohLJGReK", - "MBKjlM+JrPZLsEgoETCJI6xXTqIULwDPzEA7mvXhGTGYYRWwBq7/jVlMoZad7rmY8sGQYbQzGKCUYCZt", - "6rXEE7hwbBsIBlEZ0PcIo73Bc/tVbd8Ac9ct/4Y+TUKQOY/wOFkgoqkYwB/UJmxgamtLmhq9evsmVEiz", - "X4XJ0BYNq2wsla5UYtxFOSuTy8F8nrMiF1xvl8oFg3laxxqhorgGLZ7GmERYryfj1X4AyZBHUS5CF6Te", - "aq/I6b+j4OhN7xyWKpy6nYAWHpEY9pxxNYMzzeEobX7fQFUlUf05LprgIeECYeTRdWkkIFEOrHEDkP8u", - "yop9zFXgvdj83p0dfXwtI3DH32DvPZb7CYiITyaVA7j+ajIHeFXKxDIJ/1nP6ZEr1eqzuJjiKeNS0cgx", - "w3pl928KYWuFcPXKBql5wsWlL1tV6fcnLi7bamAWT5Q+LkXMn+FXaNvXwwPs5oc38YOB2SgrmmjuXUmr", - "01dxSkHookq62GGOEs6m+hSVhu57t8T7Wt2GwWHTl6kw/uMCNUcrISP7o6n2qidja2mC1T6yrT40L9K9", - "34N/51euEE2zhKQEqsH2DLHpzS4RlqByPpUeztDNeKU+VX46sNEFpXHpd504BHTlNmwDpPfl7Qoy1YRP", - "1+P4FZ070LoAkN+Qmcr+BF0Yb84FKniwFmgNaj66mtFoBqB+oLfq9g3mH86yiwLPePMAvYSD7MM6Q+cb", - "Bitf05rkCTFYffM0vThYrnf6/vQUPjJ4fqay6cUBcjVOi/tD6rd8kD49iwRLhX610IMbhTIOO3qhsNY3", - "i/ltWvi+Em96yEJQfoxc2QbpBF14qH4XDZBTjt++4tMHE8a6zVUCzFwUR1Z1BNokLO40xU3QJAzotz0Y", - "hMCrW4ILmmHcMbbg0mBe8WlRoaBCyjjL2pKvHSZQ8TxNV9Aw2vCQyqSKea7+IlVMhICPLXU3ETfawJGt", - "ToUvNaFaXDp3sDeB/ILRQQYyPLhUmql2uh3C8rRz8Jv91zxNO92OHY8HNX4D4X4NSGO9weUoFr0zHhLj", - "N7H8JhiLVWbvgSzWbg6rTjdL5G/MC396b6Gz2T0gGYJ8UDPifk0iqDfeqsGH8QIs8nEhQllTU01qazbV", - "uFn29PDi3JXQaROWcW4/PXdffgXa77poDTdm5KZ772EbyyN4zNmtcmk2Ey7qMELr4jm+ekL6cluyNNU2", - "FPKNNm9u52tFmFpSX2YR9oPYlFTDueIpVjSCcj7RjHPpkX2B+WsKb1nzbUGZYNwweqYNi7/QpHphDcEX", - "VpA/sEYrhP1Hto8+fG6D6cNfuEflFz95ennB8btO+AbIe6g3LiiZoAznkmi5Kk8JihaR5oqmfhPB0QxF", - "OFO5IFCajqCUMprmqQ/mrHdsjgF44mI7veiica5QgsUU9CLz0IW7RDxNCYsJWMiGbEbwnGqlTqAEK8Ki", - "RU8SKGk7J+iKi8uE4xiU/CzG4GuBkniCaAoEZOyUKBxjhUHUuNAnfmQycy6KKrdGsWbkuqSGeMhEzr43", - "MP262Qs30AtEAIeayllRDTHCMWFREJ/5/OtmY1/eGnxOVH2iDxSbcyte+pDBOr7V0w3n64jjeWQBxlzY", - "bWzD5lcIvbJZiaymNDgy+vc80maubo4P5OIplnjVKf46fDsF0X01/p2Hd+BwgeLcdOedSiDzP6tXpmAo", - "frgTpEuabbyta6Yo+1Ys84143tYf7s+TW1jTvhJO2G1U7JsKDJWT/hpYrl3VW/HcBzIjWluSZxV7QBbs", - "YqoeTHziwuNyj8XcaRm2OZoF3/a5kxIYtC/OvrHtOtu2IQe3ZdvONrvkVPcYOWU9iNIMc3Brxm1k1dZ0", - "8G+aD1KbnccyH5xFlr6De2OLJwUjNKwxw4uE4/jPEKa7woMTcSEMpgOgRDwmTFHPaugH6INtrqxc1nX5", - "ku9PTzebuIRQK3mEUI+YQ3hJMfqzNFDO//WcCEFjC72Jjk6PbcAslUjkrI9epxQK/l8SkpU5JZDX19fz", - "c+gWy1XSKzAW3Q5hSiwyTplaO4ry1bsZzKdb1Va/Zz5pQaq/OaRbO6TBsv/42BlwGciaMBNYrZkqrNYW", - "z6RswkVq5DI85rluXfMgvUx6P00RyglNiFxIRVITFzjJEzhuUPDAFrW135ld7kJUrD45JmEtIyKlUlLO", - "5JDZbI2MCN23/ly374U4BR0CChf89cwwya8jfE4PxkSMYdW0aoBDBMUyOwedLZxlWzFWuCFEyw7vM4b0", - "E8TDIblIxzyhEUoou5RoI6GXRj1Bc4kS/cfmyoC6EXz3pUv23v5k6ZU+YRMeLIhmaLYg5j9VXpVla84x", - "+ejY2kviHxbHf2Cjw2xtfVFgQXDSgyK7Do0G5Yom9KNhdboRKhWNTNIPLtbu/WnBVPtDdkqU0O9gSC5L", - "EoMpANrlViZ4tDXMB4PdKKMAabZLYHDA8Jofp9Dj0dk7kwhKUi4W3SHT/4CG3x6eGe/uBFtrgjdQWw0Y", - "nWy9XhNifA7L9G8co2cmuBI/ILjh31yCN0f5aDxDsuGI8myVqsSzP30QqZXgvtkVHqddAWCWitlsTAWO", - "QCiWs1zF/IqFbQhznuSp/of542QdWJfC0ew9vPrVSLtmOGu7cRN8FIfSzikmpmDjgzg9zII91phVvXBu", - "CiDEVKIBg7fAofozUveXN9/76/gVujvtirpiqF/N2brvm8+OwWFc+OvxWI65oTQ3E8VXW5+uMG22Pv2Y", - "8OhSWjAU32yo9TYADdc/liDP1kUIYgLkZiILImSgqojsDlnNAGkwdyTCSBGRUoaTLZizaQTgqp0VC885", - "hRTpKKGQpEZjQC1KAJMaAOj0bMBQ5RrwPLrSlovy3/GdkYqjMYl4ShyE92ZIdfsbpuonLqp43F8LX3zr", - "rT+A8mEK9vY1EOTNPX4WJPkpvoZQ6Ti3DmU3oo2XvPzRmIK6CPZm2NkdyGGni4adnXTY0TtwhMGEihXa", - "RylluSKyj46NfQuSYJ8MkCQRZ7F0SOLOgrc7kE0psYYsG/Irn8B39yn2WKqCpXxjOwmxB/0e0t9D0g7a", - "8A+cPZNxFw5djHiujLnfniv7VkwUmEc2791X652Rb7p9G07+N3t8KzwKdlmzS2/rDWfPcjkjzSa3V6Y6", - "T67GgFDtKmbKGfo7H8suYuTKWMOFVP0lvqe/PjMd3Ad6vu7qJsj5du7fYPNbwOaXaxWGSzQBlvpKdtRh", - "MBPJdcaFAhxFm+1uaAg0CcBu4BFO0OujkyGLNCsy4H6CpBy4kwX5Nrfw4d/O0YujN110DNUb0c/5eLOP", - "XrNk4WpoGx/NkBlJzDCvCDM0NlRL4tD1bMYO1HOXweK6gwcqh2xORsCz4vbKBYl3OzOCY5BI/ui84qaz", - "AO7vm1f6AAH0rvmy2PbOSuGj84YosegdThQRy82e2jwpVqBW2EvagcBZwc1AT+oOpcM+K/s0soEBp9jd", - "6QSwKj59q2Rw91U/78dLZuJETA25cQ5YnwySDHC8eFyxTHKGCuYYYoH+dV3UAmjKEra8bKWCAV02RX5/", - "RSb3lbyrgu7+73q6YKaP1tGUVfZJE3FRQ2Stp9clB88MILF1VEU4wxFViy7CSWLvKHsTFBEpvUL8HQuC", - "L2N+xfpD9qaoXmITetHR2buuc9SimMpL04L1xfbR6zkRMh8Xg0Nw0IzXGNacxEOmOIpwEuWJFjfIZEIi", - "yMWFoiSywZdbDKVzh2en7CRYQcWLas8fXeG2ME3A7pVkUae4LbPVW4JECaZpM/y3FdQg4BBCDca6Uc4Q", - "ZZPEhlRFgkuJbFM9ktApHSc2QEj20dsZQRKnZMiyBDNGBMqliYrXQ+9lgkiZmwRv3QDA5BqK6qIS2i8T", - "XNnQhIRzIU00gabw96dIKpKtILM3puVTmPMdybamcdvTAxmpa2NoNoXYV5DeEEMpZsE1HeWJC2C811B0", - "M6CHlhIfy8F/K+h0SoQ+FdgwWROOZ461W05z6CsZy41FHM+Lt9oVcSxa9bISvYy9ldBsoxLtOu7cLOov", - "0PklbUTvs49ulkX8i/6oZd/VbNXwIOyjz5zln6U2/rmXJNjWgFVS+GMzJ3kjrxzVSqLtelit1pm1d5np", - "2ho/68Fgsx4zWhaupM82KbxfHyEM7hfl4b7LnD1u2qqgXVV004aU//V49l8FBd4NkP0Do5zcAsj+q8q7", - "B6Txh8M/CR7Uh8qjr/ieXQXZPz0W/V2lzxtAeoBja0qfN1zPBq+uVJTe23faqUm2xT+TBG/jHW8gv7tl", - "/6b1t1AZvMVa54LWBE/STC1cQJv1VZZBZ5J+JP0GR3ARt3p3ruBbhHR+OfJwdNoY0PnnLPj+IDGjtngf", - "lejkOFBJ/ZFhDPpnrnKxbOlbp4dFNKNz0mx0r55gu0SZIL2MZ+Bcic2C2fVwd5nCoj/9iGzzFnPV/guq", - "PwJYPolRTAWJVLIwlTg1RzB9fCeR4FoTgOdcLJqjRMwR+Unw9NDOZs19aM+UNYaVcYbpohdjhXtzx21W", - "mNA+I7rTxVNqhocoQy9/RBvkWglTYwJNtOaD6KRYUlNCXwJNbvoD3h40WDbpRzKajtuMckW1kNe2GguK", - "cql46vb+5BhtQPWxKWF6L7SoPwFJNhN8TmMSV8bYmfPErOp2w4Le1O6qhYqidJxTLszgHkSGaXMhTT/S", - "rMoWipCYMWUYBre2Lkf1TJkkft0fpswF4Ng9cqP4doVZzW/DKTuaEqESpl1ExbmBeN78ds095mvOT4Zy", - "d1rltnPhOauN1+3yo1qmLd1F4Ycid+5+zdbvv56UHiofZTaPNZ3PC4W0yWz+dZHg4P7uh/s2l79/xCmg", - "L4lTvj1TOTSgWwwRzCuI6Y7JnCQ8S6EiObzb6XZykXQOOjOlsoOtLYj9nnGpDvaeP93tfPrw6f8PAAD/", - "//30TUjf7gEA", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9zxJkkYm7MGkQqmurd5JNigaSH", + "9ZwsTKCKQZo050PfPHZRcCS4lBAj8PLsnewaww2JASNVj2+8QBNBSBFYM8mZKTFlDTquYCXgf4zzyYSI", + "PvqRSNUjkwkX6gCQB9MsL7Rga4gxNMiZyfH6PSe1/IT95yHZ2+tmlAaMgT/p58i8UMAknP5Ylb+1nN9W", + "yz+rUA+o+RMcUTbdbE0eAQNibRrrcAhfnr17Y8sMNSE0670qShEZcOY++rUoxAW7W6Iw9QOWxTrSX0Pi", + "2dlsIWmEDb3YKhqU+QZBOD2tJfqz8kNrOg3I9eH6be6koo35NMuBT5y/6Z28fr+VxmTerYwJAv9mPCF6", + "3JseO5s79IMyS67CxeZNlhlDGLLtCffWqmAxrRfJYyiB1VFc4WQkEx6KMXqrHyJ4iDbe/2TSl/UIuiir", + "bKX+3Ycm8un7SfDEAOJ9Q7fn0GHdxFs54EFdtw6Z1qlOr9Jp6KiY1KtlmWy5eCK/rG40v1xfsM800tzv", + "kcsOqxnhreCLTBKZwXd0YRPIfGqs7NaUI0mGBVYkWdTiN6sA7GTZA06uSXSD9LQX+vVPpuhGLshIzQSR", + "M55U4yZ2u6ErKMrhBjL1OsycPEeB4ijF4hJubif4o5yZFaiy+t11eDQzpbIbTOrnt2/PjDVA6XsyqSdJ", + "yKWIgGOS4AUaE3VFCHNTwRJhP0a2nmQqG+oFCTXKiKC8uoad3UC/5yZuGk0FjggyX7myz3ZLJMQjtV1K", + "20sA4TCKiJQN+7u9an/tp5M8abfHoWFtr62SHt1kg98enbm6N0VdYbfMO8urfEZEzxw5V2B49dbuyNUV", + "mFxXzADXLgkMYwIlmWxmip846uKhoOSU/rySrOkxB+mnF9p+4BSYpeqac/6hlTRaP+4hx3+KWRySNE1C", + "gknlnwK+F0RFixx8XTQ2wVIGXcDoDX5ehSA4poxIWcuDjnKRdLqd3sTO6mBrK+ERTgDicG93+9nW6rDT", + "lfHGNrxqFNNV+qgLwjJhOi7r1uDDwaSrJLGFs6yFxc6s45r7AdjTcnwj1GTWd5sn4bmA98FgKaX7GkfK", + "VZEDE17FRYv9YwtAxJX5QIOpKQyjRe3n/vkcBH3mGVazKvlvLdE+xNFADKSmEWMeqa2jIfKPQYmKi1Cp", + "MC6UzRAcExckWdyHLgTRQe5WnHmDZ/4sn+zv7+6v40PAbGrH3J67wFTN29UcaOIttz2/tgEI56rKHO5I", + "r652qtdlDU1pjrhEUqsXlGeE3Wg99/d2d262nm0ncuLCyGp8KQQhc3R6bGSiiDOFKSMCpUThGCtcZTJg", + "+9JcBmrRYJJCGtHk+9WspSHewseEuW0hrS/lrW+oqffGQUinmNGJZsj2Tb9nOcM7+08OTOXPmEz29p/0", + "+/2bImW8KKExWm3Flgnq80Az+nL2eftwB4AYbebyR+fs8O3PmpHlUphLa0uOKTvw/l38s3wAf5h/jikL", + "A2m0KRZLJ0tFYqvxa7lFKibxASrrgTu5p008UYPxGqKZAb0nCEtXieq8O/y5gsZptczADXKJV+TWanHl", + "NUsWjWtz67qvZUV05dV79bPTWtR+pR9X++OduQvesX0awOqiLO6yJ/5WhY3lytqPS6XBMsKKao9JYv6K", + "OAMg4FDpx8oV6Z61qBwG14gtEVZ06f9Y9O79eOQPxPvdVR7zfrI1ID/cMIRmpUD6t2U5dD0XcuLomsMc", + "tj0Wt0LbersWty4YO/7Ad+Ftwsyqvb+e/tfv/0eePf379u+v3r//7/nL/zr+lf73++Ts9WdhoqyGLHxQ", + "3MEvBjVoasj7eINtSekUqyhgo9PqX8MK2yfG4qCiGVQIRWNyMGQ99IoqIky9uVqy5LCDNghoSvCVFneh", + "mI7JU9vUH58ZD6j++A8nBn+qtxHbJHZhN6TAJpH5OOYppmxzyIbMtoXcRCToBfqvGEU4M0XjKENa/12g", + "sYAKf9YlVXbeRX/gLPu0OWS2qr9B384wFEmbFFlizDmU7ahM2Kx9nRQwFSaDcciK27rA7DN+yX4J8E9J", + "Us8xaliU1fqb1ZyeDULohpD/ojcSitiAClJQtiajIjEHPRtsLutza3SMgoZWkJ913JsEycM8ZC5uSqo8", + "JjGNgK+4vMKZzTwtUjoNpVkjXib49QL25o1JdosRztVM86LIJuJHnF9S0oUt7YIfDSJF4Evj/5/xrDde", + "9GY8K0AZsDDRMdh40KtK9v/p2Yn23hNBJ7anYG69JpGA0AlHxs7MpCgW1oWlib019T+YFn3mxL5uCslI", + "UzvEeApVLpgrb0GgyCWgfxTUR0Iy+fcoSihYneSM50mMZgDQp3QzIYy9zqBIscHjKCaT+r+rIRA7+09A", + "g3X/3t1pneFqlm4VleVJQKdNHetrwbENm4QBGPFg5Azha4KW9A1oK22BnUJx+O85cg2VJ65gJMY7ZXLu", + "pK2WkEgv224zmBZlj4GFGBlhe5ra3EdLp7CSZdWiBRPNAJ8lLcBNXpjszrevzpEiInX59huR3h04JQbJ", + "okelzG3hrsOj0xeb/U4QmKniqoKtWpmFVR10AJvBRjc0BW2UNhqcki46OYbsWnutlLoYpEP8xAVKzK1Y", + "XkYHAO5RNfdgU/Lv5NgKoMmiDJEwYsuws+lazOrX2wF6U6iAuBhKkSdZ0pZrsrxMoFkbMGdyNZZar6XV", + "gn/Mqn/2PobMDKiHaHgxgEs23l/tbY4OqkpfVDUL2Y0vJD9qpdH+5e39l4Zh/vIy+u7NZHTrAx5lMyxD", + "1D3zvZrw0tK++27kKrsXzTFGlX5HkgbP1t9cVRjvGlJE33OVz0O4o/u97e2323s3N9/dFEG3Cp3lweoV", + "ILrt0W/vAkU2gAlL1agxGB3pxzb03NlF3p+iGZbsOwUPa9aR7d2nbYwS0GvbMG4/gJtPzJAKLuVwuIrw", + "Y4NIdkmTxAgwkk4ZTtBztHF+8vKXk1evNlEPvX59Wt+KVV8E9+cWYLpwC8A6moy0ABRTBUIAFbmGb9++", + "gsOVEEjXMHL45e0hdteaFltA7rrBvTx7B45/LEcu0LM5txGX+cHkmkoll1HYWsVLfw7Er/m0NIy1maRp", + "w0Z0rcUJ/rkCZBuE1du8A4BfF+y+tJwPgH37kEmFXx/u7kqk3M+Fu7V2hjtCu2280kJIsTUEhf2m2+32", + "uLV3MpwKAE2IbfkSjsv4vjVQbLdDA9muh1JfPCRGJ2dlZajSGeGar83p+U5/+8kzKG66PWjD2FMcrej7", + "9PCofeeDHXPLHODxQRQfgMJ+W5+VJWyjguDkCi+gOKBZ2mHHXJiedusdW6tItoqvWcbjvR38bl2MawDY", + "BXHWBS7JUbqyNkmLdMY6yFqaWwjJlCYJlSTiLJZVGXmGJZKZQU418duFBD9kMMAuKkolg5SCcBSJvDQ9", + "Wunayvt5Zuke6oRmnGkdAAoF/EIWEqUUnKBF9xD6KFGRNRMP2YZwGVZFKhWUCI31D5Cv0LW1x2M9NKqg", + "Fon+YMjkLFeaiW320RFnMk+JsFZZNKbgMdpEMjcqLYwXVmOhGaakMRFDpl8LYLP+UagnB08Gg8Gg2yk0", + "uV3970GImu7U+dm32MMmRxjQAZlFIQbYQZEzlLOYiKJeODHkUA+Ru6Hj9DNBh93n7cQr+3kpV4UP5jqM", + "4nbgw5+L+ApDbdDPITr0Fsr5/u1F9FY5Sk5+tdlJ9qvRTSIYCIp4nsRa4xvr284Y5EhszZCSKMOdzbtU", + "onemVmd16jb0WHGTP4Len55Wwh4EmWge0G7iwCUa9oFnN9qGnTU2krWjuYl72cPHvQ9M3Lqk4kmIXxwB", + "1/cwuqRpQ6EVw1ZFcV5lX9NaZTCLhDKzT5poVkywZsqPyXyU5yGtRD9yuCnv3p0cVygF4yfbzwbPnvee", + "jbef9PbiwXYPb+8+6e3s48FkN3q6u72zuyLjpEWa2+0z14KqaSBguAgPH7kw9YY8tWCSQE0IsIHPV5TF", + "/KpyzwQjUf3ebZTruu6XY9hbDyGY+ZJgqYydoIFlnMJtSiLdton8tqmURQ2ksEXxydvB9ueaWWBwDcz4", + "rciZ8V+a5P/CVp96A/Y3qzrO2/FWGJDLMFm3Wn7n7RdtcLD//GD/cxfNZUmsG2OdnO5xc5tCrxw4ci0N", + "w6UCegYbZwnsWOHDmM9t1kan2ykSS+BvuHVrQcvF41bZUk0HthtmI6v4d0NW80lFMYCQEAOOFx9okcAJ", + "/lBqocid17LGUcLzGHlGL4MVBg6vE09J0M2A/8nawgz2p8l60MoEgE1DSQfKNCMGR59uxKZAH6CX8C48", + "wqnRn+wgTGER38eF44UJTNHny3VttJnVQz63igx8o7UapP8F09bLYG2jq5swYtAB+pXDN4VaxXjdyGpe", + "B31m+fW6QXbD4mg7SAvozMp0B+inQo4rJEEr+W1IYv8cWYZVIslsVvL57Y53NLWUO+flpnc7ZkU73Y5b", + "KMhhX85mf1dS/dL580kxFLFFcAJnuUzGzRVNLH42zIRKRSNpszT05jbJF7bmEYlHRktpCv40GZ5Wkyk+", + "cuLL+1O0ARCJf0HWgqz/tVkEilbuup3ne8+fPN15/qQVEFI5wPUy6BHkHy8Pbq1AGmX5yBohmqZ+dPbO", + "GBkio74XQSbvT33ciUxwzXr0zF2DfufP+899/KeY5+PE8+BZsDgDNwsbFoQ4K3hRQ8Dh7zSZ08mE/f4x", + "utz5u6Dp9vUTuTPebsC1NR2F7Vsnvhd/yRhMxj1TvygM0QMEJWQjitUbImEG6JwoBPTTQzgCPaJIG7Yk", + "57Cu7IoHCWtvd3f32dP9nVZ0ZUfnHZwRWLsCl7IdgXfE4E208eb8HG15BGfadGAPAD3OrI4ZPmfIFh8e", + "VAXS/vZgN0QlDRd3STW27XnauOTvrZ5mJ2UXHbKfCx1u6ZQHV3t3d/B0b//ZfrtjbO2wI3G9msO43CCz", + "PBYh39/5DZAm3x6eIci8neCoakRxoVg3GpW60aiguoNBZb/BwJ49fbK/t7uz3Q6OLRTdYYEGKwe2yrsC", + "hy5AFIHdCCzFMuvtNt0WIXHKENgbEiWYpoeRy2Wo3T4GfX0kzGvlJrS5GKzpf+niavFtKytSYRsymTBG", + "NOAC5ayo+dFf7/v8Ii7MZq5trof1XD2U/8L06lncIFPb7BZLmQkypzyXX6Ahrkxy6iThXNzo2yaF5Q2R", + "eaKMn5FK9P70O+ApmtaQVCSr6lCWGlegK91ycjc6zxUSCRN502K12o02W79qwt2GU9tdhVxR4QaNmGax", + "5lw5Wx9leYSTKIcqN7jYTz0rAB+C3PssSxYmiD5JOGcommEG3ghha2qxKcJoxpO4Hww51U9Gk2D4Ar9C", + "CTdozJeEZLYAjBmE/kyLMHRO0IZf+syQUq0g6X5qmIwt8VGlxv00XFkRy1BWWJFzrtcTK+4BBZtPKibH", + "hE8NtpKC9IB+HZ8+w8JE/WNmChrNU6NLBiKbA0OsMfPQjWpuUj6xCq4VOSCj26ykBY0iCZ1C8Zz3p7VE", + "4RXJZUW68PrIyepgW5Cu8RwGrjKDWNW67lnofgwkznzODQk0DMl5K2ISnXEyxSyHkjAeIVuLd7913OGM", + "SzUqAKBuOFipRlDnIRekhLEr0tsLe5B7J3gvOtZ2m+WyAb63+nqJqsJNNQ2wmacGVzS8Wt2CBkNkvAyB", + "tRJ1q4TxqmM23QRFriwMQCW0Sj18MLQBySUeW/LA7TfbRKOEVVbdz5K2ast2vtobnLfFT1sNl3aG1eyE", + "TXgAZOMGLkpnibZhoRkRKYVKJygmjJLY6ZKFr9KauiAzO5EExTmxK2fkU4HtgmNzvAEogzkbGWXTGq+v", + "d9jGPGzGsLoMBPRrX2wTVyTDmatvRQ5rZQIDJcJlDmuraEsqR2F31nLDgkzzBAtkkRLbDFku0oSyyzat", + "y0U65gmNkP6g7oCe8CThVyP9SP4Ac9lsNTv9waipZtC5GZxNwDMbUuu3nMIPepabtfRfsMRsme+3AJql", + "TZhWMCT7J5oQC6P3jtFrj9CrOOl7O4OmtPSGRisJ6csQjDfl3JZkgyc+l4EkvpVSjqt2RGILXm/EniyX", + "pvBKi1vJoaM6F+DtPDrVDI3PwwA5Mvy6hgCCxgQSbNzUlrlGC7bYZirBmg+5nKG/83HVINo2vjZQSWyD", + "lVgUgkyCgfSwoysN0uaNpTXxdvcmYA/AVvVE4aMbYiisq7lWBjI18ZM3S+XHZsQuGXVzNKXIWpTWcIEW", + "BU6A7bU9YEC9UFwgMBjgYKRaQMlVqHOz8KofSjTmQgA0tJZwOHOzAXwTLfPotXYAU+jtjCwsFO2QUVYY", + "SQG1jCBG5kR46ahcaCVrSuI++pun4gGYdpqphUVpB+P5dxLxK1aMccj8QerGc6nbOWTGsihyqOJfvqSb", + "Ba1PEwqkB4MTTAkobEjVDE0EkTN/7qFillrGu+IibqwStEDuFSg+Az5WpPglYT4rK5oJqoamoZH5ajlc", + "zlSihadW/0SV4rCoXvx1dX+5JCIsJBZTKl5pFbriHRVPOTFoKwA9AkX/7F+GxRdwIy3ARcrm/+qaLH86", + "Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1VgXyzVbAxy74EtOFilV3llKok4FUwaXVPtkt5q0fl", + "u9FsSRJVe997tv/0ScsSMp/lrDMwWV/aNTdPV7jkGnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPf", + "H7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJDGkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10", + "gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WDqeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDI", + "a9q28KOac8l8XGZ3brjO0X8a33CNFp61rkQl83GTH/p1vVfjhS69Fn6MQ4sQA1kUQV82cBfzucKyEjmt", + "/44gwcGlci2ntZg3VsPb1nMOIIrFFlzzQgFDsOg1edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweM", + "vtH6HNYaf7AX4O2+Go39GnEri/BVCsqVt+7N+22RprtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJo", + "oCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiSTyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3A", + "dIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliFCl7mmdcGKd0MxgmPLk35M6iq3UcDlBLMJMoZHP6a", + "V217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6QfrU5lWAjHJNVUSAkahnQMUcwOrVKn9ameoXzNZhAfl", + "pOHSwWxhG9YN6vc4cxGt5btg4JtAxVn2kQjetagAmmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9", + "HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEBinjOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAU", + "kwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf8RvrEK4PwBgbvLLNph0/rtt4CUdScVvnqzjVI3Id", + "ERLXkTXDr7SNlbdfBmPlX2ELxlNUVLZvQ7zz8uz6d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUw", + "vj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHStazKXkbrVvzWKYRNB5oLsrZu3h3UkzPx5reqKGdD", + "1R+iqJx9604KyS3tzjlR7t1zS0aNO1StqFJxabmAf/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMp", + "GWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F0QwLWRs7o9OZShbVAJy9AGjRZ1VbFEQR5gyFbXa+", + "3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelts9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+x", + "tQmr7TSl1HnfWUd/JUrVb6HIZl4ueHslKCRFF8sklSA4PYDEmwxHBCVkAmh0RaHx9Z7FeterB28FKov8", + "UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GINrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK", + "9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRPG1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfVKxvUK4JIk", + "lJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIXIbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC9", + "1YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuq", + "Zunq26J4rcDrhrjpj1LFVWClPjqZMihj7v9chMn5SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB", + "lfzVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3", + "nqso6BcaqhRX3ssWBNjSeFk02l0dLuG2ssHl46U9rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkM", + "jcmUMtnGbrs7KAy3++mw00eHFokbdFnFi3785qE4vEcnNE1JTLWMaVT/5gyGnZa2uLoucbMiIO6rgLTW", + "D4trz9dDJKxLuFp3TfY/Ix/3s7TfdhrvKvQOsKs5FRXAuuDFLqIThFmtEihlc5zQ2CbSQ2IkxKsdOES0", + "kmQtD5ClHOjsJF005QqVKfQt7W05a7YLFuMn12BvXYGZYQhi54sAohRIXXQV+zo5RpngcR6V+aMJDLpE", + "/BB5DQtthZC/PiT3Lu0bkJg94QKtt280GTTa2Seb9rtmm9QE27zV24P1W30nRpFuJ8/i9TzMvNSOg90I", + "In1NCmLARFNd9pok6E3mQwuO/sZfwWWd19iSIy0S5ZlzsGiaWqakgLsFXAyhuN5jkhB9TS03gngSl1kS", + "VJZcdD1L3X7ybNbk4gSP1PJAfiEk07oK4B9Bfylmi+DAXH3P4i7ZGDhYbWkcXj1TF8iuVnVwT9dKYo1b", + "5Ztwm2oVGC5fs3kbvJRLz/xdgGn7otkyAopj+BUh7U0z1r790oW9NdqP78Is95BC2mvreqjhozr03gKG", + "3PVfxgJrsa5KvHsh93yILN5azbgJ87WeBepbnQ97/2OszGjUP9j64S//d+/DfwatzTW9WRLRi8kEAo0u", + "yaJnqvxoHb1fRTyFEgNamJ5aUiE4BRsSoInbw+iPd39QMI3FrzhdmgJEaHklerbXTugv/9Ec3+Qt4zvg", + "k2tJ9rMrcNxFpVLF3XW0kRIxdbHkLpFssz9kkJt2SRYSeYW/rEjjCPU7WXziRaCjCyMG9gmbX6AxhUqK", + "csi0VoujiGRam7C1ZKgpB86B+wiCE78dW4DMJX5bh6SJJyDo/ekSXO7rd29/fP3u1+PR67MXvx6ejH55", + "8d8Q4nHVMz3EPU17e/tPbBFwfyW3g4Uobl5PoY9ObZi+dfVPclBoAadLojRXOQSFkOsoySWdOwehSm5f", + "OWE5Wff2lQg+E2pXqSQUlWAhoRM6IeDXh+vEBtVQ6YiRSqiebo0blKHlG9sQzrADnNQrfh+qW6G3Irza", + "5cZWF/3JrB0LNSCkgcMOGa9Q5j6gvVAJeBUu9sN7GW1A5ogr8eoSZzdvBop6WDQYjDz8wpV8Bs+/RLXN", + "dyvLa8550tPqTUNJgqA12axFMHIemjIZCZ0mp8N0HJDhrWl3Sqc44GcI+RO+SFVMN6C1GVNL+99YHiyc", + "x3Bcr9dgjqVZqlp9gZqRQKpec5pDqqXaUVn6vxo8kzObu0q92LpqomrK1JatXhvCy4g5oIavylYuT5lD", + "R+zBR+uTcFfqVd7MvJE0782pUx9qCs6KBTrTS3M1I4J4GwEflDj4N1wym5fTAoXFVP/LiChjVl1Sj5ZK", + "wd0s0UZh+XFLUGQbL5vDV9c5OMXXRQ/gSsFyyf8I8yjrLG2//BEw6d+42pJ04pqAYdSUuzACe5WKVq2J", + "o6rlzfCpanne5v3gwbO8agX3azpbNeIs+6iQZoge/4ap+okLUAebMU/uHMgdLv+YCMCAq8O0t8I4pymJ", + "RzxXq8+/LV1vr/yi/mhZv9apvhiIOKqk8zbxAofKUY5heaX1cpAoF1QtzvV62WBuSIN0RWNhIaEj+Lns", + "GAp1fvoERuNJIGHkJWFE0AjKoOrzmGIGGhN6f+pVwzOFEZfwWkEEen10Ys0NDvIX1EeqgPRc3OXh2Umn", + "25kTYVTuzqC/2x/AYc4IwxntHHR2+9v9QQe0qhlMcQtK19v8aZtvXCiuJ7GVhH50L+kvBU6Jgi9+CyAB", + "QNyhfR1UEDz1lMgMU2G1yCwBhAJDMFR/Dbj+7kI9MLdy1yx7a5sppBlD9gvJXtvN/QCCMpwdmObOYGCB", + "zZW9fiF3xyQMbP3dRo+W/baS6uwSBWDul9Q8J1sWS/+p29kbbN9oTKuGAmc31PE7hm0SLwHtfP+GC3Gr", + "Tk+YScuzSdY2HMo/cUBI/ln77YPeM5mnKRYLt2D+amVcNgnGRCLs3jV6nJIo0qwCivH00WtGzHOEFcIm", + "clnkDGoYuw81hVZPgWnbbXIBUvQjjxdfbAkrfTgbxacqO9PH5dMSPX852inIeHkj7SOHsG2o9h4I6Edc", + "FOB+sJOyN3h+950ecTZJaKRQryBgG49MJYT8JIAX7rCHuEC/51xhVITzP6IjbWXWcUFu3fIq2vqDxp/M", + "8U5IyAx+RkSKmUmOMO+sOfRLx9m4JMrjvPJWc4QPpT3gpnIgPOaiAkGuekT9a6suDC5fR3sBBAbbp5le", + "/ICEv3cPJ9xOtqjB+pBHDipfolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ8EtSSHjlbi1d", + "CluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZL18rZ3p4X8sJ", + "27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTKWhf+zsd9ZCNS", + "Ix4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80SRXNburmei0Ms", + "F7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpbrd5uwHDexFg+XViK5mXJICz48p7zaH9GZptGNotj9k", + "by3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQjHsLaeUsYZqon", + "MxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ49LJ5SCEsBjj", + "JAkW5pgmfIyTkVmfSxLwCb6EN+yi+PX9nTeJ8ZiYWu3ZQs04M3/n45yp3Pw9FvxKEjHsbPaHDBIx7FqT", + "uFsKiOgKCrmlGdfnTPDU9Lllhrj1xyVZfOoP2WGcUuYoAj7BieSIXMN3UN8KMDMM92qgB3Oawn7wo1wq", + "nvrIp47uzDB5rrJc2YwSSVQ3hPo5ZIqjPxy246etP8oeP4GzmOBY04n3ipkSyNZNo5YjrGc/glcD7nYC", + "CzDs6IvUhHlMBWbKwHYW4JRo6m/pRlEdQR/SzfoKR5ihjGemsgQQ1Qxrkqu0AVgNOEmQgqPkvtWCO+xk", + "w3ws9F46bsTdM0BptWNEGTr90TtMg71n4fMkSSRIKKLkv85f/4rgVtZ7YF4rw7VMSgfTAgOKc3CdOp72", + "AkczZBxVUExw2KHxsFO4c+NNGGsubbhMrwc+xR/00H4w3XRp/EO/r5sy7soD9NsfppUDfZay1OCADjuf", + "ush7MKVqlo+LZx/CC9oEX3ZeYQRow1xzm8BJMAWkGe/GN1ckZjHi9hZIFgijkgP5gStjyrBYrEokDCy9", + "XUE+MZGM3mL8MYTIxWHnYOhiF4ed7rBD2Bx+swGOw86n8ApYr2Vz5Tq4zwrnZkFETwaDzfVI2HZ9Az7L", + "Fo6BL6wDNmpFRdlNvYMWhvXP5R/4t9Y/C9cPZrrzEprIKP7O+P4IHRCexO5rogEXRE3sxiwiiRO71xt6", + "7t95oDcrIkly3wT6UORZuMcKpP5HRY6wWeUxWmm+f2CKG9zXpVIx2z8M/T46+3nAem5t52TuQp3DdUoA", + "g8aq0si8jLBE5zCm3rlWvl/Ar337X6f7AabiRcKnFwdGdUcJn6KEMpsP4AUqa/HAriV8ZGBoiu8sKo0r", + "ErdhJIl//eOfMCjKpv/6xz8ttvu//vFPOO5bBl4NakxfzAgWakywujhAvxCS9XBC58RNBqrAkjkRC7Q7", + "sDZ/eIS8UvdWSpNDNmRviMoF8/ImTL02aRu0rgI9H8pyIi2Mj36RTmwxGRPbGLDbuLNslvJeT3Q3AIcI", + "M/AmoG9FRwOAJUdNoW2riXbCJlMz54rRtB6muRSst56/KHKtDPX2zABvyGBgiUPnDh7YSaON8/MXm30E", + "2pahCigYBLpD2YxVI/rfeNJ6nmQ4SpWhwCob3hThDI9pQp3JsaHaiTmCKY5mlJEyvrjAGndNHLiRah5z", + "eHaCbCBkF14dstfnW2BiVSRSuSBdywmERRgty6Fxm+cCPQD/ogqiw3r23SGbEAx5QifHhgl4INxFPmDR", + "MAMgD4hxpapSea07ZAZJ1iIX64OX8pgk8BH0P8WKXOFFFxW1bl11lAQrrRDLrn55yAzWq12DHkCVIG+Y", + "feBnZkg9F8lrc7YEmSRaNYYIfFP2G/remHCBbISzV+XfdWeSLM2w9KKlOHp9ruc3BU2QG3sgtPT63O3G", + "ZhdJjqKEAjVEmA3ZFAKBHHgvZ5VdLRLKZljEvYjrS8AHc7pk/Coh8bSJxx75RHaHkkyln8Bx+rlOro9N", + "uJgtT0AfYgNQt9pzd2zfaee6sy3+mXx3thDkDZx3xoJLDL8xq/vNkdfCkRdeN+fUC3nWjh0C491F/Jou", + "Hijg19He8pqbJ96SPYRFD204aBvwinCBzo5OEI5jQaTc/Pe29+mZGiot5T99P2pW/BChJ3YsXFjQP2tv", + "qRLIY2EHb+yoEXbzqtfX9e+3rUrxncabrqjDU155d3971Dq9yTVSCr0lrX27SdYG21IZcSgzWFJLD0Sj", + "hBTiS3FOfSpaZ1U2YbzFlbNSXLLs+eTYHcj7sy/brnNWvxvugSke1xjiAzLCaqq1XzX7MVHzu2IXHdr0", + "CvPz10Wag/uTgu7bFB0i88ekLsa1ZdNc0ACdNF6gL4ky8CZ3qafbHgITPyfCnWoz0IWZdTEt8ykyOC0w", + "IbDErNZ9T8wr7VRf096fSfOF5bmJxGKX/JuI0kLZLddqlYJ7YktA351+Cz3cSL39cmErlsACiwxW1LFz", + "O4FldQPLBYs2v0WufHGKNnGNpRIr3LxJXFiyDZpSoWfdl1x3yPx641qms3otZWiS0OnMOgFiOoFYPeXX", + "74ZR7tzDKIs62QIrYkMUH2Pe75leZOsFnhOh0OujE7P+/pW69QcEra5XlRzzWnm7vnvzqkdYxOPCedIs", + "k9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOTPx7n/L/tfNTQscCi8X/2vkJJxll5H/tHiZYEak2", + "74xYBvd10923AvOIiU/rL7S6aMCa2BQgY9cI/MVbLWV+9/6fSuw3k76R4F+s6zfZv43s7y/XSvHfbsWd", + "KgCmjwfycBXEFlptePQN0uYejKaWIj1Im4oXqQS1mXGp4NHjy2+2QeW0oDj/2mhp/S8P5Mrrw5HuyXEX", + "FhIqSkNFC5s+eE++ADeOexdubb/37wg4TMd0mvNc+pmJKVbRjEibtZuQKgN+bGJ3eT03Ct5fMZUO7vPq", + "uHe5+hvd35HEX99Qw7yNQ2+dzO/eaivz2/e1zG8QTW1msy270XUlmTYbAq0dpmlbMq5Avy4HgIfGFdJF", + "0DutqJTqAgIN4mDI/rfWP35TBKcffnAplPlgsPMEfids/uEHl0XJTh2pEKYEtRX0Dn89Bi/qFAJloche", + "mbBdH4ep2Q2k58oK/NspSKUjub2G5Kjwm4bUSkPylmu1hmT34m5VpGppknvXkRy9hRbcYor/ObWkP7l7", + "pKLByXwyoRElDAq8QGK6XIoHNJrcN8/ILROSmfVHesFEFUmktRpZcK01EnpZU/reA8lOyiJa9609uvLV", + "jzOxime2HqzV10ppoVlh+9roYXC/t9f9K2qPmcSMRrS8dJkWugOVgUxNqDQ3WWjFlyWoWB+9ffvKZZxp", + "iV+4OlOKu+JSru7mkPnFpfroRVm1y7zgWtASOYlthirk4dlyTTHBcUIZgRBdIkPJYdWScA96LL68UBmu", + "d9dKqLznY2krmD6cUPlgrOBexLeTSmFoXur4fim84rQ4EQ5OzaPiV5YBBRhPSHzawrniPZvDujXjBtgs", + "jO14luAIoB31awZ1zMIGGJhBvynAAhA8SYgwaHJZrlwNxCErBkeZV+Pd1ri50M2PcqZoctE1ETIACSIR", + "ZgsLqTRklc6wUiTNFKT2Qto6jFCQzIy4VvxRD5ryXMJbkIXrd4lwcoUXcshsMrD5HArlChIZ4MUk6aOf", + "OeAwIDzFlHmM11Qg/E4O2QWNEzKyMAoXiEokZ1wowkiMUj4nstovwSKhRMAkjrBeOYlSvAA8MwPtaNaH", + "Z8RghlXAGrj+N2YxhVp2uudiygdDhtHOYIBSgpm0qdcST+DCsW0gGERlQN8jjPYGz+1XtX0DzF23/Bv6", + "NAlB5jzC42SBiKZiAH9Qm7CBqa0taWr06u2bUCHNfhUmQ1s0rLKxVLpSiXEX5axMLgfzec6KXHC9XSoX", + "DOZpHWuEiuIatHgaYxJhvZ6MV/sBJEMeRbkIXZB6q70ip/+OgqM3vXNYqnDqdgJaeERi2HPG1QzONIej", + "tPl9A1WVRPXnuGiCh4QLhJFH16WRgEQ5sMYNQP67KCv2MVeB92Lze3d29PG1jMAdf4O991juJyAiPplU", + "DuD6q8kc4FUpE8sk/Gc9p0euVKvP4mKKp4xLRSPHDOuV3b8phK0VwtUrG6TmCReXvmxVpd+fuLhsq4FZ", + "PFH6uBQxf4ZfoW1fDw+wmx/exA8GZqOsaKK5dyWtTl/FKQWhiyrpYoc5Sjib6lNUGrrv3RLva3UbBodN", + "X6bC+I8L1BythIzsj6baq56MraUJVvvItvrQvEj3fg/+nV+5QjTNEpISqAbbM8SmN7tEWILK+VR6OEM3", + "45X6VPnpwEYXlMal33XiENCV27ANkN6XtyvIVBM+XY/jV3TuQOsCQH5DZir7E3RhvDkXqODBWqA1qPno", + "akajGYD6gd6q2zeYfzjLLgo8480D9BIOsg/rDJ1vGKx8TWuSJ8Rg9c3T9OJgud7p+9NT+Mjg+ZnKphcH", + "yNU4Le4Pqd/yQfr0LBIsFfrVQg9uFMo47OiFwlrfLOa3aeH7SrzpIQtB+TFyZRukE3ThofpdNEBOOX77", + "ik8fTBjrNlcJMHNRHFnVEWiTsLjTFDdBkzCg3/ZgEAKvbgkuaIZxx9iCS4N5xadFhYIKKeMsa0u+dphA", + "xfM0XUHDaMNDKpMq5rn6i1QxEQI+ttTdRNxoA0e2OhW+1IRqcencwd4E8gtGBxnI8OBSaaba6XYIy9PO", + "wW/2X/M07XQ7djwe1PgNhPs1II31BpejWPTOeEiM38Tym2AsVpm9B7JYuzmsOt0skb8xL/zpvYXOZveA", + "ZAjyQc2I+zWJoN54qwYfxguwyMeFCGVNTTWprdlU42bZ08OLc1dCp01Yxrn99Nx9+RVov+uiNdyYkZvu", + "vYdtLI/gMWe3yqXZTLiowwiti+f46gnpy23J0lTbUMg32ry5na8VYWpJfZlF2A9iU1IN54qnWNEIyvlE", + "M86lR/YF5q8pvGXNtwVlgnHD6Jk2LP5Ck+qFNQRfWEH+wBqtEPYf2T768LkNpg9/4R6VX/zk6eUFx+86", + "4Rsg76HeuKBkgjKcS6LlqjwlKFpEmiua+k0ERzMU4UzlgkBpOoJSymiapz6Ys96xOQbgiYvt9KKLxrlC", + "CRZT0IvMQxfuEvE0JSwmYCEbshnBc6qVOoESrAiLFj1JoKTtnKArLi4TjmNQ8rMYg68FSuIJoikQkLFT", + "onCMFQZR40Kf+JHJzLkoqtwaxZqR65Ia4iETOfvewPTrZi/cQC8QARxqKmdFNcQIx4RFQXzm86+bjX15", + "a/A5UfWJPlBszq146UMG6/hWTzecryOO55EFGHNht7ENm18h9MpmJbKa0uDI6N/zSJu5ujk+kIunWOJV", + "p/jr8O0URPfV+Hce3oHDBYpz0513KoHM/6xemYKh+OFOkC5ptvG2rpmi7FuxzDfieVt/uD9PbmFN+0o4", + "YbdRsW8qMFRO+mtguXZVb8VzH8iMaG1JnlXsAVmwi6l6MPGJC4/LPRZzp2XY5mgWfNvnTkpg0L44+8a2", + "62zbhhzclm072+ySU91j5JT1IEozzMGtGbeRVVvTwb9pPkhtdh7LfHAWWfoO7o0tnhSM0LDGDC8SjuM/", + "Q5juCg9OxIUwmA6AEvGYMEU9q6EfoA+2ubJyWdflS74/Pd1s4hJCreQRQj1iDuElxejP0kA5/9dzIgSN", + "LfQmOjo9tgGzVCKRsz56nVIo+H9JSFbmlEBeX1/Pz6FbLFdJr8BYdDuEKbHIOGVq7SjKV+9mMJ9uVVv9", + "nvmkBan+5pBu7ZAGy/7jY2fAZSBrwkxgtWaqsFpbPJOyCRepkcvwmOe6dc2D9DLp/TRFKCc0IXIhFUlN", + "XOAkT+C4QcEDW9TWfmd2uQtRsfrkmIS1jIiUSkk5k0NmszUyInTf+nPdvhfiFHQIKFzw1zPDJL+O8Dk9", + "GBMxhlXTqgEOERTL7Bx0tnCWbcVY4YYQLTu8zxjSTxAPh+QiHfOERiih7FKijYReGvUEzSVK9B+bKwPq", + "RvDdly7Ze/uTpVf6hE14sCCaodmCmP9UeVWWrTnH5KNjay+Jf1gc/4GNDrO19UWBBcFJD4rsOjQalCua", + "0I+G1elGqFQ0Mkk/uFi796cFU+0P2SlRQr+DIbksSQymAGiXW5ng0dYwHwx2o4wCpNkugcEBw2t+nEKP", + "R2fvTCIoSblYdIdM/wMafnt4Zry7E2ytCd5AbTVgdLL1ek2I8Tks079xjJ6Z4Er8gOCGf3MJ3hzlo/EM", + "yYYjyrNVqhLP/vRBpFaC+2ZXeJx2BYBZKmazMRU4AqFYznIV8ysWtiHMeZKn+h/mj5N1YF0KR7P38OpX", + "I+2a4aztxk3wURxKO6eYmIKND+L0MAv2WGNW9cK5KYAQU4kGDN4Ch+rPSN1f3nzvr+NX6O60K+qKoX41", + "Z+u+bz47Bodx4a/HYznmhtLcTBRfbX26wrTZ+vRjwqNLacFQfLOh1tsANFz/WII8WxchiAmQm4ksiJCB", + "qiKyO2Q1A6TB3JEII0VEShlOtmDOphGAq3ZWLDznFFKko4RCkhqNAbUoAUxqAKDTswFDlWvA8+hKWy7K", + "f8d3RiqOxiTiKXEQ3psh1e1vmKqfuKjicX8tfPGtt/4Ayocp2NvXQJA39/hZkOSn+BpCpePcOpTdiDZe", + "8vJHYwrqItibYWd3IIedLhp2dtJhR+/AEQYTKlZoH6WU5YrIPjo29i1Ign0yQJJEnMXSIYk7C97uQDal", + "xBqybMivfALf3afYY6kKlvKN7STEHvR7SH8PSTtowz9w9kzGXTh0MeK5MuZ+e67sWzFRYB7ZvHdfrXdG", + "vun2bTj53+zxrfAo2GXNLr2tN5w9y+WMNJvcXpnqPLkaA0K1q5gpZ+jvfCy7iJErYw0XUvWX+J7++sx0", + "cB/o+bqrmyDn27l/g81vAZtfrlUYLtEEWOor2VGHwUwk1xkXCnAUbba7oSHQJAC7gUc4Qa+PToYs0qzI", + "gPsJknLgThbk29zCh387Ry+O3nTRMVRvRD/n480+es2ShauhbXw0Q2YkMcO8IszQ2FAtiUPXsxk7UM9d", + "BovrDh6oHLI5GQHPitsrFyTe7cwIjkEi+aPzipvOAri/b17pAwTQu+bLYts7K4WPzhuixKJ3OFFELDd7", + "avOkWIFaYS9pBwJnBTcDPak7lA77rOzTyAYGnGJ3pxPAqvj0rZLB3Vf9vB8vmYkTMTXkxjlgfTJIMsDx", + "4nHFMskZKphjiAX613VRC6ApS9jyspUKBnTZFPn9FZncV/KuCrr7v+vpgpk+WkdTVtknTcRFDZG1nl6X", + "HDwzgMTWURXhDEdULboIJ4m9o+xNUESk9ArxdywIvoz5FesP2ZuieolN6EVHZ++6zlGLYiovTQvWF9tH", + "r+dEyHxcDA7BQTNeY1hzEg+Z4ijCSZQnWtwgkwmJIBcXipLIBl9uMZTOHZ6dspNgBRUvqj1/dIXbwjQB", + "u1eSRZ3itsxWbwkSJZimzfDfVlCDgEMINRjrRjlDlE0SG1IVCS4lsk31SEKndJzYACHZR29nBEmckiHL", + "EswYESiXJipeD72XCSJlbhK8dQMAk2soqotKaL9McGVDExLOhTTRBJrC358iqUi2gszemJZPYc53JNua", + "xm1PD2Skro2h2RRiX0F6QwylmAXXdJQnLoDxXkPRzYAeWkp8LAf/raDTKRH6VGDDZE04njnWbjnNoa9k", + "LDcWcTwv3mpXxLFo1ctK9DL2VkKzjUq067hzs6i/QOeXtBG9zz66WRbxL/qjln1Xs1XDg7CPPnOWf5ba", + "+OdekmBbA1ZJ4Y/NnOSNvHJUK4m262G1WmfW3mWma2v8rAeDzXrMaFm4kj7bpPB+fYQwuF+Uh/suc/a4", + "aauCdlXRTRtS/tfj2X8VFHg3QPYPjHJyCyD7ryrvHpDGHw7/JHhQHyqPvuJ7dhVk//RY9HeVPm8A6QGO", + "rSl93nA9G7y6UlF6b99ppybZFv9MEryNd7yB/O6W/ZvW30Jl8BZrnQtaEzxJM7VwAW3WV1kGnUn6kfQb", + "HMFF3OrduYJvEdL55cjD0WljQOefs+D7g8SM2uJ9VKKT40Al9UeGMeifucrFsqVvnR4W0YzOSbPRvXqC", + "7RJlgvQynoFzJTYLZtfD3WUKi/70I7LNW8xV+y+o/ghg+SRGMRUkUsnCVOLUHMH08Z1EgmtNAJ5zsWiO", + "EjFH5CfB00M7mzX3oT1T1hhWxhmmi16MFe7NHbdZYUL7jOhOF0+pGR6iDL38EW2QayVMjQk00ZoPopNi", + "SU0JfQk0uekPeHvQYNmkH8loOm4zyhXVQl7baiwoyqXiqdv7k2O0AdXHpoTpvdCi/gQk2UzwOY1JXBlj", + "Z84Ts6rbDQt6U7urFiqK0nFOuTCDexAZps2FNP1IsypbKEJixpRhGNzauhzVM2WS+HV/mDIXgGP3yI3i", + "2xVmNb8Np+xoSoRKmHYRFecG4nnz2zX3mK85PxnK3WmV286F56w2XrfLj2qZtnQXhR+K3Ln7NVu//3pS", + "eqh8lNk81nQ+LxTSJrP510WCg/u7H+7bXP7+EaeAviRO+fZM5dCAbjFEMK8gpjsmc5LwLIWK5PBup9vJ", + "RdI56MyUyg62tiD2e8alOth7/nS38+nDp/8/AAD///Z0/+oP7wEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index d69b1e2a4..2e7eeea4e 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), + metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index 14ffd7ffc..ca3702af4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." + description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." example: 59 PassthroughDevice: From 7f4a8bc1a638164574abef06508b31dd0c4d5584 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:29:25 +0000 Subject: [PATCH 15/24] Revert "Report conservative per-GPU vGPU profile capacity" This reverts commit d6adc4c41df2aff189c966df5dfdd73619f6b5d9. --- lib/devices/vendor_vfio_linux.go | 51 +-- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 492 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 259 insertions(+), 316 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index d67ddbc00..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,37 +81,22 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles reports a conservative estimate of how many instances of each -// profile are concurrently creatable. Free VFs on the same GPU share its -// framebuffer, so counting every advertising VF overreports capacity: one 48Q -// assignment can revoke the type from all sibling VFs. Each GPU instead -// contributes min(free VFs advertising the type, remaining framebuffer / -// profile framebuffer), where the largest profile still creatable on the GPU -// is a lower bound on its remaining framebuffer. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - freeVFsByGPU := make(map[string]map[string]int) - remainingFBByGPU := make(map[string]int) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if vf.Allocated { - continue - } - if freeVFsByGPU[gpu] == nil { - freeVFsByGPU[gpu] = make(map[string]int) - } - freeVFsByGPU[gpu][profile.TypeName]++ - if profile.FramebufferMB > remainingFBByGPU[gpu] { - remainingFBByGPU[gpu] = profile.FramebufferMB + if !vf.Allocated { + creatableVFs[profile.TypeName]++ } } } @@ -124,35 +109,15 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { - available := 0 - for gpu, freeVFs := range freeVFsByGPU { - available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) - } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: available, + Available: creatableVFs[profile.TypeName], }) } return profiles, nil } -// gpuProfileCapacity estimates how many instances of a profile one GPU can -// still create concurrently. When a profile's framebuffer is unknown (0), the -// free VF count is the only signal available. -func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { - if freeVFs == 0 { - return 0 - } - if profileFB <= 0 || remainingFB <= 0 { - return freeVFs - } - if byFB := remainingFB / profileFB; byFB < freeVFs { - return byFB - } - return freeVFs -} - func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b68a9f912..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,30 +196,8 @@ func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), - "small profiles stay capped by the free VF count") -} - -func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { - t.Parallel() - - // One 48G GPU with a 24Q already assigned: siblings only advertise up to - // 24Q, so at most one more 24Q fits despite two free VFs. - remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" - sysfs := newTestVendorVFIOSysfs(t) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) - - vfs, err := sysfs.discoverVFs() - require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) - require.NoError(t, err) - assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), - "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 599c8a1d4..8c66a74d3 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1006,7 +1006,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18968,251 +18968,251 @@ var swaggerSpec = []string{ "JLVWYtGUYuKRzuH11nlS+sO1hpQW687I1X0sOmT69TTZ9iTD2d2s+KqI4cLW4OXpcxtn30dwuiDIzyXo", "107auYOasEveHzKbaOThFYAjDbAETJj4jFCBuKBTWu24ame9x5jdm1CmI65bU6f/YVMMnizcgCEeYJCW", "JyYE1hQA8YFe7Z50up3zomiUZUzVpXlTFN5aWpEysn4Zc+fs3U0jbzPBJzQEhAORQPapVdRcTOqrvcF5", - "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe9zxJkkYm7MGkQqmurd5JNigaSH", - "9ZwsTKCKQZo050PfPHZRcCS4lBAj8PLsnewaww2JASNVj2+8QBNBSBFYM8mZKTFlDTquYCXgf4zzyYSI", - "PvqRSNUjkwkX6gCQB9MsL7Rga4gxNMiZyfH6PSe1/IT95yHZ2+tmlAaMgT/p58i8UMAknP5Ylb+1nN9W", - "yz+rUA+o+RMcUTbdbE0eAQNibRrrcAhfnr17Y8sMNSE0670qShEZcOY++rUoxAW7W6Iw9QOWxTrSX0Pi", - "2dlsIWmEDb3YKhqU+QZBOD2tJfqz8kNrOg3I9eH6be6koo35NMuBT5y/6Z28fr+VxmTerYwJAv9mPCF6", - "3JseO5s79IMyS67CxeZNlhlDGLLtCffWqmAxrRfJYyiB1VFc4WQkEx6KMXqrHyJ4iDbe/2TSl/UIuiir", - "bKX+3Ycm8un7SfDEAOJ9Q7fn0GHdxFs54EFdtw6Z1qlOr9Jp6KiY1KtlmWy5eCK/rG40v1xfsM800tzv", - "kcsOqxnhreCLTBKZwXd0YRPIfGqs7NaUI0mGBVYkWdTiN6sA7GTZA06uSXSD9LQX+vVPpuhGLshIzQSR", - "M55U4yZ2u6ErKMrhBjL1OsycPEeB4ijF4hJubif4o5yZFaiy+t11eDQzpbIbTOrnt2/PjDVA6XsyqSdJ", - "yKWIgGOS4AUaE3VFCHNTwRJhP0a2nmQqG+oFCTXKiKC8uoad3UC/5yZuGk0FjggyX7myz3ZLJMQjtV1K", - "20sA4TCKiJQN+7u9an/tp5M8abfHoWFtr62SHt1kg98enbm6N0VdYbfMO8urfEZEzxw5V2B49dbuyNUV", - "mFxXzADXLgkMYwIlmWxmip846uKhoOSU/rySrOkxB+mnF9p+4BSYpeqac/6hlTRaP+4hx3+KWRySNE1C", - "gknlnwK+F0RFixx8XTQ2wVIGXcDoDX5ehSA4poxIWcuDjnKRdLqd3sTO6mBrK+ERTgDicG93+9nW6rDT", - "lfHGNrxqFNNV+qgLwjJhOi7r1uDDwaSrJLGFs6yFxc6s45r7AdjTcnwj1GTWd5sn4bmA98FgKaX7GkfK", - "VZEDE17FRYv9YwtAxJX5QIOpKQyjRe3n/vkcBH3mGVazKvlvLdE+xNFADKSmEWMeqa2jIfKPQYmKi1Cp", - "MC6UzRAcExckWdyHLgTRQe5WnHmDZ/4sn+zv7+6v40PAbGrH3J67wFTN29UcaOIttz2/tgEI56rKHO5I", - "r652qtdlDU1pjrhEUqsXlGeE3Wg99/d2d262nm0ncuLCyGp8KQQhc3R6bGSiiDOFKSMCpUThGCtcZTJg", - "+9JcBmrRYJJCGtHk+9WspSHewseEuW0hrS/lrW+oqffGQUinmNGJZsj2Tb9nOcM7+08OTOXPmEz29p/0", - "+/2bImW8KKExWm3Flgnq80Az+nL2eftwB4AYbebyR+fs8O3PmpHlUphLa0uOKTvw/l38s3wAf5h/jikL", - "A2m0KRZLJ0tFYqvxa7lFKibxASrrgTu5p008UYPxGqKZAb0nCEtXieq8O/y5gsZptczADXKJV+TWanHl", - "NUsWjWtz67qvZUV05dV79bPTWtR+pR9X++OduQvesX0awOqiLO6yJ/5WhY3lytqPS6XBMsKKao9JYv6K", - "OAMg4FDpx8oV6Z61qBwG14gtEVZ06f9Y9O79eOQPxPvdVR7zfrI1ID/cMIRmpUD6t2U5dD0XcuLomsMc", - "tj0Wt0LbersWty4YO/7Ad+Ftwsyqvb+e/tfv/0eePf379u+v3r//7/nL/zr+lf73++Ts9WdhoqyGLHxQ", - "3MEvBjVoasj7eINtSekUqyhgo9PqX8MK2yfG4qCiGVQIRWNyMGQ99IoqIky9uVqy5LCDNghoSvCVFneh", - "mI7JU9vUH58ZD6j++A8nBn+qtxHbJHZhN6TAJpH5OOYppmxzyIbMtoXcRCToBfqvGEU4M0XjKENa/12g", - "sYAKf9YlVXbeRX/gLPu0OWS2qr9B384wFEmbFFlizDmU7ahM2Kx9nRQwFSaDcciK27rA7DN+yX4J8E9J", - "Us8xaliU1fqb1ZyeDULohpD/ojcSitiAClJQtiajIjEHPRtsLutza3SMgoZWkJ913JsEycM8ZC5uSqo8", - "JjGNgK+4vMKZzTwtUjoNpVkjXib49QL25o1JdosRztVM86LIJuJHnF9S0oUt7YIfDSJF4Evj/5/xrDde", - "9GY8K0AZsDDRMdh40KtK9v/p2Yn23hNBJ7anYG69JpGA0AlHxs7MpCgW1oWlib019T+YFn3mxL5uCslI", - "UzvEeApVLpgrb0GgyCWgfxTUR0Iy+fcoSihYneSM50mMZgDQp3QzIYy9zqBIscHjKCaT+r+rIRA7+09A", - "g3X/3t1pneFqlm4VleVJQKdNHetrwbENm4QBGPFg5Azha4KW9A1oK22BnUJx+O85cg2VJ65gJMY7ZXLu", - "pK2WkEgv224zmBZlj4GFGBlhe5ra3EdLp7CSZdWiBRPNAJ8lLcBNXpjszrevzpEiInX59huR3h04JQbJ", - "okelzG3hrsOj0xeb/U4QmKniqoKtWpmFVR10AJvBRjc0BW2UNhqcki46OYbsWnutlLoYpEP8xAVKzK1Y", - "XkYHAO5RNfdgU/Lv5NgKoMmiDJEwYsuws+lazOrX2wF6U6iAuBhKkSdZ0pZrsrxMoFkbMGdyNZZar6XV", - "gn/Mqn/2PobMDKiHaHgxgEs23l/tbY4OqkpfVDUL2Y0vJD9qpdH+5e39l4Zh/vIy+u7NZHTrAx5lMyxD", - "1D3zvZrw0tK++27kKrsXzTFGlX5HkgbP1t9cVRjvGlJE33OVz0O4o/u97e2323s3N9/dFEG3Cp3lweoV", - "ILrt0W/vAkU2gAlL1agxGB3pxzb03NlF3p+iGZbsOwUPa9aR7d2nbYwS0GvbMG4/gJtPzJAKLuVwuIrw", - "Y4NIdkmTxAgwkk4ZTtBztHF+8vKXk1evNlEPvX59Wt+KVV8E9+cWYLpwC8A6moy0ABRTBUIAFbmGb9++", - "gsOVEEjXMHL45e0hdteaFltA7rrBvTx7B45/LEcu0LM5txGX+cHkmkoll1HYWsVLfw7Er/m0NIy1maRp", - "w0Z0rcUJ/rkCZBuE1du8A4BfF+y+tJwPgH37kEmFXx/u7kqk3M+Fu7V2hjtCu2280kJIsTUEhf2m2+32", - "uLV3MpwKAE2IbfkSjsv4vjVQbLdDA9muh1JfPCRGJ2dlZajSGeGar83p+U5/+8kzKG66PWjD2FMcrej7", - "9PCofeeDHXPLHODxQRQfgMJ+W5+VJWyjguDkCi+gOKBZ2mHHXJiedusdW6tItoqvWcbjvR38bl2MawDY", - "BXHWBS7JUbqyNkmLdMY6yFqaWwjJlCYJlSTiLJZVGXmGJZKZQU418duFBD9kMMAuKkolg5SCcBSJvDQ9", - "Wunayvt5Zuke6oRmnGkdAAoF/EIWEqUUnKBF9xD6KFGRNRMP2YZwGVZFKhWUCI31D5Cv0LW1x2M9NKqg", - "Fon+YMjkLFeaiW320RFnMk+JsFZZNKbgMdpEMjcqLYwXVmOhGaakMRFDpl8LYLP+UagnB08Gg8Gg2yk0", - "uV3970GImu7U+dm32MMmRxjQAZlFIQbYQZEzlLOYiKJeODHkUA+Ru6Hj9DNBh93n7cQr+3kpV4UP5jqM", - "4nbgw5+L+ApDbdDPITr0Fsr5/u1F9FY5Sk5+tdlJ9qvRTSIYCIp4nsRa4xvr284Y5EhszZCSKMOdzbtU", - "onemVmd16jb0WHGTP4Len55Wwh4EmWge0G7iwCUa9oFnN9qGnTU2krWjuYl72cPHvQ9M3Lqk4kmIXxwB", - "1/cwuqRpQ6EVw1ZFcV5lX9NaZTCLhDKzT5poVkywZsqPyXyU5yGtRD9yuCnv3p0cVygF4yfbzwbPnvee", - "jbef9PbiwXYPb+8+6e3s48FkN3q6u72zuyLjpEWa2+0z14KqaSBguAgPH7kw9YY8tWCSQE0IsIHPV5TF", - "/KpyzwQjUf3ebZTruu6XY9hbDyGY+ZJgqYydoIFlnMJtSiLdton8tqmURQ2ksEXxydvB9ueaWWBwDcz4", - "rciZ8V+a5P/CVp96A/Y3qzrO2/FWGJDLMFm3Wn7n7RdtcLD//GD/cxfNZUmsG2OdnO5xc5tCrxw4ci0N", - "w6UCegYbZwnsWOHDmM9t1kan2ykSS+BvuHVrQcvF41bZUk0HthtmI6v4d0NW80lFMYCQEAOOFx9okcAJ", - "/lBqocid17LGUcLzGHlGL4MVBg6vE09J0M2A/8nawgz2p8l60MoEgE1DSQfKNCMGR59uxKZAH6CX8C48", - "wqnRn+wgTGER38eF44UJTNHny3VttJnVQz63igx8o7UapP8F09bLYG2jq5swYtAB+pXDN4VaxXjdyGpe", - "B31m+fW6QXbD4mg7SAvozMp0B+inQo4rJEEr+W1IYv8cWYZVIslsVvL57Y53NLWUO+flpnc7ZkU73Y5b", - "KMhhX85mf1dS/dL580kxFLFFcAJnuUzGzRVNLH42zIRKRSNpszT05jbJF7bmEYlHRktpCv40GZ5Wkyk+", - "cuLL+1O0ARCJf0HWgqz/tVkEilbuup3ne8+fPN15/qQVEFI5wPUy6BHkHy8Pbq1AGmX5yBohmqZ+dPbO", - "GBkio74XQSbvT33ciUxwzXr0zF2DfufP+899/KeY5+PE8+BZsDgDNwsbFoQ4K3hRQ8Dh7zSZ08mE/f4x", - "utz5u6Dp9vUTuTPebsC1NR2F7Vsnvhd/yRhMxj1TvygM0QMEJWQjitUbImEG6JwoBPTTQzgCPaJIG7Yk", - "57Cu7IoHCWtvd3f32dP9nVZ0ZUfnHZwRWLsCl7IdgXfE4E208eb8HG15BGfadGAPAD3OrI4ZPmfIFh8e", - "VAXS/vZgN0QlDRd3STW27XnauOTvrZ5mJ2UXHbKfCx1u6ZQHV3t3d/B0b//ZfrtjbO2wI3G9msO43CCz", - "PBYh39/5DZAm3x6eIci8neCoakRxoVg3GpW60aiguoNBZb/BwJ49fbK/t7uz3Q6OLRTdYYEGKwe2yrsC", - "hy5AFIHdCCzFMuvtNt0WIXHKENgbEiWYpoeRy2Wo3T4GfX0kzGvlJrS5GKzpf+niavFtKytSYRsymTBG", - "NOAC5ayo+dFf7/v8Ii7MZq5trof1XD2U/8L06lncIFPb7BZLmQkypzyXX6Ahrkxy6iThXNzo2yaF5Q2R", - "eaKMn5FK9P70O+ApmtaQVCSr6lCWGlegK91ycjc6zxUSCRN502K12o02W79qwt2GU9tdhVxR4QaNmGax", - "5lw5Wx9leYSTKIcqN7jYTz0rAB+C3PssSxYmiD5JOGcommEG3ghha2qxKcJoxpO4Hww51U9Gk2D4Ar9C", - "CTdozJeEZLYAjBmE/kyLMHRO0IZf+syQUq0g6X5qmIwt8VGlxv00XFkRy1BWWJFzrtcTK+4BBZtPKibH", - "hE8NtpKC9IB+HZ8+w8JE/WNmChrNU6NLBiKbA0OsMfPQjWpuUj6xCq4VOSCj26ykBY0iCZ1C8Zz3p7VE", - "4RXJZUW68PrIyepgW5Cu8RwGrjKDWNW67lnofgwkznzODQk0DMl5K2ISnXEyxSyHkjAeIVuLd7913OGM", - "SzUqAKBuOFipRlDnIRekhLEr0tsLe5B7J3gvOtZ2m+WyAb63+nqJqsJNNQ2wmacGVzS8Wt2CBkNkvAyB", - "tRJ1q4TxqmM23QRFriwMQCW0Sj18MLQBySUeW/LA7TfbRKOEVVbdz5K2ast2vtobnLfFT1sNl3aG1eyE", - "TXgAZOMGLkpnibZhoRkRKYVKJygmjJLY6ZKFr9KauiAzO5EExTmxK2fkU4HtgmNzvAEogzkbGWXTGq+v", - "d9jGPGzGsLoMBPRrX2wTVyTDmatvRQ5rZQIDJcJlDmuraEsqR2F31nLDgkzzBAtkkRLbDFku0oSyyzat", - "y0U65gmNkP6g7oCe8CThVyP9SP4Ac9lsNTv9waipZtC5GZxNwDMbUuu3nMIPepabtfRfsMRsme+3AJql", - "TZhWMCT7J5oQC6P3jtFrj9CrOOl7O4OmtPSGRisJ6csQjDfl3JZkgyc+l4EkvpVSjqt2RGILXm/EniyX", - "pvBKi1vJoaM6F+DtPDrVDI3PwwA5Mvy6hgCCxgQSbNzUlrlGC7bYZirBmg+5nKG/83HVINo2vjZQSWyD", - "lVgUgkyCgfSwoysN0uaNpTXxdvcmYA/AVvVE4aMbYiisq7lWBjI18ZM3S+XHZsQuGXVzNKXIWpTWcIEW", - "BU6A7bU9YEC9UFwgMBjgYKRaQMlVqHOz8KofSjTmQgA0tJZwOHOzAXwTLfPotXYAU+jtjCwsFO2QUVYY", - "SQG1jCBG5kR46ahcaCVrSuI++pun4gGYdpqphUVpB+P5dxLxK1aMccj8QerGc6nbOWTGsihyqOJfvqSb", - "Ba1PEwqkB4MTTAkobEjVDE0EkTN/7qFillrGu+IibqwStEDuFSg+Az5WpPglYT4rK5oJqoamoZH5ajlc", - "zlSihadW/0SV4rCoXvx1dX+5JCIsJBZTKl5pFbriHRVPOTFoKwA9AkX/7F+GxRdwIy3ARcrm/+qaLH86", - "Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1VgXyzVbAxy74EtOFilV3llKok4FUwaXVPtkt5q0fl", - "u9FsSRJVe997tv/0ScsSMp/lrDMwWV/aNTdPV7jkGnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPf", - "H7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJDGkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10", - "gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WDqeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDI", - "a9q28KOac8l8XGZ3brjO0X8a33CNFp61rkQl83GTH/p1vVfjhS69Fn6MQ4sQA1kUQV82cBfzucKyEjmt", - "/44gwcGlci2ntZg3VsPb1nMOIIrFFlzzQgFDsOg1edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweM", - "vtH6HNYaf7AX4O2+Go39GnEri/BVCsqVt+7N+22RprtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJo", - "oCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiSTyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3A", - "dIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliFCl7mmdcGKd0MxgmPLk35M6iq3UcDlBLMJMoZHP6a", - "V217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6QfrU5lWAjHJNVUSAkahnQMUcwOrVKn9ameoXzNZhAfl", - "pOHSwWxhG9YN6vc4cxGt5btg4JtAxVn2kQjetagAmmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9", - "HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEBinjOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAU", - "kwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf8RvrEK4PwBgbvLLNph0/rtt4CUdScVvnqzjVI3Id", - "ERLXkTXDr7SNlbdfBmPlX2ELxlNUVLZvQ7zz8uz6d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUw", - "vj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHStazKXkbrVvzWKYRNB5oLsrZu3h3UkzPx5reqKGdD", - "1R+iqJx9604KyS3tzjlR7t1zS0aNO1StqFJxabmAf/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMp", - "GWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F0QwLWRs7o9OZShbVAJy9AGjRZ1VbFEQR5gyFbXa+", - "3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelts9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+x", - "tQmr7TSl1HnfWUd/JUrVb6HIZl4ueHslKCRFF8sklSA4PYDEmwxHBCVkAmh0RaHx9Z7FeterB28FKov8", - "UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GINrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK", - "9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRPG1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfVKxvUK4JIk", - "lJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIXIbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC9", - "1YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuq", - "Zunq26J4rcDrhrjpj1LFVWClPjqZMihj7v9chMn5SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB", - "lfzVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3", - "nqso6BcaqhRX3ssWBNjSeFk02l0dLuG2ssHl46U9rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkM", - "jcmUMtnGbrs7KAy3++mw00eHFokbdFnFi3785qE4vEcnNE1JTLWMaVT/5gyGnZa2uLoucbMiIO6rgLTW", - "D4trz9dDJKxLuFp3TfY/Ix/3s7TfdhrvKvQOsKs5FRXAuuDFLqIThFmtEihlc5zQ2CbSQ2IkxKsdOES0", - "kmQtD5ClHOjsJF005QqVKfQt7W05a7YLFuMn12BvXYGZYQhi54sAohRIXXQV+zo5RpngcR6V+aMJDLpE", - "/BB5DQtthZC/PiT3Lu0bkJg94QKtt280GTTa2Seb9rtmm9QE27zV24P1W30nRpFuJ8/i9TzMvNSOg90I", - "In1NCmLARFNd9pok6E3mQwuO/sZfwWWd19iSIy0S5ZlzsGiaWqakgLsFXAyhuN5jkhB9TS03gngSl1kS", - "VJZcdD1L3X7ybNbk4gSP1PJAfiEk07oK4B9Bfylmi+DAXH3P4i7ZGDhYbWkcXj1TF8iuVnVwT9dKYo1b", - "5Ztwm2oVGC5fs3kbvJRLz/xdgGn7otkyAopj+BUh7U0z1r790oW9NdqP78Is95BC2mvreqjhozr03gKG", - "3PVfxgJrsa5KvHsh93yILN5azbgJ87WeBepbnQ97/2OszGjUP9j64S//d+/DfwatzTW9WRLRi8kEAo0u", - "yaJnqvxoHb1fRTyFEgNamJ5aUiE4BRsSoInbw+iPd39QMI3FrzhdmgJEaHklerbXTugv/9Ec3+Qt4zvg", - "k2tJ9rMrcNxFpVLF3XW0kRIxdbHkLpFssz9kkJt2SRYSeYW/rEjjCPU7WXziRaCjCyMG9gmbX6AxhUqK", - "csi0VoujiGRam7C1ZKgpB86B+wiCE78dW4DMJX5bh6SJJyDo/ekSXO7rd29/fP3u1+PR67MXvx6ejH55", - "8d8Q4nHVMz3EPU17e/tPbBFwfyW3g4Uobl5PoY9ObZi+dfVPclBoAadLojRXOQSFkOsoySWdOwehSm5f", - "OWE5Wff2lQg+E2pXqSQUlWAhoRM6IeDXh+vEBtVQ6YiRSqiebo0blKHlG9sQzrADnNQrfh+qW6G3Irza", - "5cZWF/3JrB0LNSCkgcMOGa9Q5j6gvVAJeBUu9sN7GW1A5ogr8eoSZzdvBop6WDQYjDz8wpV8Bs+/RLXN", - "dyvLa8550tPqTUNJgqA12axFMHIemjIZCZ0mp8N0HJDhrWl3Sqc44GcI+RO+SFVMN6C1GVNL+99YHiyc", - "x3Bcr9dgjqVZqlp9gZqRQKpec5pDqqXaUVn6vxo8kzObu0q92LpqomrK1JatXhvCy4g5oIavylYuT5lD", - "R+zBR+uTcFfqVd7MvJE0782pUx9qCs6KBTrTS3M1I4J4GwEflDj4N1wym5fTAoXFVP/LiChjVl1Sj5ZK", - "wd0s0UZh+XFLUGQbL5vDV9c5OMXXRQ/gSsFyyf8I8yjrLG2//BEw6d+42pJ04pqAYdSUuzACe5WKVq2J", - "o6rlzfCpanne5v3gwbO8agX3azpbNeIs+6iQZoge/4ap+okLUAebMU/uHMgdLv+YCMCAq8O0t8I4pymJ", - "RzxXq8+/LV1vr/yi/mhZv9apvhiIOKqk8zbxAofKUY5heaX1cpAoF1QtzvV62WBuSIN0RWNhIaEj+Lns", - "GAp1fvoERuNJIGHkJWFE0AjKoOrzmGIGGhN6f+pVwzOFEZfwWkEEen10Ys0NDvIX1EeqgPRc3OXh2Umn", - "25kTYVTuzqC/2x/AYc4IwxntHHR2+9v9QQe0qhlMcQtK19v8aZtvXCiuJ7GVhH50L+kvBU6Jgi9+CyAB", - "QNyhfR1UEDz1lMgMU2G1yCwBhAJDMFR/Dbj+7kI9MLdy1yx7a5sppBlD9gvJXtvN/QCCMpwdmObOYGCB", - "zZW9fiF3xyQMbP3dRo+W/baS6uwSBWDul9Q8J1sWS/+p29kbbN9oTKuGAmc31PE7hm0SLwHtfP+GC3Gr", - "Tk+YScuzSdY2HMo/cUBI/ln77YPeM5mnKRYLt2D+amVcNgnGRCLs3jV6nJIo0qwCivH00WtGzHOEFcIm", - "clnkDGoYuw81hVZPgWnbbXIBUvQjjxdfbAkrfTgbxacqO9PH5dMSPX852inIeHkj7SOHsG2o9h4I6Edc", - "FOB+sJOyN3h+950ecTZJaKRQryBgG49MJYT8JIAX7rCHuEC/51xhVITzP6IjbWXWcUFu3fIq2vqDxp/M", - "8U5IyAx+RkSKmUmOMO+sOfRLx9m4JMrjvPJWc4QPpT3gpnIgPOaiAkGuekT9a6suDC5fR3sBBAbbp5le", - "/ICEv3cPJ9xOtqjB+pBHDipfolySx3ScrIttXAohQVnuJVFfC80P7vPKskUE/oSn6LEQ8EtSSHjlbi1d", - "CluZyJlRgIMS4JsyYdF+911V+HtbPvGiZMCvoZuGchbK+FVxvOgjt6ZG6VcLgFgSBOYZL18rZ3p4X8sJ", - "27mPEwYzLjxF366pb9fUqlNuqMVNAQ6md8pb2CBuZIH489kfbmx9+GZ7aG97aGV5YOTKWhf+zsd9ZCNS", - "Ix4TJGc8T2I0JsjgHbnYE4VFf/oRYRHN6JwAqB0UacsTRTMsILIkRTFW2PjQGw0TK80SRXNburmei0Ms", - "F7iOYyHJCHD4Rk34k2UEImWMxEh/YqH7SjjBpbrd5uwHDexFg+XViK5mXJICz48p7zaH9GZptGNotj9k", - "by3Qq15ACKZ2vEaSBOBqV9h/OEN4yOwH3zsW4gLBJE5LzoUFYAZSg0xptmU5tU2PdCQjHsLaeUsYZqon", - "MxLRCY3stC7JwsZzBhtsVXdJD9iN8/1pkbCBdjbDeG0AzxgG5z0uniFLSVX/DYMg6CjJ49LJ5SCEsBjj", - "JAkW5pgmfIyTkVmfSxLwCb6EN+yi+PX9nTeJ8ZiYWu3ZQs04M3/n45yp3Pw9FvxKEjHsbPaHDBIx7FqT", - "uFsKiOgKCrmlGdfnTPDU9Lllhrj1xyVZfOoP2WGcUuYoAj7BieSIXMN3UN8KMDMM92qgB3Oawn7wo1wq", - "nvrIp47uzDB5rrJc2YwSSVQ3hPo5ZIqjPxy246etP8oeP4GzmOBY04n3ipkSyNZNo5YjrGc/glcD7nYC", - "CzDs6IvUhHlMBWbKwHYW4JRo6m/pRlEdQR/SzfoKR5ihjGemsgQQ1Qxrkqu0AVgNOEmQgqPkvtWCO+xk", - "w3ws9F46bsTdM0BptWNEGTr90TtMg71n4fMkSSRIKKLkv85f/4rgVtZ7YF4rw7VMSgfTAgOKc3CdOp72", - "AkczZBxVUExw2KHxsFO4c+NNGGsubbhMrwc+xR/00H4w3XRp/EO/r5sy7soD9NsfppUDfZay1OCADjuf", - "ush7MKVqlo+LZx/CC9oEX3ZeYQRow1xzm8BJMAWkGe/GN1ckZjHi9hZIFgijkgP5gStjyrBYrEokDCy9", - "XUE+MZGM3mL8MYTIxWHnYOhiF4ed7rBD2Bx+swGOw86n8ApYr2Vz5Tq4zwrnZkFETwaDzfVI2HZ9Az7L", - "Fo6BL6wDNmpFRdlNvYMWhvXP5R/4t9Y/C9cPZrrzEprIKP7O+P4IHRCexO5rogEXRE3sxiwiiRO71xt6", - "7t95oDcrIkly3wT6UORZuMcKpP5HRY6wWeUxWmm+f2CKG9zXpVIx2z8M/T46+3nAem5t52TuQp3DdUoA", - "g8aq0si8jLBE5zCm3rlWvl/Ar337X6f7AabiRcKnFwdGdUcJn6KEMpsP4AUqa/HAriV8ZGBoiu8sKo0r", - "ErdhJIl//eOfMCjKpv/6xz8ttvu//vFPOO5bBl4NakxfzAgWakywujhAvxCS9XBC58RNBqrAkjkRC7Q7", - "sDZ/eIS8UvdWSpNDNmRviMoF8/ImTL02aRu0rgI9H8pyIi2Mj36RTmwxGRPbGLDbuLNslvJeT3Q3AIcI", - "M/AmoG9FRwOAJUdNoW2riXbCJlMz54rRtB6muRSst56/KHKtDPX2zABvyGBgiUPnDh7YSaON8/MXm30E", - "2pahCigYBLpD2YxVI/rfeNJ6nmQ4SpWhwCob3hThDI9pQp3JsaHaiTmCKY5mlJEyvrjAGndNHLiRah5z", - "eHaCbCBkF14dstfnW2BiVSRSuSBdywmERRgty6Fxm+cCPQD/ogqiw3r23SGbEAx5QifHhgl4INxFPmDR", - "MAMgD4hxpapSea07ZAZJ1iIX64OX8pgk8BH0P8WKXOFFFxW1bl11lAQrrRDLrn55yAzWq12DHkCVIG+Y", - "feBnZkg9F8lrc7YEmSRaNYYIfFP2G/remHCBbISzV+XfdWeSLM2w9KKlOHp9ruc3BU2QG3sgtPT63O3G", - "ZhdJjqKEAjVEmA3ZFAKBHHgvZ5VdLRLKZljEvYjrS8AHc7pk/Coh8bSJxx75RHaHkkyln8Bx+rlOro9N", - "uJgtT0AfYgNQt9pzd2zfaee6sy3+mXx3thDkDZx3xoJLDL8xq/vNkdfCkRdeN+fUC3nWjh0C491F/Jou", - "Hijg19He8pqbJ96SPYRFD204aBvwinCBzo5OEI5jQaTc/Pe29+mZGiot5T99P2pW/BChJ3YsXFjQP2tv", - "qRLIY2EHb+yoEXbzqtfX9e+3rUrxncabrqjDU155d3971Dq9yTVSCr0lrX27SdYG21IZcSgzWFJLD0Sj", - "hBTiS3FOfSpaZ1U2YbzFlbNSXLLs+eTYHcj7sy/brnNWvxvugSke1xjiAzLCaqq1XzX7MVHzu2IXHdr0", - "CvPz10Wag/uTgu7bFB0i88ekLsa1ZdNc0ACdNF6gL4ky8CZ3qafbHgITPyfCnWoz0IWZdTEt8ykyOC0w", - "IbDErNZ9T8wr7VRf096fSfOF5bmJxGKX/JuI0kLZLddqlYJ7YktA351+Cz3cSL39cmErlsACiwxW1LFz", - "O4FldQPLBYs2v0WufHGKNnGNpRIr3LxJXFiyDZpSoWfdl1x3yPx641qms3otZWiS0OnMOgFiOoFYPeXX", - "74ZR7tzDKIs62QIrYkMUH2Pe75leZOsFnhOh0OujE7P+/pW69QcEra5XlRzzWnm7vnvzqkdYxOPCedIs", - "k9onX1hhMvRfyeW9/1P3CPNZqRMPmgTGz9h/E0yOTPx7n/L/tfNTQscCi8X/2vkJJxll5H/tHiZYEak2", - "74xYBvd10923AvOIiU/rL7S6aMCa2BQgY9cI/MVbLWV+9/6fSuw3k76R4F+s6zfZv43s7y/XSvHfbsWd", - "KgCmjwfycBXEFlptePQN0uYejKaWIj1Im4oXqQS1mXGp4NHjy2+2QeW0oDj/2mhp/S8P5Mrrw5HuyXEX", - "FhIqSkNFC5s+eE++ADeOexdubb/37wg4TMd0mvNc+pmJKVbRjEibtZuQKgN+bGJ3eT03Ct5fMZUO7vPq", - "uHe5+hvd35HEX99Qw7yNQ2+dzO/eaivz2/e1zG8QTW1msy270XUlmTYbAq0dpmlbMq5Avy4HgIfGFdJF", - "0DutqJTqAgIN4mDI/rfWP35TBKcffnAplPlgsPMEfids/uEHl0XJTh2pEKYEtRX0Dn89Bi/qFAJloche", - "mbBdH4ep2Q2k58oK/NspSKUjub2G5Kjwm4bUSkPylmu1hmT34m5VpGppknvXkRy9hRbcYor/ObWkP7l7", - "pKLByXwyoRElDAq8QGK6XIoHNJrcN8/ILROSmfVHesFEFUmktRpZcK01EnpZU/reA8lOyiJa9609uvLV", - "jzOxime2HqzV10ppoVlh+9roYXC/t9f9K2qPmcSMRrS8dJkWugOVgUxNqDQ3WWjFlyWoWB+9ffvKZZxp", - "iV+4OlOKu+JSru7mkPnFpfroRVm1y7zgWtASOYlthirk4dlyTTHBcUIZgRBdIkPJYdWScA96LL68UBmu", - "d9dKqLznY2krmD6cUPlgrOBexLeTSmFoXur4fim84rQ4EQ5OzaPiV5YBBRhPSHzawrniPZvDujXjBtgs", - "jO14luAIoB31awZ1zMIGGJhBvynAAhA8SYgwaHJZrlwNxCErBkeZV+Pd1ri50M2PcqZoctE1ETIACSIR", - "ZgsLqTRklc6wUiTNFKT2Qto6jFCQzIy4VvxRD5ryXMJbkIXrd4lwcoUXcshsMrD5HArlChIZ4MUk6aOf", - "OeAwIDzFlHmM11Qg/E4O2QWNEzKyMAoXiEokZ1wowkiMUj4nstovwSKhRMAkjrBeOYlSvAA8MwPtaNaH", - "Z8RghlXAGrj+N2YxhVp2uudiygdDhtHOYIBSgpm0qdcST+DCsW0gGERlQN8jjPYGz+1XtX0DzF23/Bv6", - "NAlB5jzC42SBiKZiAH9Qm7CBqa0taWr06u2bUCHNfhUmQ1s0rLKxVLpSiXEX5axMLgfzec6KXHC9XSoX", - "DOZpHWuEiuIatHgaYxJhvZ6MV/sBJEMeRbkIXZB6q70ip/+OgqM3vXNYqnDqdgJaeERi2HPG1QzONIej", - "tPl9A1WVRPXnuGiCh4QLhJFH16WRgEQ5sMYNQP67KCv2MVeB92Lze3d29PG1jMAdf4O991juJyAiPplU", - "DuD6q8kc4FUpE8sk/Gc9p0euVKvP4mKKp4xLRSPHDOuV3b8phK0VwtUrG6TmCReXvmxVpd+fuLhsq4FZ", - "PFH6uBQxf4ZfoW1fDw+wmx/exA8GZqOsaKK5dyWtTl/FKQWhiyrpYoc5Sjib6lNUGrrv3RLva3UbBodN", - "X6bC+I8L1BythIzsj6baq56MraUJVvvItvrQvEj3fg/+nV+5QjTNEpISqAbbM8SmN7tEWILK+VR6OEM3", - "45X6VPnpwEYXlMal33XiENCV27ANkN6XtyvIVBM+XY/jV3TuQOsCQH5DZir7E3RhvDkXqODBWqA1qPno", - "akajGYD6gd6q2zeYfzjLLgo8480D9BIOsg/rDJ1vGKx8TWuSJ8Rg9c3T9OJgud7p+9NT+Mjg+ZnKphcH", - "yNU4Le4Pqd/yQfr0LBIsFfrVQg9uFMo47OiFwlrfLOa3aeH7SrzpIQtB+TFyZRukE3ThofpdNEBOOX77", - "ik8fTBjrNlcJMHNRHFnVEWiTsLjTFDdBkzCg3/ZgEAKvbgkuaIZxx9iCS4N5xadFhYIKKeMsa0u+dphA", - "xfM0XUHDaMNDKpMq5rn6i1QxEQI+ttTdRNxoA0e2OhW+1IRqcencwd4E8gtGBxnI8OBSaaba6XYIy9PO", - "wW/2X/M07XQ7djwe1PgNhPs1II31BpejWPTOeEiM38Tym2AsVpm9B7JYuzmsOt0skb8xL/zpvYXOZveA", - "ZAjyQc2I+zWJoN54qwYfxguwyMeFCGVNTTWprdlU42bZ08OLc1dCp01Yxrn99Nx9+RVov+uiNdyYkZvu", - "vYdtLI/gMWe3yqXZTLiowwiti+f46gnpy23J0lTbUMg32ry5na8VYWpJfZlF2A9iU1IN54qnWNEIyvlE", - "M86lR/YF5q8pvGXNtwVlgnHD6Jk2LP5Ck+qFNQRfWEH+wBqtEPYf2T768LkNpg9/4R6VX/zk6eUFx+86", - "4Rsg76HeuKBkgjKcS6LlqjwlKFpEmiua+k0ERzMU4UzlgkBpOoJSymiapz6Ys96xOQbgiYvt9KKLxrlC", - "CRZT0IvMQxfuEvE0JSwmYCEbshnBc6qVOoESrAiLFj1JoKTtnKArLi4TjmNQ8rMYg68FSuIJoikQkLFT", - "onCMFQZR40Kf+JHJzLkoqtwaxZqR65Ia4iETOfvewPTrZi/cQC8QARxqKmdFNcQIx4RFQXzm86+bjX15", - "a/A5UfWJPlBszq146UMG6/hWTzecryOO55EFGHNht7ENm18h9MpmJbKa0uDI6N/zSJu5ujk+kIunWOJV", - "p/jr8O0URPfV+Hce3oHDBYpz0513KoHM/6xemYKh+OFOkC5ptvG2rpmi7FuxzDfieVt/uD9PbmFN+0o4", - "YbdRsW8qMFRO+mtguXZVb8VzH8iMaG1JnlXsAVmwi6l6MPGJC4/LPRZzp2XY5mgWfNvnTkpg0L44+8a2", - "62zbhhzclm072+ySU91j5JT1IEozzMGtGbeRVVvTwb9pPkhtdh7LfHAWWfoO7o0tnhSM0LDGDC8SjuM/", - "Q5juCg9OxIUwmA6AEvGYMEU9q6EfoA+2ubJyWdflS74/Pd1s4hJCreQRQj1iDuElxejP0kA5/9dzIgSN", - "LfQmOjo9tgGzVCKRsz56nVIo+H9JSFbmlEBeX1/Pz6FbLFdJr8BYdDuEKbHIOGVq7SjKV+9mMJ9uVVv9", - "nvmkBan+5pBu7ZAGy/7jY2fAZSBrwkxgtWaqsFpbPJOyCRepkcvwmOe6dc2D9DLp/TRFKCc0IXIhFUlN", - "XOAkT+C4QcEDW9TWfmd2uQtRsfrkmIS1jIiUSkk5k0NmszUyInTf+nPdvhfiFHQIKFzw1zPDJL+O8Dk9", - "GBMxhlXTqgEOERTL7Bx0tnCWbcVY4YYQLTu8zxjSTxAPh+QiHfOERiih7FKijYReGvUEzSVK9B+bKwPq", - "RvDdly7Ze/uTpVf6hE14sCCaodmCmP9UeVWWrTnH5KNjay+Jf1gc/4GNDrO19UWBBcFJD4rsOjQalCua", - "0I+G1elGqFQ0Mkk/uFi796cFU+0P2SlRQr+DIbksSQymAGiXW5ng0dYwHwx2o4wCpNkugcEBw2t+nEKP", - "R2fvTCIoSblYdIdM/wMafnt4Zry7E2ytCd5AbTVgdLL1ek2I8Tks079xjJ6Z4Er8gOCGf3MJ3hzlo/EM", - "yYYjyrNVqhLP/vRBpFaC+2ZXeJx2BYBZKmazMRU4AqFYznIV8ysWtiHMeZKn+h/mj5N1YF0KR7P38OpX", - "I+2a4aztxk3wURxKO6eYmIKND+L0MAv2WGNW9cK5KYAQU4kGDN4Ch+rPSN1f3nzvr+NX6O60K+qKoX41", - "Z+u+bz47Bodx4a/HYznmhtLcTBRfbX26wrTZ+vRjwqNLacFQfLOh1tsANFz/WII8WxchiAmQm4ksiJCB", - "qiKyO2Q1A6TB3JEII0VEShlOtmDOphGAq3ZWLDznFFKko4RCkhqNAbUoAUxqAKDTswFDlWvA8+hKWy7K", - "f8d3RiqOxiTiKXEQ3psh1e1vmKqfuKjicX8tfPGtt/4Ayocp2NvXQJA39/hZkOSn+BpCpePcOpTdiDZe", - "8vJHYwrqItibYWd3IIedLhp2dtJhR+/AEQYTKlZoH6WU5YrIPjo29i1Ign0yQJJEnMXSIYk7C97uQDal", - "xBqybMivfALf3afYY6kKlvKN7STEHvR7SH8PSTtowz9w9kzGXTh0MeK5MuZ+e67sWzFRYB7ZvHdfrXdG", - "vun2bTj53+zxrfAo2GXNLr2tN5w9y+WMNJvcXpnqPLkaA0K1q5gpZ+jvfCy7iJErYw0XUvWX+J7++sx0", - "cB/o+bqrmyDn27l/g81vAZtfrlUYLtEEWOor2VGHwUwk1xkXCnAUbba7oSHQJAC7gUc4Qa+PToYs0qzI", - "gPsJknLgThbk29zCh387Ry+O3nTRMVRvRD/n480+es2ShauhbXw0Q2YkMcO8IszQ2FAtiUPXsxk7UM9d", - "BovrDh6oHLI5GQHPitsrFyTe7cwIjkEi+aPzipvOAri/b17pAwTQu+bLYts7K4WPzhuixKJ3OFFELDd7", - "avOkWIFaYS9pBwJnBTcDPak7lA77rOzTyAYGnGJ3pxPAqvj0rZLB3Vf9vB8vmYkTMTXkxjlgfTJIMsDx", - "4nHFMskZKphjiAX613VRC6ApS9jyspUKBnTZFPn9FZncV/KuCrr7v+vpgpk+WkdTVtknTcRFDZG1nl6X", - "HDwzgMTWURXhDEdULboIJ4m9o+xNUESk9ArxdywIvoz5FesP2ZuieolN6EVHZ++6zlGLYiovTQvWF9tH", - "r+dEyHxcDA7BQTNeY1hzEg+Z4ijCSZQnWtwgkwmJIBcXipLIBl9uMZTOHZ6dspNgBRUvqj1/dIXbwjQB", - "u1eSRZ3itsxWbwkSJZimzfDfVlCDgEMINRjrRjlDlE0SG1IVCS4lsk31SEKndJzYACHZR29nBEmckiHL", - "EswYESiXJipeD72XCSJlbhK8dQMAk2soqotKaL9McGVDExLOhTTRBJrC358iqUi2gszemJZPYc53JNua", - "xm1PD2Skro2h2RRiX0F6QwylmAXXdJQnLoDxXkPRzYAeWkp8LAf/raDTKRH6VGDDZE04njnWbjnNoa9k", - "LDcWcTwv3mpXxLFo1ctK9DL2VkKzjUq067hzs6i/QOeXtBG9zz66WRbxL/qjln1Xs1XDg7CPPnOWf5ba", - "+OdekmBbA1ZJ4Y/NnOSNvHJUK4m262G1WmfW3mWma2v8rAeDzXrMaFm4kj7bpPB+fYQwuF+Uh/suc/a4", - "aauCdlXRTRtS/tfj2X8VFHg3QPYPjHJyCyD7ryrvHpDGHw7/JHhQHyqPvuJ7dhVk//RY9HeVPm8A6QGO", - "rSl93nA9G7y6UlF6b99ppybZFv9MEryNd7yB/O6W/ZvW30Jl8BZrnQtaEzxJM7VwAW3WV1kGnUn6kfQb", - "HMFF3OrduYJvEdL55cjD0WljQOefs+D7g8SM2uJ9VKKT40Al9UeGMeifucrFsqVvnR4W0YzOSbPRvXqC", - "7RJlgvQynoFzJTYLZtfD3WUKi/70I7LNW8xV+y+o/ghg+SRGMRUkUsnCVOLUHMH08Z1EgmtNAJ5zsWiO", - "EjFH5CfB00M7mzX3oT1T1hhWxhmmi16MFe7NHbdZYUL7jOhOF0+pGR6iDL38EW2QayVMjQk00ZoPopNi", - "SU0JfQk0uekPeHvQYNmkH8loOm4zyhXVQl7baiwoyqXiqdv7k2O0AdXHpoTpvdCi/gQk2UzwOY1JXBlj", - "Z84Ts6rbDQt6U7urFiqK0nFOuTCDexAZps2FNP1IsypbKEJixpRhGNzauhzVM2WS+HV/mDIXgGP3yI3i", - "2xVmNb8Np+xoSoRKmHYRFecG4nnz2zX3mK85PxnK3WmV286F56w2XrfLj2qZtnQXhR+K3Ln7NVu//3pS", - "eqh8lNk81nQ+LxTSJrP510WCg/u7H+7bXP7+EaeAviRO+fZM5dCAbjFEMK8gpjsmc5LwLIWK5PBup9vJ", - "RdI56MyUyg62tiD2e8alOth7/nS38+nDp/8/AAD///Z0/+oP7wEA", + "b/uvJmpekx9IbJSZ6KGUx7UaT/b9dhfRy7N3Z01jKqouIX90S3NaAe+zHO0yyZmp++TZVLCtx2gR1wwe", + "j+msj34kUvXIZMKFOrCBLOCQ8mq+4gUSJM41PfjQZJwhSccJnNGiVy3w61/0hAAIZJxPJkRUUQeeh2Rp", + "7+1RGjDu/aSfI/NCAXtw+mNVntZye1ut/axCDaC2T3BE2XSz9XYHDIK1aazDFXx59u6NLRvUhLisl7Io", + "LWTAlvvo16Kwll5qWaIq9QOWwjpyX0Mi2dlsIWmEE9OiqYpBmW/gg9PQWkI/Kz+0ptCAnB6ux+ZOHtqY", + "T7Mczv35m97J6/dbaUzm3cqYIJBvxhOix73psae5QzMos94qXGneZGkxhCHbnlhvrQqW0XqRPAYRWB3F", + "FU5GMuGhmKG3+iGCh2jj/U8mHVmPoIuyylbq332oIZ++nwRPDCDYN3R7Dh3WTbaVAx7UXesQaJ3q9Cqd", + "ho6KSaValrGWiyHyy+pG88v1BfhMI839Hrlsr5pR3QqyyCSFGbxGFwaBzKfGam5NM5JkWGBFkkUtHrMK", + "qE6WPdrkmkQ3SDd7oV//ZIpo5IKM1EwQOeNJNQ5it7tciFVCzPGc2NpTZk6e4V9xlGJxCTexE+RRzswK", + "VEPWd9fhy8yUym4wqZ/fvj0z2r0iYo6TetKDXPLwH5MEL9CYqCtCmJsKlgj7Ma/1pFHZUP9HqFFGBOXV", + "NezsBvo9N3HQaCpwRJD5ypVxtlsiIb6o7VLaXgKIhVFEpGzY3+1V+2s/neRJuz0ODWt7bdXz6CYb/Pbo", + "zNWxKeoEu2XeWV7lMyJ65si5gsGrt3ZHrq6o5LpiBoh2SWAYEyixZDNN/ERQF98EJaT055XkS485SD9d", + "0PYDp8AsVdec8w+tpMv6cQ858lPM4lC9ZZNgYFLzp4DXBVHOIgfRj8Ym+MmgBRg9wM+TEATHlBEpa3nN", + "US6STrfTm9hZHWxtJTzCCUAW7u1uP9taHUa6Mn7YhkuNYrpKv3RBVSbsxmXRGrw3mHSVJLZwlrWwwJl1", + "XHM/AHtajleEGsv6bvMkPBfAPhgspWhf40i5qnBgkqu4XLF/bAFYuDIfaDA1hV60qP3cP5+DoA88w2pW", + "Jf+tJdqHuBiIadQ0YswdtXU0RP4xKFFxESr9xYWyGX9j4oIei/vQhRQ6CN2Kc27wzJ/lk/393f11fAiY", + "Te2Y23MXmKp5u5rTTLzltufXNgDhWVWZwx3p1dVL9bqsoSnNEZdIavWC8oywG63n/t7uzs3Ws+1ETlxY", + "WI0vhSBhjk6PjUwUcaYwZUSglCgcY4WrTAZsWZrLQG0ZTFJIC5p8v5q1NMRP+Bgvty2M9aW87w018t44", + "SOgUMzrRDNm+6fcsZ3hn/8mBqeQZk8ne/pN+v39T5IsXJdRFq63YMkF6HghGX84+bx/uAOCizVz+6Jwd", + "vv1ZM7JcCnNpbckxZQfev4t/lg/gD/PPMWVhYIw2xV/pZKnoazUeLbfIwyQ+QGV9byf3tIkPajBGQ3Qy", + "oPEEYeYqUZp3hydX0Ditlg24QW7wilxZLa68ZsmicW1uXce1rHCuvPqtfrZZi1qu9ONq/7ozd8E7tk8D", + "QF2UuV32rN+qULFcWctxqdRXRlhRvTFJzF8RZwDsGyrlWLki3bMWlcDgGrElv4ou/R+L3r0fj/yBeL+7", + "SmLeT7am44cbhsSsFEj/tiyHrudCThxdc5jDtsfiVmhbP9fi0AVjwR/4LrxN2Fi199fT//r9/8izp3/f", + "/v3V+/f/PX/5X8e/0v9+n5y9/iyMk9UQhA+KI/jFoANNTXgfP7AtKZ1iFQVsdFr9a1hh+8RYHFQ0g4qf", + "aEwOhqyHXlFFhKkfV0t+HHbQBgFNCb7S4i4UxzF5Z5v64zPj0dQf/+HE4E/1NmKblC7shhRYIzIfxzzF", + "lG0O2ZDZtpCbiAS9QP8VowhnpggcZUjrvws0FlCxz7qYys676A+cZZ82h8xW6Tdo2hmGomeTIuuLOQex", + "HZUJg7WvkwJ2wmQkDllxWxcYfMbP2C8B+ylJ6jlDDYuyWn+zmtOzQQitEPJZ9EZCURpQQQrK1mRUJNqg", + "Z4PNZX1ujY5R0NAK8rOOeJPweJiHzMVNSZLHJKYR8BWXJzizmaRFiqahNGvEywS/XsDevDHJazHCuZpp", + "XhTZxPqI80tKurClXXCHQeQHfGn8+TOe9caL3oxnBcgCFibaBRuPeFXJ/j89O9HeeyLoxPYUzJXXJBIQ", + "OuHI2JmZlMPCurA0sbemngfTos+c2NdNYRhpaoGYUG6VC+bKVRAoWgloHgX1kZBM/j2KEgpWJznjeRKj", + "GQDuKd1MCDOvMyhSZvA4ismk/u9qSMPO/hPQYN2/d3daZ6yapVtFZXkS0GlTx/pacGzDJmEARjwYOUP4", + "miAkfQNaPy7YKRSH/54j11B54gpGYrxTJodO2uoHifSy5zaDaU72GFjIkBG2p6nNfbR0CitZUy1aMNEJ", + "8FnSAqzkhcnWfPvqHCkiUpc/vxHp3YFTYpApelTK3BbiOjw6fbHZ7wSBliquKtiqlVlV1UEHsBZstEJT", + "EEZpo8Ep6aKTY8iWtddKqYtBesNPXKDE3IrlZXQAYB1Vcw82JfxOjq0AmizKkAcjtgw7m67FrH69HaA3", + "hQqIi6EUeY8lbbkmy8sEmrUBcCb3Yqn1Wpos+Mes+mfvY8i0gPqGhhcDWGTj/dXe5uigp/RFVbOQ3fhC", + "8qNQGu1f3t5/aVjlLy+j795MRrc+4FE2wzJE3TPfqwkvLe2770ausnvRHDNU6XckafBs/c1VefGuIUX0", + "PVf5PIQjut/b3n67vXdz891NEXGrUFgeTF4BitsezfYuUGEDGK9UjRqDy5F+bEPJnV3k/SmaYcm+U/Cw", + "Zh3Z3n3axigBvbYNy/YDsvnEDKngUg5XqwgnNghjlzRJjAAj6ZThBD1HG+cnL385efVqE/XQ69en9a1Y", + "9UVwf24Bjgu3AKyjyTALQCtVIAFQkTv49u0rOFwJgfQLI4df3h4yd61psQWErhvcy7N34PjHcuQCN5tz", + "FXGZ70uuqVRyGVWtVfzz50D2mk/bFfl3kzRtlLX+V+P+/lwBpg3C5G3eAWCvC15fWs4HwLJ9yCTBrw9H", + "dyXy7efC11o7wx2h1zZeaSHk1xoiwn7T7XZ7HNo7GU4FUCbEtnwJx2Vw3xr4tduhgezVQ6kvHhKjk7Oy", + "0lPpjHDN1+b0fKe//eQZFCvdHrRh7CmOVvR9enjUvvPBjrllDvD4IIoPQGG/rc/KErZRQXByhRdQ7M8s", + "7bBjLkxPu/WOrVUkW8XXLOPr3g5Oty7GNQDmgjjrApfkKF1Za6RFemIdNC3NLSRkSpOEShJxFsuqjDzD", + "EsnMIKGamhuFBD9kMMAuKkofg5SCcBSJvDQ9Wunayvt5Zuke6n5mnGkdAID/fyELiVIKTtCiewh9lKjI", + "gomHbEO4jKkiNQpKfsb6B8g/6NrI9lgPjSqoLaI/GDI5y5VmYpt9dMSZzFMirFUWjSl4jDaRzI1KC+OF", + "1VhohilpTMSQ6dcCWKt/FOrJwZPBYDDodgpNblf/exCipjt1fvYtlrDJ+QW0P2ZRhQFGUOQM5Swmoqj/", + "TQw51EPkbug4/UwQYfd5O/HKfl7KVeGDuQ5zuB2Y8OciuMJQG/RziA69hXK+f3sRvVXOkZNfbbaR/Wp0", + "kwgGgiKeJ7HW+Mb6tjMGORJbM6QkynBn8y6V6J2pvVmdug09Vhz9nhOxQO9PTythD4JMNA9oN3HgEg37", + "wLMbbcPOGhvJ2tHcxL3s4d3eB8ZtXVLxJMQvjmjrexhdErSh0Iphq6I4r7Kvaa0ymEVCmdknTTQrJlgz", + "5cdkPsrzkFaiHzkclHfvTo4rlILxk+1ng2fPe8/G2096e/Fgu4e3d5/0dvbxYLIbPd3d3tldkXHSIm3t", + "9ploQdU0EDBchIePXJh6KHq4KUmgJgTYwOcrymJ+VblngpGofu82ynVd98sx7K2HEMx8SbBUxk7QwDJO", + "4TYlkW7bRH7b1MiiplHYovjk7WD7c80sMLgGZvxW5Mz4L00yf2GrT70B+5tVHefteCsMyGWYrFstv/P2", + "izY42H9+sP+5i+ayJNaNsU5O97i5TaFXDuy4lobhUgE9g42zBHas8GHM5zZro9PtFIkl8DfcurWg5eJx", + "q2yppgPbDbORVfy7IUv5pKIYQEiIAbuLD7RI4AR/KJ1Q5MJrWeMo4XmMPKOXwf4Ch9eJpyToZsD/ZG1h", + "BsvTZD1oZQLAo6FEA2WaEYOjTzdiU5oP0Et4Fx7h1OhPdhCmUIjv48LxwgSm6PPlujbazOohn1tFBr7R", + "Wg3S/4Jp62WwttHVTRgx6AD9yuGbQq1ivG5kNa+DPrP8et0gu2FxsR1EBXRmZboD9FMhxxWSoJX8NiSx", + "f44swyqRYTYr+fl2xzuaWsqd83LNux2zop1uxy0U5KQvZ6e/K6l+6fz5pBiK2CI4gbNcJuPmiiYWDxtm", + "QqWikbRZGnpzm+QLW8OIxCOjpTQFf5oMT6vJFB858eX9KdoAyMO/IGtB1v/aLAJFK3fdzvO950+e7jx/", + "0grYqBzgehn0CPKPlwe3ViCNsnxkjRBNUz86e2eMDJFR34sgk/enPo5EJrhmPXrmrkG/8+f95z6eU8zz", + "ceJ58Cz4m4GPhQ0LQpYVvKgh4PB3mszpZMJ+/xhd7vxd0HT7+oncGW834NSajsL2rRPfi79kDCbjnqlH", + "FIbcAYISshGV6g2RMAN0ThQC+ukhHIEeUaQNW5Jz2FV2xYOEtbe7u/vs6f5OK7qyo/MOzgisXYFL2Y7A", + "O2LwJtp4c36OtjyCM2068AaAEmdWxwyfM2SLCQ+qAml/e7AbopKGi7ukGtv2PG1c8vdWT7OTsosO2c+F", + "Drd0yoOrvbs7eLq3/2y/3TG2dtiRuF7NYVxukFkei3jv7/wGSJNvD88QZN5OcFQ1orhQrBuNSt1oVFCt", + "waCs32Bgz54+2d/b3dluB68Wiu6wwIGVA1vlXYFDFyCKwG4ElmKZ9XabbouQOGUI7A2JEkzTw8jlMtRu", + "H4OmPhLmtXIT2lwM1vS/dHG1+LaVFamwDZlMGCMacIFyVtTw6K/3fX4RF2Yz1zbXw3quHsp/YXr1LA6Q", + "qVV2i6XMBJlTnssv0BBXJjl1knAubvRtk8Lyhsg8UcbPSCV6f/od8BRNa0gqklV1KEuNK9CSbjm5G53n", + "ComEibxpsVrtRputXzXhbsOp7a5Crqhwg0aMslhzrpytj7I8wkmUQ9UaXOynnhWAbUHufZYlCxNEnySc", + "MxTNMANvhPCghdCMJ3E/GHKqn4wmwfAFfoUSbtCVLwnJbEEXMwj9mRZh6JygDb+UmSGlWoHR/dQwGVuy", + "o0qN+2m4UiKWoaywIudcrydW3AP+NZ9UTI4Jn0pQChWkB/TrePMZFibqHzNToGieGl0yENkcGGKNmYdu", + "VHOT8olVcK3IARndZiVxJLiUiCR0CsVw3p/WEoVXJJcV6cLrIyerg21BusZzGLjKDOxU6zpmofsxkDjz", + "OTck0DAk562ISXTGyRSzHEq8eIRsLd791nGHMy7VqACAuuFgpRpB3YZckBKWrkhvL+xB7p3gvehY222W", + "ywb43urrJaoKN9U0wGaeGlzR8Gp1CxoMkfEyBNZK1K0SxquO2XQTVLgS6J9KaJV6+GBoA5JLPLbkYb1t", + "tolGCausup8lbdWW4Xy1Nzhvi5+2Gi7tDKvZCZvwAMjGDVyUzhJtw0IzIlIKlUtQTBglsdMlC1+lNXVB", + "ZnYiCYpzYlfOyKcC2wXH5ngDUAZzNjLKpjVeX++wjXnYjGF1WQfo177YJq5IhjNX34oc1soEBkqEyxzW", + "VtGWVI7C7qzlhgWZ5gkWyCIfthmyXKQJZZdtWpeLdMwTGiH9Qd0BPeFJwq9G+pH8Aeay2Wp2+oNRUw2g", + "czM4m4BnNqTWbzmFH/QsN2vpv2CJ2TLfbwE0S5swrWBI9k80IRZG7x2j1x6hV3HP93YGTWnpDY1WEtKX", + "IRhvyrktyQZPfC4DSXwrpRxXvYjEFozeiD1ZLk0hlRa3kkM7dS7A23l0qhkan4cBcmT4dQ0BBI0JJNi4", + "qS1zjRZssc1UgjUccjlDf+fjqkG0bXxtoDLYBiuxKASZBAPpYUdXGqTNG0tr4u3uTcAegK3qicJHN8RQ", + "WFdDrQxkauInb5bKic2IXTLq5mhKi7UoleECLQqcANtre8CAeuG3QGAwwMFItYASqlC3ZuFVM5RozIUA", + "qGct4XDmZgP4Jlrm0WvtAKbQ2xlZIEFSTNmQUVYYSQG1jCBG5kR46ahcaCVrSuI++pun4gE4dpqphUVd", + "B+P5dxLxK1aMccj8QerGc6nbOWTGsihyqMpfvqSbBa1PEwqkB4MTTAkoVEjVDE0EkTN/7qHilFrGu+Ii", + "bqz6s0DuFSgmAz5WpPglYT4rK5oJqoamoZH5ajlczlSWhadW/0SVYq+oXsx1dX+5JCIsJBZTKl5pFbri", + "HRVPOTFoKwA9AkX87F+GxRdwIy3ARcrm/+qaLH86Kxqv/lZ7zQMQcXi+h8ZsGzTBRiZfphbsU/WkrQ1V", + "gXyzVbAxy74EtOFilV0llKok4FUkaXVPtkt5q0flu9FsSRJVe997tv/0ScuSMJ/lrDMwWV/aNTdPV7jk", + "GnbqtI3f59n+s+fPd/f2n+/cyMPiEjga9qcpicPfH7RBrpU+rMm//vHP96c1r88+BDsPbjQok8IRHlJD", + "Gkd1QO9P//WPf7pR3XpAIUazDMXd4LdvjNJJ/J10gQJVF147J9kK/f6wYiTABZtBG2QyIWAGHZl165WD", + "qeFttJOCcYYjqhYBRo6vTFh58UoNUrqNO6g62JDIa9q28KOac8l8XGZ3brjO0X8a33CNFp61riwl83GT", + "H/p1vVfjhS69Fn6MQ4sQA1kUNV82cBfzucKyEjmt/44gwcGlci2ntZg3VsPb1nMOIIrFFlDzQgFDsOg1", + "edJ+5G9/bTs9v2XFrFNf8Q8rzmHzEbyR1TdwIweMvtH6HNYaf7AX4O2+Go39mm8ri+pVCsSVt+7N+22R", + "prtckKC4wW7en5eZeJMP6+C7QI92DHbJy7a7FZJooCYv6SRgQOMJ6RWBejYjBcnceAT1mbd47oFUyeiS", + "TyZVUNn9ZhBywNeBrCrXC1ZKayZdRK6dzaKOYG3AdIadfTnsaBVg2NlOh52a2yqYp5ji65HtoAqiMliF", + "Cl7mmdcGKd0MxgmPLk05M6iS3UcDlBLMJMoZHP6aV217sNo71O1k3t4UGNzEhDgtsS0Y05jM8JxC6Qfr", + "U5lWAjHJNVUSAkahnQMUcwOrVKnlameoXzNZhAflpOHSwWxhG9YN6vc4cxGt5btg4JtABVn2kQjetagA", + "mmO/fn3aNQEMEHpoBlaJb3QTNSPQDLLoolbHoPw9HD88TsgIxl3HxU+X19FP/gbPqiCSKGmBsktyqBEB", + "injOVB0wP22nyFXzt5avpJxBsJ8N/wAANNu7IRAUkwhOpFw+i1VCvwVx1/IG7EqHEgd2QyQMhwJ8SWFf", + "8RvrEK4PwBgbvDLMph0/rtt4CUdScVu3qzjVI3IdERLXkTXDr7SNlbdfBmPlX2ELxlNUSLZvQ7zz8uz6", + "d5dJBWNtWm0/pp9x1gMYELelFrLDYPBZUJgqoVUwvj3siFEIxzT0QpvUZnK9eq1/JdcKgMjjPDHocmHS", + "tazKXkbrVvzWKYRNB5oLsrYO3h3UhzPx5reqEGdD1R+iSJx9604Kwy3tzjlR7t1zS0aNO1StqFJxabmA", + "f/dKNcbGkFIX2QsebaebNRLcm4WtIhb9tmUyJMMpGWWCTOj1CuIxLxjFuIofUh6kIoPBAHlupPga7T1F", + "0QwLWRs7o9OZShbVAJy9AGjRZ1VPFEQR5gyFbXa+3E334XK0m91Ov/WQcHzuYfAs1Q6xIuloFUD1Uelt", + "s9b5DC/AitPoJHy6uzcY7O4MboVQ7YZ1g+U6Kj+xtQar7TSl1HnfWUd/JUrVb6HIZl4uYHslKCRFF8sk", + "lSA4PYDEmwxHBCVkAmh0ReHw9Z7FeterB28FKov8UtC/2yi7b84HX61NU3Rlwb3dNDrOuVgF+/Gfr3GI", + "NrCZaAm7LpBzt9sbPHm7vXuw/+Rge/suUKWLRWrK9nj6cfvqabKDJ3vJs8XT37dnT6c76W5QD7ukpgRP", + "G1r9Rb/bGGVTXpJV0KAKS0Mbdg4ZEfXKxPWK3pIklJGeLDKk1qcpruAFxv++9vzfzM5vZrBSdjivTtIX", + "IbAqF6dCWQ8DdGUns9J3UZ/NyfHqWdwqA6k+kDC91YcC5NVuMFAKYrvzmRAIOWt5Db3zXmx9Ea3Milt3", + "FYU87HDSg7vcsOIh8q4hIHizXnWBL19yAdvplAuqZunq26J4rcDrhrjpj1LFVWClPjqZMihL7v9chMn5", + "SpT+uNPtJB/3qmfG/t4eYstC/RYEaLfalwpahJFB1fvVqwCvlIqHMJHsWlfXY/5hu7f9HOIQko97Pwx6", + "z6sRB12zWv7ybbu3K78O2qyhX2vP1Wjafn6jiGu3nqso6BcaqhRX3ssWBNjSeFkE2l0dLuG2ssHl46U9", + "rkHmNAqgnyvp2ctt5AtNMUnwIgQC7xlqZU179IkMjcmUMtnGbrs7KAy3++mw00eHFokbdNmy5H+leSj2", + "7tEJTVMSUy1jGtW/OYNhp6Utrq5L3KwIiPsqIK31w+La8/UQCesSrtZdk/3PyMf9LO23nca7Cr0D7GpO", + "RQWwLnixi+gEYVarBErZHCc0ton0kBgJ8WoHDhGtJFnLA2QpBzo7SRdNuUJlCn1Le1vOmu2CxfjJNdhb", + "V2BmGILY+SKAKAVSF13Fvk6OUSZ4nEdl/mgCgy4RP0Rew0JbIeSvD8m9S/sGJGZPuEDr7RtNBo129smm", + "/a7ZJjXBNm/19mD9Vt+JUaTbybN4PQ8zL7XjYDeCSF+Tghgw0VSXvSYJepP50IKjv/FXcFnnNbbkSItE", + "eeYcLJqmlikp4G4BF0MorveYJERfU8uNIJ7EZZYElSUXXc9St588mzW5OMEjtTyQXwjJtK4C+EfQX4rZ", + "IjgwV9+zuEs2Bg5WWxqHV8/UBbKrVR3c07WSWONW+SbcploFhsvXbN4GL+XSM38XYNq+aLaMgOIYfkVI", + "e9OMtW+/dGFvjfbjuzDLPaSQ9tq6Hmr4qA69t4Ahd/2XscBarKsS717IPR8ii7dWM27CfK1ngfpW58Pe", + "/xgrMxr1D7Z++Mv/3fvwn0Frc01vlkT0YjKBQKNLsuiZKj9aR+9XEU+hxIAWpqeWVAhOwYYEaOL2MPrj", + "3R8UTGPxK06XpgARWl6Jnu21E/rLfzTHN3nL+A745FqS/ewKHHdRqVRxdx1tpERMXSy5SyTb7A8Z5KZd", + "koVEXuEvK9I4Qv1OFp94EejowoiBfcLmF2hMoZKiHDKt1eIoIpnWJmwtGWrKgXPgPoLgxG/HFiBzid/W", + "IWniCQh6f7oEl/v63dsfX7/79Xj0+uzFr4cno19e/DeEeFz1TA9xT9Pe3v4TWwTcX8ntYCGKm9dT6KNT", + "G6ZvXf2THBRawOmSKM1VDkEh5DpKcknnzkGokttXTlhO1r19JYLPhNpVKglFJVhI6IROCPj14TqxQTVU", + "OmKkEqqnW+MGZWj5xjaEM+wAJ/WK34fqVuitCK92ubHVRX8ya8dCDQhp4LBDxiuUuQ9oL1QCXoWL/fBe", + "RhuQOeJKvLrE2c2bgaIeFg0GIw+/cCWfwfMvUW3z3crymnOe9LR601CSIGhNNmsRjJyHpkxGQqfJ6TAd", + "B2R4a9qd0ikO+BlC/oQvUhXTDWhtxtTS/jeWBwvnMRzX6zWYY2mWqlZfoGYkkKrXnOaQaql2VJb+rwbP", + "5MzmrlIvtq6aqJoytWWr14bwMmIOqOGrspXLU+bQEXvw0fok3JV6lTczbyTNe3Pq1IeagrNigc700lzN", + "iCDeRsAHJQ7+DZfM5uW0QGEx1f8yIsqYVZfUo6VScDdLtFFYftwSFNnGy+bw1XUOTvF10QO4UrBc8j/C", + "PMo6S9svfwRM+jeutiSduCZgGDXlLozAXqWiVWviqGp5M3yqWp63eT948CyvWsH9ms5WjTjLPiqkGaLH", + "v2GqfuIC1MFmzJM7B3KHyz8mAjDg6jDtrTDOaUriEc/V6vNvS9fbK7+oP1rWr3WqLwYijirpvE28wKFy", + "lGNYXmm9HCTKBVWLc71eNpgb0iBd0VhYSOgIfi47hkKdnz6B0XgSSBh5SRgRNIIyqPo8ppiBxoTen3rV", + "8ExhxCW8VhCBXh+dWHODg/wF9ZEqID0Xd3l4dtLpduZEGJW7M+jv9gdwmDPCcEY7B53d/nZ/0AGtagZT", + "3ILS9TZ/2uYbF4rrSWwloR/dS/pLgVOi4IvfAkgAEHdoXwcVBE89JTLDVFgtMksAocAQDNVfA66/u1AP", + "zK3cNcve2mYKacaQ/UKy13ZzP4CgDGcHprkzGFhgc2WvX8jdMQkDW3+30aNlv62kOrtEAZj7JTXPyZbF", + "0n/qdvYG2zca06qhwNkNdfyOYZvES0A737/hQtyq0xNm0vJskrUNh/JPHBCSf9Z++6D3TOZpisXCLZi/", + "WhmXTYIxkQi7d40epySKNKuAYjx99JoR8xxhhbCJXBY5gxrG7kNNodVTYNp2m1yAFP3I48UXW8JKH85G", + "8anKzvRx+bREz1+OdgoyXt5I+8ghbBuqvQcC+hEXBbgf7KTsDZ7ffadHnE0SGinUKwjYxiNTCSE/CeCF", + "O+whLtDvOVcYFeH8j+hIW5l1XJBbt7yKtv6g8SdzvBMSMoOfEZFiZpIjzDtrDv3ScTYuifI4r7zVHOFD", + "aQ+4qRwIj7moQJCrHlH/2qoLg8vX0V4AgcH2aaYXPyDh793DCbeTLWqwPuSRg8qXKJfkMR0n62Ibl0JI", + "UJZ7SdTXQvOD+7yybBGBP+EpeiwE/JIUEl65W0uXwlYmcmYU4KAE+KZMWLTffVcV/t6WT7woGfBr6Kah", + "nIUyflUcL/rIralR+tUCIJYEgXnGy9fKmR7e13LCdu7jhMGMC0/Rt2vq2zW16pQbanFTgIPpnfIWNogb", + "WSD+fPaHG1sfvtke2tseWlkeGLmy1oW/83Ef2YjUiMcEyRnPkxiNCTJ4Ry72RGHRn35EWEQzOicAagdF", + "2vJE0QwLiCxJUYwVNj70RsPESrNE0dyWbq7n4hDLBa7jWEgyAhy+URP+ZBmBSBkjMdKfWOi+Ek5wqW63", + "OftBA3vRYHk1oqsZl6TA82PKu80hvVka7Ria7Q/ZWwv0qhcQgqkdr5EkAbjaFfYfzhAeMvvB946FuEAw", + "idOSc2EBmIHUIFOabVlObdMjHcmIh7B23hKGmerJjER0QiM7rUuysPGcwQZb1V3SA3bjfH9aJGygnc0w", + "XhvAM4bBeY+LZ8hSUtV/wyAIOkryuHRyOQghLMY4SYKFOaYJH+NkZNbnkgR8gi/hDbsofn1/501iPCam", + "Vnu2UDPOzN/5OGcqN3+PBb+SRAw7m/0hg0QMu9Yk7pYCIrqCQm5pxvU5Ezw1fW6ZIW79cUkWn/pDdhin", + "lDmKgE9wIjki1/Ad1LcCzAzDvRrowZymsB/8KJeKpz7yqaM7M0yeqyxXNqNEEtUNoX4OmeLoD4ft+Gnr", + "j7LHT+AsJjjWdOK9YqYEsnXTqOUI69mP4NWAu53AAgw7+iI1YR5TgZkysJ0FOCWa+lu6UVRH0Id0s77C", + "EWYo45mpLAFENcOa5CptAFYDThKk4Ci5b7XgDjvZMB8LvZeOG3H3DFBa7RhRhk5/9A7TYO9Z+DxJEgkS", + "iij5r/PXvyK4lfUemNfKcC2T0sG0wIDiHFynjqe9wNEMGUcVFBMcdmg87BTu3HgTxppLGy7T64FP8Qc9", + "tB9MN10a/9Dv66aMu/IA/faHaeVAn6UsNTigw86nLvIeTKma5ePi2YfwgjbBl51XGAHaMNfcJnASTAFp", + "xrvxzRWJWYy4vQWSBcKo5EB+4MqYMiwWqxIJA0tvV5BPTCSjtxh/DCFycdg5GLrYxWGnO+wQNoffbIDj", + "sPMpvALWa9lcuQ7us8K5WRDRk8Fgcz0Stl3fgM+yhWPgC+uAjVpRUXZT76CFYf1z+Qf+rfXPwvWDme68", + "hCYyir8zvj9CB4QnsfuaaMAFURO7MYtI4sTu9Yae+3ce6M2KSJLcN4E+FHkW7rECqf9RkSNsVnmMVprv", + "H5jiBvd1qVTM9g9Dv4/Ofh6wnlvbOZm7UOdwnRLAoLGqNDIvIyzROYypd66V7xfwa9/+1+l+gKl4kfDp", + "xYFR3VHCpyihzOYDeIHKWjywawkfGRia4juLSuOKxG0YSeJf//gnDIqy6b/+8U+L7f6vf/wTjvuWgVeD", + "GtMXM4KFGhOsLg7QL4RkPZzQOXGTgSqwZE7EAu0OrM0fHiGv1L2V0uSQDdkbonLBvLwJU69N2gatq0DP", + "h7KcSAvjo1+kE1tMxsQ2Buw27iybpbzXE90NwCHCDLwJ6FvR0QBgyVFTaNtqop2wydTMuWI0rYdpLgXr", + "recvilwrQ709M8AbMhhY4tC5gwd20mjj/PzFZh+BtmWoAgoGge5QNmPViP43nrSeJxmOUmUosMqGN0U4", + "w2OaUGdybKh2Yo5giqMZZaSMLy6wxl0TB26kmsccnp0gGwjZhVeH7PX5FphYFYlULkjXcgJhEUbLcmjc", + "5rlAD8C/qILosJ59d8gmBEOe0MmxYQIeCHeRD1g0zADIA2JcqapUXusOmUGStcjF+uClPCYJfAT9T7Ei", + "V3jRRUWtW1cdJcFKK8Syq18eMoP1ategB1AlyBtmH/iZGVLPRfLanC1BJolWjSEC35T9hr43JlwgG+Hs", + "Vfl33ZkkSzMsvWgpjl6f6/lNQRPkxh4ILb0+d7ux2UWSoyihQA0RZkM2hUAgB97LWWVXi4SyGRZxL+L6", + "EvDBnC4Zv0pIPG3isUc+kd2hJFPpJ3Ccfq6T62MTLmbLE9CH2ADUrfbcHdt32rnubIt/Jt+dLQR5A+ed", + "seASw2/M6n5z5LVw5IXXzTn1Qp61Y4fAeHcRv6aLBwr4dbS3vObmibdkD2HRQxsO2ga8Ilygs6MThONY", + "ECk3/73tfXqmhkpL+U/fj5oVP0ToiR0LFxb0z9pbqgTyWNjBGztqhN286vV1/fttq1J8p/GmK+rwlFfe", + "3d8etU5vco2UQm9Ja99ukrXBtlRGHMoMltTSA9EoIYX4UpxTn4rWWZVNGG9x5awUlyx7Pjl2B/L+7Mu2", + "65zV74Z7YIrHNYb4gIywmmrtV81+TNT8rthFhza9wvz8dZHm4P6koPs2RYfI/DGpi3Ft2TQXNEAnjRfo", + "S6IMvMld6um2h8DEz4lwp9oMdGFmXUzLfIoMTgtMCCwxq3XfE/NKO9XXtPdn0nxheW4isdgl/yaitFB2", + "y7VapeCe2BLQd6ffQg83Um+/XNiKJbDAIoMVdezcTmBZ3cBywaLNb5ErX5yiTVxjqcQKN28SF5Zsg6ZU", + "6Fn3JdcdMr/euJbprF5LGZokdDqzToCYTiBWT/n1u2GUO/cwyqJOtsCK2BDFx5j3e6YX2XqB50Qo9Pro", + "xKy/f6Vu/QFBq+tVJce8Vt6u79686hEW8bhwnjTLpPbJF1aYDP1Xcnnv/9Q9wnxW6sSDJoHxM/bfBJMj", + "E//ep/x/7fyU0LHAYvG/dn7CSUYZ+V+7hwlWRKrNOyOWwX3ddPetwDxi4tP6C60uGrAmNgXI2DUCf/FW", + "S5nfvf+nEvvNpG8k+Bfr+k32byP7+8u1Uvy3W3GnCoDp44E8XAWxhVYbHn2DtLkHo6mlSA/SpuJFKkFt", + "ZlwqePT48pttUDktKM6/Nlpa/8sDufL6cKR7ctyFhYSK0lDRwqYP3pMvwI3j3oVb2+/9OwIO0zGd5jyX", + "fmZiilU0I9Jm7SakyoAfm9hdXs+NgvdXTKWD+7w67l2u/kb3dyTx1zfUMG/j0Fsn87u32sr89n0t8xtE", + "U5vZbMtudF1Jps2GQGuHadqWjCvQr8sB4KFxhXQR9E4rKqW6gECDOBiy/631j98UwemHH1wKZT4Y7DyB", + "3wmbf/jBZVGyU0cqhClBbQW9w1+PwYs6hUBZKLJXJmzXx2FqdgPpubIC/3YKUulIbq8hOSr8piG10pC8", + "5VqtIdm9uFsVqVqa5N51JEdvoQW3mOJ/Ti3pT+4eqWhwMp9MaEQJgwIvkJgul+IBjSb3zTNyy4RkZv2R", + "XjBRRRJprUYWXGuNhF7WlL73QLKTsojWfWuPrnz140ys4pmtB2v1tVJaaFbYvjZ6GNzv7XX/itpjJjGj", + "ES0vXaaF7kBlIFMTKs1NFlrxZQkq1kdv375yGWda4heuzpTirriUq7s5ZH5xqT56UVbtMi+4FrRETmKb", + "oQp5eLZcU0xwnFBGIESXyFByWLUk3IMeiy8vVIbr3bUSKu/5WNoKpg8nVD4YK7gX8e2kUhialzq+Xwqv", + "OC1OhINT86j4lWVAAcYTEp+2cK54z+awbs24ATYLYzueJTgCaEf9mkEds7ABBmbQbwqwAARPEiIMmlyW", + "K1cDcciKwVHm1Xi3NW4udPOjnCmaXHRNhAxAgkiE2cJCKg1ZpTOsFEkzBam9kLYOIxQkMyOuFX/Ug6Y8", + "l/AWZOH6XSKcXOGFHDKbDGw+h0K5gkQGeDFJ+uhnDjgMCE8xZR7jNRUIv5NDdkHjhIwsjMIFohLJGReK", + "MBKjlM+JrPZLsEgoETCJI6xXTqIULwDPzEA7mvXhGTGYYRWwBq7/jVlMoZad7rmY8sGQYbQzGKCUYCZt", + "6rXEE7hwbBsIBlEZ0PcIo73Bc/tVbd8Ac9ct/4Y+TUKQOY/wOFkgoqkYwB/UJmxgamtLmhq9evsmVEiz", + "X4XJ0BYNq2wsla5UYtxFOSuTy8F8nrMiF1xvl8oFg3laxxqhorgGLZ7GmERYryfj1X4AyZBHUS5CF6Te", + "aq/I6b+j4OhN7xyWKpy6nYAWHpEY9pxxNYMzzeEobX7fQFUlUf05LprgIeECYeTRdWkkIFEOrHEDkP8u", + "yop9zFXgvdj83p0dfXwtI3DH32DvPZb7CYiITyaVA7j+ajIHeFXKxDIJ/1nP6ZEr1eqzuJjiKeNS0cgx", + "w3pl928KYWuFcPXKBql5wsWlL1tV6fcnLi7bamAWT5Q+LkXMn+FXaNvXwwPs5oc38YOB2SgrmmjuXUmr", + "01dxSkHookq62GGOEs6m+hSVhu57t8T7Wt2GwWHTl6kw/uMCNUcrISP7o6n2qidja2mC1T6yrT40L9K9", + "34N/51euEE2zhKQEqsH2DLHpzS4RlqByPpUeztDNeKU+VX46sNEFpXHpd504BHTlNmwDpPfl7Qoy1YRP", + "1+P4FZ070LoAkN+Qmcr+BF0Yb84FKniwFmgNaj66mtFoBqB+oLfq9g3mH86yiwLPePMAvYSD7MM6Q+cb", + "Bitf05rkCTFYffM0vThYrnf6/vQUPjJ4fqay6cUBcjVOi/tD6rd8kD49iwRLhX610IMbhTIOO3qhsNY3", + "i/ltWvi+Em96yEJQfoxc2QbpBF14qH4XDZBTjt++4tMHE8a6zVUCzFwUR1Z1BNokLO40xU3QJAzotz0Y", + "hMCrW4ILmmHcMbbg0mBe8WlRoaBCyjjL2pKvHSZQ8TxNV9Aw2vCQyqSKea7+IlVMhICPLXU3ETfawJGt", + "ToUvNaFaXDp3sDeB/ILRQQYyPLhUmql2uh3C8rRz8Jv91zxNO92OHY8HNX4D4X4NSGO9weUoFr0zHhLj", + "N7H8JhiLVWbvgSzWbg6rTjdL5G/MC396b6Gz2T0gGYJ8UDPifk0iqDfeqsGH8QIs8nEhQllTU01qazbV", + "uFn29PDi3JXQaROWcW4/PXdffgXa77poDTdm5KZ772EbyyN4zNmtcmk2Ey7qMELr4jm+ekL6cluyNNU2", + "FPKNNm9u52tFmFpSX2YR9oPYlFTDueIpVjSCcj7RjHPpkX2B+WsKb1nzbUGZYNwweqYNi7/QpHphDcEX", + "VpA/sEYrhP1Hto8+fG6D6cNfuEflFz95ennB8btO+AbIe6g3LiiZoAznkmi5Kk8JihaR5oqmfhPB0QxF", + "OFO5IFCajqCUMprmqQ/mrHdsjgF44mI7veiica5QgsUU9CLz0IW7RDxNCYsJWMiGbEbwnGqlTqAEK8Ki", + "RU8SKGk7J+iKi8uE4xiU/CzG4GuBkniCaAoEZOyUKBxjhUHUuNAnfmQycy6KKrdGsWbkuqSGeMhEzr43", + "MP262Qs30AtEAIeayllRDTHCMWFREJ/5/OtmY1/eGnxOVH2iDxSbcyte+pDBOr7V0w3n64jjeWQBxlzY", + "bWzD5lcIvbJZiaymNDgy+vc80maubo4P5OIplnjVKf46fDsF0X01/p2Hd+BwgeLcdOedSiDzP6tXpmAo", + "frgTpEuabbyta6Yo+1Ys84143tYf7s+TW1jTvhJO2G1U7JsKDJWT/hpYrl3VW/HcBzIjWluSZxV7QBbs", + "YqoeTHziwuNyj8XcaRm2OZoF3/a5kxIYtC/OvrHtOtu2IQe3ZdvONrvkVPcYOWU9iNIMc3Brxm1k1dZ0", + "8G+aD1KbnccyH5xFlr6De2OLJwUjNKwxw4uE4/jPEKa7woMTcSEMpgOgRDwmTFHPaugH6INtrqxc1nX5", + "ku9PTzebuIRQK3mEUI+YQ3hJMfqzNFDO//WcCEFjC72Jjk6PbcAslUjkrI9epxQK/l8SkpU5JZDX19fz", + "c+gWy1XSKzAW3Q5hSiwyTplaO4ry1bsZzKdb1Va/Zz5pQaq/OaRbO6TBsv/42BlwGciaMBNYrZkqrNYW", + "z6RswkVq5DI85rluXfMgvUx6P00RyglNiFxIRVITFzjJEzhuUPDAFrW135ld7kJUrD45JmEtIyKlUlLO", + "5JDZbI2MCN23/ly374U4BR0CChf89cwwya8jfE4PxkSMYdW0aoBDBMUyOwedLZxlWzFWuCFEyw7vM4b0", + "E8TDIblIxzyhEUoou5RoI6GXRj1Bc4kS/cfmyoC6EXz3pUv23v5k6ZU+YRMeLIhmaLYg5j9VXpVla84x", + "+ejY2kviHxbHf2Cjw2xtfVFgQXDSgyK7Do0G5Yom9KNhdboRKhWNTNIPLtbu/WnBVPtDdkqU0O9gSC5L", + "EoMpANrlViZ4tDXMB4PdKKMAabZLYHDA8Jofp9Dj0dk7kwhKUi4W3SHT/4CG3x6eGe/uBFtrgjdQWw0Y", + "nWy9XhNifA7L9G8co2cmuBI/ILjh31yCN0f5aDxDsuGI8myVqsSzP30QqZXgvtkVHqddAWCWitlsTAWO", + "QCiWs1zF/IqFbQhznuSp/of542QdWJfC0ew9vPrVSLtmOGu7cRN8FIfSzikmpmDjgzg9zII91phVvXBu", + "CiDEVKIBg7fAofozUveXN9/76/gVujvtirpiqF/N2brvm8+OwWFc+OvxWI65oTQ3E8VXW5+uMG22Pv2Y", + "8OhSWjAU32yo9TYADdc/liDP1kUIYgLkZiILImSgqojsDlnNAGkwdyTCSBGRUoaTLZizaQTgqp0VC885", + "hRTpKKGQpEZjQC1KAJMaAOj0bMBQ5RrwPLrSlovy3/GdkYqjMYl4ShyE92ZIdfsbpuonLqp43F8LX3zr", + "rT+A8mEK9vY1EOTNPX4WJPkpvoZQ6Ti3DmU3oo2XvPzRmIK6CPZm2NkdyGGni4adnXTY0TtwhMGEihXa", + "RylluSKyj46NfQuSYJ8MkCQRZ7F0SOLOgrc7kE0psYYsG/Irn8B39yn2WKqCpXxjOwmxB/0e0t9D0g7a", + "8A+cPZNxFw5djHiujLnfniv7VkwUmEc2791X652Rb7p9G07+N3t8KzwKdlmzS2/rDWfPcjkjzSa3V6Y6", + "T67GgFDtKmbKGfo7H8suYuTKWMOFVP0lvqe/PjMd3Ad6vu7qJsj5du7fYPNbwOaXaxWGSzQBlvpKdtRh", + "MBPJdcaFAhxFm+1uaAg0CcBu4BFO0OujkyGLNCsy4H6CpBy4kwX5Nrfw4d/O0YujN110DNUb0c/5eLOP", + "XrNk4WpoGx/NkBlJzDCvCDM0NlRL4tD1bMYO1HOXweK6gwcqh2xORsCz4vbKBYl3OzOCY5BI/ui84qaz", + "AO7vm1f6AAH0rvmy2PbOSuGj84YosegdThQRy82e2jwpVqBW2EvagcBZwc1AT+oOpcM+K/s0soEBp9jd", + "6QSwKj59q2Rw91U/78dLZuJETA25cQ5YnwySDHC8eFyxTHKGCuYYYoH+dV3UAmjKEra8bKWCAV02RX5/", + "RSb3lbyrgu7+73q6YKaP1tGUVfZJE3FRQ2Stp9clB88MILF1VEU4wxFViy7CSWLvKHsTFBEpvUL8HQuC", + "L2N+xfpD9qaoXmITetHR2buuc9SimMpL04L1xfbR6zkRMh8Xg0Nw0IzXGNacxEOmOIpwEuWJFjfIZEIi", + "yMWFoiSywZdbDKVzh2en7CRYQcWLas8fXeG2ME3A7pVkUae4LbPVW4JECaZpM/y3FdQg4BBCDca6Uc4Q", + "ZZPEhlRFgkuJbFM9ktApHSc2QEj20dsZQRKnZMiyBDNGBMqliYrXQ+9lgkiZmwRv3QDA5BqK6qIS2i8T", + "XNnQhIRzIU00gabw96dIKpKtILM3puVTmPMdybamcdvTAxmpa2NoNoXYV5DeEEMpZsE1HeWJC2C811B0", + "M6CHlhIfy8F/K+h0SoQ+FdgwWROOZ461W05z6CsZy41FHM+Lt9oVcSxa9bISvYy9ldBsoxLtOu7cLOov", + "0PklbUTvs49ulkX8i/6oZd/VbNXwIOyjz5zln6U2/rmXJNjWgFVS+GMzJ3kjrxzVSqLtelit1pm1d5np", + "2ho/68Fgsx4zWhaupM82KbxfHyEM7hfl4b7LnD1u2qqgXVV004aU//V49l8FBd4NkP0Do5zcAsj+q8q7", + "B6Txh8M/CR7Uh8qjr/ieXQXZPz0W/V2lzxtAeoBja0qfN1zPBq+uVJTe23faqUm2xT+TBG/jHW8gv7tl", + "/6b1t1AZvMVa54LWBE/STC1cQJv1VZZBZ5J+JP0GR3ARt3p3ruBbhHR+OfJwdNoY0PnnLPj+IDGjtngf", + "lejkOFBJ/ZFhDPpnrnKxbOlbp4dFNKNz0mx0r55gu0SZIL2MZ+Bcic2C2fVwd5nCoj/9iGzzFnPV/guq", + "PwJYPolRTAWJVLIwlTg1RzB9fCeR4FoTgOdcLJqjRMwR+Unw9NDOZs19aM+UNYaVcYbpohdjhXtzx21W", + "mNA+I7rTxVNqhocoQy9/RBvkWglTYwJNtOaD6KRYUlNCXwJNbvoD3h40WDbpRzKajtuMckW1kNe2GguK", + "cql46vb+5BhtQPWxKWF6L7SoPwFJNhN8TmMSV8bYmfPErOp2w4Le1O6qhYqidJxTLszgHkSGaXMhTT/S", + "rMoWipCYMWUYBre2Lkf1TJkkft0fpswF4Ng9cqP4doVZzW/DKTuaEqESpl1ExbmBeN78ds095mvOT4Zy", + "d1rltnPhOauN1+3yo1qmLd1F4Ycid+5+zdbvv56UHiofZTaPNZ3PC4W0yWz+dZHg4P7uh/s2l79/xCmg", + "L4lTvj1TOTSgWwwRzCuI6Y7JnCQ8S6EiObzb6XZykXQOOjOlsoOtLYj9nnGpDvaeP93tfPrw6f8PAAD/", + "//30TUjf7gEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 2e7eeea4e..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index ca3702af4..14ffd7ffc 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1775,7 +1775,7 @@ components: example: 1024 available: type: integer - description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From 5dcf26b49cdb4989a0a12692f4a21ff3e4e249c6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:23:13 +0000 Subject: [PATCH 16/24] Retain vendor vGPU assignments after rollback failure --- lib/devices/types.go | 10 ++++++++++ lib/devices/vendor_vfio_linux.go | 27 ++++++++++++++++----------- lib/devices/vendor_vfio_linux_test.go | 18 +++++++++++++++--- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/lib/devices/types.go b/lib/devices/types.go index c76d239ed..31ebd90ef 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -100,6 +100,16 @@ type VGPUDevice struct { MdevUUID string } +// VGPUCreateCleanupPendingError reports a failed create whose assignment could +// not be rolled back. Device identifies the assignment that still needs release. +type VGPUCreateCleanupPendingError struct { + Device VGPUDevice + Err error +} + +func (e *VGPUCreateCleanupPendingError) Error() string { return e.Err.Error() } +func (e *VGPUCreateCleanupPendingError) Unwrap() error { return e.Err } + // MdevDevice represents an active mediated device (vGPU instance) type MdevDevice struct { UUID string `json:"uuid"` // e.g., "aa618089-8b16-4d01-a136-25a0f3c73123" diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..6615e035c 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -159,14 +159,21 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) } + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + } currentType, err := readCurrentVGPUType(currentTypePath) if err != nil { verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } if currentType != requested.TypeName { verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } s.owners[targetVF] = instanceID @@ -175,13 +182,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str "vf", targetVF, "instance_id", instanceID, ) - return &VGPUDevice{ - Framework: VGPUFrameworkVendorVFIO, - VFAddress: targetVF, - ProfileType: requested.TypeName, - ProfileName: profileName, - SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), - }, nil + return &device, nil } func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { @@ -475,9 +476,13 @@ func framebufferFromProfileName(name string) int { return gb * 1024 } -func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { +func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + s.owners[vfAddress] = instanceID + return &VGPUCreateCleanupPendingError{ + Device: device, + Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), + } } return verifyErr } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b45884640 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -428,21 +428,33 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Parallel() verifyErr := errors.New("verification failed") + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assertFileValue(t, currentTypePath, "0") + assert.Empty(t, sysfs.owners) }) - t.Run("surfaces rollback error", func(t *testing.T) { + t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + var pending *VGPUCreateCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, device, pending.Device) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) }) } From 3de1649b5c8ce3ac4cccec708dab4659eb31bc22 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:39:33 +0000 Subject: [PATCH 17/24] Degrade vendor VFIO discovery per VF instead of failing the host A single unreadable current_vgpu_type failed discoverVFs wholesale, and GetGPUStatus turns a discovery error into a host with no GPU, so one flaky sysfs read blanked out the host's entire GPU capacity for admission and monitoring. Skip unreadable VFs with a warning and keep the readable inventory: a skipped VF is never selected for placement and never reconciled, both safe directions. When no VF is readable, discovery still fails so a wholesale sysfs outage cannot demote a vGPU host to passthrough while assignments exist. Also document that vendor VFIO vGPUs are known broken on Cloud Hypervisor upstream and QEMU is the required hypervisor for GPU instances. --- lib/devices/GPU.md | 7 +++++++ lib/devices/vendor_vfio_linux.go | 19 ++++++++++++++++-- lib/devices/vendor_vfio_linux_test.go | 28 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index cdd269dc8..0cba075fa 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -99,6 +99,13 @@ Instance Stop/Delete → Release profile → VF available again Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +### Hypervisor Support + +Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: + +- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts. +- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor. + ## Passthrough Mode Passthrough mode assigns entire physical GPUs to instances via VFIO. diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6615e035c..bdd026ea8 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" @@ -50,6 +51,7 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { } vfs := make([]VirtualFunction, 0) + var vfErrs []error for _, entry := range entries { vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) nvidiaPath := filepath.Join(vfPath, "nvidia") @@ -57,12 +59,14 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { if os.IsNotExist(err) { continue } - return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err)) + continue } currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) if err != nil { - return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err)) + continue } parentGPU := "" @@ -76,6 +80,17 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { ProfileType: currentType, }) } + if len(vfErrs) > 0 { + // One unreadable VF must not blank out the host's GPU capacity: skip + // it and keep the readable inventory. A skipped VF is never selected + // for placement and never reconciled, both safe directions. Only when + // no VF is readable does discovery fail, so a wholesale sysfs outage + // cannot demote a vGPU host to passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable vendor VFIO VFs", "error", errors.Join(vfErrs...)) + } sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) return vfs, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b45884640..0879af3e7 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -29,6 +29,34 @@ func TestParseCreatableVGPUTypes(t *testing.T) { assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) } +func TestVendorVFIODiscoverSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + // Replace one VF's current_vgpu_type with a directory so reading it fails. + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to passthrough. + _, err := sysfs.discoverVFs() + require.Error(t, err) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From 46476c45234a27c78386d5e7cf7f7d0333c43a6a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:10:26 +0000 Subject: [PATCH 18/24] Degrade listProfiles per VF and document deliberate probe strictness listProfiles failed wholesale when one VF's creatable_vgpu_types read failed, blanking every advertised profile while discoverVFs directly above it already skips unreadable VFs for exactly that reason. Skip and warn instead; underreporting is the safe direction for status and admission. Also document why openVFIOPaths stays strict where mdev's scan is lax (it authorizes clearing a reused VF path), and the 0Q/0B parsing caveat in framebufferFromProfileName. --- lib/devices/vendor_vfio_linux.go | 15 ++++++++++++++- lib/devices/vendor_vfio_linux_test.go | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index bdd026ea8..dbb9ed1f7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -106,7 +106,11 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror discoverVFs: one unreadable VF must not blank the host's + // advertised capacity. Skipping only underreports availability, + // the safe direction for status and admission. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range creatable { profilesByType[profile.TypeName] = profile @@ -411,6 +415,11 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] return false, nil } +// openVFIOPaths hard-fails on any unreadable /proc entry, unlike mdev's +// isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes +// clearing current_vgpu_type on a VF path that is reused across assignments, +// so an incomplete scan must fail the release (which retains metadata for a +// later retry) rather than risk a false "not in use" answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { processes, err := os.ReadDir(s.procPath) if err != nil { @@ -475,6 +484,10 @@ func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { return profiles, nil } +// framebufferFromProfileName parses the framebuffer size from names like +// "NVIDIA L40S-12Q". NVIDIA's 512 MB 0Q/0B profiles parse as 0 and would read +// as free VRAM in least-loaded placement; no supported GPU exposes them today, +// so placement only skews if such a profile ever appears. func framebufferFromProfileName(name string) int { series := strings.LastIndexAny(name, "ABCQ") if series <= 0 { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 0879af3e7..50e4d9f00 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -57,6 +57,24 @@ func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { require.Error(t, err) } +func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + require.NotEmpty(t, profiles, "readable VFs must keep advertising capacity") + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "only the readable VF counts toward availability") +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From b68c2c322a4190bc7375d71f4cea920900a79759 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:20:06 +0000 Subject: [PATCH 19/24] Skip unreadable VFs in create placement like listProfiles does listProfiles skips an unreadable VF but create still failed placement wholesale when profileMetadata or selectLeastLoadedVF hit the same VF, so /resources could advertise capacity a create then failed to use. Skip the VF in both loops; it simply stops being a placement candidate. --- lib/devices/vendor_vfio_linux.go | 9 +++++++-- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index dbb9ed1f7..566a0d772 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -322,7 +322,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType } profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return "", err + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { if profile.TypeName == profileType { @@ -356,7 +358,10 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada for _, vf := range vfs { profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror listProfiles: an unreadable VF must not fail placement + // while /resources still advertises the remaining capacity. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { profilesByType[profile.TypeName] = profile diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 50e4d9f00..4f2e40844 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -75,6 +75,20 @@ func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { "only the readable VF counts toward availability") } +func TestVendorVFIOCreateSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + // Create places on the readable VF instead of failing the placement. + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "inst-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", device.VFAddress) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From 821d1775cc7718e7fd58977d9643282dca55f933 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:42:26 +0000 Subject: [PATCH 20/24] Degrade mdev discovery per VF instead of failing the host --- lib/devices/mdev_linux.go | 16 ++++++++++++++- lib/devices/vgpu_linux_test.go | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 61e55b599..6ef9f6c60 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -5,7 +5,9 @@ package devices import ( "bufio" "context" + "errors" "fmt" + "log/slog" "os" "os/exec" "path/filepath" @@ -112,6 +114,7 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice } var vfs []VirtualFunction + var vfErrs []error for _, entry := range entries { vfAddr := entry.Name() types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) @@ -119,7 +122,8 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice if os.IsNotExist(err) { continue } - return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + vfErrs = append(vfErrs, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err)) + continue } usable := false for _, typ := range types { @@ -149,6 +153,16 @@ func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice Allocated: hasMdev, }) } + if len(vfErrs) > 0 { + // Mirror vendor VFIO discovery: one unreadable VF must not blank out + // the host's GPU capacity. Only when no VF is readable does discovery + // fail, so a wholesale sysfs outage cannot demote an mdev host to + // vendor VFIO or passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable mdev VFs", "error", errors.Join(vfErrs...)) + } return vfs, nil } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 40b987394..7b03d9c26 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -73,3 +73,40 @@ func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { assert.Equal(t, VGPUFrameworkNone, framework) assert.Nil(t, vfs) } + +func TestDiscoverMdevVFsSkipsUnreadableVF(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types", "nvidia-556"), 0755)) + // A regular file makes the supported-types read fail without IsNotExist. + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.5"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.5", "mdev_supported_types"), nil, 0644)) + + vfs, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestDiscoverMdevVFsFailsWhenNoVFIsReadable(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + pciPath := filepath.Join(root, "sys", "bus", "pci", "devices") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), nil, 0644)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to vendor VFIO or + // passthrough. + _, err := discoverMdevVFsWith(busPath, pciPath, func() ([]MdevDevice, error) { + return nil, nil + }) + require.Error(t, err) +} From 283be51091fbbefd141c6c69a88346d22257a8ee Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:42:26 +0000 Subject: [PATCH 21/24] Warn when reconcile preserves an unclaimed in-use vGPU --- lib/devices/vendor_vfio_linux.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 566a0d772..112030896 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -292,6 +292,7 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map continue } if inUse { + log.WarnContext(ctx, "preserving vendor VFIO vGPU held open without a live instance claim", "vf", vf.PCIAddress) continue } if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { From 00bf116724f4b010cf950f4cda2b6a459475f072 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 17 Aug 2026 21:10:15 +0000 Subject: [PATCH 22/24] Document wedged-VF forensics and SR-IOV cycle remediation --- lib/devices/GPU.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 0cba075fa..3a1504428 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -271,6 +271,44 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` +### Guest driver init times out on one VF (vendor VFIO) + +A VF can be wedged inside the NVIDIA stack while its sysfs interface stays +healthy: assignment succeeds, the host plugin logs `display_init inst: 0 +successful`, but the guest driver loops on + +``` +NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884) +``` + +(0x65 = timeout; the guest's init requests are never answered, and +`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because +placement is deterministic least-loaded, an idle host re-picks the same VF for +every request, so one wedged VF presents as all vGPU instances failing while +`/resources` reports full capacity. + +The wedge itself leaves no host-side log: no kernel error, no XID, no plugin +crash. In the observed case it followed a period of heavy attach/teardown +churn on the VF, including QEMU processes that exited within seconds of +opening the VFIO device (failed start attempts that were then retried), so +suspect any workload that repeatedly kills the VMM mid-device-init. + +Confirm by assigning the same profile on a different VF: if that guest +initializes, the VF is wedged, not the driver stack. Remediate by cycling +SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it +requires no vGPU assignments on that GPU): + +```bash +/usr/lib/nvidia/sriov-manage -d +/usr/lib/nvidia/sriov-manage -e +``` + +Do not unbind/rebind the VF from the nvidia driver — it breaks the +nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`) +and the VF stops accepting assignments entirely until the SR-IOV cycle. +Services holding the GPU (DCGM, persistenced) must be stopped for the cycle +to obtain the unbind lock. + ### vGPU assignment fails Check the files for the framework detected on the host: From 227e6871c4e0d2d0b9023472db402499b9d1f024 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:10 +0000 Subject: [PATCH 23/24] Tolerate processes exiting mid-scan in the open-VFIO-handle scan A process that exits between the /proc listing and its fd walk surfaces ENOENT or ESRCH; it holds nothing open, so skipping it cannot produce a false "not in use" answer. Everything else still fails the scan closed. --- lib/devices/vendor_vfio_linux.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 112030896..a6d37e80d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "syscall" "github.com/kernel/hypeman/lib/logger" ) @@ -425,7 +426,9 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] // isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes // clearing current_vgpu_type on a VF path that is reused across assignments, // so an incomplete scan must fail the release (which retains metadata for a -// later retry) rather than risk a false "not in use" answer. +// later retry) rather than risk a false "not in use" answer. The exception +// is a process that exits mid-scan (ENOENT/ESRCH): a dead process holds +// nothing open, so skipping it cannot produce that false answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { processes, err := os.ReadDir(s.procPath) if err != nil { @@ -440,7 +443,7 @@ func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { fdPath := filepath.Join(s.procPath, process.Name(), "fd") fds, err := os.ReadDir(fdPath) if err != nil { - if os.IsNotExist(err) { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { continue } return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) @@ -448,7 +451,7 @@ func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { for _, fd := range fds { target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) if err != nil { - if os.IsNotExist(err) { + if os.IsNotExist(err) || errors.Is(err, syscall.ESRCH) { continue } return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) From 6f980556795c687fbe0e8280606dd9f24491464b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:10:10 +0000 Subject: [PATCH 24/24] Skip mdev release for an assignment with no device An mdev assignment carrying neither a UUID nor a device path would resolve to DestroyMdev("."); release nothing instead. --- lib/devices/vgpu_linux.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72fe3b944..b3c497899 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -92,6 +92,9 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { case VGPUFrameworkMdev: mdevUUID := assignment.MdevUUID if mdevUUID == "" { + if assignment.DevicePath == "" { + return nil + } mdevUUID = filepath.Base(assignment.DevicePath) } return DestroyMdev(ctx, mdevUUID)