Integrate vendor VFIO vGPUs into the instance lifecycle

cmd/api/api/instances.go

@@ -362,7 +362,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+ var vgpuPending *instances.VGPUCleanupPendingError
  switch {
+ // Checked first: it wraps the original create error, so a later
+ // errors.Is case would match the cause and hide the pending vGPU cleanup.
+ case errors.As(err, &vgpuPending):
+ log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+ message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry")
+ return oapi.CreateInstance500JSONResponse{
+ Code: "vgpu_cleanup_pending",
+ Message: message,
+ InnerError: inner,
+ }, nil
  case errors.Is(err, instances.ErrImageNotReady):
  return oapi.CreateInstance400JSONResponse{
  Code: "image_not_ready",
@@ -424,6 +435,22 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil
 }
 
+// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message
+// and inner error detail shared by the create and start handlers. The
+// retained guidance names the verb-specific way to release the assignment.
+func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) {
+	message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance)
+	innerCode := "vgpu_retained_instance"
+	if !pending.Retained {
+ message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID)
+ innerCode = "vgpu_unretained_instance"
+	}
+	return message, &oapi.ErrorDetail{
+ Code: lo.ToPtr(innerCode),
+ Message: lo.ToPtr(pending.InstanceID),
+	}
+}
+
 // GetInstance gets instance details
 // The id parameter can be an instance ID, name, or ID prefix
 // Note: Resolution is handled by ResolveResource middleware
@@ -823,7 +850,18 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan
 
 	result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq)
 	if err != nil {
+ var vgpuPending *instances.VGPUCleanupPendingError
  switch {
+ // Checked first: it wraps the original start error, so a later
+ // errors.Is case would match the cause and hide the pending vGPU cleanup.
+ case errors.As(err, &vgpuPending):
+ log.ErrorContext(ctx, "failed to start instance", "error", err)
+ message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it")
+ return oapi.StartInstance500JSONResponse{
+ Code: "vgpu_cleanup_pending",
+ Message: message,
+ InnerError: inner,
+ }, nil
  case errors.Is(err, instances.ErrInvalidState):
  return oapi.StartInstance409JSONResponse{
  Code: "invalid_state",
@@ -1258,6 +1296,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
  if inst.GPUMdevUUID != "" {
  gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID)
  }
+ if inst.GPUDevicePath != "" {
+ gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath)
+ }
  oapiInst.Gpu = gpu
 	}
 

cmd/api/api/instances_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -47,6 +48,73 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+type createErrorInstanceManager struct {
+	instances.Manager
+	err error
+}
+
+func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) {
+	return nil, m.err
+}
+
+// A retained-assignment error must win over the mapping of the create error
+// it wraps, or the response omits the instance the caller has to delete.
+func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+ InstanceID: "inst-1",
+ Retained: true,
+ Err: network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+ Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+ "the underlying create failure must survive the cleanup guidance")
+	assert.Contains(t, pending.Message, "delete it to retry")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
+}
+
+func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+ InstanceID: "inst-1",
+ Err: network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+ Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+ "the underlying create failure must survive the cleanup guidance")
+	assert.Contains(t, pending.Message, "startup reconcile")
+	assert.NotContains(t, pending.Message, "delete")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
@@ -923,6 +991,68 @@ func (m *errActionInstanceManager) RestoreSnapshot(context.Context, string, stri
 	return nil, m.err
 }
 
+// A retained-assignment error must win over the mapping of the start error
+// it wraps, or the response omits the pending vGPU cleanup the caller has to
+// resolve.
+func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+
+	resolved := &instances.Instance{
+ StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"},
+ State: instances.StateStopped,
+	}
+
+	t.Run("retained", func(t *testing.T) {
+ t.Parallel()
+ svc := newTestService(t)
+ svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+ InstanceID: "inst-1",
+ Retained: true,
+ Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+ }}
+
+ resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+ require.NoError(t, rerr)
+
+ pending, ok := resp.(oapi.StartInstance500JSONResponse)
+ require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+ assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+ assert.Contains(t, pending.Message, "inst-1")
+ assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(),
+ "the underlying start failure must survive the cleanup guidance")
+ assert.Contains(t, pending.Message, "delete it or retry start")
+ require.NotNil(t, pending.InnerError)
+ require.NotNil(t, pending.InnerError.Code)
+ assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+ require.NotNil(t, pending.InnerError.Message)
+ assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+
+	t.Run("unretained", func(t *testing.T) {
+ t.Parallel()
+ svc := newTestService(t)
+ svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+ InstanceID: "inst-1",
+ Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+ }}
+
+ resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+ require.NoError(t, rerr)
+
+ pending, ok := resp.(oapi.StartInstance500JSONResponse)
+ require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+ assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+ assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+ assert.Contains(t, pending.Message, "startup reconcile")
+ assert.NotContains(t, pending.Message, "delete")
+ require.NotNil(t, pending.InnerError)
+ require.NotNil(t, pending.InnerError.Code)
+ assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+ require.NotNil(t, pending.InnerError.Message)
+ assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+}
+
 func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) {
 	t.Parallel()
 

cmd/api/main.go

@@ -185,6 +185,63 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) {
+	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
+	if err != nil {
+ return nil, 0, err
+	}
+	protected := make(map[string]struct{})
+	var retryAfter time.Duration
+	for _, inst := range allInstances {
+ if inst.GPUDevicePath == "" {
+ continue
+ }
+ if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) {
+ protected[inst.GPUDevicePath] = struct{}{}
+ continue
+ }
+ if inst.GPUAssignedAt == nil {
+ continue
+ }
+ remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt)
+ if remaining <= 0 {
+ continue
+ }
+ protected[inst.GPUDevicePath] = struct{}{}
+ if retryAfter == 0 || remaining < retryAfter {
+ retryAfter = remaining
+ }
+	}
+	return protected, retryAfter, nil
+}
+
+func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) {
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager)
+	if err != nil {
+ // Operator-actionable: vendor VFIO reconciliation stays disabled
+ // host-wide (and releases fail closed on the same inventory) until
+ // the unreadable instance metadata is repaired.
+ logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err)
+ protected = nil
+ retryAfter = 0
+	}
+	if err := devices.ReconcileVGPUs(ctx, protected); err != nil {
+ logger.Warn("failed to reconcile vGPU devices", "error", err)
+	}
+	if retryAfter <= 0 {
+ return
+	}
+	go func() {
+ timer := time.NewTimer(retryAfter)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ case <-timer.C:
+ reconcileVGPUs(ctx, instanceManager, logger)
+ }
+	}()
+}
+
 func run() error {
 	startupStarted := time.Now()
 	slog.Info("starting hypeman initialization")
@@ -384,12 +441,9 @@ func run() error {
  return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
- // Log but don't fail - mdev cleanup is best-effort
- logger.Warn("failed to reconcile mdev devices", "error", err)
-	}
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	reconcileVGPUs(ctx, app.InstanceManager, logger)
 
 	// Wire up resource validator for aggregate limit checking
 	// This enables the instance manager to validate CPU, memory, network, and GPU

cmd/api/main_test.go

@@ -2,15 +2,18 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/kernel/hypeman/lib/instances"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware"
@@ -338,3 +341,38 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) {
  })
 	}
 }
+
+type vgpuReconcileManagerStub struct {
+	instances.Manager
+	list []instances.Instance
+}
+
+func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) {
+	return s.list, nil
+}
+
+func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) {
+	dead := exec.Command("true")
+	require.NoError(t, dead.Run())
+	deadPID := dead.Process.Pid
+	recent := time.Now().Add(-time.Minute)
+	stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute)
+
+	manager := vgpuReconcileManagerStub{list: []instances.Instance{
+ {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}},
+ {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}},
+ {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}},
+ {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}},
+ {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}},
+	}}
+
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	require.NoError(t, err)
+	require.Positive(t, retryAfter)
+	require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod)
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7")
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8")
+}

integration/vgpu_test.go

@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
  return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
- // CreateVGPU rejects vendor VFIO until the instance lifecycle
- // integration lands.
- return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()

lib/builds/manager_test.go

@@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc
 	return result, nil
 }
 
+func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) {
+	return m.ListInstances(ctx, nil)
+}
+
 func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) {
 	return nil, nil
 }