featuregate healthcheck

Signed-off-by: Swati Gupta <[email protected]>

Author: guptaNswati

cmd/gpu-kubelet-plugin/device_health.go

@@ -74,7 +74,8 @@ func newDeviceHealthMonitor(ctx context.Context, config *Config, allocatable All
 }
 
 func (m *deviceHealthMonitor) registerDevicesForEvents() {
-	// TODO: add a list of xids to ignore
+	// SWATI: 1. add a list of xids to ignore
+	// 2. Skip ECC errors
 	eventMask := uint64(nvml.EventTypeXidCriticalError | nvml.EventTypeDoubleBitEccError | nvml.EventTypeSingleBitEccError)
 
 	processedUUIDs := make(map[string]bool)

cmd/gpu-kubelet-plugin/device_state.go

@@ -610,6 +610,7 @@ func GetOpaqueDeviceConfigs(
 }
 
 func (s *DeviceState) MarkDeviceUnhealthy(device *AllocatableDevice) {
+	// SWATI: check if a mig device is marked properly
 	s.Lock()
 	defer s.Unlock()
 

cmd/gpu-kubelet-plugin/driver.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/dynamic-resource-allocation/resourceslice"
 	"k8s.io/klog/v2"
 
+	"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/featuregates"
 	"github.com/NVIDIA/k8s-dra-driver-gpu/pkg/flock"
 )
 
@@ -98,19 +99,21 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) {
 	}
 	driver.healthcheck = healthcheck
 
-	deviceHealthMonitor, err := newDeviceHealthMonitor(ctx, config, state.allocatable, state.nvdevlib)
-	if err != nil {
-		return nil, fmt.Errorf("start deviceHealthMonitor: %w", err)
-	}
+	if featuregates.Enabled(featuregates.DeviceHealthCheck) {
+
+		deviceHealthMonitor, err := newDeviceHealthMonitor(ctx, config, state.allocatable, state.nvdevlib)
+		if err != nil {
+			return nil, fmt.Errorf("start deviceHealthMonitor: %w", err)
+		}
 
-	driver.deviceHealthMonitor = deviceHealthMonitor
+		driver.deviceHealthMonitor = deviceHealthMonitor
+		go driver.deviceHealthEvents(ctx, config.flags.nodeName)
+	}
 
 	if err := driver.pluginhelper.PublishResources(ctx, resources); err != nil {
 		return nil, err
 	}
 
-	go driver.deviceHealthEvents(ctx, config.flags.nodeName)
-
 	return driver, nil
 }
 

deployments/helm/nvidia-dra-driver-gpu/templates/rbac.yaml

@@ -22,7 +22,11 @@ rules:
   verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: ["resource.k8s.io"]
   resources: ["resourceclaims/status"]
-  verbs: ["update"]
+  verbs:
+  - "update"
+  {{- if .Values.featureGates.DeviceHealthCheck }}
+  - "patch"
+  {{- end }}
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "list", "watch", "update"]

deployments/helm/nvidia-dra-driver-gpu/values.yaml

@@ -67,7 +67,8 @@ resources:
 #   ContextualLogging: true            # Kubernetes logging feature (enabled by default)
 #   LoggingAlphaOptions: false         # Kubernetes logging alpha features
 #   LoggingBetaOptions: true           # Kubernetes logging beta features
-featureGates: {}
+featureGates:
+  DeviceHealthCheck: false
 
 # Webhook configuration
 webhook:

pkg/featuregates/featuregates.go

@@ -33,6 +33,9 @@ const (
 
 	// MPSSupport allows MPS (Multi-Process Service) settings to be specified.
 	MPSSupport featuregate.Feature = "MPSSupport"
+
+	// DeviceHealthCheck allows Device Health Checking
+	DeviceHealthCheck featuregate.Feature = "DeviceHealthCheck"
 )
 
 // FeatureGates is a singleton representing the set of all feature gates and their values.
@@ -56,6 +59,13 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
 			Version:    version.MajorMinor(25, 8),
 		},
 	},
+	DeviceHealthCheck: {
+		{
+			Default:    false,
+			PreRelease: featuregate.Alpha,
+			Version:    version.MajorMinor(25, 8),
+		},
+	},
 }
 
 // init instantiates and sets the singleton 'FeatureGates' variable with newFeatureGates().