[LEP-3790] fix(nvidia/fabric-manager): skip V3 fabric API on older drivers to prevent crash (#1183)

nvmlDeviceGetGpuFabricInfoV was introduced in driver 550. Calling it on
older drivers (e.g., 535.x) causes a symbol lookup error that crashes
the process with exit code 127 at the dynamic linker level.

This fix:
- Adds MinDriverVersionForV3FabricAPI constant (550) with documentation
- Passes driver major version to device creation via WithDriverMajor()
- Checks driver version before calling V3 API in GetFabricState()
- Logs a warning (once) when skipping V3 API due to old driver
- Falls back to V1 API (GetGpuFabricInfo) on older drivers

The V1 API provides basic fabric info but lacks health metrics
(HealthMask, HealthSummary) which are set to defaults.

Reference: https://docs.nvidia.com/deploy/nvml-api/change-log.html

Before

> gpud scan
>
>
{"level":"info","ts":"2025-12-18T16:17:35.540+0530","caller":"fabric-manager/component.go:279","msg":"checking
nvidia fabric manager"}
> gpud: symbol lookup error: gpud: undefined symbol:
nvmlDeviceGetGpuFabricInfoV

After

> gpud scan
>
>
{"level":"info","ts":"2025-12-18T17:30:41.418+0530","caller":"fabric-manager/component.go:279","msg":"checking
nvidia fabric manager"}

{"level":"warn","ts":"2025-12-18T17:30:41.467+0530","caller":"device/fabric_state.go:311","msg":"skipping
fabric state V3 API (nvmlDeviceGetGpuFabricInfoV) due to old driver
version; V3 API requires driver >=
550","driverMajor":535,"minRequired":550}

health states:

```json
    "component": "accelerator-nvidia-fabric-manager",
    "states": [
      {
        "time": "2025-12-18T12:01:59Z",
        "component": "accelerator-nvidia-fabric-manager",
        "name": "accelerator-nvidia-fabric-manager",
        "health": "Healthy",
        "reason": "fabric manager found and active",
```

Signed-off-by: Gyuho Lee <[email protected]>

Author: gyuho

pkg/nvidia-query/nvml/device/device.go

@@ -20,10 +20,19 @@ type Device interface {
 
 var _ Device = &nvDevice{}
 
+// MinDriverVersionForV3FabricAPI is the minimum NVIDIA driver major version
+// required for nvmlDeviceGetGpuFabricInfoV (V3 fabric state API).
+// This API was introduced in driver 550 (see NVML changelog:
+// https://docs.nvidia.com/deploy/nvml-api/change-log.html).
+// Calling this function on older drivers (e.g., 535.x) causes a symbol lookup
+// error that crashes the process.
+const MinDriverVersionForV3FabricAPI = 550
+
 type nvDevice struct {
 	device.Device
-	busID string
-	uuid  string
+	busID       string
+	uuid        string
+	driverMajor int // Driver major version, used to gate V3 fabric API calls
 }
 
 func (d *nvDevice) PCIBusID() string {
@@ -45,7 +54,7 @@ func New(dev device.Device, busID string, opts ...OpOption) Device {
 	}
 
 	// Create the base device
-	baseDevice := &nvDevice{Device: dev, busID: busID, uuid: uuid}
+	baseDevice := &nvDevice{Device: dev, busID: busID, uuid: uuid, driverMajor: op.DriverMajor}
 
 	// If ANY test flags are set, wrap with testDevice
 	if op.GPULost || op.GPURequiresReset || op.FabricHealthUnhealthy {
@@ -62,6 +71,9 @@ func New(dev device.Device, busID string, opts ...OpOption) Device {
 
 // Op struct holds options for device creation
 type Op struct {
+	// DriverMajor is the major version of the NVIDIA driver.
+	// Used to gate V3 fabric API calls which require driver >= 550.
+	DriverMajor int
 	// GPULost indicates that all device methods should return nvml.ERROR_GPU_IS_LOST
 	GPULost bool
 	// GPURequiresReset indicates that all device methods should return nvml.ERROR_RESET_REQUIRED
@@ -100,3 +112,11 @@ func WithFabricHealthUnhealthy() OpOption {
 		op.FabricHealthUnhealthy = true
 	}
 }
+
+// WithDriverMajor returns an OpOption that sets the driver major version.
+// This is used to gate V3 fabric API calls which require driver >= 550.
+func WithDriverMajor(major int) OpOption {
+	return func(op *Op) {
+		op.DriverMajor = major
+	}
+}

pkg/nvidia-query/nvml/device/device_test.go

@@ -0,0 +1,179 @@
+package device
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMinDriverVersionForV3FabricAPI(t *testing.T) {
+	t.Parallel()
+
+	// Verify the constant is set correctly per NVIDIA documentation
+	// nvmlDeviceGetGpuFabricInfoV was introduced in driver 550
+	// See: https://docs.nvidia.com/deploy/nvml-api/change-log.html
+	assert.Equal(t, 550, MinDriverVersionForV3FabricAPI,
+		"MinDriverVersionForV3FabricAPI should be 550 per NVIDIA NVML changelog")
+}
+
+func TestWithDriverMajor(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name          string
+		driverMajor   int
+		expectedMajor int
+	}{
+		{
+			name:          "driver version 535 (old driver)",
+			driverMajor:   535,
+			expectedMajor: 535,
+		},
+		{
+			name:          "driver version 550 (minimum for V3 API)",
+			driverMajor:   550,
+			expectedMajor: 550,
+		},
+		{
+			name:          "driver version 560 (newer driver)",
+			driverMajor:   560,
+			expectedMajor: 560,
+		},
+		{
+			name:          "driver version 0 (uninitialized)",
+			driverMajor:   0,
+			expectedMajor: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			op := &Op{}
+			opt := WithDriverMajor(tt.driverMajor)
+			opt(op)
+
+			assert.Equal(t, tt.expectedMajor, op.DriverMajor)
+		})
+	}
+}
+
+func TestOpApplyOpts(t *testing.T) {
+	t.Parallel()
+
+	t.Run("multiple options applied correctly", func(t *testing.T) {
+		t.Parallel()
+
+		op := &Op{}
+		opts := []OpOption{
+			WithDriverMajor(550),
+			WithGPULost(),
+			WithGPURequiresReset(),
+			WithFabricHealthUnhealthy(),
+		}
+		op.applyOpts(opts)
+
+		assert.Equal(t, 550, op.DriverMajor)
+		assert.True(t, op.GPULost)
+		assert.True(t, op.GPURequiresReset)
+		assert.True(t, op.FabricHealthUnhealthy)
+	})
+
+	t.Run("driver major can be overwritten", func(t *testing.T) {
+		t.Parallel()
+
+		op := &Op{}
+		opts := []OpOption{
+			WithDriverMajor(535),
+			WithDriverMajor(550), // overwrite
+		}
+		op.applyOpts(opts)
+
+		assert.Equal(t, 550, op.DriverMajor)
+	})
+}
+
+func TestNvDeviceDriverMajorField(t *testing.T) {
+	t.Parallel()
+
+	// Test that the driverMajor field is correctly set on nvDevice
+	// We can't directly test nvDevice since it's private, but we can verify
+	// the Op struct correctly stores the value
+
+	tests := []struct {
+		name        string
+		driverMajor int
+		expectV3API bool // whether V3 API should be attempted
+	}{
+		{
+			name:        "driver 535 should skip V3 API",
+			driverMajor: 535,
+			expectV3API: false,
+		},
+		{
+			name:        "driver 549 should skip V3 API",
+			driverMajor: 549,
+			expectV3API: false,
+		},
+		{
+			name:        "driver 550 should use V3 API",
+			driverMajor: 550,
+			expectV3API: true,
+		},
+		{
+			name:        "driver 560 should use V3 API",
+			driverMajor: 560,
+			expectV3API: true,
+		},
+		{
+			name:        "driver 0 (uninitialized) should skip V3 API",
+			driverMajor: 0,
+			expectV3API: false,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// Verify the logic matches expectations
+			shouldUseV3 := tt.driverMajor >= MinDriverVersionForV3FabricAPI
+			assert.Equal(t, tt.expectV3API, shouldUseV3,
+				"driver major %d: expected V3 API usage = %v", tt.driverMajor, tt.expectV3API)
+		})
+	}
+}
+
+func TestDriverVersionBoundaryConditions(t *testing.T) {
+	t.Parallel()
+
+	// Test boundary conditions around the minimum driver version
+	tests := []struct {
+		driverMajor int
+		expectV3    bool
+		description string
+	}{
+		{driverMajor: 548, expectV3: false, description: "two versions below minimum"},
+		{driverMajor: 549, expectV3: false, description: "one version below minimum"},
+		{driverMajor: 550, expectV3: true, description: "exactly at minimum"},
+		{driverMajor: 551, expectV3: true, description: "one version above minimum"},
+		{driverMajor: 552, expectV3: true, description: "two versions above minimum"},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.description, func(t *testing.T) {
+			t.Parallel()
+
+			shouldUseV3 := tt.driverMajor >= MinDriverVersionForV3FabricAPI
+			require.Equal(t, tt.expectV3, shouldUseV3,
+				"driver %d: V3 API should be %s",
+				tt.driverMajor,
+				map[bool]string{true: "enabled", false: "disabled"}[tt.expectV3])
+		})
+	}
+}

pkg/nvidia-query/nvml/device/fabric_state.go

@@ -5,13 +5,20 @@ import (
 	"fmt"
 	"io"
 	"sort"
+	"sync"
 
 	"github.com/NVIDIA/go-nvml/pkg/nvml"
 	"github.com/olekukonko/tablewriter"
 
+	"github.com/leptonai/gpud/pkg/log"
 	nvmlerrors "github.com/leptonai/gpud/pkg/nvidia/errors"
 )
 
+var (
+	// logV3SkipOnce ensures we only log the V3 API skip warning once
+	logV3SkipOnce sync.Once
+)
+
 // FabricState represents fabric state information for a GPU device.
 // This struct encapsulates all fabric-related data from V3 and V1 APIs.
 type FabricState struct {
@@ -274,21 +281,39 @@ func fabricTriStateStatus(val, notSupported, trueValue, falseValue uint32) strin
 // GetFabricState retrieves fabric state information from the device.
 // It attempts V3 API first for detailed health metrics, falling back to V1 API if needed.
 // This method properly handles GPU lost and reset required errors.
+//
+// IMPORTANT: The V3 API (nvmlDeviceGetGpuFabricInfoV) was introduced in driver 550.
+// Calling it on older drivers (e.g., 535.x) causes a symbol lookup error that crashes
+// the process. We skip V3 API calls for drivers < MinDriverVersionForV3FabricAPI.
+// See: https://docs.nvidia.com/deploy/nvml-api/change-log.html
 func (d *nvDevice) GetFabricState() (FabricState, error) {
-	// Try V3 API first (provides detailed health metrics)
-	handler := d.GetGpuFabricInfoV()
-	info, ret := handler.V3()
-	if ret == nvml.SUCCESS {
-		return FabricState{
-			CliqueID:      info.CliqueId,
-			ClusterUUID:   formatClusterUUID(info.ClusterUuid),
-			State:         info.State,
-			Status:        nvml.Return(info.Status),
-			HealthMask:    info.HealthMask,
-			HealthSummary: info.HealthSummary,
-		}, nil
+	// Only try V3 API if driver supports it (>= 550).
+	// Calling GetGpuFabricInfoV() on older drivers (e.g., 535.x) causes:
+	// "symbol lookup error: undefined symbol: nvmlDeviceGetGpuFabricInfoV"
+	// which crashes the process (exit code 127) at the dynamic linker level.
+	if d.driverMajor >= MinDriverVersionForV3FabricAPI {
+		handler := d.GetGpuFabricInfoV()
+		info, ret := handler.V3()
+		if ret == nvml.SUCCESS {
+			return FabricState{
+				CliqueID:      info.CliqueId,
+				ClusterUUID:   formatClusterUUID(info.ClusterUuid),
+				State:         info.State,
+				Status:        nvml.Return(info.Status),
+				HealthMask:    info.HealthMask,
+				HealthSummary: info.HealthSummary,
+			}, nil
+		}
+		// V3 failed, fall through to V1
+	} else {
+		// Log warning once when skipping V3 API due to old driver
+		logV3SkipOnce.Do(func() {
+			log.Logger.Warnw("skipping fabric state V3 API (nvmlDeviceGetGpuFabricInfoV) due to old driver version; V3 API requires driver >= 550",
+				"driverMajor", d.driverMajor,
+				"minRequired", MinDriverVersionForV3FabricAPI,
+			)
+		})
 	}
-	// V3 failed, fall through to V1
 
 	// Try V1 API (basic fabric information)
 	infoV1, ret := d.GetGpuFabricInfo()