feat: Enhanced Health Checks with Dependency Status

   - Add /readyz endpoint for Kubernetes readiness probes
   - Add /healthz/detailed endpoint for component-level status
   - Implement GossipChecker, LedgerChecker, and OrdererChecker
   - Support OK/DEGRADED/UNAVAILABLE status semantics
   - DEGRADED components don't fail readiness (only UNAVAILABLE does)
   - Safe defaults: minPeers=0, failOnLag=false, orderer disabled
   - Add comprehensive unit and integration tests
   - Update configuration and documentation

Author: remo-lab

core/operations/detailed_health_handler.go

@@ -0,0 +1,90 @@
+/*
+Copyright IBM Corp All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package operations
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	libhealthz "github.com/hyperledger/fabric-lib-go/healthz"
+	"github.com/hyperledger/fabric/core/operations/healthz"
+)
+
+// DetailedHealthHandler handles /healthz/detailed. Access should be restricted
+// via TLS/client authentication.
+type DetailedHealthHandler struct {
+	readinessHandler *healthz.ReadinessHandler
+	healthHandler    HealthHandler
+	timeout          time.Duration
+}
+
+type HealthHandler interface {
+	RunChecks(context.Context) []libhealthz.FailedCheck
+}
+
+func NewDetailedHealthHandler(readinessHandler *healthz.ReadinessHandler, healthHandler HealthHandler, timeout time.Duration) *DetailedHealthHandler {
+	return &DetailedHealthHandler{
+		readinessHandler: readinessHandler,
+		healthHandler:    healthHandler,
+		timeout:          timeout,
+	}
+}
+
+func (h *DetailedHealthHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	if req.Method != "GET" {
+		rw.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(req.Context(), h.timeout)
+	defer cancel()
+
+	detailedStatus := h.readinessHandler.GetDetailedStatus(ctx)
+
+	livenessChecks := h.healthHandler.RunChecks(ctx)
+	if len(livenessChecks) > 0 {
+		for _, check := range livenessChecks {
+			detailedStatus.FailedChecks = append(detailedStatus.FailedChecks, healthz.FailedCheck{
+				Component: check.Component,
+				Reason:    check.Reason,
+			})
+			if _, exists := detailedStatus.Components[check.Component]; !exists {
+				detailedStatus.Components[check.Component] = healthz.ComponentStatus{
+					Status:  healthz.StatusUnavailable,
+					Message: check.Reason,
+				}
+			}
+		}
+		if detailedStatus.Status == healthz.StatusOK {
+			detailedStatus.Status = healthz.StatusUnavailable
+		}
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	
+	var statusCode int
+	switch detailedStatus.Status {
+	case healthz.StatusOK:
+		statusCode = http.StatusOK
+	case healthz.StatusDegraded:
+		statusCode = http.StatusOK
+	default:
+		statusCode = http.StatusServiceUnavailable
+	}
+
+	resp, err := json.Marshal(detailedStatus)
+	if err != nil {
+		rw.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	rw.WriteHeader(statusCode)
+	rw.Write(resp)
+}
+

core/operations/healthcheckers/gossip_checker.go

@@ -0,0 +1,85 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package healthcheckers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/hyperledger/fabric/core/operations/healthz"
+	"github.com/hyperledger/fabric/gossip/service"
+)
+
+// GossipChecker verifies gossip service connectivity. When minPeers = 0 (default),
+// only checks that gossip is initialized, avoiding false negatives in dev setups.
+// When minPeers > 0, enforces minimum peer count. DEGRADED at minimum is informational.
+type GossipChecker struct {
+	gossipService *service.GossipService
+	minPeers      int
+	timeout       time.Duration
+}
+
+// NewGossipChecker creates a new GossipChecker. If minPeers is 0, only verifies
+// gossip is initialized.
+func NewGossipChecker(gossipService *service.GossipService, minPeers int, timeout time.Duration) *GossipChecker {
+	return &GossipChecker{
+		gossipService: gossipService,
+		minPeers:      minPeers,
+		timeout:       timeout,
+	}
+}
+
+func (g *GossipChecker) ReadinessCheck(ctx context.Context) error {
+	if g.gossipService == nil {
+		return fmt.Errorf("gossip service not initialized")
+	}
+
+	peers := g.gossipService.Peers()
+	connectedCount := len(peers)
+
+	if g.minPeers > 0 && connectedCount < g.minPeers {
+		return fmt.Errorf("insufficient peers connected: %d (minimum: %d)", connectedCount, g.minPeers)
+	}
+
+	return nil
+}
+
+func (g *GossipChecker) GetStatus() healthz.ComponentStatus {
+	if g.gossipService == nil {
+		return healthz.ComponentStatus{
+			Status:  healthz.StatusUnavailable,
+			Message: "Gossip service not initialized",
+		}
+	}
+
+	peers := g.gossipService.Peers()
+	connectedCount := len(peers)
+	
+	status := healthz.StatusOK
+	message := fmt.Sprintf("Connected to %d peers", connectedCount)
+	
+	if g.minPeers > 0 && connectedCount < g.minPeers {
+		status = healthz.StatusUnavailable
+		message = fmt.Sprintf("Insufficient peers: %d (minimum: %d)", connectedCount, g.minPeers)
+	} else if g.minPeers > 0 && connectedCount == g.minPeers {
+		status = healthz.StatusDegraded
+		message = fmt.Sprintf("Connected to minimum required peers: %d", connectedCount)
+	}
+
+	details := map[string]interface{}{
+		"connected_peers": connectedCount,
+		"min_peers":       g.minPeers,
+	}
+
+	return healthz.ComponentStatus{
+		Status:  status,
+		Message: message,
+		Details: details,
+	}
+}
+

core/operations/healthcheckers/gossip_checker_test.go

@@ -0,0 +1,87 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package healthcheckers
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hyperledger/fabric/core/operations/healthz"
+	"github.com/hyperledger/fabric/gossip/service"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGossipChecker_ReadinessCheck(t *testing.T) {
+		tests := []struct {
+		name        string
+		gossip      *service.GossipService
+		minPeers    int
+		expectError bool
+		errorMsg    string
+	}{
+		{
+			name:        "nil gossip service",
+			gossip:      nil,
+			minPeers:    0,
+			expectError: true,
+			errorMsg:    "gossip service not initialized",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			checker := &GossipChecker{
+				gossipService: tt.gossip,
+				minPeers:      tt.minPeers,
+				timeout:       5,
+			}
+
+			err := checker.ReadinessCheck(context.Background())
+
+			if tt.expectError {
+				require.Error(t, err)
+				if tt.errorMsg != "" {
+					assert.Contains(t, err.Error(), tt.errorMsg)
+				}
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestGossipChecker_GetStatus(t *testing.T) {
+		tests := []struct {
+		name           string
+		gossip         *service.GossipService
+		minPeers       int
+		expectedStatus string
+	}{
+		{
+			name:           "nil gossip service",
+			gossip:         nil,
+			minPeers:       0,
+			expectedStatus: healthz.StatusUnavailable,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			checker := &GossipChecker{
+				gossipService: tt.gossip,
+				minPeers:      tt.minPeers,
+				timeout:       5,
+			}
+
+			status := checker.GetStatus()
+			assert.Equal(t, tt.expectedStatus, status.Status)
+			assert.NotEmpty(t, status.Message)
+		})
+	}
+}
+