feat: Enhanced Health Checks with Dependency Status

   - Add /readyz endpoint for Kubernetes readiness probes
   - Add /healthz/detailed endpoint for component-level status
   - Implement GossipChecker, LedgerChecker, and OrdererChecker
   - Support OK/DEGRADED/UNAVAILABLE status semantics
   - DEGRADED components don't fail readiness (only UNAVAILABLE does)
   - Safe defaults: minPeers=0, failOnLag=false, orderer disabled
   - Add comprehensive unit and integration tests
   - Update configuration and documentation

Signed-off-by: remo-lab <[email protected]>

Author: remo-lab

core/operations/detailed_health_handler.go

@@ -0,0 +1,90 @@
+/*
+Copyright IBM Corp All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package operations
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	libhealthz "github.com/hyperledger/fabric-lib-go/healthz"
+	"github.com/hyperledger/fabric/core/operations/healthz"
+)
+
+// DetailedHealthHandler handles /healthz/detailed. Access should be restricted
+// via TLS/client authentication.
+type DetailedHealthHandler struct {
+	readinessHandler *healthz.ReadinessHandler
+	healthHandler    HealthHandler
+	timeout          time.Duration
+}
+
+type HealthHandler interface {
+	RunChecks(context.Context) []libhealthz.FailedCheck
+}
+
+func NewDetailedHealthHandler(readinessHandler *healthz.ReadinessHandler, healthHandler HealthHandler, timeout time.Duration) *DetailedHealthHandler {
+	return &DetailedHealthHandler{
+		readinessHandler: readinessHandler,
+		healthHandler:    healthHandler,
+		timeout:          timeout,
+	}
+}
+
+func (h *DetailedHealthHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	if req.Method != "GET" {
+		rw.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(req.Context(), h.timeout)
+	defer cancel()
+
+	detailedStatus := h.readinessHandler.GetDetailedStatus(ctx)
+
+	livenessChecks := h.healthHandler.RunChecks(ctx)
+	if len(livenessChecks) > 0 {
+		for _, check := range livenessChecks {
+			detailedStatus.FailedChecks = append(detailedStatus.FailedChecks, healthz.FailedCheck{
+				Component: check.Component,
+				Reason:    check.Reason,
+			})
+			if _, exists := detailedStatus.Components[check.Component]; !exists {
+				detailedStatus.Components[check.Component] = healthz.ComponentStatus{
+					Status:  healthz.StatusUnavailable,
+					Message: check.Reason,
+				}
+			}
+		}
+		if detailedStatus.Status == healthz.StatusOK {
+			detailedStatus.Status = healthz.StatusUnavailable
+		}
+	}
+
+	rw.Header().Set("Content-Type", "application/json")
+	
+	var statusCode int
+	switch detailedStatus.Status {
+	case healthz.StatusOK:
+		statusCode = http.StatusOK
+	case healthz.StatusDegraded:
+		statusCode = http.StatusOK
+	default:
+		statusCode = http.StatusServiceUnavailable
+	}
+
+	resp, err := json.Marshal(detailedStatus)
+	if err != nil {
+		rw.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	rw.WriteHeader(statusCode)
+	rw.Write(resp)
+}
+

core/operations/system.go

@@ -8,6 +8,7 @@ package operations
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"strings"
 	"time"
@@ -20,9 +21,10 @@ import (
 	"github.com/hyperledger/fabric-lib-go/common/metrics/prometheus"
 	"github.com/hyperledger/fabric-lib-go/common/metrics/statsd"
 	"github.com/hyperledger/fabric-lib-go/common/metrics/statsd/goruntime"
-	"github.com/hyperledger/fabric-lib-go/healthz"
+	libhealthz "github.com/hyperledger/fabric-lib-go/healthz"
 	"github.com/hyperledger/fabric/common/fabhttp"
 	"github.com/hyperledger/fabric/common/metadata"
+	"github.com/hyperledger/fabric/core/operations/healthz"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -55,13 +57,14 @@ type System struct {
 	*fabhttp.Server
 	metrics.Provider
 
-	logger          Logger
-	healthHandler   *healthz.HealthHandler
-	options         Options
-	statsd          *kitstatsd.Statsd
-	collectorTicker *time.Ticker
-	sendTicker      *time.Ticker
-	versionGauge    metrics.Gauge
+	logger           Logger
+	healthHandler    *libhealthz.HealthHandler
+	readinessHandler *healthz.ReadinessHandler
+	options          Options
+	statsd           *kitstatsd.Statsd
+	collectorTicker  *time.Ticker
+	sendTicker       *time.Ticker
+	versionGauge     metrics.Gauge
 }
 
 func NewSystem(o Options) *System {
@@ -109,10 +112,21 @@ func (s *System) Stop() error {
 	return s.Server.Stop()
 }
 
-func (s *System) RegisterChecker(component string, checker healthz.HealthChecker) error {
+func (s *System) RegisterChecker(component string, checker libhealthz.HealthChecker) error {
 	return s.healthHandler.RegisterChecker(component, checker)
 }
 
+// RegisterReadinessChecker registers a ReadinessChecker for a named component.
+// Readiness checkers verify that dependencies are available and the service
+// is ready to accept traffic. These checks are separate from liveness checks
+// and are used by Kubernetes readiness probes.
+func (s *System) RegisterReadinessChecker(component string, checker healthz.ReadinessChecker) error {
+	if s.readinessHandler == nil {
+		return fmt.Errorf("readiness handler not initialized")
+	}
+	return s.readinessHandler.RegisterChecker(component, checker)
+}
+
 func (s *System) initializeMetricsProvider() error {
 	m := s.options.Metrics
 	providerType := m.Provider
@@ -180,7 +194,7 @@ func (s *System) initializeLoggingHandler() {
 }
 
 func (s *System) initializeHealthCheckHandler() {
-	s.healthHandler = healthz.NewHealthHandler()
+	s.healthHandler = libhealthz.NewHealthHandler()
 	// swagger:operation GET /healthz operations healthz
 	// ---
 	// summary: Retrieves all registered health checkers for the process.
@@ -190,6 +204,40 @@ func (s *System) initializeHealthCheckHandler() {
 	//     '503':
 	//        description: Service unavailable.
 	s.RegisterHandler("/healthz", s.healthHandler, false)
+
+	// Initialize readiness handler for Kubernetes readiness probes
+	s.readinessHandler = healthz.NewReadinessHandler()
+	// swagger:operation GET /readyz operations readyz
+	// ---
+	// summary: Retrieves readiness status for the process. Used by Kubernetes readiness probes.
+	// responses:
+	//     '200':
+	//        description: Ready.
+	//     '503':
+	//        description: Not ready.
+	s.RegisterHandler("/readyz", s.readinessHandler, false)
+}
+
+// SetDetailedHealthEnabled enables or disables the detailed health endpoint.
+func (s *System) SetDetailedHealthEnabled(enabled bool) {
+	if enabled && s.readinessHandler != nil {
+		s.logger.Warn("Detailed health endpoint enabled; exposes internal component status. Ensure access is restricted via TLS and client authentication.")
+		
+		detailedHandler := NewDetailedHealthHandler(
+			s.readinessHandler,
+			&healthHandlerAdapter{handler: s.healthHandler},
+			30*time.Second,
+		)
+		s.RegisterHandler("/healthz/detailed", detailedHandler, s.options.TLS.Enabled)
+	}
+}
+
+type healthHandlerAdapter struct {
+	handler *libhealthz.HealthHandler
+}
+
+func (a *healthHandlerAdapter) RunChecks(ctx context.Context) []libhealthz.FailedCheck {
+	return a.handler.RunChecks(ctx)
 }
 
 func (s *System) initializeVersionInfoHandler() {

core/peer/config.go

@@ -48,6 +48,35 @@ type ExternalBuilder struct {
 	Path                 string   `yaml:"path"`
 }
 
+// HealthCheckConfig represents the configuration for readiness health checks
+type HealthCheckConfig struct {
+	ReadinessTimeout time.Duration
+	Gossip           GossipHealthCheckConfig
+	Ledger           LedgerHealthCheckConfig
+	Orderer          OrdererHealthCheckConfig
+}
+
+// GossipHealthCheckConfig represents configuration for gossip readiness checks
+type GossipHealthCheckConfig struct {
+	Enabled  bool
+	MinPeers int
+	Timeout  time.Duration
+}
+
+// LedgerHealthCheckConfig represents configuration for ledger readiness checks
+type LedgerHealthCheckConfig struct {
+	Enabled   bool
+	Timeout   time.Duration
+	FailOnLag bool
+	MaxLag    uint64
+}
+
+// OrdererHealthCheckConfig represents configuration for orderer connectivity checks
+type OrdererHealthCheckConfig struct {
+	Enabled bool
+	Timeout time.Duration
+}
+
 // Config is the struct that defines the Peer configurations.
 type Config struct {
 	// LocalMSPID is the identifier of the local MSP.
@@ -187,6 +216,12 @@ type Config struct {
 	// trust for client authentication.
 	OperationsTLSClientRootCAs []string
 
+	// OperationsHealthCheck provides configuration for readiness health checks
+	OperationsHealthCheck HealthCheckConfig
+	
+	// OperationsDetailedHealth enables the detailed health endpoint
+	OperationsDetailedHealthEnabled bool
+
 	// ----- Metrics config -----
 	// TODO: create separate sub-struct for Metrics config.
 
@@ -321,6 +356,44 @@ func (c *Config) load() error {
 		c.OperationsTLSClientRootCAs = append(c.OperationsTLSClientRootCAs, config.TranslatePath(configDir, rca))
 	}
 
+	// Load health check configuration
+	c.OperationsHealthCheck.ReadinessTimeout = viper.GetDuration("operations.healthCheck.readinessTimeout")
+	if c.OperationsHealthCheck.ReadinessTimeout == 0 {
+		c.OperationsHealthCheck.ReadinessTimeout = 10 * time.Second
+	}
+
+	c.OperationsHealthCheck.Gossip.Enabled = viper.GetBool("operations.healthCheck.gossip.enabled")
+	c.OperationsHealthCheck.Gossip.MinPeers = viper.GetInt("operations.healthCheck.gossip.minPeers")
+	// Default minPeers to 0 to avoid false negatives in dev and single-node setups.
+	// minPeers = 0 means "gossip initialized but no connectivity requirement".
+	// Only enforce peer count when minPeers > 0.
+	if !viper.IsSet("operations.healthCheck.gossip.minPeers") {
+		c.OperationsHealthCheck.Gossip.MinPeers = 0
+	}
+	c.OperationsHealthCheck.Gossip.Timeout = viper.GetDuration("operations.healthCheck.gossip.timeout")
+	if c.OperationsHealthCheck.Gossip.Timeout == 0 {
+		c.OperationsHealthCheck.Gossip.Timeout = 5 * time.Second
+	}
+
+	c.OperationsHealthCheck.Ledger.Enabled = viper.GetBool("operations.healthCheck.ledger.enabled")
+	c.OperationsHealthCheck.Ledger.Timeout = viper.GetDuration("operations.healthCheck.ledger.timeout")
+	if c.OperationsHealthCheck.Ledger.Timeout == 0 {
+		c.OperationsHealthCheck.Ledger.Timeout = 5 * time.Second
+	}
+	c.OperationsHealthCheck.Ledger.FailOnLag = viper.GetBool("operations.healthCheck.ledger.failOnLag")
+	c.OperationsHealthCheck.Ledger.MaxLag = viper.GetUint64("operations.healthCheck.ledger.maxLag")
+	if c.OperationsHealthCheck.Ledger.MaxLag == 0 {
+		c.OperationsHealthCheck.Ledger.MaxLag = 10 // Default max lag
+	}
+
+	c.OperationsHealthCheck.Orderer.Enabled = viper.GetBool("operations.healthCheck.orderer.enabled")
+	c.OperationsHealthCheck.Orderer.Timeout = viper.GetDuration("operations.healthCheck.orderer.timeout")
+	if c.OperationsHealthCheck.Orderer.Timeout == 0 {
+		c.OperationsHealthCheck.Orderer.Timeout = 5 * time.Second
+	}
+
+	c.OperationsDetailedHealthEnabled = viper.GetBool("operations.healthCheck.detailedHealth.enabled")
+
 	c.MetricsProvider = viper.GetString("metrics.provider")
 	c.StatsdNetwork = viper.GetString("metrics.statsd.network")
 	c.StatsdAaddress = viper.GetString("metrics.statsd.address")