invalid memory address with WithLabelValues() #1877
Description
Environment:
golang: 1.24.2
prometheus/client_golang: v1.23.2
code
type Metric struct {
API string
Status Status
Cache string
Cost time.Duration
}
func CollectAppMetrics() func(ctx context.Context) {
return func(ctx context.Context) {
v := FromContext(ctx) // v type is *Metric from ctx, always has value
defer func() {
if rec := recover(); rec != nil {
err, ok := rec.(error)
if !ok {
err = fmt.Errorf("%v", rec)
}
stack := make([]byte, 1<<12)
length := runtime.Stack(stack, true)
panicLogger.Error(ctx, fmt.Sprintf("err: %v", err), logit.ByteString("stack", stack[:length]),
logit.String("api", v.API),
logit.String("cache", v.Cache),
logit.String("status", string(v.Status)),
logit.Int("api_len", len(v.API)),
logit.Int("cache_len", len(v.Cache)),
logit.Int("status_len", len(string(v.Status))),
logit.String("api_bytes", fmt.Sprintf("%v", []byte(v.API))),
logit.String("cache_bytes", fmt.Sprintf("%v", []byte(v.Cache))),
logit.String("status_bytes", fmt.Sprintf("%v", []byte(v.Status))),
)
}
}()
apiMetrics.WithLabelValues(v.API, v.Cache, string(v.Status)).Observe(float64(v.Cost.Milliseconds()))
}
}The CollectAppMetrics function returns an HTTP middleware that is executed post-request to collect API metrics.
apiMetrics is a HistogramVec instance.
This issue is commonly observed after a ctx timeout. In such cases, the final statement of the middleware function may intermittently raise an error, though the behavior is not consistently reproducible.
panic log:
ERROR: 2025-09-19 18:32:56.978031391 example/myserver/model/metrics/service.go:552 qcl_cost[0.030] qlc_emsg[nil] err[1] status[fail] error[context canceled] write_err[nil] logid[3474584649] stack[goroutine 29004693 [running]:
icode.example.com/example/myserver/servers/httpserver.httpRouter.CollectAppMetrics.func3.1()
/root/ONLINE_SERVICE/other/ferry/task_workspace/example/myserver/model/metrics/service.go:551 +0x134
panic({0x19788a0?, 0x2dda830?})
/home/opt/deck/2.0/go-1.24.2/src/runtime/panic.go:792 +0x132
unicode/utf8.ValidString({0x0?, 0xc02aa62f90?})
/home/opt/deck/2.0/go-1.24.2/src/unicode/utf8/utf8.go:508 +0x4f
github.com/prometheus/client_golang/prometheus.validateLabelValues({0xc01d933260?, 0x0?, 0x0?}, 0x0?)
/root/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/labels.go:178 +0x1b2
github.com/prometheus/client_golang/prometheus.(*MetricVec).hashLabelValues(0xc00097a6f0, {0xc01d933260, 0x3, 0x0?})
/root/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/vec.go:251 +0x45
github.com/prometheus/client_golang/prometheus.(*MetricVec).GetMetricWithLabelValues(0xc00097a6f0, {0xc01d933260?, 0x18b7140?, 0x1f7fc50?})
/root/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/vec.go:214 +0x59
github.com/prometheus/client_golang/prometheus.(*HistogramVec).GetMetricWithLabelValues(...)
/root/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/histogram.go:1226
github.com/prometheus/client_golang/prometheus.(*HistogramVec).WithLabelValues(0x1f9dda8?, {0xc01d933260?, 0x7461a0?, 0xc01d932e40?})
/root/go/pkg/mod/github.com/prometheus/[email protected]/prometheus/histogram.go:1259 +0x1f
icode.example.com/example/myserver/servers/httpserver.httpRouter.CollectAppMetrics.func3({0x1f9dda8, 0xc0053956e0})
/root/ONLINE_SERVICE/other/ferry/task_workspace/example/myserver/model/metrics/service.go:565 +0x128
icode.example.com/example/myserver/servers/httpserver.httpRouter.MetricsMiddleware.func4({0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730}, {0x1f854e0, 0xc00cf2a828})
/root/ONLINE_SERVICE/other/ferry/task_workspace/example/myserver/httpserver/middleware/custom_metric.go:47 +0x243
icode.example.com/example/framework/ghttp.middleWareFuncs.Next({0xc0048aa5a0?, 0x1ca5d25?, 0xc01d933438?}, {0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730})
/root/go/pkg/mod/icode.example.com/example/framework/[email protected]/middleware.go:65 +0xdd
icode.example.com/example/myserver/servers/httpserver.httpRouter.DebugMiddleware.func2({0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730}, {0x1f854e0, 0xc00cf2a810})
/root/ONLINE_SERVICE/other/ferry/task_workspace/example/myserver/httpserver/middleware/debug.go:27 +0x115
icode.example.com/example/framework/ghttp.middleWareFuncs.Next({0xc0048aa598?, 0x10?, 0x1?}, {0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730})
/root/go/pkg/mod/icode.example.com/example/framework/[email protected]/middleware.go:65 +0xdd
icode.example.com/example/framework/metrics/gmetrics.(*Metrics).CollectHTTPServer.(*HTTPServerCollector).MiddleWareFunc.(*HTTPServerCollector).init.func1({0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730}, {0x1f854e0, 0xc00cf2a7f8})
/root/go/pkg/mod/icode.example.com/example/framework/[email protected]/gmetrics/httpserver.go:188 +0x2d7
icode.example.com/example/framework/ghttp.middleWareFuncs.Next({0xc0048aa590?, 0xc0053952c0?, 0xc021f77ee8?}, {0x1f9dda8, 0xc005395560}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730})
/root/go/pkg/mod/icode.example.com/example/framework/[email protected]/middleware.go:65 +0xdd
icode.example.com/example/myserver/servers/httpserver.httpRouter.NewAccessLogMiddleWareFunc.func1({0x1f9dda8?, 0xc005395350?}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730}, {0x1f854e0, 0xc00cf2a7e0})
/root/ONLINE_SERVICE/other/ferry/task_workspace/example/myserver/httpserver/middleware/custom_logger.go:36 +0x1e2
icode.example.com/example/framework/ghttp.middleWareFuncs.Next({0xc0048aa588?, 0x47bddc?, 0x7f726433cf40?}, {0x1f9dda8, 0xc005395350}, {0x1fa7720, 0xc0053952f0}, {0x1fb2258, 0xc0287d6730})
/root/go/pkg/mod/icode.example.com/example/framework/[email protected]/middleware.go:65 +0xdd
icode.example.com/bai] api[/llb/searchbar/hotel] cache[] api_len[20] cache_len[0] status_len[4] api_bytes[[47 108 108 98 47 115 101 97 114 99 104 98 97 114 47 104 111 116 101 108]] cache_bytes[[]] status_bytes[[102 97 105 108]] message[err: runtime error: invalid memory address or nil pointer dereference]