PMM-14368 PostgreSQL 18 support (#336)

Introduce support for PostgreSQL 18, adding enhanced metrics for checkpointer, vacuum/analyze timings, and parallel worker activity. Implement new collectors for `pg_stat_io` and `pg_backend_memory_contexts` with version-specific queries, along with comprehensive tests. Update README and CI to reflect PostgreSQL 18 compatibility.

Refactored IO and statistics queries to accommodate PostgreSQL 18 features, including byte-level statistics and WAL I/O. Updated tests to validate new columns and ensure query expectation checks are correctly enforced. Improved test coverage across different PostgreSQL versions. Updated CI matrix.
---------

Co-authored-by: Nurlan Moldomurov <[email protected]>

Author: artemgavrilov

.github/workflows/go.yml

@@ -14,11 +14,12 @@ jobs:
     strategy:
       matrix:
         postgresql-image:
-          - postgres:10
-          - postgres:11
-          - postgres:12
           - postgres:13
           - postgres:14
+          - postgres:15
+          - postgres:16
+          - postgres:17
+          - postgres:18
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code

CHANGELOG.md

@@ -1,3 +1,12 @@
+## [Unreleased]
+
+* [ENHANCEMENT] Add PostgreSQL 18 support:
+  * Add parallel worker activity metrics
+  * Add vacuum/analyze timing metrics
+  * Add enhanced checkpointer metrics
+  * Add `pg_stat_io` collector with byte statistics and WAL I/O activity tracking
+* [ENHANCEMENT] Update CI tested PostgreSQL versions to include PostgreSQL 18
+
 ## 0.15.0 / 2023-10-27
 
 * [ENHANCEMENT] Add 1kB and 2kB units #915

README.md

@@ -7,7 +7,7 @@
 
 Prometheus exporter for PostgreSQL server metrics.
 
-CI Tested PostgreSQL versions: `11`, `12`, `13`, `14`, `15`, `16`
+CI Tested PostgreSQL versions: `13`, `14`, `15`, `16`, `18`
 
 ## Quick Start
 This package is available for Docker:

collector/pg_stat_bgwriter.go

@@ -16,6 +16,7 @@ package collector
 import (
 	"context"
 	"database/sql"
+
 	"github.com/blang/semver/v4"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -46,6 +47,12 @@ var (
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	)
+	statBGWriterCheckpointsDoneDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_done_total"),
+		"Number of completed checkpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	)
 	statBGWriterCheckpointsReqTimeDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_write_time_total"),
 		"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds",
@@ -94,6 +101,12 @@ var (
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	)
+	statBGWriterCheckpointsSlruWrittenDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "slru_written_total"),
+		"Number of SLRU buffers written during checkpoints and restartpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	)
 	statBGWriterStatsResetDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "stats_reset_total"),
 		"Time at which these statistics were last reset",
@@ -114,6 +127,12 @@ var statBGWriter = map[string]*prometheus.Desc{
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	),
+	"percona_checkpoints_done": prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_done"),
+		"Number of completed checkpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	),
 	"percona_checkpoint_write_time": prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_write_time"),
 		"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds",
@@ -162,6 +181,12 @@ var statBGWriter = map[string]*prometheus.Desc{
 		[]string{"collector", "server"},
 		prometheus.Labels{},
 	),
+	"percona_slru_written": prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, bgWriterSubsystem, "slru_written"),
+		"Number of SLRU buffers written during checkpoints and restartpoints",
+		[]string{"collector", "server"},
+		prometheus.Labels{},
+	),
 	"percona_stats_reset": prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, bgWriterSubsystem, "stats_reset"),
 		"Time at which these statistics were last reset",
@@ -191,25 +216,50 @@ const statBGWriterQueryPost17 = `SELECT
 		,stats_reset
 	FROM pg_stat_bgwriter;`
 
-const statCheckpointerQuery = `SELECT
-		num_timed
-		,num_requested
-		,restartpoints_timed
-		,restartpoints_req
-		,restartpoints_done
-		,write_time
-		,sync_time
-		,buffers_written
-		,stats_reset
-	FROM pg_stat_checkpointer;`
+const statCheckpointerQueryPrePG18 = `
+		SELECT
+			num_timed,
+			num_requested,
+			NULL::bigint as num_done,
+			restartpoints_timed,
+			restartpoints_req,
+			restartpoints_done,
+			write_time,
+			sync_time,
+			buffers_written,
+			NULL::bigint as slru_written,
+			stats_reset
+		FROM pg_stat_checkpointer;`
+
+const statCheckpointerQueryPostPG18 = `
+		SELECT
+			num_timed,
+			num_requested,
+			num_done,
+			restartpoints_timed,
+			restartpoints_req,
+			restartpoints_done,
+			write_time,
+			sync_time,
+			buffers_written,
+			slru_written,
+			stats_reset
+		FROM pg_stat_checkpointer;`
 
 func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
 	db := instance.getDB()
 
-	var cpt, cpr, bcp, bc, mwc, bb, bbf, ba sql.NullInt64
+	var cpt, cpr, cpd, bcp, bc, mwc, bb, bbf, ba, slruw sql.NullInt64
 	var cpwt, cpst sql.NullFloat64
 	var sr sql.NullTime
 
+	v18plus := instance.version.GTE(semver.Version{Major: 18})
+
+	statCheckpointerQuery := statCheckpointerQueryPrePG18
+	if v18plus {
+		statCheckpointerQuery = statCheckpointerQueryPostPG18
+	}
+
 	if instance.version.GE(semver.MustParse("17.0.0")) {
 		row := db.QueryRowContext(ctx,
 			statBGWriterQueryPost17)
@@ -219,10 +269,9 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		}
 		var rpt, rpr, rpd sql.NullInt64
 		var csr sql.NullTime
-		// these variables are not used, but I left them here for reference
-		row = db.QueryRowContext(ctx,
-			statCheckpointerQuery)
-		err = row.Scan(&cpt, &cpr, &rpt, &rpr, &rpd, &cpwt, &cpst, &bcp, &csr)
+
+		row = db.QueryRowContext(ctx, statCheckpointerQuery)
+		err = row.Scan(&cpt, &cpr, &cpd, &rpt, &rpr, &rpd, &cpwt, &cpst, &bcp, &slruw, &csr)
 		if err != nil {
 			return err
 		}
@@ -257,6 +306,21 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+
+	cpdMetric := 0.0
+	if v18plus {
+		if cpd.Valid {
+			cpdMetric = float64(cpd.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriterCheckpointsDoneDesc,
+			prometheus.CounterValue,
+			cpdMetric,
+			"exporter",
+			instance.name,
+		)
+	}
+
 	cpwtMetric := 0.0
 	if cpwt.Valid {
 		cpwtMetric = float64(cpwt.Float64)
@@ -345,6 +409,19 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	slruwMetric := 0.0
+	if v18plus {
+		if slruw.Valid {
+			slruwMetric = float64(slruw.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriterCheckpointsSlruWrittenDesc,
+			prometheus.CounterValue,
+			slruwMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	srMetric := 0.0
 	if sr.Valid {
 		srMetric = float64(sr.Time.Unix())
@@ -373,6 +450,15 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	if v18plus {
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriter["percona_checkpoints_done"],
+			prometheus.CounterValue,
+			cpdMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	ch <- prometheus.MustNewConstMetric(
 		statBGWriter["percona_checkpoint_write_time"],
 		prometheus.CounterValue,
@@ -429,6 +515,15 @@ func (p PGStatBGWriterCollector) Update(ctx context.Context, instance *instance,
 		"exporter",
 		instance.name,
 	)
+	if v18plus {
+		ch <- prometheus.MustNewConstMetric(
+			statBGWriter["percona_slru_written"],
+			prometheus.CounterValue,
+			slruwMetric,
+			"exporter",
+			instance.name,
+		)
+	}
 	ch <- prometheus.MustNewConstMetric(
 		statBGWriter["percona_stats_reset"],
 		prometheus.CounterValue,