Merge pull request lightningnetwork#9017 from jharveyb/log_comress_zstd

log: bump logrotate dep, switch to zstd compressor

Author: guggero

build/log.go

@@ -36,6 +36,32 @@ func (t LogType) String() string {
 	}
 }
 
+// Declare the supported log file compressors as exported consts for easier use
+// from other projects.
+const (
+	// Gzip is the default compressor.
+	Gzip = "gzip"
+
+	// Zstd is a modern compressor that compresses better than Gzip, in less
+	// time.
+	Zstd = "zstd"
+)
+
+// logCompressors maps the identifier for each supported compression algorithm
+// to the extension used for the compressed log files.
+var logCompressors = map[string]string{
+	Gzip: "gz",
+	Zstd: "zst",
+}
+
+// SuportedLogCompressor returns whether or not logCompressor is a supported
+// compression algorithm for log files.
+func SuportedLogCompressor(logCompressor string) bool {
+	_, ok := logCompressors[logCompressor]
+
+	return ok
+}
+
 // LogWriter is a stub type whose behavior can be changed using the build flags
 // "stdlog" and "nolog". The default behavior is to write to both stdout and the
 // RotatorPipe. Passing "stdlog" will cause it only to write to stdout, and

build/logrotator.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+	"compress/gzip"
 	"fmt"
 	"io"
 	"os"
@@ -9,6 +10,7 @@ import (
 
 	"github.com/btcsuite/btclog"
 	"github.com/jrick/logrotate/rotator"
+	"github.com/klauspost/compress/zstd"
 )
 
 // RotatingLogWriter is a wrapper around the LogWriter that supports log file
@@ -58,8 +60,8 @@ func (r *RotatingLogWriter) RegisterSubLogger(subsystem string,
 // InitLogRotator initializes the log file rotator to write logs to logFile and
 // create roll files in the same directory. It should be called as early on
 // startup and possible and must be closed on shutdown by calling `Close`.
-func (r *RotatingLogWriter) InitLogRotator(logFile string, maxLogFileSize int,
-	maxLogFiles int) error {
+func (r *RotatingLogWriter) InitLogRotator(logFile, logCompressor string,
+	maxLogFileSize int, maxLogFiles int) error {
 
 	logDir, _ := filepath.Split(logFile)
 	err := os.MkdirAll(logDir, 0700)
@@ -73,6 +75,27 @@ func (r *RotatingLogWriter) InitLogRotator(logFile string, maxLogFileSize int,
 		return fmt.Errorf("failed to create file rotator: %w", err)
 	}
 
+	// Reject unknown compressors.
+	if !SuportedLogCompressor(logCompressor) {
+		return fmt.Errorf("unknown log compressor: %v", logCompressor)
+	}
+
+	var c rotator.Compressor
+	switch logCompressor {
+	case Gzip:
+		c = gzip.NewWriter(nil)
+
+	case Zstd:
+		c, err = zstd.NewWriter(nil)
+		if err != nil {
+			return fmt.Errorf("failed to create zstd compressor: "+
+				"%w", err)
+		}
+	}
+
+	// Apply the compressor and its file suffix to the log rotator.
+	r.logRotator.SetCompressor(c, logCompressors[logCompressor])
+
 	// Run rotator as a goroutine now but make sure we catch any errors
 	// that happen in case something with the rotation goes wrong during
 	// runtime (like running out of disk space or not being allowed to

config.go

@@ -59,6 +59,7 @@ const (
 	defaultLogLevel           = "info"
 	defaultLogDirname         = "logs"
 	defaultLogFilename        = "lnd.log"
+	defaultLogCompressor      = build.Gzip
 	defaultRPCPort            = 10009
 	defaultRESTPort           = 8080
 	defaultPeerPort           = 9735
@@ -315,6 +316,7 @@ type Config struct {
 	ReadMacPath     string        `long:"readonlymacaroonpath" description:"Path to write the read-only macaroon for lnd's RPC and REST services if it doesn't exist"`
 	InvoiceMacPath  string        `long:"invoicemacaroonpath" description:"Path to the invoice-only macaroon for lnd's RPC and REST services if it doesn't exist"`
 	LogDir          string        `long:"logdir" description:"Directory to log output."`
+	LogCompressor   string        `long:"logcompressor" description:"Compression algorithm to use when rotating logs." choice:"gzip" choice:"zstd"`
 	MaxLogFiles     int           `long:"maxlogfiles" description:"Maximum logfiles to keep (0 for no rotation)"`
 	MaxLogFileSize  int           `long:"maxlogfilesize" description:"Maximum logfile size in MB"`
 	AcceptorTimeout time.Duration `long:"acceptortimeout" description:"Time after which an RPCAcceptor will time out and return false if it hasn't yet received a response"`
@@ -560,6 +562,7 @@ func DefaultConfig() Config {
 		LetsEncryptDir:    defaultLetsEncryptDir,
 		LetsEncryptListen: defaultLetsEncryptListen,
 		LogDir:            defaultLogDir,
+		LogCompressor:     defaultLogCompressor,
 		MaxLogFiles:       defaultMaxLogFiles,
 		MaxLogFileSize:    defaultMaxLogFileSize,
 		AcceptorTimeout:   defaultAcceptorTimeout,
@@ -1446,11 +1449,16 @@ func ValidateConfig(cfg Config, interceptor signal.Interceptor, fileParser,
 		os.Exit(0)
 	}
 
+	if !build.SuportedLogCompressor(cfg.LogCompressor) {
+		return nil, mkErr("invalid log compressor: %v",
+			cfg.LogCompressor)
+	}
+
 	// Initialize logging at the default logging level.
 	SetupLoggers(cfg.LogWriter, interceptor)
 	err = cfg.LogWriter.InitLogRotator(
 		filepath.Join(cfg.LogDir, defaultLogFilename),
-		cfg.MaxLogFileSize, cfg.MaxLogFiles,
+		cfg.LogCompressor, cfg.MaxLogFileSize, cfg.MaxLogFiles,
 	)
 	if err != nil {
 		str := "log rotation setup failed: %v"

docs/release-notes/release-notes-0.19.0.md

@@ -36,6 +36,8 @@
 # Improvements
 ## Functional Updates
 
+* [Allow](https://github.com/lightningnetwork/lnd/pull/9017) the compression of logs during rotation with ZSTD via the `logcompressor` startup argument.
+
 ## RPC Updates
 
 ## lncli Updates
@@ -45,6 +47,8 @@
 ## Breaking Changes
 ## Performance Improvements
 
+* Log rotation can now use ZSTD 
+
 # Technical and Architectural Updates
 ## BOLT Spec Updates
 

go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad
 	github.com/jedib0t/go-pretty/v6 v6.2.7
 	github.com/jessevdk/go-flags v1.4.0
-	github.com/jrick/logrotate v1.0.0
+	github.com/jrick/logrotate v1.1.2
 	github.com/kkdai/bstream v1.0.0
 	github.com/lightninglabs/neutrino v0.16.1-0.20240425105051-602843d34ffd
 	github.com/lightninglabs/neutrino/cache v1.1.2
@@ -118,6 +118,7 @@ require (
 	github.com/json-iterator/go v1.1.11 // indirect
 	github.com/juju/loggo v0.0.0-20210728185423-eebad3a902c4 // indirect
 	github.com/juju/testing v0.0.0-20220203020004-a0ff61f03494 // indirect
+	github.com/klauspost/compress v1.17.9
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

go.sum

@@ -384,8 +384,9 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS
 github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
-github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
+github.com/jrick/logrotate v1.1.2 h1:6ePk462NCX7TfKtNp5JJ7MbA2YIslkpfgP03TlTYMN0=
+github.com/jrick/logrotate v1.1.2/go.mod h1:f9tdWggSVK3iqavGpyvegq5IhNois7KXmasU6/N96OQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
@@ -418,6 +419,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/kkdai/bstream v1.0.0 h1:Se5gHwgp2VT2uHfDrkbbgbgEvV9cimLELwrPJctSjg8=
 github.com/kkdai/bstream v1.0.0/go.mod h1:FDnDOHt5Yx4p3FaHcioFT0QjDOtgUpvjeZqAs+NVZZA=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=

sample-lnd.conf

@@ -36,6 +36,9 @@
 ; Max log file size in MB before it is rotated.
 ; maxlogfilesize=10
 
+; Compression algorithm to use when rotating logs.
+; logcompressor=gzip
+
 ; Time after which an RPCAcceptor will time out and return false if
 ; it hasn't yet received a response.
 ; acceptortimeout=15s