Merge pull request #83 from nlnwa/feat/warc-date-precision

feat: Write WARC-Date with up to nanosecond precision

Author: maeb

go.mod

@@ -1,25 +1,23 @@
 module github.com/nlnwa/gowarc/v2
 
-go 1.22.0
+go 1.23.0
 
-toolchain go1.22.7
+toolchain go1.23.7
 
 require (
 	github.com/google/uuid v1.6.0
-	github.com/klauspost/compress v1.17.11
-	github.com/nlnwa/whatwg-url v0.5.0
-	github.com/prometheus/prometheus v0.55.0
-	github.com/stretchr/testify v1.9.0
+	github.com/klauspost/compress v1.18.0
+	github.com/nlnwa/whatwg-url v0.6.2
+	github.com/prometheus/prometheus v0.302.1
+	github.com/stretchr/testify v1.10.0
 )
 
 require (
-	github.com/bits-and-blooms/bitset v1.14.3 // indirect
+	github.com/bits-and-blooms/bitset v1.22.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/kr/pretty v0.3.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -42,7 +42,7 @@ type HttpResponseBlock interface {
 	HttpHeader() *http.Header
 }
 
-var missingEndOfHeaders = errors.New("missing line separator at end of http headers")
+var errMissingEndOfHeaders = errors.New("missing line separator at end of http headers")
 
 type httpRequestBlock struct {
 	opts                *warcRecordOptions
@@ -335,7 +335,7 @@ func headerBytes(r buffer) ([]byte, error) {
 	}
 	var err error
 	if !sepFound {
-		err = missingEndOfHeaders
+		err = errMissingEndOfHeaders
 	}
 	return result.Bytes(), err
 }
@@ -369,7 +369,7 @@ func newHttpBlock(opts *warcRecordOptions, wf *WarcFields, r io.Reader, blockDig
 		}
 	}
 
-	if herr == missingEndOfHeaders && opts.fixSyntaxErrors {
+	if herr == errMissingEndOfHeaders && opts.fixSyntaxErrors {
 		// Fix header and update content-length field
 		hb = append(hb, '\r', '\n')
 		l, _ := wf.GetInt64(ContentLength)
@@ -396,7 +396,7 @@ func newHttpBlock(opts *warcRecordOptions, wf *WarcFields, r io.Reader, blockDig
 			payloadDigest:   payloadDigest,
 		}
 
-		if herr == missingEndOfHeaders && !opts.fixSyntaxErrors {
+		if herr == errMissingEndOfHeaders && !opts.fixSyntaxErrors {
 			// We have to fix the header for parsing even if we don't fix the record
 			hb = append(hb, '\r', '\n')
 		}
@@ -418,7 +418,7 @@ func newHttpBlock(opts *warcRecordOptions, wf *WarcFields, r io.Reader, blockDig
 			payloadDigest:   payloadDigest,
 		}
 
-		if herr == missingEndOfHeaders && !opts.fixSyntaxErrors {
+		if herr == errMissingEndOfHeaders && !opts.fixSyntaxErrors {
 			// We have to fix the header for parsing even if we don't fix the record
 			hb = append(hb, '\r', '\n')
 		}

httpblock.go

@@ -34,14 +34,6 @@ func From14ToTime(s string) (time.Time, error) {
 	return t, err
 }
 
-func UTC(t time.Time) time.Time {
-	return t.In(time.UTC)
-}
-
 func UTC14(t time.Time) string {
-	return t.In(time.UTC).Format("20060102150405")
-}
-
-func UTCW3cIso8601(t time.Time) string {
-	return t.In(time.UTC).Format(time.RFC3339)
+	return t.UTC().Format("20060102150405")
 }

internal/timestamp/timestamp.go

@@ -75,26 +75,10 @@ func TestFrom14ToTimeSucceedsOnValidString(t *testing.T) {
 	}
 }
 
-func TestUTC(t *testing.T) {
-	data := createTestData()
-
-	if ts := timestamp.UTC(data.time); ts != data.time {
-		t.Errorf("UTC() = %s, want %s", ts, data.time)
-	}
-}
-
 func TestUTC14(t *testing.T) {
 	data := createTestData()
 
 	if ts := timestamp.UTC14(data.time); ts != data.gowarc14Date {
 		t.Errorf("UTC14() = %s, want %s", ts, data.gowarc14Date)
 	}
 }
-
-func TestUTCW3cIso8601(t *testing.T) {
-	data := createTestData()
-
-	if ts := timestamp.UTCW3cIso8601(data.time); ts != data.iso8601Date {
-		t.Errorf("UTCW3cIso8601() = %s, want %s", ts, data.iso8601Date)
-	}
-}

internal/timestamp/timestamp_test.go

@@ -80,7 +80,11 @@ func (rb *recordBuilder) AddWarcHeaderInt64(name string, value int64) {
 
 // AddWarcHeaderTime adds a new WARC header field with the given name and a time.Time value to the record
 func (rb *recordBuilder) AddWarcHeaderTime(name string, value time.Time) {
-	rb.headers.AddTime(name, value)
+	if rb.version.id == V1_0.id {
+		rb.headers.AddTime(name, value)
+	} else {
+		rb.headers.AddTimeNano(name, value)
+	}
 }
 
 // Close releases resources used by the WarcRecordBuilder

recordbuilder.go

@@ -195,7 +195,7 @@ func TestRecordBuilder(t *testing.T) {
 					"Referer: http://example.com/foo.html\n" +
 					"Connection: close\n" +
 					"User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36\n",
-				&Validation{missingEndOfHeaders},
+				&Validation{errMissingEndOfHeaders},
 				true,
 			},
 			false,

recordbuilder_test.go

@@ -836,7 +836,7 @@ func Test_unmarshaler_Unmarshal(t *testing.T) {
 					"Referer: http://www.archive.org/\r\n" +
 					"Host: www.archive.org\r\n" +
 					"Cookie: PHPSESSID=009d7bb11022f80605aa87e18224d824\r\n",
-				&Validation{missingEndOfHeaders},
+				&Validation{errMissingEndOfHeaders},
 				true,
 			},
 			0,

unmarshaler_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -141,6 +142,14 @@ func (wf *WarcFields) AddTime(name string, value time.Time) {
 	wf.Add(name, value.UTC().Format(time.RFC3339))
 }
 
+// AddTimeNano adds the key, value pair to the header.
+// It appends to any existing values associated with key. The key is case-insensitive.
+//
+// The value is formatted as RFC 3339 with up to nanosecond precision.
+func (wf *WarcFields) AddTimeNano(name string, value time.Time) {
+	wf.Add(name, value.UTC().Format(time.RFC3339Nano))
+}
+
 // AddId adds the key, value pair to the header.
 // It appends to any existing values associated with key. The key is case-insensitive.
 //
@@ -163,7 +172,7 @@ func (wf *WarcFields) Set(name string, value string) {
 	for idx, nv := range *wf {
 		if nv.Name == name {
 			if isSet {
-				*wf = append((*wf)[:idx], (*wf)[idx+1:]...)
+				*wf = slices.Delete(*wf, idx, idx+1)
 			} else {
 				nv.Value = value
 				isSet = true

warcfields.go

@@ -25,8 +25,8 @@ import (
 )
 
 var (
-	colon        = []byte{':'}
-	endOfHeaders = errors.New("EOH")
+	colon           = []byte{':'}
+	errEndOfHeaders = errors.New("EOH")
 )
 
 type warcfieldsParser struct {
@@ -66,7 +66,7 @@ func (p *warcfieldsParser) readLine(r *bufio.Reader, pos *position) (line []byte
 	line, err = r.ReadBytes('\n')
 	if err != nil {
 		if err == io.EOF {
-			err = endOfHeaders
+			err = errEndOfHeaders
 		}
 		line = bytes.Trim(line, sphtcrlf)
 		return
@@ -97,7 +97,7 @@ func (p *warcfieldsParser) Parse(r *bufio.Reader, validation *Validation, pos *p
 	for {
 		line, nc, err := p.readLine(r, pos.incrLineNumber())
 		if err != nil {
-			if err == endOfHeaders {
+			if err == errEndOfHeaders {
 				eoh = true
 				if len(line) == 0 {
 					return &wf, nil