Merge pull request #9720 from dolthub/macneale4/conjoin-prep

[no-release-notes] conjoin prep

Author: macneale4

go/libraries/doltcore/remotesrv/grpc.go

@@ -192,6 +192,10 @@ func (rs *RemoteChunkStore) GetDownloadLocations(ctx context.Context, req *remot
 
 		var ranges []*remotesapi.RangeChunk
 		for h, r := range hashToRange {
+			if r.DictLength != 0 {
+				return nil, status.Error(codes.Unknown, "upgrade your dolt client; it is too old to read these files")
+			}
+
 			hCpy := h
 			ranges = append(ranges, &remotesapi.RangeChunk{Hash: hCpy[:], Offset: r.Offset, Length: r.Length})
 		}

go/store/nbs/archive.go

@@ -69,12 +69,12 @@ Footer:
        is that we load the larger footer for all versions, but ignore the first 4 bytes for versions 1 and 2.
 
    CheckSums:
-   +----------------------------+-------------------+----------------------+
-   | (64) Sha512 ByteSpan 1 - N | (64) Sha512 Index | (64) Sha512 Metadata |
-   +----------------------------+-------------------+----------------------+
-   - The Sha512 checksums of the ByteSpans, Index, and Metadata. Currently unused, but may be used in the future. Leaves
-     the opening to verify integrity manually at least, but could be used in the future to allow to break the file into
-     parts, and ensure we can verify the integrity of each part.
+   +------------------+
+   | (192) DEAD SPACE |
+   +------------------+
+   - The Sha512 checksums of the ByteSpans, Index, and Metadata were in initial design, but were never used. The ability
+     to calculate was thrown out to support archive conjoins, and leaving the 192 bytes in the foorer allows us to avoid
+     a format bump.
 
 Index:
    The Index is a concatenation of 4 sections, all of which are stored in raw form on disk.
@@ -131,6 +131,7 @@ Index:
 
      - Each Hash Suffix is the last 12 bytes of a Chunk in this Table.
      - Hash Suffix M must correspond to Prefix M and Chunk Record M
+     - The ID, or name, of the artifact is calculated using the truncated Sha512 (first 20 bytes) of the Suffix data.
 
 Metadata:
    The Metadata section is intended to be used for additional information about the Archive. This may include the version

go/store/nbs/archive_build.go

@@ -16,23 +16,21 @@ package nbs
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
 	"math/rand"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
-	"time"
 
 	"github.com/dolthub/gozstd"
 	lru "github.com/hashicorp/golang-lru/v2"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/dolthub/dolt/go/cmd/dolt/doltversion"
 	"github.com/dolthub/dolt/go/store/chunks"
 	"github.com/dolthub/dolt/go/store/hash"
 )
@@ -336,42 +334,10 @@ func convertTableFileToArchive(
 	return arcW.finalPath, name, chunkCount, err
 }
 
-func indexFinalize(arcW *archiveWriter, originTableFile hash.Hash) error {
-	err := arcW.finalizeByteSpans()
-	if err != nil {
-		return err
-	}
-
-	err = arcW.writeIndex()
-	if err != nil {
-		return err
-	}
-
-	meta := map[string]string{
-		amdkDoltVersion:    doltversion.Version,
-		amdkConversionTime: time.Now().UTC().Format(time.RFC3339),
-	}
-	if !originTableFile.IsEmpty() {
-		meta[amdkOriginTableFile] = originTableFile.String()
-	}
-
-	jsonData, err := json.Marshal(meta)
-	if err != nil {
-		return err
-	}
-
-	err = arcW.writeMetadata(jsonData)
-	if err != nil {
-		return err
-	}
-
-	return arcW.writeFooter()
-}
-
 // indexAndFinalizeArchive writes the index, metadata, and footer to the archive file. It also flushes the archive writer
 // to the directory provided. The name is calculated from the footer, and can be obtained by calling getName on the archive.
 func indexFinalizeFlushArchive(arcW *archiveWriter, archivePath string, originTableFile hash.Hash) error {
-	err := indexFinalize(arcW, originTableFile)
+	err := arcW.indexFinalize(originTableFile)
 	if err != nil {
 		return err
 	}
@@ -600,7 +566,13 @@ func verifyAllChunks(ctx context.Context, idx tableIndex, archiveFile string, pr
 		return err
 	}
 
-	index, err := newArchiveReader(ctx, fra, uint64(fra.sz), stats)
+	id := strings.TrimSuffix(filepath.Base(archiveFile), ArchiveFileSuffix)
+	name, ok := hash.MaybeParse(id)
+	if !ok {
+		return fmt.Errorf("invalid archive file path: %s", archiveFile)
+	}
+
+	index, err := newArchiveReader(ctx, fra, name, uint64(fra.sz), stats)
 	if err != nil {
 		return err
 	}

go/store/nbs/archive_chunk_source.go

@@ -17,8 +17,10 @@ package nbs
 import (
 	"context"
 	"encoding/binary"
+	"fmt"
 	"io"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -42,7 +44,7 @@ func newArchiveChunkSource(ctx context.Context, dir string, h hash.Hash, chunkCo
 		return archiveChunkSource{}, err
 	}
 
-	aRdr, err := newArchiveReader(ctx, fra, uint64(fra.sz), stats)
+	aRdr, err := newArchiveReader(ctx, fra, h, uint64(fra.sz), stats)
 	if err != nil {
 		return archiveChunkSource{}, err
 	}
@@ -64,7 +66,13 @@ func newAWSArchiveChunkSource(ctx context.Context,
 		return emptyChunkSource{}, err
 	}
 
-	aRdr, err := newArchiveReaderFromFooter(ctx, &s3TableReaderAt{s3, name}, sz, footer, stats)
+	id := strings.TrimSuffix(filepath.Base(name), ArchiveFileSuffix)
+	hashId, ok := hash.MaybeParse(id)
+	if !ok {
+		return emptyChunkSource{}, fmt.Errorf("invalid archive file path: %s", name)
+	}
+
+	aRdr, err := newArchiveReaderFromFooter(ctx, &s3TableReaderAt{s3, name}, hashId, sz, footer, stats)
 	if err != nil {
 		return emptyChunkSource{}, err
 	}

go/store/nbs/archive_reader.go

@@ -113,8 +113,8 @@ func (f archiveFooter) metadataSpan() byteSpan {
 	return byteSpan{offset: f.fileSize - f.actualFooterSize() - uint64(f.metadataSize), length: uint64(f.metadataSize)}
 }
 
-func newArchiveMetadata(ctx context.Context, reader tableReaderAt, fileSize uint64, stats *Stats) (*ArchiveMetadata, error) {
-	aRdr, err := newArchiveReader(ctx, reader, fileSize, stats)
+func newArchiveMetadata(ctx context.Context, reader tableReaderAt, name hash.Hash, fileSize uint64, stats *Stats) (*ArchiveMetadata, error) {
+	aRdr, err := newArchiveReader(ctx, reader, name, fileSize, stats)
 	if err != nil {
 		return nil, err
 	}
@@ -180,21 +180,21 @@ func newArchiveMetadata(ctx context.Context, reader tableReaderAt, fileSize uint
 	}, nil
 }
 
-func newArchiveReaderFromFooter(ctx context.Context, reader tableReaderAt, fileSz uint64, footer []byte, stats *Stats) (archiveReader, error) {
+func newArchiveReaderFromFooter(ctx context.Context, reader tableReaderAt, name hash.Hash, fileSz uint64, footer []byte, stats *Stats) (archiveReader, error) {
 	if uint64(len(footer)) != archiveFooterSize {
 		return archiveReader{}, errors.New("runtime error: invalid footer.")
 	}
 
-	ftr, err := buildFooter(fileSz, footer)
+	ftr, err := buildFooter(name, fileSz, footer)
 	if err != nil {
 		return archiveReader{}, err
 	}
 
 	return buildArchiveReader(ctx, reader, ftr, stats)
 }
 
-func newArchiveReader(ctx context.Context, reader tableReaderAt, fileSize uint64, stats *Stats) (archiveReader, error) {
-	footer, err := loadFooter(ctx, reader, fileSize, stats)
+func newArchiveReader(ctx context.Context, reader tableReaderAt, name hash.Hash, fileSize uint64, stats *Stats) (archiveReader, error) {
+	footer, err := loadFooter(ctx, reader, name, fileSize, stats)
 	if err != nil {
 		return archiveReader{}, fmt.Errorf("Failed to loadFooter: %w", err)
 	}
@@ -355,17 +355,17 @@ func newSectionReader(ctx context.Context, rd ReaderAtWithStats, off, len int64,
 	return io.NewSectionReader(readerAtWithStatsBridge{rd, ctx, stats}, off, len)
 }
 
-func loadFooter(ctx context.Context, reader ReaderAtWithStats, fileSize uint64, stats *Stats) (f archiveFooter, err error) {
+func loadFooter(ctx context.Context, reader ReaderAtWithStats, name hash.Hash, fileSize uint64, stats *Stats) (f archiveFooter, err error) {
 	section := newSectionReader(ctx, reader, int64(fileSize-archiveFooterSize), int64(archiveFooterSize), stats)
 	buf := make([]byte, archiveFooterSize)
 	_, err = io.ReadFull(section, buf)
 	if err != nil {
 		return
 	}
-	return buildFooter(fileSize, buf)
+	return buildFooter(name, fileSize, buf)
 }
 
-func buildFooter(fileSize uint64, buf []byte) (f archiveFooter, err error) {
+func buildFooter(name hash.Hash, fileSize uint64, buf []byte) (f archiveFooter, err error) {
 	f.formatVersion = buf[afrVersionOffset]
 	f.fileSignature = string(buf[afrSigOffset:])
 	// Verify File Signature
@@ -404,14 +404,7 @@ func buildFooter(fileSize uint64, buf []byte) (f archiveFooter, err error) {
 	f.metaCheckSum = sha512Sum(buf[afrMetaChkSumOffset : afrMetaChkSumOffset+sha512.Size])
 	f.fileSize = fileSize
 
-	// calculate the hash of the footer. We don't currently verify that this is what was used to load the content.
-	sha := sha512.New()
-	if smallFooter {
-		buf = buf[4:]
-	}
-
-	sha.Write(buf)
-	f.hash = hash.New(sha.Sum(nil)[:hash.ByteLen])
+	f.hash = name
 
 	return
 }
@@ -583,22 +576,6 @@ func (ar archiveReader) getMetadata(ctx context.Context, stats *Stats) ([]byte,
 	return ar.readByteSpan(ctx, ar.footer.metadataSpan(), stats)
 }
 
-// verifyDataCheckSum verifies the checksum of the data section of the archive. Note - this requires a fully read of
-// the data section, which could be sizable.
-func (ar archiveReader) verifyDataCheckSum(ctx context.Context, stats *Stats) error {
-	return verifyCheckSum(ctx, ar.reader, ar.footer.dataSpan(), ar.footer.dataCheckSum, stats)
-}
-
-// verifyIndexCheckSum verifies the checksum of the index section of the archive.
-func (ar archiveReader) verifyIndexCheckSum(ctx context.Context, stats *Stats) error {
-	return verifyCheckSum(ctx, ar.reader, ar.footer.totalIndexSpan(), ar.footer.indexCheckSum, stats)
-}
-
-// verifyMetaCheckSum verifies the checksum of the metadata section of the archive.
-func (ar archiveReader) verifyMetaCheckSum(ctx context.Context, stats *Stats) error {
-	return verifyCheckSum(ctx, ar.reader, ar.footer.metadataSpan(), ar.footer.metaCheckSum, stats)
-}
-
 func (ar archiveReader) iterate(ctx context.Context, cb func(chunks.Chunk) error, stats *Stats) error {
 	for i := uint32(0); i < ar.footer.chunkCount; i++ {
 		var hasBytes [hash.ByteLen]byte