File system image structure, version 2, part 1

Author: mhx

include/dwarfs/fstypes.h

@@ -32,6 +32,7 @@
 #include <folly/small_vector.h>
 
 #include "dwarfs/block_compressor.h" // TODO: or the other way round?
+#include "dwarfs/checksum.h"
 
 namespace dwarfs {
 
@@ -64,7 +65,7 @@ struct iovec_read_buf {
 };
 
 constexpr uint8_t MAJOR_VERSION = 2;
-constexpr uint8_t MINOR_VERSION = 1;
+constexpr uint8_t MINOR_VERSION = 2;
 
 enum class section_type : uint16_t {
   BLOCK = 0,
@@ -93,6 +94,26 @@ struct section_header {
   void dump(std::ostream& os) const;
 };
 
+struct section_header_v2 {
+  char magic[6];            //  [0] "DWARFS" / file_header no longer needed
+  uint8_t major;            //  [6] major version
+  uint8_t minor;            //  [7] minor version
+  uint8_t sha2_512_256[32]; //  [8] SHA2-512/256 starting from next field
+  uint64_t xxh3_64;         // [40] XXH3-64 starting from next field
+  uint32_t number;          // [48] section number
+  uint16_t type;            // [52] section type
+  uint16_t compression;     // [54] compression
+  uint64_t length;          // [56] length of section
+
+  static_assert(checksum::digest_size(checksum::algorithm::XXH3_64) ==
+                sizeof(xxh3_64));
+  static_assert(checksum::digest_size(checksum::algorithm::SHA2_512_256) ==
+                sizeof(sha2_512_256));
+
+  std::string to_string() const;
+  void dump(std::ostream& os) const;
+};
+
 std::string get_compression_name(compression_type type);
 
 std::string get_section_name(section_type type);

src/dwarfs/filesystem_writer.cpp

@@ -31,6 +31,7 @@
 #include <folly/system/ThreadName.h>
 
 #include "dwarfs/block_compressor.h"
+#include "dwarfs/checksum.h"
 #include "dwarfs/filesystem_writer.h"
 #include "dwarfs/fstypes.h"
 #include "dwarfs/logger.h"
@@ -155,7 +156,6 @@ class filesystem_writer_ : public filesystem_writer::impl {
   template <typename T>
   void write(const T& obj);
   void write(const std::vector<uint8_t>& data);
-  void write_file_header();
   void writer_thread();
   size_t mem_used() const;
 
@@ -172,6 +172,7 @@ class filesystem_writer_ : public filesystem_writer::impl {
   std::condition_variable cond_;
   volatile bool flush_;
   std::thread writer_thread_;
+  uint32_t section_number_{0};
 };
 
 template <typename LoggerPolicy>
@@ -188,9 +189,7 @@ filesystem_writer_<LoggerPolicy>::filesystem_writer_(
     , max_queue_size_(max_queue_size)
     , log_(lgr)
     , flush_(false)
-    , writer_thread_(&filesystem_writer_::writer_thread, this) {
-  write_file_header();
-}
+    , writer_thread_(&filesystem_writer_::writer_thread, this) {}
 
 template <typename LoggerPolicy>
 filesystem_writer_<LoggerPolicy>::~filesystem_writer_() noexcept {
@@ -268,24 +267,31 @@ void filesystem_writer_<LoggerPolicy>::write(const std::vector<uint8_t>& data) {
   write(reinterpret_cast<const char*>(&data[0]), data.size());
 }
 
-template <typename LoggerPolicy>
-void filesystem_writer_<LoggerPolicy>::write_file_header() {
-  file_header hdr;
-  ::memcpy(&hdr.magic[0], "DWARFS", 6);
-  hdr.major = MAJOR_VERSION;
-  hdr.minor = MINOR_VERSION;
-  write(hdr);
-}
-
 template <typename LoggerPolicy>
 void filesystem_writer_<LoggerPolicy>::write(section_type type,
                                              compression_type compression,
                                              const std::vector<uint8_t>& data) {
-  section_header sh;
-  sh.type = type;
-  sh.compression = compression;
-  sh.unused = 0;
+  section_header_v2 sh;
+  ::memcpy(&sh.magic[0], "DWARFS", 6);
+  sh.major = MAJOR_VERSION;
+  sh.minor = MINOR_VERSION;
+  sh.number = section_number_++;
+  sh.type = static_cast<uint16_t>(type);
+  sh.compression = static_cast<uint16_t>(compression);
   sh.length = data.size();
+
+  checksum xxh(checksum::algorithm::XXH3_64);
+  xxh.update(&sh.number,
+             sizeof(section_header_v2) - offsetof(section_header_v2, number));
+  xxh.update(data.data(), data.size());
+  DWARFS_CHECK(xxh.finalize(&sh.xxh3_64), "XXH3-64 checksum failed");
+
+  checksum sha(checksum::algorithm::SHA2_512_256);
+  sha.update(&sh.xxh3_64,
+             sizeof(section_header_v2) - offsetof(section_header_v2, xxh3_64));
+  sha.update(data.data(), data.size());
+  DWARFS_CHECK(sha.finalize(&sh.sha2_512_256), "SHA512/256 checksum failed");
+
   write(sh);
   write(data);