Add experimental stream archive support

src/archive.cpp

@@ -15,7 +15,6 @@ namespace
 {
 
 constexpr char kArchiveMagic[kMagicSize] = {'M', '2', 'P', 'A', 'C', 'K', '2', '\0'};
-constexpr std::uint32_t kArchiveVersion = 1;
 constexpr std::size_t kCompressionMinSavingsBytes = 64;
 constexpr std::size_t kCompressionMinSavingsPercent = 5;
 
@@ -46,6 +45,35 @@ Compression select_compression_mode(
     return Compression::Zstd;
 }
 
+bool supports_archive_version(std::uint32_t version)
+{
+    return version == kArchiveVersionAead || version == kArchiveVersionStream;
+}
+
+std::vector<std::uint8_t> decrypt_payload_for_entry(
+    std::uint32_t version,
+    const std::vector<std::uint8_t>& ciphertext,
+    const ManifestEntry& entry,
+    const std::array<std::uint8_t, kAeadKeySize>& master_key)
+{
+    if (version == kArchiveVersionAead)
+    {
+        return decrypt_payload(ciphertext, master_key, entry.nonce, entry.path);
+    }
+
+    if (version == kArchiveVersionStream)
+    {
+        auto payload = decrypt_payload_stream(ciphertext, master_key, entry.nonce);
+        if (hash_bytes(payload) != entry.payload_hash)
+        {
+            fail("Payload hash mismatch: " + entry.path);
+        }
+        return payload;
+    }
+
+    fail("Unsupported archive version");
+}
+
 }  // namespace
 
 std::string normalize_path(const std::filesystem::path& root, const std::filesystem::path& file)
@@ -74,7 +102,7 @@ std::vector<std::filesystem::path> collect_files(const std::filesystem::path& ro
     return files;
 }
 
-std::vector<std::uint8_t> serialize_manifest(const std::vector<ManifestEntry>& entries)
+std::vector<std::uint8_t> serialize_manifest(const std::vector<ManifestEntry>& entries, std::uint32_t version)
 {
     std::vector<std::uint8_t> bytes;
     const ManifestFixedHeader fixed {
@@ -90,23 +118,45 @@ std::vector<std::uint8_t> serialize_manifest(const std::vector<ManifestEntry>& e
             fail("Path too long for manifest entry: " + entry.path);
         }
 
-        ManifestEntryFixed pod {};
+        if (version == kArchiveVersionAead)
-        pod.path_size = static_cast<std::uint16_t>(entry.path.size());
+        {
-        pod.compression = static_cast<std::uint8_t>(entry.compression);
+            ManifestEntryFixed pod {};
-        pod.flags = 0;
+            pod.path_size = static_cast<std::uint16_t>(entry.path.size());
-        pod.data_offset = entry.data_offset;
+            pod.compression = static_cast<std::uint8_t>(entry.compression);
-        pod.original_size = entry.original_size;
+            pod.flags = 0;
-        pod.stored_size = entry.stored_size;
+            pod.data_offset = entry.data_offset;
-        std::memcpy(pod.nonce, entry.nonce.data(), entry.nonce.size());
+            pod.original_size = entry.original_size;
-        std::memcpy(pod.plaintext_hash, entry.plaintext_hash.data(), entry.plaintext_hash.size());
+            pod.stored_size = entry.stored_size;
-        append_pod(bytes, pod);
+            std::memcpy(pod.nonce, entry.nonce.data(), entry.nonce.size());
+            std::memcpy(pod.plaintext_hash, entry.plaintext_hash.data(), entry.plaintext_hash.size());
+            append_pod(bytes, pod);
+        }
+        else if (version == kArchiveVersionStream)
+        {
+            ManifestEntryFixedV2 pod {};
+            pod.path_size = static_cast<std::uint16_t>(entry.path.size());
+            pod.compression = static_cast<std::uint8_t>(entry.compression);
+            pod.flags = 0;
+            pod.data_offset = entry.data_offset;
+            pod.original_size = entry.original_size;
+            pod.stored_size = entry.stored_size;
+            std::memcpy(pod.nonce, entry.nonce.data(), entry.nonce.size());
+            std::memcpy(pod.payload_hash, entry.payload_hash.data(), entry.payload_hash.size());
+            std::memcpy(pod.plaintext_hash, entry.plaintext_hash.data(), entry.plaintext_hash.size());
+            append_pod(bytes, pod);
+        }
+        else
+        {
+            fail("Unsupported archive version");
+        }
+
         bytes.insert(bytes.end(), entry.path.begin(), entry.path.end());
     }
 
     return bytes;
 }
 
-std::vector<ManifestEntry> parse_manifest(const std::vector<std::uint8_t>& bytes)
+std::vector<ManifestEntry> parse_manifest(const std::vector<std::uint8_t>& bytes, std::uint32_t version)
 {
     std::size_t offset = 0;
     const auto fixed = read_pod<ManifestFixedHeader>(bytes, offset);
@@ -116,21 +166,44 @@ std::vector<ManifestEntry> parse_manifest(const std::vector<std::uint8_t>& bytes
 
     for (std::uint32_t i = 0; i < fixed.entry_count; ++i)
     {
-        const auto pod = read_pod<ManifestEntryFixed>(bytes, offset);
+        ManifestEntry entry;
-        if (offset + pod.path_size > bytes.size())
+        std::uint16_t path_size = 0;
+
+        if (version == kArchiveVersionAead)
+        {
+            const auto pod = read_pod<ManifestEntryFixed>(bytes, offset);
+            path_size = pod.path_size;
+            entry.compression = static_cast<Compression>(pod.compression);
+            entry.data_offset = pod.data_offset;
+            entry.original_size = pod.original_size;
+            entry.stored_size = pod.stored_size;
+            std::memcpy(entry.nonce.data(), pod.nonce, entry.nonce.size());
+            std::memcpy(entry.plaintext_hash.data(), pod.plaintext_hash, entry.plaintext_hash.size());
+        }
+        else if (version == kArchiveVersionStream)
+        {
+            const auto pod = read_pod<ManifestEntryFixedV2>(bytes, offset);
+            path_size = pod.path_size;
+            entry.compression = static_cast<Compression>(pod.compression);
+            entry.data_offset = pod.data_offset;
+            entry.original_size = pod.original_size;
+            entry.stored_size = pod.stored_size;
+            std::memcpy(entry.nonce.data(), pod.nonce, entry.nonce.size());
+            std::memcpy(entry.payload_hash.data(), pod.payload_hash, entry.payload_hash.size());
+            std::memcpy(entry.plaintext_hash.data(), pod.plaintext_hash, entry.plaintext_hash.size());
+        }
+        else
+        {
+            fail("Unsupported archive version");
+        }
+
+        if (offset + path_size > bytes.size())
         {
             fail("Manifest path data exceeds buffer");
         }
 
-        ManifestEntry entry;
+        entry.path.assign(reinterpret_cast<const char*>(bytes.data() + offset), path_size);
-        entry.path.assign(reinterpret_cast<const char*>(bytes.data() + offset), pod.path_size);
+        offset += path_size;
-        offset += pod.path_size;
-        entry.compression = static_cast<Compression>(pod.compression);
-        entry.data_offset = pod.data_offset;
-        entry.original_size = pod.original_size;
-        entry.stored_size = pod.stored_size;
-        std::memcpy(entry.nonce.data(), pod.nonce, entry.nonce.size());
-        std::memcpy(entry.plaintext_hash.data(), pod.plaintext_hash, entry.plaintext_hash.size());
         entries.push_back(std::move(entry));
     }
 
@@ -171,10 +244,23 @@ BuildResult build_archive(
         entry.data_offset = payload_bytes.size();
         entry.original_size = plain.size();
         entry.nonce = random_nonce();
+        const auto& payload = entry.compression == Compression::Zstd ? compressed : plain;
+        entry.payload_hash = hash_bytes(payload);
         entry.plaintext_hash = hash_bytes(plain);
 
-        const auto& payload = entry.compression == Compression::Zstd ? compressed : plain;
+        std::vector<std::uint8_t> encrypted;
-        const auto encrypted = encrypt_payload(payload, require_master_key(keys), entry.nonce, entry.path);
+        if (kCurrentArchiveVersion == kArchiveVersionAead)
+        {
+            encrypted = encrypt_payload(payload, require_master_key(keys), entry.nonce, entry.path);
+        }
+        else if (kCurrentArchiveVersion == kArchiveVersionStream)
+        {
+            encrypted = encrypt_payload_stream(payload, require_master_key(keys), entry.nonce);
+        }
+        else
+        {
+            fail("Unsupported archive version");
+        }
         entry.stored_size = encrypted.size();
 
         payload_bytes.insert(payload_bytes.end(), encrypted.begin(), encrypted.end());
@@ -185,13 +271,13 @@ BuildResult build_archive(
         result.total_stored_bytes += encrypted.size();
     }
 
-    const auto manifest_bytes = serialize_manifest(manifest_entries);
+    const auto manifest_bytes = serialize_manifest(manifest_entries, kCurrentArchiveVersion);
     const auto manifest_hash = hash_bytes(manifest_bytes);
     const auto signature = sign_detached(manifest_bytes, *keys.signing_secret_key);
 
     ArchiveHeader header {};
     std::memcpy(header.magic, kArchiveMagic, sizeof(kArchiveMagic));
-    header.version = kArchiveVersion;
+    header.version = kCurrentArchiveVersion;
     header.flags = 0;
     header.key_id = keys.key_id;
     header.manifest_offset = sizeof(ArchiveHeader) + payload_bytes.size();
@@ -239,7 +325,7 @@ LoadedArchive load_archive(const std::filesystem::path& archive_path)
     {
         fail("Archive magic mismatch");
     }
-    if (archive.header.version != kArchiveVersion)
+    if (!supports_archive_version(archive.header.version))
     {
         fail("Unsupported archive version");
     }
@@ -252,7 +338,7 @@ LoadedArchive load_archive(const std::filesystem::path& archive_path)
         archive.file_bytes.begin() + static_cast<std::ptrdiff_t>(archive.header.manifest_offset),
         archive.file_bytes.begin() + static_cast<std::ptrdiff_t>(archive.header.manifest_offset + archive.header.manifest_size));
 
-    archive.entries = parse_manifest(archive.manifest_bytes);
+    archive.entries = parse_manifest(archive.manifest_bytes, archive.header.version);
     return archive;
 }
 
@@ -308,11 +394,14 @@ bool verify_archive(
             for (const auto& entry : archive.entries)
             {
                 const auto plain = extract_entry(archive, entry, keys.master_key);
-                const auto hash = hash_bytes(plain);
+                if (archive.header.version == kArchiveVersionAead || archive.header.version == kArchiveVersionStream)
-                if (hash != entry.plaintext_hash)
                 {
-                    error = "Plaintext hash mismatch: " + entry.path;
+                    const auto hash = hash_bytes(plain);
-                    return false;
+                    if (hash != entry.plaintext_hash)
+                    {
+                        error = "Plaintext hash mismatch: " + entry.path;
+                        return false;
+                    }
                 }
             }
         }
@@ -337,13 +426,13 @@ std::vector<std::uint8_t> extract_entry(
         archive.file_bytes.begin() + static_cast<std::ptrdiff_t>(begin),
         archive.file_bytes.begin() + static_cast<std::ptrdiff_t>(end));
 
-    const auto compressed = decrypt_payload(ciphertext, master_key, entry.nonce, entry.path);
+    const auto payload = decrypt_payload_for_entry(archive.header.version, ciphertext, entry, master_key);
     switch (entry.compression)
     {
     case Compression::None:
-        return compressed;
+        return payload;
     case Compression::Zstd:
-        return decompress_zstd(compressed, static_cast<std::size_t>(entry.original_size));
+        return decompress_zstd(payload, static_cast<std::size_t>(entry.original_size));
     default:
         fail("Unsupported compression mode");
     }

src/archive.h

@@ -16,6 +16,9 @@ constexpr std::size_t kHashSize = 32;
 constexpr std::size_t kSignatureSize = 64;
 constexpr std::size_t kAeadKeySize = 32;
 constexpr std::size_t kAeadNonceSize = 24;
+constexpr std::uint32_t kArchiveVersionAead = 1;
+constexpr std::uint32_t kArchiveVersionStream = 2;
+constexpr std::uint32_t kCurrentArchiveVersion = kArchiveVersionAead;
 
 enum class Compression : std::uint8_t
 {
@@ -54,6 +57,19 @@ struct ManifestEntryFixed
     std::uint8_t nonce[kAeadNonceSize];
     std::uint8_t plaintext_hash[kHashSize];
 };
+
+struct ManifestEntryFixedV2
+{
+    std::uint16_t path_size;
+    std::uint8_t compression;
+    std::uint8_t flags;
+    std::uint64_t data_offset;
+    std::uint64_t original_size;
+    std::uint64_t stored_size;
+    std::uint8_t nonce[kAeadNonceSize];
+    std::uint8_t payload_hash[kHashSize];
+    std::uint8_t plaintext_hash[kHashSize];
+};
 #pragma pack(pop)
 
 struct ManifestEntry
@@ -64,6 +80,7 @@ struct ManifestEntry
     std::uint64_t original_size = 0;
     std::uint64_t stored_size = 0;
     std::array<std::uint8_t, kAeadNonceSize> nonce {};
+    std::array<std::uint8_t, kHashSize> payload_hash {};
     std::array<std::uint8_t, kHashSize> plaintext_hash {};
 };
 
@@ -116,7 +133,7 @@ void extract_all(
     const std::filesystem::path& output_dir,
     const std::array<std::uint8_t, kAeadKeySize>& master_key);
 
-std::vector<std::uint8_t> serialize_manifest(const std::vector<ManifestEntry>& entries);
+std::vector<std::uint8_t> serialize_manifest(const std::vector<ManifestEntry>& entries, std::uint32_t version);
-std::vector<ManifestEntry> parse_manifest(const std::vector<std::uint8_t>& bytes);
+std::vector<ManifestEntry> parse_manifest(const std::vector<std::uint8_t>& bytes, std::uint32_t version);
 
 }  // namespace m2pack

src/crypto.cpp

@@ -205,6 +205,42 @@ std::vector<std::uint8_t> decrypt_payload(
     return plaintext;
 }
 
+std::vector<std::uint8_t> encrypt_payload_stream(
+    const std::vector<std::uint8_t>& plaintext,
+    const std::array<std::uint8_t, kAeadKeySize>& key,
+    const std::array<std::uint8_t, kAeadNonceSize>& nonce)
+{
+    std::vector<std::uint8_t> ciphertext(plaintext.size());
+    if (!plaintext.empty())
+    {
+        crypto_stream_xchacha20_xor(
+            ciphertext.data(),
+            plaintext.data(),
+            plaintext.size(),
+            nonce.data(),
+            key.data());
+    }
+    return ciphertext;
+}
+
+std::vector<std::uint8_t> decrypt_payload_stream(
+    const std::vector<std::uint8_t>& ciphertext,
+    const std::array<std::uint8_t, kAeadKeySize>& key,
+    const std::array<std::uint8_t, kAeadNonceSize>& nonce)
+{
+    std::vector<std::uint8_t> plaintext(ciphertext.size());
+    if (!ciphertext.empty())
+    {
+        crypto_stream_xchacha20_xor(
+            plaintext.data(),
+            ciphertext.data(),
+            ciphertext.size(),
+            nonce.data(),
+            key.data());
+    }
+    return plaintext;
+}
+
 std::vector<std::uint8_t> sign_detached(
     const std::vector<std::uint8_t>& data,
     const std::vector<std::uint8_t>& secret_key)

src/crypto.h

@@ -38,6 +38,16 @@ std::vector<std::uint8_t> decrypt_payload(
     const std::array<std::uint8_t, kAeadNonceSize>& nonce,
     std::string_view associated_data);
 
+std::vector<std::uint8_t> encrypt_payload_stream(
+    const std::vector<std::uint8_t>& plaintext,
+    const std::array<std::uint8_t, kAeadKeySize>& key,
+    const std::array<std::uint8_t, kAeadNonceSize>& nonce);
+
+std::vector<std::uint8_t> decrypt_payload_stream(
+    const std::vector<std::uint8_t>& ciphertext,
+    const std::array<std::uint8_t, kAeadKeySize>& key,
+    const std::array<std::uint8_t, kAeadNonceSize>& nonce);
+
 std::vector<std::uint8_t> sign_detached(
     const std::vector<std::uint8_t>& data,
     const std::vector<std::uint8_t>& secret_key);