aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Engelbrecht <[email protected]>2022-11-18 12:02:41 +0100
committerDan Engelbrecht <[email protected]>2022-11-25 10:17:34 +0100
commit5041f3521c2b3074032c06ed04a391ed90a06c7e (patch)
treefbe6c18c79de5abff79394b69d65e38004d9e39e
parent0.1.9 (diff)
downloadzen-5041f3521c2b3074032c06ed04a391ed90a06c7e.tar.xz
zen-5041f3521c2b3074032c06ed04a391ed90a06c7e.zip
reduce parsing of compressed headers
Diffstat
-rw-r--r--zen/cmds/exportproject.cpp2
-rw-r--r--zen/cmds/print.cpp11
-rw-r--r--zencore/compactbinarypackage.cpp26
-rw-r--r--zencore/compactbinaryvalidation.cpp9
-rw-r--r--zencore/compress.cpp81
-rw-r--r--zencore/include/zencore/compress.h20
-rw-r--r--zenhttp/httpshared.cpp40
-rw-r--r--zenserver-test/zenserver-test.cpp26
-rw-r--r--zenserver/cache/structuredcache.cpp204
-rw-r--r--zenserver/cache/structuredcachestore.cpp33
-rw-r--r--zenserver/cidstore.cpp24
-rw-r--r--zenserver/compute/function.cpp6
-rw-r--r--zenserver/projectstore.cpp48
-rw-r--r--zenserver/testing/launch.cpp6
-rw-r--r--zenserver/upstream/hordecompute.cpp8
-rw-r--r--zenserver/upstream/upstreamcache.cpp62
-rw-r--r--zenstore/cidstore.cpp12
-rw-r--r--zenstore/compactcas.cpp13
-rw-r--r--zenstore/filecas.cpp6
-rw-r--r--zenstore/gc.cpp5
-rw-r--r--zenstore/include/zenstore/cidstore.h2
-rw-r--r--zenutil/cache/cacherequests.cpp77
22 files changed, 424 insertions, 297 deletions
diff --git a/zen/cmds/exportproject.cpp b/zen/cmds/exportproject.cpp
index 5d8c7d536..a9a796085 100644
--- a/zen/cmds/exportproject.cpp
+++ b/zen/cmds/exportproject.cpp
@@ -147,7 +147,7 @@ ExportProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
return 1;
}
zen::IoBuffer CompressedPayload(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size());
- zen::IoBuffer Payload = zen::CompressedBuffer::FromCompressed(zen::SharedBuffer(CompressedPayload)).Decompress().AsIoBuffer();
+ zen::IoBuffer Payload = zen::CompressedBuffer::FromCompressedNoValidate(std::move(CompressedPayload)).Decompress().AsIoBuffer();
OplogResponses.emplace_back(zen::ParsePackageMessage(Payload));
zen::CbPackage& ResponsePackage = OplogResponses.back();
diff --git a/zen/cmds/print.cpp b/zen/cmds/print.cpp
index a8b2215a2..bd037dc99 100644
--- a/zen/cmds/print.cpp
+++ b/zen/cmds/print.cpp
@@ -79,12 +79,11 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
IoBuffer Data = Fc.Flatten();
- if (CompressedBuffer Compressed{CompressedBuffer::FromCompressed(SharedBuffer(Data))})
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed{CompressedBuffer::FromCompressed(SharedBuffer(Data), RawHash, RawSize)})
{
- zen::ConsoleLog().info("Compressed binary: size {}, raw size {}, hash: {}",
- Compressed.GetCompressedSize(),
- Compressed.GetRawSize(),
- IoHash::FromBLAKE3(Compressed.GetRawHash()));
+ zen::ConsoleLog().info("Compressed binary: size {}, raw size {}, hash: {}", Compressed.GetCompressedSize(), RawSize, RawHash);
}
else if (IsPackageMessage(Data))
{
@@ -108,7 +107,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
AttachmentType = "Compressed";
AttachmentSize = fmt::format("{} ({} uncompressed)",
Attachment.AsCompressedBinary().GetCompressedSize(),
- Attachment.AsCompressedBinary().GetRawSize());
+ Attachment.AsCompressedBinary().DecodeRawSize());
}
else if (Attachment.IsBinary())
{
diff --git a/zencore/compactbinarypackage.cpp b/zencore/compactbinarypackage.cpp
index 19675b9cf..a4fa38a1d 100644
--- a/zencore/compactbinarypackage.cpp
+++ b/zencore/compactbinarypackage.cpp
@@ -135,10 +135,12 @@ CbAttachment::TryLoad(CbFieldIterator& Fields)
if (BinaryView.GetSize() > 0)
{
// Is a compressed binary blob
+ IoHash RawHash;
+ uint64_t RawSize;
CompressedBuffer Compressed =
- CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer())).MakeOwned();
+ CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView, Fields.GetOuterBuffer()), RawHash, RawSize).MakeOwned();
Value.emplace<CompressedBuffer>(Compressed);
- Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
+ Hash = RawHash;
++Fields;
}
else
@@ -191,8 +193,10 @@ TryLoad_ArchiveFieldIntoAttachment(CbAttachment& TargetAttachment, CbField&& Fie
if (Buffer.GetSize() > 0)
{
// Is a compressed binary blob
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer));
- TargetAttachment = CbAttachment(Compressed, IoHash::FromBLAKE3(Compressed.GetRawHash()));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(std::move(Buffer), RawHash, RawSize);
+ TargetAttachment = CbAttachment(Compressed, RawHash);
}
else
{
@@ -715,9 +719,11 @@ namespace legacy {
{
return false;
}
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Buffer, RawHash, RawSize))
{
- if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+ if (RawHash != Hash)
{
return false;
}
@@ -747,8 +753,14 @@ namespace legacy {
ZEN_ASSERT(Mapper);
if (SharedBuffer AttachmentData = (*Mapper)(Hash))
{
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(AttachmentData, RawHash, RawSize))
{
+ if (RawHash != Hash)
+ {
+ return false;
+ }
Package.AddAttachment(CbAttachment(Compressed, Hash));
}
else
diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp
index a787e88ab..02148d96a 100644
--- a/zencore/compactbinaryvalidation.cpp
+++ b/zencore/compactbinaryvalidation.cpp
@@ -463,17 +463,18 @@ ValidateCbPackageAttachment(CbFieldView& Value, MemoryView& View, CbValidateMode
{
if (BinaryView.GetSize() > 0)
{
- CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView));
+ IoHash DecodedHash;
+ uint64_t DecodedRawSize;
+ CompressedBuffer Buffer = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(BinaryView), DecodedHash, DecodedRawSize);
if (EnumHasAnyFlags(Mode, CbValidateMode::Package) && Buffer.IsNull())
{
AddError(Error, CbValidateError::NullPackageAttachment);
}
- if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) &&
- (IoHash::FromBLAKE3(Buffer.GetRawHash()) != IoHash::HashBuffer(Buffer.DecompressToComposite())))
+ if (EnumHasAnyFlags(Mode, CbValidateMode::PackageHash) && (DecodedHash != IoHash::HashBuffer(Buffer.DecompressToComposite())))
{
AddError(Error, CbValidateError::InvalidPackageHash);
}
- return IoHash::FromBLAKE3(Buffer.GetRawHash());
+ return DecodedHash;
}
else
{
diff --git a/zencore/compress.cpp b/zencore/compress.cpp
index 15cc5f6a7..89d35b2fb 100644
--- a/zencore/compress.cpp
+++ b/zencore/compress.cpp
@@ -6,6 +6,7 @@
#include <zencore/compositebuffer.h>
#include <zencore/crc32.h>
#include <zencore/endian.h>
+#include <zencore/iohash.h>
#include <zencore/testing.h>
#include "../thirdparty/Oodle/include/oodle2.h"
@@ -56,8 +57,11 @@ struct BufferHeader
BLAKE3 RawHash; // The hash of the uncompressed data
/** Checks validity of the buffer based on the magic number, method, and CRC-32. */
- static bool IsValid(const CompositeBuffer& CompressedData);
- static bool IsValid(const SharedBuffer& CompressedData) { return IsValid(CompositeBuffer(CompressedData)); }
+ static bool IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize);
+ static bool IsValid(const SharedBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
+ {
+ return IsValid(CompositeBuffer(CompressedData), OutRawHash, OutRawSize);
+ }
/** Read a header from a buffer that is at least sizeof(BufferHeader) without any validation. */
static BufferHeader Read(const CompositeBuffer& CompressedData)
@@ -65,6 +69,10 @@ struct BufferHeader
BufferHeader Header;
if (sizeof(BufferHeader) <= CompressedData.GetSize())
{
+ // if (CompressedData.GetSegments()[0].AsIoBuffer().IsWholeFile())
+ // {
+ // ZEN_ASSERT(true);
+ // }
CompositeBuffer::Iterator It;
CompressedData.CopyTo(MakeMutableMemoryView(&Header, &Header + 1), It);
Header.ByteSwap();
@@ -664,7 +672,7 @@ GetDecoder(CompressionMethod Method)
//////////////////////////////////////////////////////////////////////////
bool
-BufferHeader::IsValid(const CompositeBuffer& CompressedData)
+BufferHeader::IsValid(const CompositeBuffer& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
if (sizeof(BufferHeader) <= CompressedData.GetSize())
{
@@ -679,6 +687,8 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
const MemoryView HeaderView = HeaderCopy.GetView();
if (Header.Crc32 == BufferHeader::CalculateCrc32(HeaderView))
{
+ OutRawHash = IoHash::FromBLAKE3(Header.RawHash);
+ OutRawSize = Header.TotalRawSize;
return true;
}
}
@@ -691,9 +701,10 @@ BufferHeader::IsValid(const CompositeBuffer& CompressedData)
template<typename BufferType>
inline CompositeBuffer
-ValidBufferOrEmpty(BufferType&& CompressedData)
+ValidBufferOrEmpty(BufferType&& CompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
- return BufferHeader::IsValid(CompressedData) ? CompositeBuffer(std::forward<BufferType>(CompressedData)) : CompositeBuffer();
+ return BufferHeader::IsValid(CompressedData, OutRawHash, OutRawSize) ? CompositeBuffer(std::forward<BufferType>(CompressedData))
+ : CompositeBuffer();
}
CompositeBuffer
@@ -826,34 +837,34 @@ CompressedBuffer::Compress(const SharedBuffer& RawData,
}
CompressedBuffer
-CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
CompressedBuffer Local;
- Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+ Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
return Local;
}
CompressedBuffer
-CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(CompositeBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
CompressedBuffer Local;
- Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+ Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
return Local;
}
CompressedBuffer
-CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData)
+CompressedBuffer::FromCompressed(const SharedBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
CompressedBuffer Local;
- Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData);
+ Local.CompressedData = detail::ValidBufferOrEmpty(InCompressedData, OutRawHash, OutRawSize);
return Local;
}
CompressedBuffer
-CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData)
+CompressedBuffer::FromCompressed(SharedBuffer&& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize)
{
CompressedBuffer Local;
- Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData));
+ Local.CompressedData = detail::ValidBufferOrEmpty(std::move(InCompressedData), OutRawHash, OutRawSize);
return Local;
}
@@ -882,13 +893,13 @@ CompressedBuffer::FromCompressedNoValidate(CompositeBuffer&& InCompressedData)
}
uint64_t
-CompressedBuffer::GetRawSize() const
+CompressedBuffer::DecodeRawSize() const
{
return CompressedData ? detail::BufferHeader::Read(CompressedData).TotalRawSize : 0;
}
BLAKE3
-CompressedBuffer::GetRawHash() const
+CompressedBuffer::DecodeRawHash() const
{
return CompressedData ? detail::BufferHeader::Read(CompressedData).RawHash : BLAKE3();
}
@@ -1005,18 +1016,20 @@ TEST_CASE("CompressedBuffer")
OodleCompressor::NotSet,
OodleCompressionLevel::None);
- CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+ CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(detail::BufferHeader)));
CompositeBuffer Compressed = Buffer.GetCompressed();
- CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);
+ IoHash DecodedHash;
+ uint64_t DecodedRawSize;
+ CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
CHECK(BufferD.IsNull() == false);
CompositeBuffer Decomp = BufferD.DecompressToComposite();
- CHECK(Decomp.GetSize() == Buffer.GetRawSize());
- CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+ CHECK(Decomp.GetSize() == DecodedRawSize);
+ CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
}
{
@@ -1025,53 +1038,59 @@ TEST_CASE("CompressedBuffer")
OodleCompressor::NotSet,
OodleCompressionLevel::None);
- CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+ CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
CHECK(Buffer.GetCompressedSize() == (sizeof(Zeroes) + sizeof(Ones) + sizeof(detail::BufferHeader)));
CompositeBuffer Compressed = Buffer.GetCompressed();
- CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);
+ IoHash DecodedHash;
+ uint64_t DecodedRawSize;
+ CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
CHECK(BufferD.IsNull() == false);
CompositeBuffer Decomp = BufferD.DecompressToComposite();
- CHECK(Decomp.GetSize() == Buffer.GetRawSize());
- CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+ CHECK(Decomp.GetSize() == DecodedRawSize);
+ CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
}
{
CompressedBuffer Buffer = CompressedBuffer::Compress(CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes))));
- CHECK(Buffer.GetRawSize() == sizeof(Zeroes));
+ CHECK(Buffer.DecodeRawSize() == sizeof(Zeroes));
CHECK(Buffer.GetCompressedSize() < sizeof(Zeroes));
CompositeBuffer Compressed = Buffer.GetCompressed();
- CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);
+ IoHash DecodedHash;
+ uint64_t DecodedRawSize;
+ CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
CHECK(BufferD.IsNull() == false);
CompositeBuffer Decomp = BufferD.DecompressToComposite();
- CHECK(Decomp.GetSize() == Buffer.GetRawSize());
- CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+ CHECK(Decomp.GetSize() == DecodedRawSize);
+ CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
}
{
CompressedBuffer Buffer = CompressedBuffer::Compress(
CompositeBuffer(SharedBuffer::MakeView(MakeMemoryView(Zeroes)), SharedBuffer::MakeView(MakeMemoryView(Ones))));
- CHECK(Buffer.GetRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
+ CHECK(Buffer.DecodeRawSize() == (sizeof(Zeroes) + sizeof(Ones)));
CHECK(Buffer.GetCompressedSize() < (sizeof(Zeroes) + sizeof(Ones)));
CompositeBuffer Compressed = Buffer.GetCompressed();
- CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed);
+ IoHash DecodedHash;
+ uint64_t DecodedRawSize;
+ CompressedBuffer BufferD = CompressedBuffer::FromCompressed(Compressed, DecodedHash, DecodedRawSize);
CHECK(BufferD.IsNull() == false);
CompositeBuffer Decomp = BufferD.DecompressToComposite();
- CHECK(Decomp.GetSize() == Buffer.GetRawSize());
- CHECK(BLAKE3::HashBuffer(Decomp) == BufferD.GetRawHash());
+ CHECK(Decomp.GetSize() == DecodedRawSize);
+ CHECK(IoHash::HashBuffer(Decomp) == DecodedHash);
}
auto GenerateData = [](uint64_t N) -> std::vector<uint64_t> {
diff --git a/zencore/include/zencore/compress.h b/zencore/include/zencore/compress.h
index 92dc1fb76..8d1e4841f 100644
--- a/zencore/include/zencore/compress.h
+++ b/zencore/include/zencore/compress.h
@@ -71,10 +71,18 @@ public:
*
* @return A compressed buffer, or null on error, such as an invalid format or corrupt header.
*/
- [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData);
- [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData);
- [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData);
- [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const CompositeBuffer& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(CompositeBuffer&& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(const SharedBuffer& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
+ [[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressed(SharedBuffer&& CompressedData,
+ IoHash& OutRawHash,
+ uint64_t& OutRawSize);
[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(IoBuffer&& CompressedData);
[[nodiscard]] ZENCORE_API static CompressedBuffer FromCompressedNoValidate(CompositeBuffer&& CompressedData);
@@ -102,10 +110,10 @@ public:
[[nodiscard]] inline uint64_t GetCompressedSize() const { return CompressedData.GetSize(); }
/** Returns the size of the raw data. Zero on error or if this is empty or null. */
- [[nodiscard]] ZENCORE_API uint64_t GetRawSize() const;
+ [[nodiscard]] ZENCORE_API uint64_t DecodeRawSize() const;
/** Returns the hash of the raw data. Zero on error or if this is null. */
- [[nodiscard]] ZENCORE_API BLAKE3 GetRawHash() const;
+ [[nodiscard]] ZENCORE_API BLAKE3 DecodeRawHash() const;
[[nodiscard]] ZENCORE_API CompressedBuffer CopyRange(uint64_t RawOffset, uint64_t RawSize = ~uint64_t(0)) const;
diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp
index e2f061a87..23d9f9a30 100644
--- a/zenhttp/httpshared.cpp
+++ b/zenhttp/httpshared.cpp
@@ -344,7 +344,9 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
{
if (i == 0)
{
- CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize));
if (!CompBuf)
{
throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for CbObject", i));
@@ -360,16 +362,22 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint
else
{
// Make a copy of the buffer so we attachements don't reference the entire payload
- IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize);
- ZEN_ASSERT(AttachmentBufferCopy);
- ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize);
- AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView());
-
- CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBufferCopy)));
+ // IoBuffer AttachmentBufferCopy = CreateBuffer(Entry.AttachmentHash, AttachmentSize);
+ // ZEN_ASSERT(AttachmentBufferCopy);
+ // ZEN_ASSERT(AttachmentBufferCopy.Size() == AttachmentSize);
+ // AttachmentBufferCopy.GetMutableView().CopyFrom(AttachmentBuffer.GetView());
+
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer CompBuf(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize));
if (!CompBuf)
{
throw std::runtime_error(fmt::format("invalid format for chunk #{} expected compressed buffer for attachment", i));
}
+ if (RawHash != Entry.AttachmentHash)
+ {
+ throw std::runtime_error(fmt::format("invalid hash for chunk #{} for attachment", i));
+ }
Attachments.emplace_back(CbAttachment(std::move(CompBuf), Entry.AttachmentHash));
}
@@ -531,7 +539,13 @@ CbPackageReader::Finalize()
}
else if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
{
- m_RootObject = LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer)));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentBuffer), RawHash, RawSize);
+ if (RawHash == Entry.AttachmentHash)
+ {
+ m_RootObject = LoadCompactBinaryObject(Compressed);
+ }
}
else
{
@@ -549,13 +563,19 @@ CbPackageReader::Finalize()
if (Entry.Flags & CbAttachmentEntry::kIsCompressed)
{
- m_Attachments.push_back(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference)), Entry.AttachmentHash));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkReference), RawHash, RawSize);
+ if (RawHash == Entry.AttachmentHash)
+ {
+ m_Attachments.push_back(CbAttachment(Compressed, Entry.AttachmentHash));
+ }
}
else
{
CompressedBuffer Compressed =
CompressedBuffer::Compress(SharedBuffer(ChunkReference), OodleCompressor::NotSet, OodleCompressionLevel::None);
- m_Attachments.push_back(CbAttachment(std::move(Compressed), IoHash::FromBLAKE3(Compressed.GetRawHash())));
+ m_Attachments.push_back(CbAttachment(std::move(Compressed), IoHash::FromBLAKE3(Compressed.DecodeRawHash())));
}
}
diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 1d0596c29..c72410173 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -524,7 +524,7 @@ TEST_CASE("project.basic")
uint8_t AttachData[] = {1, 2, 3};
zen::CompressedBuffer Attachment = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone(zen::MemoryView{AttachData, 3}));
- zen::CbAttachment Attach{Attachment, IoHash::FromBLAKE3(Attachment.GetRawHash())};
+ zen::CbAttachment Attach{Attachment, IoHash::FromBLAKE3(Attachment.DecodeRawHash())};
zen::CbObjectWriter OpWriter;
OpWriter << "key"
@@ -765,7 +765,7 @@ TEST_CASE("zcache.cbpackage")
auto Data = zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
auto CompressedData = zen::CompressedBuffer::Compress(Data);
- OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
+ OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.DecodeRawHash());
zen::CbWriter Obj;
Obj.BeginObject("obj"sv);
@@ -988,7 +988,7 @@ TEST_CASE("zcache.policy")
auto GeneratePackage = [](zen::IoHash& OutRecordKey, zen::IoHash& OutAttachmentKey) -> zen::CbPackage {
auto Data = zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
auto CompressedData = zen::CompressedBuffer::Compress(Data);
- OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
+ OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.DecodeRawHash());
zen::CbWriter Writer;
Writer.BeginObject("obj"sv);
@@ -1961,8 +1961,8 @@ TEST_CASE("zcache.rpc.allpolicies")
{
Builder.BeginObject();
Builder.AddObjectId("Id"sv, ValueIds[ValueIndex]);
- Builder.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(KeyData.BufferValues[ValueIndex].GetRawHash()));
- Builder.AddInteger("RawSize"sv, KeyData.BufferValues[ValueIndex].GetRawSize());
+ Builder.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(KeyData.BufferValues[ValueIndex].DecodeRawHash()));
+ Builder.AddInteger("RawSize"sv, KeyData.BufferValues[ValueIndex].DecodeRawSize());
Builder.EndObject();
}
Builder.EndArray();
@@ -2138,9 +2138,9 @@ TEST_CASE("zcache.rpc.allpolicies")
WriteToString<32> ValueName("Get(", KeyData->KeyIndex, ",", ExpectedValueIndex, ")");
CompressedBuffer ExpectedValue = KeyData->BufferValues[ExpectedValueIndex];
- CHECK_MESSAGE(Value.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+ CHECK_MESSAGE(Value.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
WriteToString<32>(ValueName, " RawHash did not match.").c_str());
- CHECK_MESSAGE(Value.RawSize == ExpectedValue.GetRawSize(),
+ CHECK_MESSAGE(Value.RawSize == ExpectedValue.DecodeRawSize(),
WriteToString<32>(ValueName, " RawSize did not match.").c_str());
if (KeyData->GetRequestsData)
@@ -2203,9 +2203,9 @@ TEST_CASE("zcache.rpc.allpolicies")
if (!KeyData->ForceMiss && Succeeded)
{
CompressedBuffer ExpectedValue = KeyData->BufferValues[0];
- CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+ CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
WriteToString<32>(Name, " RawHash did not match.").c_str());
- CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(),
+ CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.DecodeRawSize(),
WriteToString<32>(Name, " RawSize did not match.").c_str());
if (KeyData->GetRequestsData)
@@ -2277,9 +2277,9 @@ TEST_CASE("zcache.rpc.allpolicies")
if (KeyData->ShouldBeHit && Succeeded)
{
CompressedBuffer ExpectedValue = KeyData->BufferValues[ValueIndex];
- CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+ CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
WriteToString<32>(Name, " had unexpected RawHash.").c_str());
- CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(),
+ CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.DecodeRawSize(),
WriteToString<32>(Name, " had unexpected RawSize.").c_str());
if (KeyData->GetRequestsData)
@@ -2659,11 +2659,11 @@ TEST_CASE("http.package")
zen::CompressedBuffer AttachmentData1 = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone({Data1, 4}),
zen::OodleCompressor::NotSet,
zen::OodleCompressionLevel::None);
- zen::CbAttachment Attach1{AttachmentData1, IoHash::FromBLAKE3(AttachmentData1.GetRawHash())};
+ zen::CbAttachment Attach1{AttachmentData1, IoHash::FromBLAKE3(AttachmentData1.DecodeRawHash())};
zen::CompressedBuffer AttachmentData2 = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone({Data2, 8}),
zen::OodleCompressor::NotSet,
zen::OodleCompressionLevel::None);
- zen::CbAttachment Attach2{AttachmentData2, IoHash::FromBLAKE3(AttachmentData2.GetRawHash())};
+ zen::CbAttachment Attach2{AttachmentData2, IoHash::FromBLAKE3(AttachmentData2.DecodeRawHash())};
zen::CbObjectWriter Writer;
diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index dfb69c0fe..d1630a4af 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -530,8 +530,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
{
if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
{
- Package.AddAttachment(
- CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)), AttachmentHash.AsHash()));
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
+ Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash()));
}
else
{
@@ -588,12 +588,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else if (!EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote))
{
- ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}",
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- ToString(AcceptType),
- NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
+ ZEN_INFO("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(AcceptType),
+ NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
m_CacheStats.MissCount++;
return Request.WriteResponse(HttpResponseCode::NotFound);
}
@@ -655,13 +655,15 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
AttachmentCount Count;
CacheRecord.IterateAttachments([this, &Package, &Ref, &Count, QueryLocal, StoreLocal](CbFieldView HashView) {
- if (const CbAttachment* Attachment = Package.FindAttachment(HashView.AsHash()))
+ IoHash Hash = HashView.AsHash();
+ if (const CbAttachment* Attachment = Package.FindAttachment(Hash))
{
if (CompressedBuffer Compressed = Attachment->AsCompressedBinary())
{
if (StoreLocal)
{
- auto InsertResult = m_CidStore.AddChunk(Compressed);
+ auto InsertResult =
+ m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
if (InsertResult.New)
{
Count.New++;
@@ -672,7 +674,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
else
{
ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'",
- HashView.AsHash(),
+ Hash,
Ref.BucketSegment,
Ref.HashKey);
Count.Invalid++;
@@ -680,10 +682,9 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else if (QueryLocal)
{
- if (IoBuffer Chunk = m_CidStore.FindChunkByCid(HashView.AsHash()))
+ if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash))
{
- Package.AddAttachment(
- CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk)), HashView.AsHash()));
+ Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)), Hash));
Count.Valid++;
}
}
@@ -762,12 +763,12 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
}
else
{
- ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}",
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- ToString(AcceptType),
- NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000));
+ ZEN_INFO("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ ToString(AcceptType),
+ NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000));
m_CacheStats.MissCount++;
AsyncRequest.WriteResponse(HttpResponseCode::NotFound);
}
@@ -889,7 +890,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request
if (Attachment->IsCompressedBinary())
{
CompressedBuffer Chunk = Attachment->AsCompressedBinary();
- CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk);
+ CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), Hash);
ValidAttachments.emplace_back(Hash);
@@ -992,10 +993,19 @@ HttpStructuredCacheService::HandleGetCacheChunk(zen::HttpServerRequest& Request,
m_UpstreamCache.GetCacheChunk(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId);
UpstreamResult.Status.Success)
{
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value)))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(UpstreamResult.Value), RawHash, RawSize))
{
- m_CidStore.AddChunk(Compressed);
- Source = UpstreamResult.Source;
+ if (RawHash == Ref.ValueContentId)
+ {
+ m_CidStore.AddChunk(UpstreamResult.Value, RawHash);
+ Source = UpstreamResult.Source;
+ }
+ else
+ {
+ ZEN_WARN("got missmatching upstream cache value");
+ }
}
else
{
@@ -1007,13 +1017,13 @@ HttpStructuredCacheService::HandleGetCacheChunk(zen::HttpServerRequest& Request,
if (!Value)
{
- ZEN_DEBUG("GETCACHECHUNK MISS - '{}/{}/{}/{}' '{}' in {}",
- Ref.Namespace,
- Ref.BucketSegment,
- Ref.HashKey,
- Ref.ValueContentId,
- ToString(Request.AcceptContentType()),
- NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
+ ZEN_INFO("GETCACHECHUNK MISS - '{}/{}/{}/{}' '{}' in {}",
+ Ref.Namespace,
+ Ref.BucketSegment,
+ Ref.HashKey,
+ Ref.ValueContentId,
+ ToString(Request.AcceptContentType()),
+ NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
m_CacheStats.MissCount++;
return Request.WriteResponse(HttpResponseCode::NotFound);
}
@@ -1061,21 +1071,23 @@ HttpStructuredCacheService::HandlePutCacheChunk(zen::HttpServerRequest& Request,
Body.SetContentType(Request.RequestContentType());
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Body));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Body), RawHash, RawSize);
if (!Compressed)
{
return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"sv);
}
- if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Ref.ValueContentId)
+ if (RawHash != Ref.ValueContentId)
{
return Request.WriteResponse(HttpResponseCode::BadRequest,
HttpContentType::kText,
"ValueContentId does not match attachment hash"sv);
}
- CidStore::InsertResult Result = m_CidStore.AddChunk(Compressed);
+ CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash);
ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}",
Ref.Namespace,
@@ -1247,8 +1259,9 @@ HttpStructuredCacheService::PutCacheRecord(PutRequestData& Request, const CbPack
{
if (Attachment->IsCompressedBinary())
{
- CompressedBuffer Chunk = Attachment->AsCompressedBinary();
- CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk);
+ CompressedBuffer Chunk = Attachment->AsCompressedBinary();
+ CidStore::InsertResult InsertResult =
+ m_CidStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
ValidAttachments.emplace_back(ValueHash);
@@ -1448,7 +1461,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Value.ContentId))
{
ZEN_ASSERT(Chunk.GetSize() > 0);
- Value.Payload = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+ Value.Payload = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk));
Value.Exists = true;
}
else
@@ -1560,7 +1573,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
Value.Exists = true;
if (EnumHasAllFlags(ValuePolicy, CachePolicy::StoreLocal))
{
- m_CidStore.AddChunk(Compressed);
+ m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash());
}
if (!EnumHasAllFlags(ValuePolicy, CachePolicy::SkipData))
{
@@ -1607,7 +1620,7 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
{
if (!EnumHasAllFlags(Value.DownstreamPolicy, CachePolicy::SkipData) && Value.Payload)
{
- ResponsePackage.AddAttachment(CbAttachment(Value.Payload, IoHash::FromBLAKE3(Value.Payload.GetRawHash())));
+ ResponsePackage.AddAttachment(CbAttachment(Value.Payload, Value.ContentId));
}
}
@@ -1637,13 +1650,13 @@ HttpStructuredCacheService::HandleRpcGetCacheRecords(zen::HttpServerRequest& Htt
}
else
{
- ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}'{} ({}) in {}",
- *Namespace,
- Key.Bucket,
- Key.Hash,
- Request.RecordObject ? ""sv : " (PARTIAL)"sv,
- Request.Source ? Request.Source->Url : "LOCAL"sv,
- NiceLatencyNs(Request.ElapsedTimeUs * 1000));
+ ZEN_INFO("GETCACHERECORD MISS - '{}/{}/{}'{} ({}) in {}",
+ *Namespace,
+ Key.Bucket,
+ Key.Hash,
+ Request.RecordObject ? " (PARTIAL)"sv : ""sv,
+ Request.Source ? Request.Source->Url : "LOCAL"sv,
+ NiceLatencyNs(Request.ElapsedTimeUs * 1000));
m_CacheStats.MissCount++;
}
}
@@ -1839,25 +1852,24 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
PolicyText = RequestObject["Policy"sv].AsString();
Request.Policy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy;
- CacheKey& Key = Request.Key;
- CachePolicy Policy = Request.Policy;
- CompressedBuffer& Result = Request.Result;
+ CacheKey& Key = Request.Key;
+ CachePolicy Policy = Request.Policy;
ZenCacheValue CacheValue;
if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal))
{
if (m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
{
- Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value));
+ Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), Request.RawHash, Request.RawSize);
}
}
- if (Result)
+ if (Request.Result)
{
ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
*Namespace,
Key.Bucket,
Key.Hash,
- NiceBytes(Result.GetCompressed().GetSize()),
+ NiceBytes(Request.Result.GetCompressed().GetSize()),
"LOCAL"sv,
NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
m_CacheStats.HitCount++;
@@ -1873,12 +1885,12 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
}
else
{
- ZEN_DEBUG("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}",
- *Namespace,
- Key.Bucket,
- Key.Hash,
- "LOCAL"sv,
- NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
+ ZEN_INFO("GETCACHEVALUES MISS - '{}/{}/{}' ({}) in {}",
+ *Namespace,
+ Key.Bucket,
+ Key.Hash,
+ "LOCAL"sv,
+ NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000));
m_CacheStats.MissCount++;
}
}
@@ -1916,7 +1928,7 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
{
if (HasData && !SkipData)
{
- Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value));
+ Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), Request.RawHash, Request.RawSize);
}
if (HasData && StoreData)
@@ -1961,15 +1973,14 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(zen::HttpServerRequest& Http
const CompressedBuffer& Result = Request.Result;
if (Result)
{
- IoHash Hash = IoHash::FromBLAKE3(Result.GetRawHash());
- ResponseObject.AddHash("RawHash"sv, Hash);
+ ResponseObject.AddHash("RawHash"sv, Request.RawHash);
if (!EnumHasAllFlags(Request.Policy, CachePolicy::SkipData))
{
- RpcResponse.AddAttachment(CbAttachment(Result, Hash));
+ RpcResponse.AddAttachment(CbAttachment(Result, Request.RawHash));
}
else
{
- ResponseObject.AddInteger("RawSize"sv, Result.GetRawSize());
+ ResponseObject.AddInteger("RawSize"sv, Request.RawSize);
}
}
else if (Request.RawHash != IoHash::Zero)
@@ -2026,12 +2037,12 @@ namespace cache::detail {
RecordBody* Record = nullptr;
CompressedBuffer Value;
const UpstreamEndpointInfo* Source = nullptr;
- uint64_t TotalSize = 0;
+ uint64_t RawSize = 0;
uint64_t RequestedSize = 0;
uint64_t RequestedOffset = 0;
CachePolicy DownstreamPolicy;
bool Exists = false;
- bool TotalSizeKnown = false;
+ bool RawSizeKnown = false;
bool IsRecordRequest = false;
uint64_t ElapsedTimeUs = 0;
};
@@ -2303,9 +2314,9 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::string_view Namespac
{
if (Value.ValueId == Request->Key->ValueId)
{
- Request->Key->ChunkId = Value.ContentId;
- Request->TotalSize = Value.RawSize;
- Request->TotalSizeKnown = true;
+ Request->Key->ChunkId = Value.ContentId;
+ Request->RawSize = Value.RawSize;
+ Request->RawSizeKnown = true;
break;
}
}
@@ -2316,7 +2327,7 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::string_view Namespac
{
if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::QueryLocal))
{
- if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->TotalSizeKnown)
+ if (EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData) && Request->RawSizeKnown)
{
if (m_CidStore.ContainsChunk(Request->Key->ChunkId))
{
@@ -2325,16 +2336,16 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::string_view Namespac
}
else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId))
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload));
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload));
if (Compressed)
{
if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
{
Request->Value = Compressed;
}
- Request->Exists = true;
- Request->TotalSize = Compressed.GetRawSize();
- Request->TotalSizeKnown = true;
+ Request->Exists = true;
+ Request->RawSize = Compressed.DecodeRawSize();
+ Request->RawSizeKnown = true;
}
}
}
@@ -2365,17 +2376,19 @@ HttpStructuredCacheService::GetLocalCacheValues(std::string_view Namespa
{
if (IsCompressedBinary(CacheValue.Value.GetContentType()))
{
- CompressedBuffer Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), RawHash, RawSize);
if (Result)
{
if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
{
Request->Value = Result;
}
- Request->Key->ChunkId = IoHash::FromBLAKE3(Result.GetRawHash());
- Request->Exists = true;
- Request->TotalSize = Result.GetRawSize();
- Request->TotalSizeKnown = true;
+ Request->Key->ChunkId = RawHash;
+ Request->Exists = true;
+ Request->RawSize = RawSize;
+ Request->RawSizeKnown = true;
}
}
}
@@ -2417,9 +2430,10 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Names
if (EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::StoreLocal) ||
!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value));
- if (!Compressed || Compressed.GetRawSize() != Params.RawSize ||
- IoHash::FromBLAKE3(Compressed.GetRawHash()) != Params.RawHash)
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), RawHash, RawSize);
+ if (!Compressed || RawSize != Params.RawSize || RawHash != Params.RawHash)
{
return;
}
@@ -2428,7 +2442,7 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Names
{
if (Request.IsRecordRequest)
{
- m_CidStore.AddChunk(Compressed);
+ m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), RawHash);
}
else
{
@@ -2440,11 +2454,11 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view Names
Request.Value = std::move(Compressed);
}
}
- Key.ChunkId = Params.RawHash;
- Request.Exists = true;
- Request.TotalSize = Params.RawSize;
- Request.TotalSizeKnown = true;
- Request.Source = Params.Source;
+ Key.ChunkId = Params.RawHash;
+ Request.Exists = true;
+ Request.RawSize = Params.RawSize;
+ Request.RawSizeKnown = true;
+ Request.Source = Params.Source;
m_CacheStats.UpstreamHitCount++;
};
@@ -2482,7 +2496,7 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(uint32_t Accept
}
else
{
- Writer.AddInteger("RawSize"sv, Request.TotalSize);
+ Writer.AddInteger("RawSize"sv, Request.RawSize);
}
ZEN_DEBUG("GETCACHECHUNKS HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}",
@@ -2490,7 +2504,7 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(uint32_t Accept
Request.Key->Key.Bucket,
Request.Key->Key.Hash,
Request.Key->ValueId,
- NiceBytes(Request.TotalSize),
+ NiceBytes(Request.RawSize),
Request.IsRecordRequest ? "Record"sv : "Value"sv,
Request.Source ? Request.Source->Url : "LOCAL"sv,
NiceLatencyNs(Request.ElapsedTimeUs * 1000));
@@ -2507,12 +2521,12 @@ HttpStructuredCacheService::WriteGetCacheChunksResponse(uint32_t Accept
}
else
{
- ZEN_DEBUG("GETCACHECHUNKS MISS - '{}/{}/{}/{}' in {}",
- Namespace,
- Request.Key->Key.Bucket,
- Request.Key->Key.Hash,
- Request.Key->ValueId,
- NiceLatencyNs(Request.ElapsedTimeUs * 1000));
+ ZEN_INFO("GETCACHECHUNKS MISS - '{}/{}/{}/{}' in {}",
+ Namespace,
+ Request.Key->Key.Bucket,
+ Request.Key->Key.Hash,
+ Request.Key->ValueId,
+ NiceLatencyNs(Request.ElapsedTimeUs * 1000));
m_CacheStats.MissCount++;
}
}
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 9773c2ed2..833394dd0 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -491,7 +491,7 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
std::vector<IoHash> BadHashes;
- auto ValidateEntry = [](ZenContentType ContentType, IoBuffer Buffer) {
+ auto ValidateEntry = [](const IoHash& Hash, ZenContentType ContentType, IoBuffer Buffer) {
if (ContentType == ZenContentType::kCbObject)
{
CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
@@ -499,7 +499,13 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
if (ContentType == ZenContentType::kCompressedBinary)
{
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); !Compressed)
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize); !Compressed)
+ {
+ return false;
+ }
+ if (Hash != RawHash)
{
return false;
}
@@ -509,7 +515,7 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
for (auto& Kv : m_CacheMap)
{
- if (!ValidateEntry(Kv.second.Payload.GetContentType(), Kv.second.Payload))
+ if (!ValidateEntry(Kv.first, Kv.second.Payload.GetContentType(), Kv.second.Payload))
{
BadHashes.push_back(Kv.first);
}
@@ -1021,7 +1027,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
std::vector<BlockStoreLocation> ChunkLocations;
std::vector<IoHash> ChunkIndexToChunkHash;
- auto ValidateEntry = [](ZenContentType ContentType, IoBuffer Buffer) {
+ auto ValidateEntry = [](const IoHash& Hash, ZenContentType ContentType, IoBuffer Buffer) {
if (ContentType == ZenContentType::kCbObject)
{
CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All);
@@ -1029,7 +1035,13 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
}
if (ContentType == ZenContentType::kCompressedBinary)
{
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); !Compressed)
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize); !Compressed)
+ {
+ return false;
+ }
+ if (RawHash != Hash)
{
return false;
}
@@ -1077,7 +1089,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
BadKeys.push_back(HashKey);
continue;
}
- if (!ValidateEntry(Loc.GetContentType(), Value.Value))
+ if (!ValidateEntry(HashKey, Loc.GetContentType(), Value.Value))
{
BadKeys.push_back(HashKey);
continue;
@@ -1108,7 +1120,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
return;
}
ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
- if (!ValidateEntry(ContentType, Buffer))
+ if (!ValidateEntry(Hash, ContentType, Buffer))
{
BadKeys.push_back(Hash);
return;
@@ -1127,7 +1139,7 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
return;
}
ZenContentType ContentType = m_Index.at(Hash).Location.GetContentType();
- if (!ValidateEntry(ContentType, Buffer))
+ if (!ValidateEntry(Hash, ContentType, Buffer))
{
BadKeys.push_back(Hash);
return;
@@ -3091,7 +3103,8 @@ TEST_CASE("z$.scrub")
{
IoBuffer AttachmentData = CreateBinaryCacheValue(AttachmentSizes[Index]);
CompressedBuffer CompressedAttachmentData = CompressedBuffer::Compress(SharedBuffer(AttachmentData));
- Record.AddBinaryAttachment(fmt::format("attachment-{}", Index), IoHash::FromBLAKE3(CompressedAttachmentData.GetRawHash()));
+ Record.AddBinaryAttachment(fmt::format("attachment-{}", Index),
+ IoHash::FromBLAKE3(CompressedAttachmentData.DecodeRawHash()));
Result.Attachments[Index] = CompressedAttachmentData;
}
Result.Record = Record.Save().GetBuffer().AsIoBuffer();
@@ -3133,7 +3146,7 @@ TEST_CASE("z$.scrub")
Zcs.Put("mybucket", Cid, {.Value = Record.Record});
for (const CompressedBuffer& Attachment : Record.Attachments)
{
- CidStore.AddChunk(Attachment);
+ CidStore.AddChunk(Attachment.GetCompressed().Flatten().AsIoBuffer(), IoHash::FromBLAKE3(Attachment.DecodeRawHash()));
}
}
};
diff --git a/zenserver/cidstore.cpp b/zenserver/cidstore.cpp
index 5de347a17..2c760f7be 100644
--- a/zenserver/cidstore.cpp
+++ b/zenserver/cidstore.cpp
@@ -39,22 +39,22 @@ HttpCidService::HttpCidService(CidStore& Store) : m_CidStore(Store)
case HttpVerb::kPut:
{
- IoBuffer Payload = ServerRequest.ReadPayload();
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload));
+ IoBuffer Payload = ServerRequest.ReadPayload();
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
if (!Compressed)
{
return ServerRequest.WriteResponse(HttpResponseCode::UnsupportedMediaType);
}
- IoHash PayloadHash = IoHash::FromBLAKE3(Compressed.GetRawHash());
-
// URI hash must match content hash
- if (PayloadHash != Hash)
+ if (RawHash != Hash)
{
return ServerRequest.WriteResponse(HttpResponseCode::BadRequest);
}
- m_CidStore.AddChunk(Compressed);
+ m_CidStore.AddChunk(Payload, RawHash);
return ServerRequest.WriteResponse(HttpResponseCode::OK);
}
@@ -85,18 +85,18 @@ HttpCidService::HandleRequest(zen::HttpServerRequest& Request)
case HttpVerb::kPut:
case HttpVerb::kPost:
{
- IoBuffer Payload = Request.ReadPayload();
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload));
+ IoBuffer Payload = Request.ReadPayload();
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
if (!Compressed)
{
return Request.WriteResponse(HttpResponseCode::UnsupportedMediaType);
}
- IoHash PayloadHash = IoHash::FromBLAKE3(Compressed.GetRawHash());
-
- ZEN_DEBUG("CID POST request for {} ({} bytes)", PayloadHash, Payload.Size());
+ ZEN_DEBUG("CID POST request for {} ({} bytes)", RawHash, Payload.Size());
- auto InsertResult = m_CidStore.AddChunk(Compressed);
+ auto InsertResult = m_CidStore.AddChunk(Payload, RawHash);
if (InsertResult.New)
{
diff --git a/zenserver/compute/function.cpp b/zenserver/compute/function.cpp
index d7316ac64..493e2666e 100644
--- a/zenserver/compute/function.cpp
+++ b/zenserver/compute/function.cpp
@@ -162,7 +162,8 @@ HttpFunctionService::HttpFunctionService(CidStore& InCidStore,
TotalAttachmentBytes += Buffer.GetCompressedSize();
++AttachmentCount;
- const CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Buffer);
+ const CidStore::InsertResult InsertResult =
+ m_CidStore.AddChunk(Buffer.GetCompressed().Flatten().AsIoBuffer(), DataHash);
if (InsertResult.New)
{
@@ -404,7 +405,8 @@ HttpFunctionService::HttpFunctionService(CidStore& InCidStore,
TotalAttachmentBytes += CompressedSize;
++AttachmentCount;
- const CidStore::InsertResult InsertResult = m_CidStore.AddChunk(DataView);
+ const CidStore::InsertResult InsertResult =
+ m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash);
if (InsertResult.New)
{
diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 87118991e..5f804bd77 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -638,8 +638,8 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
ZEN_ASSERT(Attach.IsCompressedBinary());
CompressedBuffer AttachmentData = Attach.AsCompressedBinary();
- const uint64_t AttachmentSize = AttachmentData.GetRawSize();
- CidStore::InsertResult InsertResult = m_CidStore.AddChunk(AttachmentData);
+ const uint64_t AttachmentSize = AttachmentData.DecodeRawSize();
+ CidStore::InsertResult InsertResult = m_CidStore.AddChunk(AttachmentData.GetCompressed().Flatten().AsIoBuffer(), Attach.GetHash());
if (InsertResult.New)
{
@@ -1407,9 +1407,11 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
uint64_t ChunkSize = Chunk.GetSize();
if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
ZEN_ASSERT(!Compressed.IsNull());
- ChunkSize = Compressed.GetRawSize();
+ ChunkSize = RawSize;
}
CbObjectWriter Response;
@@ -1464,12 +1466,13 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
ZEN_ASSERT(!Compressed.IsNull());
if (IsOffset)
{
- uint64_t RawSize = Compressed.GetRawSize();
if ((Offset + Size) > RawSize)
{
Size = RawSize - Offset;
@@ -1539,7 +1542,7 @@ ProjectStore::GetChunk(const std::string_view Cid, ZenContentType AcceptType, Io
if (AcceptType == HttpContentType::kBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(OutChunk));
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
OutChunk = Compressed.Decompress().AsIoBuffer();
OutChunk.SetContentType(HttpContentType::kBinary);
}
@@ -1821,7 +1824,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
if (AcceptType == HttpContentType::kBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Value));
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value));
Value = Compressed.Decompress().AsIoBuffer();
ContentType = HttpContentType::kBinary;
}
@@ -2066,7 +2069,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
break;
case ZenContentType::kCompressedBinary:
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload)))
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)))
{
Package.AddAttachment(CbAttachment(Compressed, AttachmentHash));
}
@@ -2199,7 +2202,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
{
ZEN_INFO("Importing oplog '{}/{}'", ProjectId, OplogId);
IoBuffer CompressedPayload = HttpReq.ReadPayload();
- IoBuffer Payload = CompressedBuffer::FromCompressed(SharedBuffer(CompressedPayload)).Decompress().AsIoBuffer();
+ IoBuffer Payload =
+ CompressedBuffer::FromCompressedNoValidate(std::move(CompressedPayload)).Decompress().AsIoBuffer();
CbPackage RequestPackage = ParsePackageMessage(Payload);
CbObject Request = RequestPackage.GetObject();
@@ -2252,7 +2256,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
JobCount.fetch_add(1);
WorkerPool.ScheduleWork([this, &Attachment, &JobCount]() {
CompressedBuffer AttachmentBody = Attachment.AsCompressedBinary();
- m_CidStore.AddChunk(AttachmentBody, CidStore::InsertMode::kCopyOnly);
+ m_CidStore.AddChunk(AttachmentBody.GetCompressed().Flatten().AsIoBuffer(),
+ Attachment.GetHash(),
+ CidStore::InsertMode::kCopyOnly);
JobCount.fetch_add(-1);
});
}
@@ -2558,7 +2564,7 @@ namespace testutils {
Object.BeginArray("bulkdata");
for (const auto& Attachment : Attachments)
{
- CbAttachment Attach(Attachment.second, IoHash::FromBLAKE3(Attachment.second.GetRawHash()));
+ CbAttachment Attach(Attachment.second, IoHash::FromBLAKE3(Attachment.second.DecodeRawHash()));
Object.BeginObject();
Object << "id"sv << Attachment.first;
Object << "type"sv
@@ -2814,11 +2820,13 @@ TEST_CASE("project.store.partial.read")
}
{
IoBuffer Chunk;
- CHECK(ProjectStore.GetChunk(IoHash::FromBLAKE3(Attachments[OpIds[1]][0].second.GetRawHash()).ToHexString(),
+ CHECK(ProjectStore.GetChunk(IoHash::FromBLAKE3(Attachments[OpIds[1]][0].second.DecodeRawHash()).ToHexString(),
HttpContentType::kCompressedBinary,
Chunk) == HttpResponseCode::OK);
- CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
- CHECK(Attachment.GetRawSize() == Attachments[OpIds[1]][0].second.GetRawSize());
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
+ CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
}
IoBuffer ChunkResult;
@@ -2830,7 +2838,8 @@ TEST_CASE("project.store.partial.read")
HttpContentType::kCompressedBinary,
ChunkResult) == HttpResponseCode::OK);
CHECK(ChunkResult);
- CHECK(CompressedBuffer::FromCompressed(SharedBuffer(ChunkResult)).GetRawSize() == Attachments[OpIds[2]][1].second.GetRawSize());
+ CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
+ Attachments[OpIds[2]][1].second.DecodeRawSize());
IoBuffer PartialChunkResult;
CHECK(ProjectStore.GetChunk("proj1"sv,
@@ -2841,8 +2850,11 @@ TEST_CASE("project.store.partial.read")
HttpContentType::kCompressedBinary,
PartialChunkResult) == HttpResponseCode::OK);
CHECK(PartialChunkResult);
- CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult));
- CHECK(PartialCompressedResult.GetRawSize() >= 1773);
+ IoHash PartialRawHash;
+ uint64_t PartialRawSize;
+ CompressedBuffer PartialCompressedResult =
+ CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult), PartialRawHash, PartialRawSize);
+ CHECK(PartialRawSize >= 1773);
uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed);
diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp
index 0e46fff94..b26f9e437 100644
--- a/zenserver/testing/launch.cpp
+++ b/zenserver/testing/launch.cpp
@@ -477,7 +477,11 @@ HttpLaunchService::HttpLaunchService(CidStore& Store, const std::filesystem::pat
{
std::filesystem::path FullPath = SandboxDir / FileName;
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
+ ZEN_ASSERT(Compressed);
+ ZEN_ASSERT(FileHash == RawHash);
CompositeBuffer CompositeBuffer = Compressed.DecompressToComposite();
std::span<const SharedBuffer> Segments = CompositeBuffer.GetSegments();
std::vector<IoBuffer> Chunks(Segments.size());
diff --git a/zenserver/upstream/hordecompute.cpp b/zenserver/upstream/hordecompute.cpp
index 988386726..64d9fff72 100644
--- a/zenserver/upstream/hordecompute.cpp
+++ b/zenserver/upstream/hordecompute.cpp
@@ -983,10 +983,12 @@ namespace detail {
return;
}
+ IoHash RawHash;
+ uint64_t RawSize;
CompressedBuffer AttachmentBuffer =
- CompressedBuffer::FromCompressed(SharedBuffer(BinaryData[CompressedId]));
+ CompressedBuffer::FromCompressed(SharedBuffer(BinaryData[CompressedId]), RawHash, RawSize);
- if (!AttachmentBuffer)
+ if (!AttachmentBuffer || RawHash != DecompressedId)
{
Log().warn(
"Invalid output encountered (not valid CompressedBuffer format) {} compressed {} uncompressed",
@@ -997,7 +999,7 @@ namespace detail {
}
ApplyResult.TotalAttachmentBytes += AttachmentBuffer.GetCompressedSize();
- ApplyResult.TotalRawAttachmentBytes += AttachmentBuffer.GetRawSize();
+ ApplyResult.TotalRawAttachmentBytes += RawSize;
CbAttachment Attachment(AttachmentBuffer, DecompressedId);
OutputPackage.AddAttachment(Attachment);
diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp
index bc06653b9..92dac12de 100644
--- a/zenserver/upstream/upstreamcache.cpp
+++ b/zenserver/upstream/upstreamcache.cpp
@@ -214,7 +214,10 @@ namespace detail {
Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds;
Result.ErrorCode = AttachmentResult.ErrorCode;
- if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response)))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Chunk =
+ CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response), RawHash, RawSize))
{
Result.Response = AttachmentResult.Response;
++NumAttachments;
@@ -251,7 +254,10 @@ namespace detail {
Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds;
Result.ErrorCode = AttachmentResult.ErrorCode;
- if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response)))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Chunk =
+ CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response), RawHash, RawSize))
{
Package.AddAttachment(CbAttachment(Chunk, AttachmentHash.AsHash()));
}
@@ -335,7 +341,10 @@ namespace detail {
if (BlobResult.ErrorCode == 0)
{
- if (CompressedBuffer Chunk = CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response)))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Chunk =
+ CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response), RawHash, RawSize))
{
Package.AddAttachment(CbAttachment(Chunk, AttachmentHash.AsHash()));
}
@@ -398,6 +407,8 @@ namespace detail {
{
CacheChunkRequest& Request = *RequestPtr;
IoBuffer Payload;
+ IoHash RawHash = IoHash::Zero;
+ uint64_t RawSize = 0;
double ElapsedSeconds = 0.0;
CompressedBuffer Compressed;
@@ -416,15 +427,15 @@ namespace detail {
m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason);
if (Payload && IsCompressedBinary(Payload.GetContentType()))
{
- Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload));
+ Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
}
}
if (Compressed)
{
OnComplete({.Request = Request,
- .RawHash = IoHash::FromBLAKE3(Compressed.GetRawHash()),
- .RawSize = Compressed.GetRawSize(),
+ .RawHash = RawHash,
+ .RawSize = RawSize,
.Value = Payload,
.ElapsedSeconds = ElapsedSeconds,
.Source = &m_Info});
@@ -451,6 +462,8 @@ namespace detail {
{
CacheValueRequest& Request = *RequestPtr;
IoBuffer Payload;
+ IoHash RawHash = IoHash::Zero;
+ uint64_t RawSize = 0;
double ElapsedSeconds = 0.0;
CompressedBuffer Compressed;
@@ -470,12 +483,12 @@ namespace detail {
{
if (IsCompressedBinary(Payload.GetContentType()))
{
- Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload));
+ Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
}
else
{
- Compressed = CompressedBuffer::Compress(SharedBuffer(Payload));
- IoHash RawHash = IoHash::FromBLAKE3(Compressed.GetRawHash());
+ Compressed = CompressedBuffer::Compress(SharedBuffer(Payload));
+ RawHash = IoHash::FromBLAKE3(Compressed.DecodeRawHash());
if (RawHash != PayloadHash)
{
ZEN_WARN("Horde request for inline payload of {}/{}/{} has hash {}, expected hash {} from header",
@@ -493,8 +506,8 @@ namespace detail {
if (Compressed)
{
OnComplete({.Request = Request,
- .RawHash = IoHash::FromBLAKE3(Compressed.GetRawHash()),
- .RawSize = Compressed.GetRawSize(),
+ .RawHash = RawHash,
+ .RawSize = RawSize,
.Value = Payload,
.ElapsedSeconds = ElapsedSeconds,
.Source = &m_Info});
@@ -543,17 +556,17 @@ namespace detail {
}
else if (CacheRecord.Type == ZenContentType::kCompressedBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue), RawHash, RawSize);
if (!Compressed)
{
return {.Reason = std::string("Invalid compressed value buffer"), .Success = false};
}
- IoHash RawHash = IoHash::FromBLAKE3(Compressed.GetRawHash());
-
CbObjectWriter ReferencingObject;
ReferencingObject.AddBinaryAttachment("RawHash", RawHash);
- ReferencingObject.AddInteger("RawSize", Compressed.GetRawSize());
+ ReferencingObject.AddInteger("RawSize", RawSize);
return PerformStructuredPut(
Session,
@@ -1053,7 +1066,7 @@ namespace detail {
{
Payload = Compressed.GetCompressed().Flatten().AsIoBuffer();
Payload.SetContentType(ZenContentType::kCompressedBinary);
- RawSize = Compressed.GetRawSize();
+ RawSize = Compressed.DecodeRawSize();
Success = true;
}
}
@@ -1189,7 +1202,7 @@ namespace detail {
{
Payload = Compressed.GetCompressed().Flatten().AsIoBuffer();
Payload.SetContentType(ZenContentType::kCompressedBinary);
- RawSize = Compressed.GetRawSize();
+ RawSize = Compressed.DecodeRawSize();
Success = true;
}
}
@@ -1252,9 +1265,11 @@ namespace detail {
for (const IoBuffer& Value : Values)
{
- if (CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Value)))
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Value), RawHash, RawSize))
{
- Package.AddAttachment(CbAttachment(AttachmentBuffer, IoHash::FromBLAKE3(AttachmentBuffer.GetRawHash())));
+ Package.AddAttachment(CbAttachment(AttachmentBuffer, RawHash));
}
else
{
@@ -1282,7 +1297,9 @@ namespace detail {
}
else if (CacheRecord.Type == ZenContentType::kCompressedBinary)
{
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue));
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue), RawHash, RawSize);
if (!Compressed)
{
return {.Reason = std::string("Invalid value compressed buffer"), .Success = false};
@@ -1312,9 +1329,8 @@ namespace detail {
}
BatchWriter.EndObject();
// Policy unspecified and expected to be Default
- IoHash Hash = IoHash::FromBLAKE3(Compressed.GetRawHash());
- BatchWriter.AddBinaryAttachment("RawHash"sv, Hash);
- BatchPackage.AddAttachment(CbAttachment(Compressed, Hash));
+ BatchWriter.AddBinaryAttachment("RawHash"sv, RawHash);
+ BatchPackage.AddAttachment(CbAttachment(Compressed, RawHash));
}
BatchWriter.EndObject();
}
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 8b2797ce9..6319c2d2e 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -23,14 +23,12 @@ struct CidStore::Impl
void Initialize(const CidStoreConfiguration& Config) { m_CasStore.Initialize(Config); }
- CidStore::InsertResult AddChunk(const CompressedBuffer& ChunkData, CidStore::InsertMode Mode)
+ CidStore::InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, CidStore::InsertMode Mode)
{
- const IoHash DecompressedId = IoHash::FromBLAKE3(ChunkData.GetRawHash());
- IoBuffer Payload = ChunkData.GetCompressed().Flatten().AsIoBuffer();
-
+ IoBuffer Payload(ChunkData);
Payload.SetContentType(ZenContentType::kCompressedBinary);
- CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, DecompressedId, static_cast<CasStore::InsertMode>(Mode));
+ CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, RawHash, static_cast<CasStore::InsertMode>(Mode));
return {.New = Result.New};
}
@@ -78,9 +76,9 @@ CidStore::Initialize(const CidStoreConfiguration& Config)
}
CidStore::InsertResult
-CidStore::AddChunk(const CompressedBuffer& ChunkData, InsertMode Mode)
+CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode)
{
- return m_Impl->AddChunk(ChunkData, Mode);
+ return m_Impl->AddChunk(ChunkData, RawHash, Mode);
}
IoBuffer
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index d0c2f59ac..9e181eb87 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -269,9 +269,11 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
}
IoBuffer Buffer(IoBuffer::Wrap, Data, Size);
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); Compressed)
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize); Compressed)
{
- if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+ if (RawHash != Hash)
{
// Hash mismatch
BadKeys.push_back(Hash);
@@ -295,10 +297,13 @@ CasContainerStrategy::Scrub(ScrubContext& Ctx)
const IoHash& Hash = ChunkIndexToChunkHash[ChunkIndex];
IoBuffer Buffer(IoBuffer::BorrowedFile, File.GetBasicFile().Handle(), Offset, Size);
+
+ IoHash RawHash;
+ uint64_t RawSize;
// TODO: Add API to verify compressed buffer without having to memorymap the whole file
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); Compressed)
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize); Compressed)
{
- if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+ if (RawHash != Hash)
{
// Hash mismatch
BadKeys.push_back(Hash);
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 1b53c405b..70ee67d93 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -789,9 +789,11 @@ FileCasStrategy::Scrub(ScrubContext& Ctx)
ChunkBytes += Payload.FileSize();
IoBuffer Buffer(IoBuffer::BorrowedFile, Payload.Handle(), 0, Payload.FileSize());
- if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer)); Compressed)
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize); Compressed)
{
- if (IoHash::FromBLAKE3(Compressed.GetRawHash()) != Hash)
+ if (RawHash != Hash)
{
// Hash mismatch
BadHashes.push_back(Hash);
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index 8aac65bb4..e0cddb291 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -701,7 +701,8 @@ TEST_CASE("gc.basic")
IoBuffer Chunk = CreateChunk(128);
auto CompressedChunk = Compress(Chunk);
- const auto InsertResult = CidStore.AddChunk(CompressedChunk);
+ const auto InsertResult =
+ CidStore.AddChunk(CompressedChunk.GetCompressed().Flatten().AsIoBuffer(), IoHash::FromBLAKE3(CompressedChunk.DecodeRawHash()));
CHECK(InsertResult.New);
GcContext GcCtx;
@@ -710,7 +711,7 @@ TEST_CASE("gc.basic")
CidStore.Flush();
Gc.CollectGarbage(GcCtx);
- CHECK(!CidStore.ContainsChunk(IoHash::FromBLAKE3(CompressedChunk.GetRawHash())));
+ CHECK(!CidStore.ContainsChunk(IoHash::FromBLAKE3(CompressedChunk.DecodeRawHash())));
}
TEST_CASE("gc.full")
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index e8984a83d..16ca78225 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -70,7 +70,7 @@ public:
};
void Initialize(const CidStoreConfiguration& Config);
- InsertResult AddChunk(const CompressedBuffer& ChunkData, InsertMode Mode = InsertMode::kMayBeMovedInPlace);
+ InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace);
IoBuffer FindChunkByCid(const IoHash& DecompressedId);
bool ContainsChunk(const IoHash& DecompressedId);
void FilterChunks(HashKeySet& InOutChunks);
diff --git a/zenutil/cache/cacherequests.cpp b/zenutil/cache/cacherequests.cpp
index 1ba2721b6..e6df4b4aa 100644
--- a/zenutil/cache/cacherequests.cpp
+++ b/zenutil/cache/cacherequests.cpp
@@ -227,7 +227,7 @@ namespace cacherequests {
PutCacheRecordRequestValue& Value = Request.Values[It->second];
ZEN_ASSERT(Value.RawHash == ValueHash);
Value.Body = Attachment->AsCompressedBinary();
- ZEN_ASSERT(IoHash::FromBLAKE3(Value.Body.GetRawHash()) == Value.RawHash);
+ ZEN_ASSERT_SLOW(IoHash::FromBLAKE3(Value.Body.DecodeRawHash()) == Value.RawHash);
}
}
});
@@ -268,10 +268,10 @@ namespace cacherequests {
const CompressedBuffer& Buffer = Value.Body;
if (Buffer)
{
- IoHash AttachmentHash = IoHash::FromBLAKE3(Buffer.GetRawHash());
+ IoHash AttachmentHash = IoHash::FromBLAKE3(Buffer.DecodeRawHash()); // TODO: Slow!
Writer.AddBinaryAttachment("RawHash", AttachmentHash);
OutPackage.AddAttachment(CbAttachment(Buffer, AttachmentHash));
- Writer.AddInteger("RawSize", Buffer.GetRawSize());
+ Writer.AddInteger("RawSize", Buffer.DecodeRawSize()); // TODO: Slow!
}
else
{
@@ -500,12 +500,12 @@ namespace cacherequests {
Writer.BeginArray("Values");
for (const GetCacheRecordResultValue& Value : RecordResult->Values)
{
- IoHash AttachmentHash = Value.Body ? IoHash::FromBLAKE3(Value.Body.GetRawHash()) : Value.RawHash;
+ IoHash AttachmentHash = Value.Body ? IoHash::FromBLAKE3(Value.Body.DecodeRawHash()) : Value.RawHash;
Writer.BeginObject();
{
Writer.AddObjectId("Id", Value.Id);
Writer.AddHash("RawHash", AttachmentHash);
- Writer.AddInteger("RawSize", Value.Body ? Value.Body.GetRawSize() : Value.RawSize);
+ Writer.AddInteger("RawSize", Value.Body ? Value.Body.DecodeRawSize() : Value.RawSize);
}
Writer.EndObject();
if (Value.Body)
@@ -591,12 +591,11 @@ namespace cacherequests {
WriteCacheRequestKey(Writer, ValueRequest.Key);
if (ValueRequest.Body)
{
- if (ValueRequest.RawHash != IoHash::Zero &&
- IoHash::FromBLAKE3(ValueRequest.Body.GetRawHash()) != ValueRequest.RawHash)
+ IoHash AttachmentHash = IoHash::FromBLAKE3(ValueRequest.Body.DecodeRawHash());
+ if (ValueRequest.RawHash != IoHash::Zero && AttachmentHash != ValueRequest.RawHash)
{
return false;
}
- IoHash AttachmentHash = IoHash::FromBLAKE3(ValueRequest.Body.GetRawHash());
Writer.AddBinaryAttachment("RawHash", AttachmentHash);
OutPackage.AddAttachment(CbAttachment(ValueRequest.Body, AttachmentHash));
}
@@ -776,7 +775,7 @@ namespace cacherequests {
ValueResult.Body = Attachment ? Attachment->AsCompressedBinary() : CompressedBuffer();
if (ValueResult.Body)
{
- ValueResult.RawSize = ValueResult.Body.GetRawSize();
+ ValueResult.RawSize = ValueResult.Body.DecodeRawSize();
}
else
{
@@ -1042,8 +1041,8 @@ namespace cacherequests {
static bool operator==(const PutCacheRecordRequestValue& Lhs, const PutCacheRecordRequestValue& Rhs)
{
- const IoHash LhsRawHash = Lhs.RawHash != IoHash::Zero ? Lhs.RawHash : IoHash::FromBLAKE3(Lhs.Body.GetRawHash());
- const IoHash RhsRawHash = Rhs.RawHash != IoHash::Zero ? Rhs.RawHash : IoHash::FromBLAKE3(Rhs.Body.GetRawHash());
+ const IoHash LhsRawHash = Lhs.RawHash != IoHash::Zero ? Lhs.RawHash : IoHash::FromBLAKE3(Lhs.Body.DecodeRawHash());
+ const IoHash RhsRawHash = Rhs.RawHash != IoHash::Zero ? Rhs.RawHash : IoHash::FromBLAKE3(Rhs.Body.DecodeRawHash());
return Lhs.Id == Rhs.Id && LhsRawHash == RhsRawHash &&
Lhs.Body.GetCompressed().Flatten().GetView().EqualBytes(Rhs.Body.GetCompressed().Flatten().GetView());
}
@@ -1112,7 +1111,7 @@ namespace cacherequests {
{
return false;
}
- if (bool(Lhs.Body) && Lhs.Body.GetRawHash() != Rhs.Body.GetRawHash())
+ if (bool(Lhs.Body) && Lhs.Body.DecodeRawHash() != Rhs.Body.DecodeRawHash())
{
return false;
}
@@ -1313,31 +1312,31 @@ namespace cacherequests {
GetCacheRecordsResult FullResult = {
{GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[0].Key,
.Values = {{.Id = FullPutRequestCopy.Requests[0].Values[0].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[0].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[0].Values[0].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[0].Values[0].Body.DecodeRawSize(),
.Body = FullPutRequestCopy.Requests[0].Values[0].Body},
{.Id = FullPutRequestCopy.Requests[0].Values[1].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[1].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[0].Values[1].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[0].Values[1].Body.DecodeRawSize(),
.Body = FullPutRequestCopy.Requests[0].Values[1].Body},
{.Id = FullPutRequestCopy.Requests[0].Values[2].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[2].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[0].Values[2].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[0].Values[2].Body.DecodeRawSize(),
.Body = FullPutRequestCopy.Requests[0].Values[2].Body}}},
{}, // Simulate not have!
GetCacheRecordResult{.Key = FullPutRequestCopy.Requests[2].Key,
.Values = {{.Id = FullPutRequestCopy.Requests[2].Values[0].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[0].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[2].Values[0].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[2].Values[0].Body.DecodeRawSize(),
.Body = FullPutRequestCopy.Requests[2].Values[0].Body},
{.Id = FullPutRequestCopy.Requests[2].Values[1].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[1].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[2].Values[1].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[2].Values[1].Body.DecodeRawSize(),
.Body = {}}, // Simulate not have
{.Id = FullPutRequestCopy.Requests[2].Values[2].Id,
- .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[2].Body.GetRawHash()),
- .RawSize = FullPutRequestCopy.Requests[2].Values[2].Body.GetRawSize(),
+ .RawHash = IoHash::FromBLAKE3(FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawHash()),
+ .RawSize = FullPutRequestCopy.Requests[2].Values[2].Body.DecodeRawSize(),
.Body = FullPutRequestCopy.Requests[2].Values[2].Body}}}}};
CbPackage FullResponsePackage;
CHECK(FullResult.Format(FullResponsePackage));
@@ -1369,15 +1368,15 @@ namespace cacherequests {
.DefaultPolicy = CachePolicy::StoreLocal,
.Namespace = "other_namespace",
.Requests = {{.Key = {.Bucket = "finebucket", .Hash = IoHash::FromHexString("d1df59fcab06793a5f2c372d795bb907a15cab15")},
- .RawHash = IoHash::FromBLAKE3(Buffers[0].GetRawHash()),
+ .RawHash = IoHash::FromBLAKE3(Buffers[0].DecodeRawHash()),
.Body = Buffers[0],
.Policy = CachePolicy::Local},
{.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("177030568fdd461bf4fe5ddbf4d463e514e8178e")},
- .RawHash = IoHash::FromBLAKE3(Buffers[1].GetRawHash()),
+ .RawHash = IoHash::FromBLAKE3(Buffers[1].DecodeRawHash()),
.Body = Buffers[1],
.Policy = CachePolicy::Remote},
{.Key = {.Bucket = "badbucket", .Hash = IoHash::FromHexString("e1ce9e1ac8a6f5953dc14c1fa9512b804ed689df")},
- .RawHash = IoHash::FromBLAKE3(Buffers[2].GetRawHash())}}};
+ .RawHash = IoHash::FromBLAKE3(Buffers[2].DecodeRawHash())}}};
CbPackage FullRequestPackage;
CHECK(FullRequest.Format(FullRequestPackage));
@@ -1441,12 +1440,12 @@ namespace cacherequests {
{MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}};
CacheValuesResult FullResult = {
.Results = {
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].GetRawHash()), .Body = Buffers[0][0]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].GetRawHash()), .Body = Buffers[0][1]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].GetRawHash()), .Body = Buffers[0][2]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].GetRawHash()), .Body = Buffers[2][0]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].GetRawHash()), .Body = Buffers[2][1]},
- CacheValueResult{.RawSize = Buffers[2][2].GetRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].GetRawHash())}}};
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].DecodeRawHash()), .Body = Buffers[0][0]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].DecodeRawHash()), .Body = Buffers[0][1]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].DecodeRawHash()), .Body = Buffers[0][2]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].DecodeRawHash()), .Body = Buffers[2][0]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].DecodeRawHash()), .Body = Buffers[2][1]},
+ CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].DecodeRawHash())}}};
CbPackage FullResponsePackage;
CHECK(FullResult.Format(FullResponsePackage));
CacheValuesResult FullResultCopy;
@@ -1511,12 +1510,12 @@ namespace cacherequests {
{MakeCompressedBuffer(5123), MakeCompressedBuffer(2321), MakeCompressedBuffer(2333)}};
GetCacheChunksResult FullResult = {
.Results = {
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].GetRawHash()), .Body = Buffers[0][0]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].GetRawHash()), .Body = Buffers[0][1]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].GetRawHash()), .Body = Buffers[0][2]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].GetRawHash()), .Body = Buffers[2][0]},
- CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].GetRawHash()), .Body = Buffers[2][1]},
- CacheValueResult{.RawSize = Buffers[2][2].GetRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].GetRawHash())}}};
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][0].DecodeRawHash()), .Body = Buffers[0][0]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][1].DecodeRawHash()), .Body = Buffers[0][1]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[0][2].DecodeRawHash()), .Body = Buffers[0][2]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][0].DecodeRawHash()), .Body = Buffers[2][0]},
+ CacheValueResult{.RawSize = 0, .RawHash = IoHash::FromBLAKE3(Buffers[2][1].DecodeRawHash()), .Body = Buffers[2][1]},
+ CacheValueResult{.RawSize = Buffers[2][2].DecodeRawSize(), .RawHash = IoHash::FromBLAKE3(Buffers[2][2].DecodeRawHash())}}};
CbPackage FullResponsePackage;
CHECK(FullResult.Format(FullResponsePackage));
GetCacheChunksResult FullResultCopy;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-26 19:49:19 +0000