reduce parsing of compressed headers

1 files changed, 30 insertions, 18 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index 87118991e..5f804bd77 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -638,8 +638,8 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
 		ZEN_ASSERT(Attach.IsCompressedBinary());
 
 		CompressedBuffer	   AttachmentData = Attach.AsCompressedBinary();
-		const uint64_t		   AttachmentSize = AttachmentData.GetRawSize();
-		CidStore::InsertResult InsertResult	  = m_CidStore.AddChunk(AttachmentData);
+		const uint64_t		   AttachmentSize = AttachmentData.DecodeRawSize();
+		CidStore::InsertResult InsertResult = m_CidStore.AddChunk(AttachmentData.GetCompressed().Flatten().AsIoBuffer(), Attach.GetHash());
 
 		if (InsertResult.New)
 		{
@@ -1407,9 +1407,11 @@ ProjectStore::GetChunkInfo(const std::string_view ProjectId,
 	uint64_t ChunkSize = Chunk.GetSize();
 	if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
 	{
-		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+		IoHash			 RawHash;
+		uint64_t		 RawSize;
+		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
 		ZEN_ASSERT(!Compressed.IsNull());
-		ChunkSize = Compressed.GetRawSize();
+		ChunkSize = RawSize;
 	}
 
 	CbObjectWriter Response;
@@ -1464,12 +1466,13 @@ ProjectStore::GetChunk(const std::string_view ProjectId,
 
 	if (Chunk.GetContentType() == HttpContentType::kCompressedBinary)
 	{
-		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
+		IoHash			 RawHash;
+		uint64_t		 RawSize;
+		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
 		ZEN_ASSERT(!Compressed.IsNull());
 
 		if (IsOffset)
 		{
-			uint64_t RawSize = Compressed.GetRawSize();
 			if ((Offset + Size) > RawSize)
 			{
 				Size = RawSize - Offset;
@@ -1539,7 +1542,7 @@ ProjectStore::GetChunk(const std::string_view Cid, ZenContentType AcceptType, Io
 
 	if (AcceptType == HttpContentType::kBinary)
 	{
-		CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(OutChunk));
+		CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk));
 		OutChunk					= Compressed.Decompress().AsIoBuffer();
 		OutChunk.SetContentType(HttpContentType::kBinary);
 	}
@@ -1821,7 +1824,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 			if (AcceptType == HttpContentType::kBinary)
 			{
-				CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Value));
+				CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value));
 				Value						= Compressed.Decompress().AsIoBuffer();
 				ContentType					= HttpContentType::kBinary;
 			}
@@ -2066,7 +2069,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 										break;
 
 									case ZenContentType::kCompressedBinary:
-										if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload)))
+										if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload)))
 										{
 											Package.AddAttachment(CbAttachment(Compressed, AttachmentHash));
 										}
@@ -2199,7 +2202,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 					{
 						ZEN_INFO("Importing oplog '{}/{}'", ProjectId, OplogId);
 						IoBuffer CompressedPayload = HttpReq.ReadPayload();
-						IoBuffer Payload = CompressedBuffer::FromCompressed(SharedBuffer(CompressedPayload)).Decompress().AsIoBuffer();
+						IoBuffer Payload =
+							CompressedBuffer::FromCompressedNoValidate(std::move(CompressedPayload)).Decompress().AsIoBuffer();
 
 						CbPackage RequestPackage = ParsePackageMessage(Payload);
 						CbObject  Request		 = RequestPackage.GetObject();
@@ -2252,7 +2256,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 							JobCount.fetch_add(1);
 							WorkerPool.ScheduleWork([this, &Attachment, &JobCount]() {
 								CompressedBuffer AttachmentBody = Attachment.AsCompressedBinary();
-								m_CidStore.AddChunk(AttachmentBody, CidStore::InsertMode::kCopyOnly);
+								m_CidStore.AddChunk(AttachmentBody.GetCompressed().Flatten().AsIoBuffer(),
+													Attachment.GetHash(),
+													CidStore::InsertMode::kCopyOnly);
 								JobCount.fetch_add(-1);
 							});
 						}
@@ -2558,7 +2564,7 @@ namespace testutils {
 			Object.BeginArray("bulkdata");
 			for (const auto& Attachment : Attachments)
 			{
-				CbAttachment Attach(Attachment.second, IoHash::FromBLAKE3(Attachment.second.GetRawHash()));
+				CbAttachment Attach(Attachment.second, IoHash::FromBLAKE3(Attachment.second.DecodeRawHash()));
 				Object.BeginObject();
 				Object << "id"sv << Attachment.first;
 				Object << "type"sv
@@ -2814,11 +2820,13 @@ TEST_CASE("project.store.partial.read")
 	}
 	{
 		IoBuffer Chunk;
-		CHECK(ProjectStore.GetChunk(IoHash::FromBLAKE3(Attachments[OpIds[1]][0].second.GetRawHash()).ToHexString(),
+		CHECK(ProjectStore.GetChunk(IoHash::FromBLAKE3(Attachments[OpIds[1]][0].second.DecodeRawHash()).ToHexString(),
 									HttpContentType::kCompressedBinary,
 									Chunk) == HttpResponseCode::OK);
-		CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk));
-		CHECK(Attachment.GetRawSize() == Attachments[OpIds[1]][0].second.GetRawSize());
+		IoHash			 RawHash;
+		uint64_t		 RawSize;
+		CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize);
+		CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize());
 	}
 
 	IoBuffer ChunkResult;
@@ -2830,7 +2838,8 @@ TEST_CASE("project.store.partial.read")
 								HttpContentType::kCompressedBinary,
 								ChunkResult) == HttpResponseCode::OK);
 	CHECK(ChunkResult);
-	CHECK(CompressedBuffer::FromCompressed(SharedBuffer(ChunkResult)).GetRawSize() == Attachments[OpIds[2]][1].second.GetRawSize());
+	CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() ==
+		  Attachments[OpIds[2]][1].second.DecodeRawSize());
 
 	IoBuffer PartialChunkResult;
 	CHECK(ProjectStore.GetChunk("proj1"sv,
@@ -2841,8 +2850,11 @@ TEST_CASE("project.store.partial.read")
 								HttpContentType::kCompressedBinary,
 								PartialChunkResult) == HttpResponseCode::OK);
 	CHECK(PartialChunkResult);
-	CompressedBuffer PartialCompressedResult = CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult));
-	CHECK(PartialCompressedResult.GetRawSize() >= 1773);
+	IoHash			 PartialRawHash;
+	uint64_t		 PartialRawSize;
+	CompressedBuffer PartialCompressedResult =
+		CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult), PartialRawHash, PartialRawSize);
+	CHECK(PartialRawSize >= 1773);
 
 	uint64_t	   RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5);
 	SharedBuffer   PartialDecompressed			= PartialCompressedResult.Decompress(RawOffsetInPartialCompressed);