optimizations (#200)

* Use direct file read and direct buffer allocation for small IoBuffer materalization
* Reduce range of materialized data in CompositeBuffer reading
CompressedBuffer header reading often only need a small part and not the whole file
* reduce lock contention in IoBuffer::Materialize
* Reduce parsing of compressed headers
Validate header type at decompression
* faster CreateDirectories - start from leaf going up and recurse back
* optimized BufferHeader::IsValid
* Add ValidateCompressedHeader to use when we don't need the actual compressed data
Validate that we always get compressed data in CidStore::AddChunk
* changelog

1 files changed, 13 insertions, 13 deletions

diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp
index 1d0596c29..c72410173 100644
--- a/zenserver-test/zenserver-test.cpp
+++ b/zenserver-test/zenserver-test.cpp
@@ -524,7 +524,7 @@ TEST_CASE("project.basic")
 			uint8_t AttachData[] = {1, 2, 3};
 
 			zen::CompressedBuffer Attachment = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone(zen::MemoryView{AttachData, 3}));
-			zen::CbAttachment	  Attach{Attachment, IoHash::FromBLAKE3(Attachment.GetRawHash())};
+			zen::CbAttachment	  Attach{Attachment, IoHash::FromBLAKE3(Attachment.DecodeRawHash())};
 
 			zen::CbObjectWriter OpWriter;
 			OpWriter << "key"
@@ -765,7 +765,7 @@ TEST_CASE("zcache.cbpackage")
 		auto Data			= zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
 		auto CompressedData = zen::CompressedBuffer::Compress(Data);
 
-		OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
+		OutAttachmentKey = zen::IoHash::FromBLAKE3(CompressedData.DecodeRawHash());
 
 		zen::CbWriter Obj;
 		Obj.BeginObject("obj"sv);
@@ -988,7 +988,7 @@ TEST_CASE("zcache.policy")
 	auto GeneratePackage = [](zen::IoHash& OutRecordKey, zen::IoHash& OutAttachmentKey) -> zen::CbPackage {
 		auto Data			= zen::SharedBuffer::Clone(zen::MakeMemoryView<uint8_t>({1, 2, 3, 4, 5, 6, 7, 8, 9}));
 		auto CompressedData = zen::CompressedBuffer::Compress(Data);
-		OutAttachmentKey	= zen::IoHash::FromBLAKE3(CompressedData.GetRawHash());
+		OutAttachmentKey	= zen::IoHash::FromBLAKE3(CompressedData.DecodeRawHash());
 
 		zen::CbWriter Writer;
 		Writer.BeginObject("obj"sv);
@@ -1961,8 +1961,8 @@ TEST_CASE("zcache.rpc.allpolicies")
 			{
 				Builder.BeginObject();
 				Builder.AddObjectId("Id"sv, ValueIds[ValueIndex]);
-				Builder.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(KeyData.BufferValues[ValueIndex].GetRawHash()));
-				Builder.AddInteger("RawSize"sv, KeyData.BufferValues[ValueIndex].GetRawSize());
+				Builder.AddBinaryAttachment("RawHash"sv, IoHash::FromBLAKE3(KeyData.BufferValues[ValueIndex].DecodeRawHash()));
+				Builder.AddInteger("RawSize"sv, KeyData.BufferValues[ValueIndex].DecodeRawSize());
 				Builder.EndObject();
 			}
 			Builder.EndArray();
@@ -2138,9 +2138,9 @@ TEST_CASE("zcache.rpc.allpolicies")
 					WriteToString<32> ValueName("Get(", KeyData->KeyIndex, ",", ExpectedValueIndex, ")");
 
 					CompressedBuffer ExpectedValue = KeyData->BufferValues[ExpectedValueIndex];
-					CHECK_MESSAGE(Value.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+					CHECK_MESSAGE(Value.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
 								  WriteToString<32>(ValueName, " RawHash did not match.").c_str());
-					CHECK_MESSAGE(Value.RawSize == ExpectedValue.GetRawSize(),
+					CHECK_MESSAGE(Value.RawSize == ExpectedValue.DecodeRawSize(),
 								  WriteToString<32>(ValueName, " RawSize did not match.").c_str());
 
 					if (KeyData->GetRequestsData)
@@ -2203,9 +2203,9 @@ TEST_CASE("zcache.rpc.allpolicies")
 			if (!KeyData->ForceMiss && Succeeded)
 			{
 				CompressedBuffer ExpectedValue = KeyData->BufferValues[0];
-				CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+				CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
 							  WriteToString<32>(Name, " RawHash did not match.").c_str());
-				CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(),
+				CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.DecodeRawSize(),
 							  WriteToString<32>(Name, " RawSize did not match.").c_str());
 
 				if (KeyData->GetRequestsData)
@@ -2277,9 +2277,9 @@ TEST_CASE("zcache.rpc.allpolicies")
 			if (KeyData->ShouldBeHit && Succeeded)
 			{
 				CompressedBuffer ExpectedValue = KeyData->BufferValues[ValueIndex];
-				CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.GetRawHash()),
+				CHECK_MESSAGE(ValueResult.RawHash == IoHash::FromBLAKE3(ExpectedValue.DecodeRawHash()),
 							  WriteToString<32>(Name, " had unexpected RawHash.").c_str());
-				CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.GetRawSize(),
+				CHECK_MESSAGE(ValueResult.RawSize == ExpectedValue.DecodeRawSize(),
 							  WriteToString<32>(Name, " had unexpected RawSize.").c_str());
 
 				if (KeyData->GetRequestsData)
@@ -2659,11 +2659,11 @@ TEST_CASE("http.package")
 	zen::CompressedBuffer AttachmentData1 = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone({Data1, 4}),
 																			zen::OodleCompressor::NotSet,
 																			zen::OodleCompressionLevel::None);
-	zen::CbAttachment	  Attach1{AttachmentData1, IoHash::FromBLAKE3(AttachmentData1.GetRawHash())};
+	zen::CbAttachment	  Attach1{AttachmentData1, IoHash::FromBLAKE3(AttachmentData1.DecodeRawHash())};
 	zen::CompressedBuffer AttachmentData2 = zen::CompressedBuffer::Compress(zen::SharedBuffer::Clone({Data2, 8}),
 																			zen::OodleCompressor::NotSet,
 																			zen::OodleCompressionLevel::None);
-	zen::CbAttachment	  Attach2{AttachmentData2, IoHash::FromBLAKE3(AttachmentData2.GetRawHash())};
+	zen::CbAttachment	  Attach2{AttachmentData2, IoHash::FromBLAKE3(AttachmentData2.DecodeRawHash())};
 
 	zen::CbObjectWriter Writer;