store cache rawhash and rawsize for unstructured cache values (#234)

* refactored MemoryCacheBucket to allow for storing RawHash/RawSize.
* remove redundant conversions in AccessTime
* reduce max count for memory cache bucket to 32-bit value
* refactored DiskCacheBucket to allow for storing RawHash/RawSize.
* Use CompressedBuffer::ValidateCompressedHeader when applicable
* Make sure we rewrite the snapshot if we read an legacy existing index/log
* changelog

1 files changed, 57 insertions, 30 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index 1ef880396..154676d77 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -973,7 +973,22 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con
 
 	if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary)
 	{
-		m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body});
+		IoHash	 RawHash = IoHash::Zero;
+		uint64_t RawSize = Body.GetSize();
+		if (ContentType == HttpContentType::kCompressedBinary)
+		{
+			if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize))
+			{
+				return Request.WriteResponse(HttpResponseCode::BadRequest,
+											 HttpContentType::kText,
+											 "Payload is not a valid compressed binary"sv);
+			}
+		}
+		else
+		{
+			RawHash = IoHash::HashBuffer(SharedBuffer(Body));
+		}
+		m_CacheStore.Put(Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body, .RawSize = RawSize, .RawHash = RawHash});
 
 		if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote))
 		{
@@ -1965,6 +1980,7 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CbPackage& BatchReques
 		PolicyText					= RequestObject["Policy"sv].AsString();
 		CachePolicy Policy			= !PolicyText.empty() ? ParseCachePolicy(PolicyText) : DefaultPolicy;
 		IoHash		RawHash			= RequestObject["RawHash"sv].AsBinaryAttachment();
+		uint64_t	RawSize			= RequestObject["RawSize"sv].AsUInt64();
 		bool		Succeeded		= false;
 		uint64_t	TransferredSize = 0;
 
@@ -1984,7 +2000,11 @@ HttpStructuredCacheService::HandleRpcPutCacheValues(const CbPackage& BatchReques
 				{
 					IoBuffer Value = Chunk.GetCompressed().Flatten().AsIoBuffer();
 					Value.SetContentType(ZenContentType::kCompressedBinary);
-					m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = Value});
+					if (RawSize == 0)
+					{
+						RawSize = Chunk.DecodeRawSize();
+					}
+					m_CacheStore.Put(*Namespace, Key.Bucket, Key.Hash, {.Value = Value, .RawSize = RawSize, .RawHash = RawHash});
 					TransferredSize = Chunk.GetCompressedSize();
 				}
 				Succeeded = true;
@@ -2089,7 +2109,9 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(CbObjectView RpcRequest)
 		{
 			if (m_CacheStore.Get(*Namespace, Key.Bucket, Key.Hash, CacheValue) && IsCompressedBinary(CacheValue.Value.GetContentType()))
 			{
-				Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), Request.RawHash, Request.RawSize);
+				Request.RawHash = CacheValue.RawHash;
+				Request.RawSize = CacheValue.RawSize;
+				Request.Result	= CompressedBuffer::FromCompressedNoValidate(std::move(CacheValue.Value));
 			}
 		}
 		if (Request.Result)
@@ -2157,12 +2179,15 @@ HttpStructuredCacheService::HandleRpcGetCacheValues(CbObjectView RpcRequest)
 					{
 						if (HasData && !SkipData)
 						{
-							Request.Result = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), Request.RawHash, Request.RawSize);
+							Request.Result = CompressedBuffer::FromCompressedNoValidate(IoBuffer(Params.Value));
 						}
 
 						if (HasData && StoreData)
 						{
-							m_CacheStore.Put(*Namespace, Request.Key.Bucket, Request.Key.Hash, ZenCacheValue{Params.Value});
+							m_CacheStore.Put(*Namespace,
+											 Request.Key.Bucket,
+											 Request.Key.Hash,
+											 ZenCacheValue{.Value = Params.Value, .RawSize = Request.RawSize, .RawHash = Request.RawHash});
 						}
 
 						ZEN_DEBUG("GETCACHEVALUES HIT - '{}/{}/{}' {} ({}) in {}",
@@ -2531,16 +2556,23 @@ HttpStructuredCacheService::GetLocalCacheRecords(std::string_view							Namespac
 				}
 				else if (IoBuffer Payload = m_CidStore.FindChunkByCid(Request->Key->ChunkId))
 				{
-					IoHash			 RawHash;
-					CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, Request->RawSize);
-					if (Compressed)
+					if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
 					{
-						if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
+						Request->Value = CompressedBuffer::FromCompressedNoValidate(std::move(Payload));
+						if (Request->Value)
 						{
-							Request->Value = Compressed;
+							Request->Exists		  = true;
+							Request->RawSizeKnown = false;
+						}
+					}
+					else
+					{
+						IoHash _;
+						if (CompressedBuffer::ValidateCompressedHeader(Payload, _, Request->RawSize))
+						{
+							Request->Exists		  = true;
+							Request->RawSizeKnown = true;
 						}
-						Request->Exists		  = true;
-						Request->RawSizeKnown = true;
 					}
 				}
 			}
@@ -2571,19 +2603,13 @@ HttpStructuredCacheService::GetLocalCacheValues(std::string_view						   Namespa
 			{
 				if (IsCompressedBinary(CacheValue.Value.GetContentType()))
 				{
-					IoHash			 RawHash;
-					uint64_t		 RawSize;
-					CompressedBuffer Result = CompressedBuffer::FromCompressed(SharedBuffer(CacheValue.Value), RawHash, RawSize);
-					if (Result)
+					Request->Key->ChunkId = CacheValue.RawHash;
+					Request->Exists		  = true;
+					Request->RawSize	  = CacheValue.RawSize;
+					Request->RawSizeKnown = true;
+					if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
 					{
-						if (!EnumHasAllFlags(Request->DownstreamPolicy, CachePolicy::SkipData))
-						{
-							Request->Value = Result;
-						}
-						Request->Key->ChunkId = RawHash;
-						Request->Exists		  = true;
-						Request->RawSize	  = RawSize;
-						Request->RawSizeKnown = true;
+						Request->Value = CompressedBuffer::FromCompressedNoValidate(std::move(CacheValue.Value));
 					}
 				}
 			}
@@ -2625,10 +2651,8 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view							 Names
 			if (EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::StoreLocal) ||
 				!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
 			{
-				IoHash			 RawHash;
-				uint64_t		 RawSize;
-				CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Params.Value), RawHash, RawSize);
-				if (!Compressed || RawSize != Params.RawSize || RawHash != Params.RawHash)
+				CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(IoBuffer(Params.Value));
+				if (!Compressed)
 				{
 					return;
 				}
@@ -2637,11 +2661,14 @@ HttpStructuredCacheService::GetUpstreamCacheChunks(std::string_view							 Names
 				{
 					if (Request.IsRecordRequest)
 					{
-						m_CidStore.AddChunk(Params.Value, RawHash);
+						m_CidStore.AddChunk(Params.Value, Params.RawHash);
 					}
 					else
 					{
-						m_CacheStore.Put(Namespace, Key.Key.Bucket, Key.Key.Hash, {.Value = Params.Value});
+						m_CacheStore.Put(Namespace,
+										 Key.Key.Bucket,
+										 Key.Key.Hash,
+										 {.Value = Params.Value, .RawSize = Params.RawSize, .RawHash = Params.RawHash});
 					}
 				}
 				if (!EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))