re-enable partial cache chunks (#21)v5.4.2-pre9v5.4.2-pre12v5.4.2-pre11v5.4.2-pre10

* Separate chunk raw hash from section hash (how to find the fragment attachment)
* fix partial get cache value tests

4 files changed, 40 insertions, 28 deletions

diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 691047f90..412a2d26a 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -1912,12 +1912,12 @@ TEST_CASE("zcache.rpc.partialchunks")
 		bool CanGetPartial = ((uint16_t)Options.AcceptOptions & (uint16_t)RpcAcceptOptions::kAllowPartialCacheChunks);
 		if (!CanGetPartial)
 		{
-			CHECK(Result.Results[0].RawOffset == 0);
+			CHECK(Result.Results[0].FragmentOffset == 0);
 			CHECK(Result.Results[0].Body.GetCompressedSize() == VerifyData.GetCompressedSize());
 		}
 		IoBuffer SourceDecompressed = VerifyData.Decompress(Options.Offset, Options.Size).AsIoBuffer();
 		IoBuffer ReceivedDecompressed =
-			Result.Results[0].Body.Decompress(Options.Offset - Result.Results[0].RawOffset, Options.Size).AsIoBuffer();
+			Result.Results[0].Body.Decompress(Options.Offset - Result.Results[0].FragmentOffset, Options.Size).AsIoBuffer();
 		CHECK(SourceDecompressed.GetView().EqualBytes(ReceivedDecompressed.GetView()));
 	};
 
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 84ca5c5e3..3cb1b03a5 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -1549,13 +1549,7 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest
 
 	using namespace cache::detail;
 
-	const bool AcceptsPartialChunks = EnumHasAnyFlags(AcceptOptions, RpcAcceptOptions::kAllowPartialCacheChunks) && false;
-	// TODO: We need to redesign how we respond with partial chunks as we can get requests for multiple parts of the same chunk
-	// and we currently identify chunks by the full chunks RawHash making it impossible to distinguish between sections for partial chunks.
-	// Also, UE currently does the lookup by checking the RawHash *inside* the attachment which will break as sections of compressed
-	// buffers has a RawHash of zero (as we don't want to re-hash the section)
-	// zen side handles this as it separates the lookup hash from the hash inside the attachment, but it still can't tell two partial
-	// section from the same chunk apart.
+	const bool AcceptsPartialChunks = EnumHasAnyFlags(AcceptOptions, RpcAcceptOptions::kAllowPartialCacheChunks);
 
 	CbPackage	   RpcResponse;
 	CbObjectWriter Writer;
@@ -1572,9 +1566,16 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest
 				Writer.AddHash("RawHash"sv, Request.Key->ChunkId);
 				if (Request.Value && !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData))
 				{
-					if (AcceptsPartialChunks && (Request.RequestedOffset != 0 || Request.RequestedSize < Request.RawSize))
+					auto IsPartialRangeRequest = [](const ChunkRequest& Request) {
+						if ((Request.RequestedOffset != 0) || (Request.RequestedSize != ~uint64_t(0)))
+						{
+							return true;
+						}
+						return Request.RequestedSize < Request.Value.DecodeRawSize();
+					};
+					if (AcceptsPartialChunks && IsPartialRangeRequest(Request))
 					{
-						uint64_t RawOffset = 0;
+						uint64_t FragmentRawOffset = 0;
 						if (Request.RequestedOffset > 0)
 						{
 							OodleCompressor		  Compressor;
@@ -1584,22 +1585,30 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest
 							ZEN_ASSERT(bOk);
 							if (BlockSize > 0)
 							{
-								RawOffset = (Request.RequestedOffset / BlockSize) * BlockSize;
+								FragmentRawOffset = (Request.RequestedOffset / BlockSize) * BlockSize;
 							}
 							else
 							{
-								RawOffset = Request.RequestedOffset;
+								FragmentRawOffset = Request.RequestedOffset;
 							}
 						}
-						// Technically the receiver can figure this offset out based on the assumption that we reply with
-						// a set of blocks that encapsulates the requested range, but since the UE side was not initially
-						// written to handle this we need to indicate that we do indeed reply with a partial response.
-						// And as we already need to add a field, why not provide some useful information so the client
-						// don't need to calculate this for us...
-						Writer.AddInteger("RawOffset", RawOffset);
-						Request.Value = Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize);
+						Request.Value			= Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize);
+						uint64_t FragmentLength = Request.Value.DecodeRawSize();
+
+						IoHashStream FragmentHashStream;
+						FragmentHashStream.Append(Request.Key->ChunkId.Hash, sizeof(Request.Key->ChunkId.Hash));
+						FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset));
+						FragmentHashStream.Append(&FragmentLength, sizeof(FragmentLength));
+						IoHash FragmentHash = FragmentHashStream.GetHash();
+
+						Writer.AddHash("FragmentHash", FragmentHash);
+						Writer.AddInteger("FragmentOffset", FragmentRawOffset);
+						RpcResponse.AddAttachment(CbAttachment(Request.Value, FragmentHash));
+					}
+					else
+					{
+						RpcResponse.AddAttachment(CbAttachment(Request.Value, Request.Key->ChunkId));
 					}
-					RpcResponse.AddAttachment(CbAttachment(Request.Value, Request.Key->ChunkId));
 				}
 				else
 				{
diff --git a/src/zenutil/cache/cacherequests.cpp b/src/zenutil/cache/cacherequests.cpp
index 442cf0dfc..7c6f493f2 100644
--- a/src/zenutil/cache/cacherequests.cpp
+++ b/src/zenutil/cache/cacherequests.cpp
@@ -792,9 +792,11 @@ namespace cacherequests {
 			bool Succeeded			 = !RawHashField.HasError();
 			if (Succeeded)
 			{
-				ValueResult.RawOffset		   = RecordObject["RawOffset"].AsUInt64(0);
-				const CbAttachment* Attachment = Package.FindAttachment(ValueResult.RawHash);
-				ValueResult.Body			   = Attachment ? Attachment->AsCompressedBinary().MakeOwned() : CompressedBuffer();
+				ValueResult.FragmentOffset = RecordObject["FragmentOffset"].AsUInt64(0);
+				ValueResult.FragmentHash   = RecordObject["FragmentHash"].AsHash();
+				const CbAttachment* Attachment =
+					Package.FindAttachment(ValueResult.FragmentHash == IoHash::Zero ? ValueResult.RawHash : ValueResult.FragmentHash);
+				ValueResult.Body = Attachment ? Attachment->AsCompressedBinary().MakeOwned() : CompressedBuffer();
 				if (ValueResult.Body)
 				{
 					ValueResult.RawSize = ValueResult.Body.DecodeRawSize();
diff --git a/src/zenutil/include/zenutil/cache/cacherequests.h b/src/zenutil/include/zenutil/cache/cacherequests.h
index 0913efc65..fbf3e08cc 100644
--- a/src/zenutil/include/zenutil/cache/cacherequests.h
+++ b/src/zenutil/include/zenutil/cache/cacherequests.h
@@ -195,10 +195,11 @@ namespace cacherequests {
 
 	struct CacheValueResult
 	{
-		uint64_t		 RawSize   = 0;
-		uint64_t		 RawOffset = 0;
-		IoHash			 RawHash   = IoHash::Zero;
-		CompressedBuffer Body	   = CompressedBuffer::Null;
+		uint64_t		 RawSize		= 0;
+		uint64_t		 FragmentOffset = 0;
+		IoHash			 FragmentHash	= IoHash::Zero;
+		IoHash			 RawHash		= IoHash::Zero;
+		CompressedBuffer Body			= CompressedBuffer::Null;
 	};
 
 	struct CacheValuesResult