diff options
| -rw-r--r-- | src/zenserver-test/zenserver-test.cpp | 4 | ||||
| -rw-r--r-- | src/zenstore/cache/cacherpc.cpp | 47 | ||||
| -rw-r--r-- | src/zenutil/cache/cacherequests.cpp | 8 | ||||
| -rw-r--r-- | src/zenutil/include/zenutil/cache/cacherequests.h | 9 |
4 files changed, 40 insertions, 28 deletions
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 691047f90..412a2d26a 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -1912,12 +1912,12 @@ TEST_CASE("zcache.rpc.partialchunks") bool CanGetPartial = ((uint16_t)Options.AcceptOptions & (uint16_t)RpcAcceptOptions::kAllowPartialCacheChunks); if (!CanGetPartial) { - CHECK(Result.Results[0].RawOffset == 0); + CHECK(Result.Results[0].FragmentOffset == 0); CHECK(Result.Results[0].Body.GetCompressedSize() == VerifyData.GetCompressedSize()); } IoBuffer SourceDecompressed = VerifyData.Decompress(Options.Offset, Options.Size).AsIoBuffer(); IoBuffer ReceivedDecompressed = - Result.Results[0].Body.Decompress(Options.Offset - Result.Results[0].RawOffset, Options.Size).AsIoBuffer(); + Result.Results[0].Body.Decompress(Options.Offset - Result.Results[0].FragmentOffset, Options.Size).AsIoBuffer(); CHECK(SourceDecompressed.GetView().EqualBytes(ReceivedDecompressed.GetView())); }; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 84ca5c5e3..3cb1b03a5 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -1549,13 +1549,7 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest using namespace cache::detail; - const bool AcceptsPartialChunks = EnumHasAnyFlags(AcceptOptions, RpcAcceptOptions::kAllowPartialCacheChunks) && false; - // TODO: We need to redesign how we respond with partial chunks as we can get requests for multiple parts of the same chunk - // and we currently identify chunks by the full chunks RawHash making it impossible to distinguish between sections for partial chunks. - // Also, UE currently does the lookup by checking the RawHash *inside* the attachment which will break as sections of compressed - // buffers has a RawHash of zero (as we don't want to re-hash the section) - // zen side handles this as it separates the lookup hash from the hash inside the attachment, but it still can't tell two partial - // section from the same chunk apart. + const bool AcceptsPartialChunks = EnumHasAnyFlags(AcceptOptions, RpcAcceptOptions::kAllowPartialCacheChunks); CbPackage RpcResponse; CbObjectWriter Writer; @@ -1572,9 +1566,16 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest Writer.AddHash("RawHash"sv, Request.Key->ChunkId); if (Request.Value && !EnumHasAllFlags(Request.DownstreamPolicy, CachePolicy::SkipData)) { - if (AcceptsPartialChunks && (Request.RequestedOffset != 0 || Request.RequestedSize < Request.RawSize)) + auto IsPartialRangeRequest = [](const ChunkRequest& Request) { + if ((Request.RequestedOffset != 0) || (Request.RequestedSize != ~uint64_t(0))) + { + return true; + } + return Request.RequestedSize < Request.Value.DecodeRawSize(); + }; + if (AcceptsPartialChunks && IsPartialRangeRequest(Request)) { - uint64_t RawOffset = 0; + uint64_t FragmentRawOffset = 0; if (Request.RequestedOffset > 0) { OodleCompressor Compressor; @@ -1584,22 +1585,30 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest ZEN_ASSERT(bOk); if (BlockSize > 0) { - RawOffset = (Request.RequestedOffset / BlockSize) * BlockSize; + FragmentRawOffset = (Request.RequestedOffset / BlockSize) * BlockSize; } else { - RawOffset = Request.RequestedOffset; + FragmentRawOffset = Request.RequestedOffset; } } - // Technically the receiver can figure this offset out based on the assumption that we reply with - // a set of blocks that encapsulates the requested range, but since the UE side was not initially - // written to handle this we need to indicate that we do indeed reply with a partial response. - // And as we already need to add a field, why not provide some useful information so the client - // don't need to calculate this for us... - Writer.AddInteger("RawOffset", RawOffset); - Request.Value = Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize); + Request.Value = Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize); + uint64_t FragmentLength = Request.Value.DecodeRawSize(); + + IoHashStream FragmentHashStream; + FragmentHashStream.Append(Request.Key->ChunkId.Hash, sizeof(Request.Key->ChunkId.Hash)); + FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset)); + FragmentHashStream.Append(&FragmentLength, sizeof(FragmentLength)); + IoHash FragmentHash = FragmentHashStream.GetHash(); + + Writer.AddHash("FragmentHash", FragmentHash); + Writer.AddInteger("FragmentOffset", FragmentRawOffset); + RpcResponse.AddAttachment(CbAttachment(Request.Value, FragmentHash)); + } + else + { + RpcResponse.AddAttachment(CbAttachment(Request.Value, Request.Key->ChunkId)); } - RpcResponse.AddAttachment(CbAttachment(Request.Value, Request.Key->ChunkId)); } else { diff --git a/src/zenutil/cache/cacherequests.cpp b/src/zenutil/cache/cacherequests.cpp index 442cf0dfc..7c6f493f2 100644 --- a/src/zenutil/cache/cacherequests.cpp +++ b/src/zenutil/cache/cacherequests.cpp @@ -792,9 +792,11 @@ namespace cacherequests { bool Succeeded = !RawHashField.HasError(); if (Succeeded) { - ValueResult.RawOffset = RecordObject["RawOffset"].AsUInt64(0); - const CbAttachment* Attachment = Package.FindAttachment(ValueResult.RawHash); - ValueResult.Body = Attachment ? Attachment->AsCompressedBinary().MakeOwned() : CompressedBuffer(); + ValueResult.FragmentOffset = RecordObject["FragmentOffset"].AsUInt64(0); + ValueResult.FragmentHash = RecordObject["FragmentHash"].AsHash(); + const CbAttachment* Attachment = + Package.FindAttachment(ValueResult.FragmentHash == IoHash::Zero ? ValueResult.RawHash : ValueResult.FragmentHash); + ValueResult.Body = Attachment ? Attachment->AsCompressedBinary().MakeOwned() : CompressedBuffer(); if (ValueResult.Body) { ValueResult.RawSize = ValueResult.Body.DecodeRawSize(); diff --git a/src/zenutil/include/zenutil/cache/cacherequests.h b/src/zenutil/include/zenutil/cache/cacherequests.h index 0913efc65..fbf3e08cc 100644 --- a/src/zenutil/include/zenutil/cache/cacherequests.h +++ b/src/zenutil/include/zenutil/cache/cacherequests.h @@ -195,10 +195,11 @@ namespace cacherequests { struct CacheValueResult { - uint64_t RawSize = 0; - uint64_t RawOffset = 0; - IoHash RawHash = IoHash::Zero; - CompressedBuffer Body = CompressedBuffer::Null; + uint64_t RawSize = 0; + uint64_t FragmentOffset = 0; + IoHash FragmentHash = IoHash::Zero; + IoHash RawHash = IoHash::Zero; + CompressedBuffer Body = CompressedBuffer::Null; }; struct CacheValuesResult |