RawOffset can be anything and we expect an empty buffer to be returned along with RawSize = 0 if the offset was out of bounds for the value. (#666)

3 files changed, 224 insertions, 21 deletions

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index d9f381811..84af83119 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -1995,8 +1995,15 @@ CompressedBuffer::CopyRange(uint64_t RawOffset, uint64_t RawSize) const
 		UniqueBuffer		 RawHeaderData;
 		if (ReadHeader(CompressedData, Header, &RawHeaderData))
 		{
-			const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset;
-			Range.CompressedData		= CopyCompressedRange(Header, RawHeaderData.GetView(), CompressedData, RawOffset, TotalRawSize);
+			if (RawOffset < Header.TotalRawSize)
+			{
+				const uint64_t MaxRawSize	= RawOffset == ~uint64_t(0) ? Header.TotalRawSize : Header.TotalRawSize - RawOffset;
+				const uint64_t TotalRawSize = RawSize == ~uint64_t(0) ? MaxRawSize : Min(RawSize, MaxRawSize);
+				if (TotalRawSize > 0)
+				{
+					Range.CompressedData = CopyCompressedRange(Header, RawHeaderData.GetView(), CompressedData, RawOffset, TotalRawSize);
+				}
+			}
 		}
 	}
 	return Range;
@@ -2012,8 +2019,16 @@ CompressedBuffer::GetRange(uint64_t RawOffset, uint64_t RawSize) const
 		UniqueBuffer		 RawHeaderData;
 		if (ReadHeader(CompressedData, Header, &RawHeaderData))
 		{
-			const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset;
-			Range.CompressedData		= GetCompressedRange(Header, RawHeaderData.GetView(), CompressedData, RawOffset, TotalRawSize);
+			if (RawOffset < Header.TotalRawSize)
+			{
+				const uint64_t MaxRawSize	= RawOffset == ~uint64_t(0) ? Header.TotalRawSize : Header.TotalRawSize - RawOffset;
+				const uint64_t TotalRawSize = RawSize == ~uint64_t(0) ? MaxRawSize : Min(RawSize, MaxRawSize);
+
+				if (TotalRawSize > 0)
+				{
+					Range.CompressedData = GetCompressedRange(Header, RawHeaderData.GetView(), CompressedData, RawOffset, TotalRawSize);
+				}
+			}
 		}
 	}
 	return Range;
diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp
index 1ce5f3be4..ef801833e 100644
--- a/src/zenserver-test/cache-tests.cpp
+++ b/src/zenserver-test/cache-tests.cpp
@@ -2361,6 +2361,188 @@ TEST_CASE("zcache.rpc.allpolicies")
 	}
 }
 
+TEST_CASE("zcache.GetChunksRanges")
+{
+	using namespace std::literals;
+	using namespace utils;
+
+	std::string_view TestBucket	   = "allpoliciestest"sv;
+	std::string_view TestNamespace = "ue4.ddc"sv;
+
+	ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+	ZenServerInstance UpstreamServer(TestEnv);
+	SpawnServer(UpstreamServer, UpstreamCfg);
+
+	ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+	ZenServerInstance LocalServer(TestEnv);
+	SpawnServer(LocalServer, LocalCfg);
+
+	const auto BaseUri = fmt::format("http://localhost:{}/z$", LocalServer.GetBasePort());
+	HttpClient Http{BaseUri};
+
+	const uint64_t BlobSize = 500u * 1024u;
+	IoHash		   RawHash;
+	uint64_t	   BlockSize = 0;
+
+	{
+		cacherequests::PutCacheValuesRequest PutRequest		= {.AcceptMagic	  = kCbPkgMagic,
+															   .DefaultPolicy = CachePolicy::Default,
+															   .Namespace	  = std::string(TestNamespace)};
+		CompressedBuffer					 CompressedBlob = CreateSemiRandomBlob(BlobSize);
+		RawHash												= CompressedBlob.DecodeRawHash();
+		OodleCompressor		  Compressor;
+		OodleCompressionLevel CompressionLevel;
+		CHECK(CompressedBlob.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize));
+		CHECK(BlockSize > 0);
+
+		PutRequest.Requests.push_back(
+			cacherequests::PutCacheValueRequest{.Key = {.Bucket = std::string(TestBucket), .Hash = RawHash}, .Body = CompressedBlob});
+
+		CbPackage Package;
+		CHECK(PutRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "PutCacheValues unexpectedly failed.");
+	}
+
+	// GetCacheChunks full
+	{
+		cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {
+			.AcceptMagic   = kCbPkgMagic,
+			.AcceptOptions = (uint16_t)(RpcAcceptOptions::kAllowPartialCacheChunks),
+			.DefaultPolicy = CachePolicy::Default,
+			.Namespace	   = std::string(TestNamespace)};
+
+		GetCacheChunksRequest.Requests.push_back({.Key = {.Bucket = std::string(TestBucket), .Hash = RawHash},
+												  //.ValueId = OutOfRangeRequest.ValueId,
+												  .ChunkId = RawHash});
+
+		CbPackage Package;
+		CHECK(GetCacheChunksRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheChunks unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load.");
+		cacherequests::GetCacheChunksResult GetCacheChunksResult;
+		CHECK(GetCacheChunksResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheChunksResult.Results.size() == 1, "GetCacheChunks response count did not match request count.");
+		CHECK_EQ(GetCacheChunksResult.Results[0].RawHash, RawHash);
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentHash, IoHash::Zero);
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentOffset, 0u);
+	}
+
+	// GetCacheChunks in range
+	{
+		cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {
+			.AcceptMagic   = kCbPkgMagic,
+			.AcceptOptions = (uint16_t)(RpcAcceptOptions::kAllowPartialCacheChunks),
+			.DefaultPolicy = CachePolicy::Default,
+			.Namespace	   = std::string(TestNamespace)};
+
+		const uint64_t RawOffset = BlobSize / 4u;
+		const uint64_t RawSize	 = BlobSize / 8u;
+
+		GetCacheChunksRequest.Requests.push_back({.Key = {.Bucket = std::string(TestBucket), .Hash = RawHash},
+												  //.ValueId = OutOfRangeRequest.ValueId,
+												  .ChunkId	 = RawHash,
+												  .RawOffset = RawOffset,
+												  .RawSize	 = RawSize});
+
+		CbPackage Package;
+		CHECK(GetCacheChunksRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheChunks unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load.");
+		cacherequests::GetCacheChunksResult GetCacheChunksResult;
+		CHECK(GetCacheChunksResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheChunksResult.Results.size() == 1, "GetCacheChunks response count did not match request count.");
+		CHECK_EQ(GetCacheChunksResult.Results[0].RawHash, RawHash);
+		CHECK_NE(GetCacheChunksResult.Results[0].FragmentHash, IoHash::Zero);
+		CHECK_NE(GetCacheChunksResult.Results[0].FragmentHash, RawHash);
+		const uint64_t ExpectedRawOffset = (RawOffset / BlockSize) * BlockSize;
+		const uint64_t ExpectedRawSize	 = RoundUp(RawSize, BlockSize);
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentOffset, ExpectedRawOffset);
+		CHECK_EQ(GetCacheChunksResult.Results[0].Body.DecodeRawSize(), ExpectedRawSize);
+	}
+
+	// GetCacheChunks partially out of bounds
+	{
+		cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {
+			.AcceptMagic   = kCbPkgMagic,
+			.AcceptOptions = (uint16_t)(RpcAcceptOptions::kAllowPartialCacheChunks),
+			.DefaultPolicy = CachePolicy::Default,
+			.Namespace	   = std::string(TestNamespace)};
+		const uint64_t RawOffset = BlobSize - 512u;
+		const uint64_t RawSize	 = 1u * 1024u;
+		GetCacheChunksRequest.Requests.push_back({.Key = {.Bucket = std::string(TestBucket), .Hash = RawHash},
+												  //.ValueId = OutOfRangeRequest.ValueId,
+												  .ChunkId	 = RawHash,
+												  .RawOffset = RawOffset,
+												  .RawSize	 = RawSize});
+
+		CbPackage Package;
+		CHECK(GetCacheChunksRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheChunks unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load.");
+		cacherequests::GetCacheChunksResult GetCacheChunksResult;
+		CHECK(GetCacheChunksResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheChunksResult.Results.size() == 1, "GetCacheChunks response count did not match request count.");
+		CHECK_EQ(GetCacheChunksResult.Results[0].RawHash, RawHash);
+		CHECK_NE(GetCacheChunksResult.Results[0].FragmentHash, IoHash::Zero);
+		CHECK_NE(GetCacheChunksResult.Results[0].FragmentHash, RawHash);
+		const uint64_t ExpectedRawOffset = (RawOffset / BlockSize) * BlockSize;
+		const uint64_t ExpectedRawSize	 = BlobSize - ExpectedRawOffset;
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentOffset, ExpectedRawOffset);
+		CHECK_EQ(GetCacheChunksResult.Results[0].Body.DecodeRawSize(), ExpectedRawSize);
+	}
+
+	// GetCacheChunks out of bounds
+	{
+		cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {
+			.AcceptMagic   = kCbPkgMagic,
+			.AcceptOptions = (uint16_t)(RpcAcceptOptions::kAllowPartialCacheChunks),
+			.DefaultPolicy = CachePolicy::Default,
+			.Namespace	   = std::string(TestNamespace)};
+
+		GetCacheChunksRequest.Requests.push_back({.Key = {.Bucket = std::string(TestBucket), .Hash = RawHash},
+												  //.ValueId = OutOfRangeRequest.ValueId,
+												  .ChunkId	 = RawHash,
+												  .RawOffset = BlobSize,
+												  .RawSize	 = 1u * 1024u});
+
+		CbPackage Package;
+		CHECK(GetCacheChunksRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheChunks unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load.");
+		cacherequests::GetCacheChunksResult GetCacheChunksResult;
+		CHECK(GetCacheChunksResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheChunksResult.Results.size() == 1, "GetCacheChunks response count did not match request count.");
+		CHECK_EQ(GetCacheChunksResult.Results[0].RawHash, RawHash);
+		CHECK_EQ(GetCacheChunksResult.Results[0].RawSize, 0u);
+		CHECK_EQ(!!GetCacheChunksResult.Results[0].Body, false);
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentHash, IoHash::Zero);
+		CHECK_EQ(GetCacheChunksResult.Results[0].FragmentOffset, 0u);
+	}
+}
+
 }  // namespace zen::tests
 
 #endif
diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp
index 4c44f43ca..09a3d0afc 100644
--- a/src/zenstore/cache/cacherpc.cpp
+++ b/src/zenstore/cache/cacherpc.cpp
@@ -2004,24 +2004,30 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest
 							}
 						}
 
-						CompressedBuffer PartialValue =
-							Ok ? Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize) : CompressedBuffer::Null;
-
-						if (PartialValue)
+						if (Ok)
 						{
-							uint64_t FragmentRawLength = PartialValue.DecodeRawSize();
-							ZEN_ASSERT(FragmentRawOffset <= Request.RequestedOffset);
-							ZEN_ASSERT(FragmentRawOffset + FragmentRawLength >= Request.RequestedOffset + Request.RequestedSize);
-
-							IoHashStream FragmentHashStream;
-							FragmentHashStream.Append(Request.Key->ChunkId.Hash, sizeof(Request.Key->ChunkId.Hash));
-							FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset));
-							FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength));
-							IoHash FragmentHash = FragmentHashStream.GetHash();
-
-							Writer.AddHash("FragmentHash", FragmentHash);
-							Writer.AddInteger("FragmentOffset", FragmentRawOffset);
-							RpcResponse.AddAttachment(CbAttachment(PartialValue, FragmentHash));
+							CompressedBuffer PartialValue = Request.Value.GetRange(Request.RequestedOffset, Request.RequestedSize);
+
+							if (PartialValue)
+							{
+								uint64_t FragmentRawLength = PartialValue.DecodeRawSize();
+								ZEN_ASSERT(FragmentRawOffset <= Request.RequestedOffset);
+
+								IoHashStream FragmentHashStream;
+								FragmentHashStream.Append(Request.Key->ChunkId.Hash, sizeof(Request.Key->ChunkId.Hash));
+								FragmentHashStream.Append(&FragmentRawOffset, sizeof(FragmentRawOffset));
+								FragmentHashStream.Append(&FragmentRawLength, sizeof(FragmentRawLength));
+								IoHash FragmentHash = FragmentHashStream.GetHash();
+
+								Writer.AddHash("FragmentHash", FragmentHash);
+								Writer.AddInteger("FragmentOffset", FragmentRawOffset);
+								RpcResponse.AddAttachment(CbAttachment(PartialValue, FragmentHash));
+							}
+							else
+							{
+								Writer.AddInteger("RawSize", 0);
+								Request.Exists = true;
+							}
 						}
 						else
 						{