batch op not in destructor (#676)

* use fixed vectors for batch requests
* refactor cache batch value put/get to not execute code that can throw execeptions in destructor
* extend test with multi-bucket requests

1 files changed, 125 insertions, 0 deletions

diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp
index ef801833e..b8f73618b 100644
--- a/src/zenserver-test/cache-tests.cpp
+++ b/src/zenserver-test/cache-tests.cpp
@@ -2543,6 +2543,131 @@ TEST_CASE("zcache.GetChunksRanges")
 	}
 }
 
+TEST_CASE("zcache.batchoperations")
+{
+	using namespace std::literals;
+	using namespace utils;
+
+	std::vector<std::string_view> TestBuckets	= {"batchtest1"sv, "batchtest2"sv};
+	std::string_view			  TestNamespace = "ue4.ddc"sv;
+
+	ZenConfig		  LocalCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
+	ZenServerInstance LocalServer(TestEnv);
+	SpawnServer(LocalServer, LocalCfg);
+
+	const auto BaseUri = fmt::format("http://localhost:{}/z$", LocalServer.GetBasePort());
+	HttpClient Http{BaseUri};
+
+	const uint64_t BlobSize = 24u * 1024u;
+	IoHash		   RawHash;
+	uint64_t	   BlobCount = 128u;
+
+	std::vector<IoHash> Hashes;
+	Hashes.reserve(BlobCount);
+
+	{
+		cacherequests::PutCacheValuesRequest PutRequest = {.AcceptMagic	  = kCbPkgMagic,
+														   .DefaultPolicy = CachePolicy::Default,
+														   .Namespace	  = std::string(TestNamespace)};
+		PutRequest.Requests.reserve(BlobCount * TestBuckets.size());
+		for (const std::string_view& TestBucket : TestBuckets)
+		{
+			for (uint64_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++)
+			{
+				CompressedBuffer CompressedBlob = CreateSemiRandomBlob(BlobSize);
+				RawHash							= CompressedBlob.DecodeRawHash();
+				Hashes.push_back(RawHash);
+
+				PutRequest.Requests.push_back(
+					cacherequests::PutCacheValueRequest{.Key  = {.Bucket = std::string(TestBucket), .Hash = RawHash},
+														.Body = CompressedBlob});
+			}
+		}
+
+		CbPackage Package;
+		CHECK(PutRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "PutCacheValues unexpectedly failed.");
+	}
+
+	{
+		cacherequests::GetCacheChunksRequest GetCacheChunksRequest = {.AcceptMagic	 = kCbPkgMagic,
+																	  .AcceptOptions = (uint16_t)(RpcAcceptOptions::kNone),
+																	  .DefaultPolicy = CachePolicy::Default,
+																	  .Namespace	 = std::string(TestNamespace)};
+		GetCacheChunksRequest.Requests.reserve(BlobCount * TestBuckets.size());
+		uint64_t BlobIndexOffset = 0;
+		for (const std::string_view& TestBucket : TestBuckets)
+		{
+			for (uint64_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++)
+			{
+				GetCacheChunksRequest.Requests.push_back(
+					{.Key	  = {.Bucket = std::string(TestBucket), .Hash = Hashes[BlobIndex + BlobIndexOffset]},
+					 .ChunkId = Hashes[BlobIndex + BlobIndexOffset]});
+			}
+			BlobIndexOffset += BlobCount;
+		}
+		CbPackage Package;
+		CHECK(GetCacheChunksRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheChunks unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheChunks response failed to load.");
+		cacherequests::GetCacheChunksResult GetCacheChunksResult;
+		CHECK(GetCacheChunksResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheChunksResult.Results.size() == BlobCount * TestBuckets.size(),
+					  "GetCacheChunks response count did not match request count.");
+
+		for (uint64_t BlobIndex = 0; BlobIndex < BlobCount * TestBuckets.size(); BlobIndex++)
+		{
+			CHECK_EQ(GetCacheChunksResult.Results[BlobIndex].RawHash, Hashes[BlobIndex]);
+			CHECK(GetCacheChunksResult.Results[BlobIndex].Body);
+		}
+	}
+
+	{
+		cacherequests::GetCacheValuesRequest GetCacheValuesRequest = {.AcceptMagic	 = kCbPkgMagic,
+																	  .AcceptOptions = (uint16_t)(RpcAcceptOptions::kNone),
+																	  .DefaultPolicy = CachePolicy::Default,
+																	  .Namespace	 = std::string(TestNamespace)};
+		GetCacheValuesRequest.Requests.reserve(BlobCount * TestBuckets.size());
+		uint64_t BlobIndexOffset = 0;
+		for (const std::string_view& TestBucket : TestBuckets)
+		{
+			for (uint64_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++)
+			{
+				GetCacheValuesRequest.Requests.push_back(cacherequests::GetCacheValueRequest{
+					.Key = {.Bucket = std::string(TestBucket), .Hash = Hashes[BlobIndex + BlobIndexOffset]}});
+			}
+			BlobIndexOffset += BlobCount;
+		}
+		CbPackage Package;
+		CHECK(GetCacheValuesRequest.Format(Package));
+
+		IoBuffer			 Body	= FormatPackageBody(Package);
+		HttpClient::Response Result = Http.Post("/$rpc", Body, {{"Accept", "application/x-ue-cbpkg"}});
+		CHECK_MESSAGE(Result.StatusCode == HttpResponseCode::OK, "GetCacheValues unexpectedly failed.");
+		CbPackage Response = ParsePackageMessage(Result.ResponsePayload);
+		bool	  Loaded   = !Response.IsNull();
+		CHECK_MESSAGE(Loaded, "GetCacheValues response failed to load.");
+		cacherequests::GetCacheValuesResult GetCacheValuesResult;
+		CHECK(GetCacheValuesResult.Parse(Response));
+		CHECK_MESSAGE(GetCacheValuesResult.Results.size() == BlobCount * TestBuckets.size(),
+					  "GetCacheValues response count did not match request count.");
+
+		for (uint64_t BlobIndex = 0; BlobIndex < BlobCount * TestBuckets.size(); BlobIndex++)
+		{
+			CHECK_EQ(GetCacheValuesResult.Results[BlobIndex].RawHash, Hashes[BlobIndex]);
+			CHECK(GetCacheValuesResult.Results[BlobIndex].Body);
+		}
+	}
+}
+
 }  // namespace zen::tests
 
 #endif