remote project store stats (#44)

* add remote oplog store statistics
* block chunking when uploading oplog to zenserver (mirroring)
* make sure we can move temporary dechunked file into cas store

1 files changed, 88 insertions, 65 deletions

diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 325b15e3f..3cf96f1cc 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -490,6 +490,52 @@ namespace utils {
 
 	void SpawnServer(ZenServerInstance& Server, ZenConfig& Cfg) { Cfg.Spawn(Server); }
 
+	CompressedBuffer CreateSemiRandomBlob(size_t AttachmentSize, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast)
+	{
+		// Convoluted way to get a compressed buffer whose result it large enough to be a separate file
+		// but also does actually compress
+		const size_t			  PartCount = (AttachmentSize / (1u * 1024u * 64)) + 1;
+		const size_t			  PartSize	= AttachmentSize / PartCount;
+		auto					  Part		= SharedBuffer(CreateRandomBlob(PartSize));
+		std::vector<SharedBuffer> Parts(PartCount, Part);
+		size_t					  RemainPartSize = AttachmentSize - (PartSize * PartCount);
+		if (RemainPartSize > 0)
+		{
+			Parts.push_back(SharedBuffer(CreateRandomBlob(RemainPartSize)));
+		}
+		CompressedBuffer Value = CompressedBuffer::Compress(CompositeBuffer(std::move(Parts)), OodleCompressor::Mermaid, CompressionLevel);
+		if (CompressionLevel != OodleCompressionLevel::VeryFast)
+		{
+			ZEN_INFO("{}", Value.GetCompressedSize());
+		}
+		return Value;
+	};
+
+	std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(const std::span<const size_t>& Sizes)
+	{
+		std::vector<std::pair<Oid, CompressedBuffer>> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)));
+			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
+		}
+		return Result;
+	}
+
+	std::vector<std::pair<Oid, CompressedBuffer>> CreateSemiRandomAttachments(const std::span<const size_t>& Sizes)
+	{
+		std::vector<std::pair<Oid, CompressedBuffer>> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			CompressedBuffer Compressed =
+				CreateSemiRandomBlob(Size, Size > 1024u * 1024u ? OodleCompressionLevel::None : OodleCompressionLevel::VeryFast);
+			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
+		}
+		return Result;
+	}
+
 }  // namespace utils
 
 TEST_CASE("zcache.basic")
@@ -1788,27 +1834,11 @@ TEST_CASE("zcache.rpc.partialchunks")
 		return CacheKey::Create(Bucket, KeyHash);
 	};
 
-	auto CreateSemiRandomBlob = [](size_t AttachmentSize) -> CompressedBuffer {
-		// Convoluted way to get a compressed buffer whose result it large enough to be a separate file
-		// but also does actually compress
-		const size_t			  PartCount = (AttachmentSize / (1u * 1024u * 64)) + 1;
-		const size_t			  PartSize	= AttachmentSize / PartCount;
-		auto					  Part		= SharedBuffer(CreateRandomBlob(PartSize));
-		std::vector<SharedBuffer> Parts(PartCount, Part);
-		size_t					  RemainPartSize = AttachmentSize - (PartSize * PartCount);
-		if (RemainPartSize > 0)
-		{
-			Parts.push_back(SharedBuffer(CreateRandomBlob(RemainPartSize)));
-		}
-		CompressedBuffer Value = CompressedBuffer::Compress(CompositeBuffer(std::move(Parts)));
-		return Value;
-	};
-
-	auto AppendCacheRecord = [&CreateSemiRandomBlob](cacherequests::PutCacheRecordsRequest& Request,
-													 const CacheKey&						CacheKey,
-													 size_t									AttachmentCount,
-													 size_t									AttachmentsSize,
-													 CachePolicy RecordPolicy) -> std::vector<std::pair<Oid, CompressedBuffer>> {
+	auto AppendCacheRecord = [](cacherequests::PutCacheRecordsRequest& Request,
+								const CacheKey&						   CacheKey,
+								size_t								   AttachmentCount,
+								size_t								   AttachmentsSize,
+								CachePolicy							   RecordPolicy) -> std::vector<std::pair<Oid, CompressedBuffer>> {
 		std::vector<std::pair<Oid, CompressedBuffer>>		   AttachmentBuffers;
 		std::vector<cacherequests::PutCacheRecordRequestValue> Attachments;
 		for (size_t AttachmentIndex = 0; AttachmentIndex < AttachmentCount; AttachmentIndex++)
@@ -2651,19 +2681,6 @@ CreateOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, Compresse
 	return Package;
 };
 
-std::vector<std::pair<Oid, CompressedBuffer>>
-CreateAttachments(const std::span<const size_t>& Sizes)
-{
-	std::vector<std::pair<Oid, CompressedBuffer>> Result;
-	Result.reserve(Sizes.size());
-	for (size_t Size : Sizes)
-	{
-		CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size)));
-		Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
-	}
-	return Result;
-}
-
 cpr::Body
 AsBody(const IoBuffer& Payload)
 {
@@ -2704,48 +2721,50 @@ operator<<(CbWriter& Writer, CbWriterMeta Meta)
 TEST_CASE("project.remote")
 {
 	using namespace std::literals;
+	using namespace utils;
 
 	ZenServerTestHelper Servers("remote", 3);
 	Servers.SpawnServers("--debug");
 
 	std::vector<Oid> OpIds;
-	OpIds.reserve(24);
-	for (size_t I = 0; I < 24; ++I)
+	const size_t	 OpCount = 24;
+	OpIds.reserve(OpCount);
+	for (size_t I = 0; I < OpCount; ++I)
 	{
 		OpIds.emplace_back(Oid::NewOid());
 	}
 
 	std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments;
 	{
-		std::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566,  3656, 6006, 24,	3466, 1093, 4269,
-												  2257, 3685, 3489, 7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759,
-												  1916, 8210, 2235, 4024, 1582, 5251, 491,	5464, 4607, 8135, 3767, 4045,
-												  4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225});
-		auto					 It = AttachmentSizes.begin();
-		Attachments[OpIds[0]]		= {};
-		Attachments[OpIds[1]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[2]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
-		Attachments[OpIds[3]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[4]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
-		Attachments[OpIds[5]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
-		Attachments[OpIds[6]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[7]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
-		Attachments[OpIds[8]]		= CreateAttachments(std::initializer_list<size_t>{});
-		Attachments[OpIds[9]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
-		Attachments[OpIds[10]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[11]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
-		Attachments[OpIds[12]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
-		Attachments[OpIds[13]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[14]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++});
-		Attachments[OpIds[15]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++});
-		Attachments[OpIds[16]]		= CreateAttachments(std::initializer_list<size_t>{});
-		Attachments[OpIds[17]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++});
-		Attachments[OpIds[18]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++});
-		Attachments[OpIds[19]]		= CreateAttachments(std::initializer_list<size_t>{});
-		Attachments[OpIds[20]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[21]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
-		Attachments[OpIds[22]]		= CreateAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
-		Attachments[OpIds[23]]		= CreateAttachments(std::initializer_list<size_t>{*It++});
+		std::vector<std::size_t> AttachmentSizes(
+			{7633,	6825,  5738, 8031,	 7225, 566,	 3656, 6006,  24,	33466, 1093, 4269,	 2257,	3685, 13489, 97194,
+			 6151,	5482,  6217, 3511,	 6738, 5061, 7537, 2759,  1916, 8210,  2235, 224024, 51582, 5251, 491,	 2u * 1024u * 1024u + 124u,
+			 74607, 18135, 3767, 154045, 4415, 5007, 8876, 96761, 3359, 8526,  4097, 4855,	 48225});
+		auto It				   = AttachmentSizes.begin();
+		Attachments[OpIds[0]]  = {};
+		Attachments[OpIds[1]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[2]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
+		Attachments[OpIds[3]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[4]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
+		Attachments[OpIds[5]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
+		Attachments[OpIds[6]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[7]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
+		Attachments[OpIds[8]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{});
+		Attachments[OpIds[9]]  = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
+		Attachments[OpIds[10]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[11]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
+		Attachments[OpIds[12]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++, *It++});
+		Attachments[OpIds[13]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[14]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++});
+		Attachments[OpIds[15]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++});
+		Attachments[OpIds[16]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{});
+		Attachments[OpIds[17]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++});
+		Attachments[OpIds[18]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++});
+		Attachments[OpIds[19]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{});
+		Attachments[OpIds[20]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[21]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
+		Attachments[OpIds[22]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++, *It++, *It++});
+		Attachments[OpIds[23]] = CreateSemiRandomAttachments(std::initializer_list<size_t>{*It++});
 		ZEN_ASSERT(It == AttachmentSizes.end());
 	}
 
@@ -2914,6 +2933,7 @@ TEST_CASE("project.remote")
 				{
 					Writer << "maxblocksize"sv << 3072u;
 					Writer << "maxchunkembedsize"sv << 1296u;
+					Writer << "chunkfilesizelimit"sv << 5u * 1024u;
 					Writer << "force"sv << false;
 					Writer << "file"sv << BeginObject;
 					{
@@ -2977,6 +2997,7 @@ TEST_CASE("project.remote")
 				{
 					Writer << "maxblocksize"sv << 3072u;
 					Writer << "maxchunkembedsize"sv << 1296u;
+					Writer << "chunkfilesizelimit"sv << 5u * 1024u;
 					Writer << "force"sv << false;
 					Writer << "file"sv << BeginObject;
 					{
@@ -3038,6 +3059,7 @@ TEST_CASE("project.remote")
 				{
 					Writer << "maxblocksize"sv << 3072u;
 					Writer << "maxchunkembedsize"sv << 1296u;
+					Writer << "chunkfilesizelimit"sv << 5u * 1024u;
 					Writer << "force"sv << false;
 					Writer << "file"sv << BeginObject;
 					{
@@ -3105,6 +3127,7 @@ TEST_CASE("project.remote")
 				{
 					Writer << "maxblocksize"sv << 3072u;
 					Writer << "maxchunkembedsize"sv << 1296u;
+					Writer << "chunkfilesizelimit"sv << 5u * 1024u;
 					Writer << "force"sv << false;
 					Writer << "zen"sv << BeginObject;
 					{