move chunking code to zenremotestore lib (#545)

19 files changed, 86 insertions, 47 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index 9eb94f024..fd1487468 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -26,11 +26,11 @@
 #include <zenremotestore/builds/buildstoragecache.h>
 #include <zenremotestore/builds/filebuildstorage.h>
 #include <zenremotestore/builds/jupiterbuildstorage.h>
+#include <zenremotestore/chunking/chunkblock.h>
+#include <zenremotestore/chunking/chunkedcontent.h>
+#include <zenremotestore/chunking/chunkedfile.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
 #include <zenutil/bufferedwritefilecache.h>
-#include <zenutil/chunkblock.h>
-#include <zenutil/chunkedcontent.h>
-#include <zenutil/chunkedfile.h>
-#include <zenutil/chunkingcontroller.h>
 #include <zenutil/jupiter/jupiterhost.h>
 #include <zenutil/jupiter/jupitersession.h>
 #include <zenutil/parallelwork.h>
diff --git a/src/zenremotestore-test/zenremotestore-test.cpp b/src/zenremotestore-test/zenremotestore-test.cpp
index 7c15e20d9..a49c6d273 100644
--- a/src/zenremotestore-test/zenremotestore-test.cpp
+++ b/src/zenremotestore-test/zenremotestore-test.cpp
@@ -19,7 +19,6 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
 #if ZEN_WITH_TESTS
 	zen::zenremotestore_forcelinktests();
-	zen::remoteprojectstore_forcelink();
 
 #	if ZEN_PLATFORM_LINUX
 	zen::IgnoreChildSignals();
diff --git a/src/zenutil/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp
index abfc0fb63..05ae13de1 100644
--- a/src/zenutil/chunkblock.cpp
+++ b/src/zenremotestore/chunking/chunkblock.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <zenutil/chunkblock.h>
+#include <zenremotestore/chunking/chunkblock.h>
 
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/fmtutils.h>
@@ -8,6 +8,13 @@
 
 #include <vector>
 
+#if ZEN_WITH_TESTS
+#	include <zencore/testing.h>
+#	include <zencore/testutils.h>
+
+#	include <unordered_map>
+#endif	// ZEN_WITH_TESTS
+
 namespace zen {
 
 using namespace std::literals;
@@ -254,4 +261,60 @@ IterateChunkBlock(const SharedBuffer&														  BlockPayload,
 	return true;
 };
 
+#if ZEN_WITH_TESTS
+
+namespace testutils {
+	static std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(
+		const std::span<const size_t>& Sizes,
+		OodleCompressionLevel		   CompressionLevel = OodleCompressionLevel::VeryFast,
+		uint64_t					   BlockSize		= 0)
+	{
+		std::vector<std::pair<Oid, CompressedBuffer>> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			CompressedBuffer Compressed =
+				CompressedBuffer::Compress(SharedBuffer(CreateSemiRandomBlob(Size)), OodleCompressor::Mermaid, CompressionLevel, BlockSize);
+			Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed));
+		}
+		return Result;
+	}
+
+}  // namespace testutils
+
+TEST_CASE("project.store.block")
+{
+	using namespace std::literals;
+	using namespace testutils;
+
+	std::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566,  3656, 6006, 24,	3466, 1093, 4269, 2257, 3685, 3489,
+											  7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251,
+											  491,	5464, 4607, 8135, 3767, 4045, 4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225});
+
+	std::vector<std::pair<Oid, CompressedBuffer>>  AttachmentsWithId = CreateAttachments(AttachmentSizes);
+	std::vector<std::pair<IoHash, FetchChunkFunc>> Chunks;
+	Chunks.reserve(AttachmentSizes.size());
+	for (const auto& It : AttachmentsWithId)
+	{
+		Chunks.push_back(
+			std::make_pair(It.second.DecodeRawHash(), [Buffer = It.second](const IoHash&) -> std::pair<uint64_t, CompressedBuffer> {
+				return {Buffer.DecodeRawSize(), Buffer};
+			}));
+	}
+	ChunkBlockDescription Block;
+	CompressedBuffer	  BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block);
+	uint64_t			  HeaderSize;
+	CHECK(IterateChunkBlock(
+		BlockBuffer.Decompress(),
+		[](CompressedBuffer&&, const IoHash&) {},
+		HeaderSize));
+}
+
+void
+chunkblock_forcelink()
+{
+}
+
+#endif	// ZEN_WITH_TESTS
+
 }  // namespace zen
diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenremotestore/chunking/chunkedcontent.cpp
index 757bcfae5..e97dcff15 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenremotestore/chunking/chunkedcontent.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <zenutil/chunkedcontent.h>
+#include <zenremotestore/chunking/chunkedcontent.h>
 
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
@@ -9,8 +9,8 @@
 #include <zencore/timer.h>
 #include <zencore/trace.h>
 
-#include <zenutil/chunkedfile.h>
-#include <zenutil/chunkingcontroller.h>
+#include <zenremotestore/chunking/chunkedfile.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
 #include <zenutil/parallelwork.h>
 #include <zenutil/workerpools.h>
 
diff --git a/src/zenutil/chunkedfile.cpp b/src/zenremotestore/chunking/chunkedfile.cpp
index a2c041ffd..652110605 100644
--- a/src/zenutil/chunkedfile.cpp
+++ b/src/zenremotestore/chunking/chunkedfile.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <zenutil/chunkedfile.h>
+#include <zenremotestore/chunking/chunkedfile.h>
 
 #include <zencore/basicfile.h>
 #include <zencore/trace.h>
diff --git a/src/zenutil/chunking.cpp b/src/zenremotestore/chunking/chunking.cpp
index 71f0a06e4..71f0a06e4 100644
--- a/src/zenutil/chunking.cpp
+++ b/src/zenremotestore/chunking/chunking.cpp
diff --git a/src/zenutil/chunking.h b/src/zenremotestore/chunking/chunking.h
index 09c56454f..09c56454f 100644
--- a/src/zenutil/chunking.h
+++ b/src/zenremotestore/chunking/chunking.h
diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenremotestore/chunking/chunkingcontroller.cpp
index 6fb4182c0..49332c2ce 100644
--- a/src/zenutil/chunkingcontroller.cpp
+++ b/src/zenremotestore/chunking/chunkingcontroller.cpp
@@ -1,6 +1,6 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <zenutil/chunkingcontroller.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
 
 #include <zencore/basicfile.h>
 #include <zencore/compactbinarybuilder.h>
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h
index 46ecd0a11..ee0ddcaa4 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h
@@ -3,7 +3,7 @@
 #pragma once
 
 #include <zencore/compactbinary.h>
-#include <zenutil/chunkblock.h>
+#include <zenremotestore/chunking/chunkblock.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
index e6ca2c5e4..2e8024915 100644
--- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
+++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h
@@ -6,7 +6,7 @@
 
 #include <zencore/compactbinary.h>
 #include <zencore/compositebuffer.h>
-#include <zenutil/chunkblock.h>
+#include <zenremotestore/chunking/chunkblock.h>
 
 namespace zen {
 
diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
index 277580c74..b0d8ef24c 100644
--- a/src/zenutil/include/zenutil/chunkblock.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h
@@ -37,4 +37,6 @@ bool				  IterateChunkBlock(const SharedBuffer&															BlockPayload,
 										uint64_t&																	OutHeaderSize);
 std::vector<uint32_t> ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize);
 
+void chunkblock_forcelink();
+
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
index 306a5d990..306a5d990 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h
diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h
index 4cec80fdb..4cec80fdb 100644
--- a/src/zenutil/include/zenutil/chunkedfile.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedfile.h
diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h
index 315502265..2d1ba36aa 100644
--- a/src/zenutil/include/zenutil/chunkingcontroller.h
+++ b/src/zenremotestore/include/zenremotestore/chunking/chunkingcontroller.h
@@ -4,7 +4,7 @@
 
 #include <zencore/compactbinary.h>
 
-#include <zenutil/chunkedfile.h>
+#include <zenremotestore/chunking/chunkedfile.h>
 
 #include <atomic>
 #include <filesystem>
diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
index 11cc58e4d..7e5af5e6b 100644
--- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
+++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h
@@ -5,7 +5,7 @@
 #include <zencore/jobqueue.h>
 #include <zenstore/projectstore.h>
 
-#include <zenutil/chunkblock.h>
+#include <zenremotestore/chunking/chunkblock.h>
 
 #include <unordered_set>
 
diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp
index 822f0b29e..c2e270909 100644
--- a/src/zenremotestore/projectstore/remoteprojectstore.cpp
+++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp
@@ -13,8 +13,8 @@
 #include <zencore/timer.h>
 #include <zencore/workthreadpool.h>
 #include <zenhttp/httpcommon.h>
+#include <zenremotestore/chunking/chunkedfile.h>
 #include <zenstore/cidstore.h>
-#include <zenutil/chunkedfile.h>
 #include <zenutil/workerpools.h>
 
 #include <unordered_map>
diff --git a/src/zenremotestore/zenremotestore.cpp b/src/zenremotestore/zenremotestore.cpp
index 2fa3ac6a4..c019bc71d 100644
--- a/src/zenremotestore/zenremotestore.cpp
+++ b/src/zenremotestore/zenremotestore.cpp
@@ -2,6 +2,9 @@
 
 #include <zenremotestore/zenremotestore.h>
 
+#include <zenremotestore/chunking/chunkedfile.h>
+#include <zenremotestore/projectstore/remoteprojectstore.h>
+
 #if ZEN_WITH_TESTS
 
 namespace zen {
@@ -9,6 +12,9 @@ namespace zen {
 void
 zenremotestore_forcelinktests()
 {
+	chunkblock_forcelink();
+	chunkedfile_forcelink();
+	remoteprojectstore_forcelink();
 }
 
 }  // namespace zen
diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp
index 9a02cc7f0..7e2fbcaff 100644
--- a/src/zenstore/projectstore.cpp
+++ b/src/zenstore/projectstore.cpp
@@ -29,7 +29,6 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #if ZEN_WITH_TESTS
 #	include <zencore/testing.h>
 #	include <zencore/testutils.h>
-#	include <zenutil/chunkblock.h>
 
 #	include <unordered_map>
 #endif	// ZEN_WITH_TESTS
@@ -7905,34 +7904,6 @@ TEST_CASE("project.store.partial.read")
 	}
 }
 
-TEST_CASE("project.store.block")
-{
-	using namespace std::literals;
-	using namespace testutils;
-
-	std::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566,  3656, 6006, 24,	3466, 1093, 4269, 2257, 3685, 3489,
-											  7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251,
-											  491,	5464, 4607, 8135, 3767, 4045, 4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225});
-
-	std::vector<std::pair<Oid, CompressedBuffer>>  AttachmentsWithId = CreateAttachments(AttachmentSizes);
-	std::vector<std::pair<IoHash, FetchChunkFunc>> Chunks;
-	Chunks.reserve(AttachmentSizes.size());
-	for (const auto& It : AttachmentsWithId)
-	{
-		Chunks.push_back(
-			std::make_pair(It.second.DecodeRawHash(), [Buffer = It.second](const IoHash&) -> std::pair<uint64_t, CompressedBuffer> {
-				return {Buffer.DecodeRawSize(), Buffer};
-			}));
-	}
-	ChunkBlockDescription Block;
-	CompressedBuffer	  BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block);
-	uint64_t			  HeaderSize;
-	CHECK(IterateChunkBlock(
-		BlockBuffer.Decompress(),
-		[](CompressedBuffer&&, const IoHash&) {},
-		HeaderSize));
-}
-
 TEST_CASE("project.store.iterateoplog")
 {
 	using namespace std::literals;
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
index 37b229c49..88be8a244 100644
--- a/src/zenutil/zenutil.cpp
+++ b/src/zenutil/zenutil.cpp
@@ -6,7 +6,6 @@
 
 #	include <zenutil/cache/cacherequests.h>
 #	include <zenutil/cache/rpcrecording.h>
-#	include <zenutil/chunkedfile.h>
 #	include <zenutil/commandlineoptions.h>
 #	include <zenutil/parallelwork.h>
 #	include <zenutil/wildcard.h>
@@ -19,7 +18,6 @@ zenutil_forcelinktests()
 	cachepolicy_forcelink();
 	cache::rpcrecord_forcelink();
 	cacherequests_forcelink();
-	chunkedfile_forcelink();
 	commandlineoptions_forcelink();
 	parallellwork_forcelink();
 	wildcard_forcelink();