Merge branch 'main' of https://github.com/EpicGames/zen

23 files changed, 277 insertions, 47 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1409b276c..55f6410e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,7 @@
 - Feature: CI build on GitHub now uploads junit test reports as artifact to the check for PR validation and mainline validation
 - Feature: Payloads from zenserver can now be sent using duplicated file handles if caller requests provides client ProcessId (Windows only).
 - Bugfix: Make sure async responses are sent async correctly in httpsys
+- Bugfix: Don't delete manifest file in cas root when initializing a new filecas folder
 - Improvement: FileCas now keeps an up to date index of all the entries improving performance when getting cache misses on large payloads
 - Improvement: Structured cache now keeps RawHash and RawSize in memory avoiding materialization of cache values before sending response
 - Changed: Exit with failure code on port conflict rather than reporting crash to Sentry
diff --git a/xmake.lua b/xmake.lua
index 7b5822371..1b6724044 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -32,6 +32,7 @@ if not is_arch("arm64") then
 end
 
 add_rules("mode.debug", "mode.release")
+--add_rules("c++.unity_build")
 
 if is_mode("release") then
     set_optimize("smallest")
diff --git a/zen/xmake.lua b/zen/xmake.lua
index 86d95406e..b83999efc 100644
--- a/zen/xmake.lua
+++ b/zen/xmake.lua
@@ -4,6 +4,7 @@ target("zen")
     set_kind("binary")
     add_headerfiles("**.h")
     add_files("**.cpp")
+    add_files("zen.cpp", {unity_ignored  = true })
     add_deps("zencore", "zenhttp", "zenutil")
     add_includedirs(".")
     set_symbols("debug")
diff --git a/zencore/include/zencore/scopeguard.h b/zencore/include/zencore/scopeguard.h
index 00836f181..13fed4ac5 100644
--- a/zencore/include/zencore/scopeguard.h
+++ b/zencore/include/zencore/scopeguard.h
@@ -1,5 +1,7 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
+#pragma once
+
 #include <type_traits>
 #include "zencore.h"
 
diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h
index 7ea8c029f..ab111ff81 100644
--- a/zencore/include/zencore/string.h
+++ b/zencore/include/zencore/string.h
@@ -691,7 +691,7 @@ template<Integral T>
 std::optional<T>
 ParseInt(const std::string_view& Input)
 {
-	T							 Out;
+	T							 Out	= 0;
 	const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out);
 	if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range)
 	{
diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp
index 450b5a1fc..f270c9d2b 100644
--- a/zenhttp/httpasio.cpp
+++ b/zenhttp/httpasio.cpp
@@ -7,6 +7,7 @@
 
 #include <deque>
 #include <memory>
+#include <string_view>
 #include <vector>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -41,6 +42,7 @@ static constinit uint32_t HashAccept		= HashStringAsLowerDjb2("Accept"sv);
 static constinit uint32_t HashExpect		= HashStringAsLowerDjb2("Expect"sv);
 static constinit uint32_t HashSession		= HashStringAsLowerDjb2("UE-Session"sv);
 static constinit uint32_t HashRequest		= HashStringAsLowerDjb2("UE-Request"sv);
+static constinit uint32_t HashRange			= HashStringAsLowerDjb2("Range"sv);
 
 inline spdlog::logger&
 InitLogger()
@@ -103,6 +105,7 @@ public:
 	virtual void	 WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override;
 	virtual void	 WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) override;
 	virtual void	 WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler) override;
+	virtual bool	 TryGetRanges(HttpRanges& Ranges) override;
 
 	using HttpServerRequest::WriteResponse;
 
@@ -151,6 +154,8 @@ struct HttpRequest
 	Oid SessionId() const { return m_SessionId; }
 	int RequestId() const { return m_RequestId; }
 
+	std::string_view RangeHeader() const { return m_RangeHeaderIndex != -1 ? m_Headers[m_RangeHeaderIndex].Value : std::string_view(); }
+
 private:
 	struct HeaderEntry
 	{
@@ -176,6 +181,7 @@ private:
 	int8_t					 m_ContentLengthHeaderIndex;
 	int8_t					 m_AcceptHeaderIndex;
 	int8_t					 m_ContentTypeHeaderIndex;
+	int8_t					 m_RangeHeaderIndex;
 	HttpVerb				 m_RequestVerb;
 	bool					 m_KeepAlive		 = false;
 	bool					 m_Expect100Continue = false;
@@ -740,6 +746,10 @@ HttpRequest::AppendCurrentHeader()
 			ZEN_INFO("Unexpected expect - Expect: {}", HeaderValue);
 		}
 	}
+	else if (HeaderHash == HashRange)
+	{
+		m_RangeHeaderIndex = (int8_t)m_Headers.size();
+	}
 
 	m_Headers.emplace_back(HeaderName, HeaderValue);
 }
@@ -912,6 +922,7 @@ HttpRequest::ResetState()
 	m_ContentLengthHeaderIndex = -1;
 	m_AcceptHeaderIndex		   = -1;
 	m_ContentTypeHeaderIndex   = -1;
+	m_RangeHeaderIndex		   = -1;
 	m_Expect100Continue		   = false;
 	m_BodyBuffer			   = {};
 	m_BodyPosition			   = 0;
@@ -1178,6 +1189,12 @@ HttpAsioServerRequest::WriteResponseAsync(std::function<void(HttpServerRequest&)
 	ContinuationHandler(*this);
 }
 
+bool
+HttpAsioServerRequest::TryGetRanges(HttpRanges& Ranges)
+{
+	return TryParseHttpRangeHeader(m_Request.RangeHeader(), Ranges);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 HttpAsioServerImpl::HttpAsioServerImpl()
diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp
index 840b90931..3576d9b3d 100644
--- a/zenhttp/httpserver.cpp
+++ b/zenhttp/httpserver.cpp
@@ -4,7 +4,6 @@
 
 #include "httpasio.h"
 #include "httpnull.h"
-#include "httpsys.h"
 
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
@@ -185,6 +184,70 @@ ParseContentTypeInit(const std::string_view& ContentTypeString)
 
 HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString) = &ParseContentTypeInit;
 
+bool
+TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges)
+{
+	if (RangeHeader.empty())
+	{
+		return false;
+	}
+
+	const size_t Count = Ranges.size();
+
+	std::size_t UnitDelim = RangeHeader.find_first_of('=');
+	if (UnitDelim == std::string_view::npos)
+	{
+		return false;
+	}
+
+	// only bytes for now
+	std::string_view Unit = RangeHeader.substr(0, UnitDelim);
+	if (Unit != "bytes"sv)
+	{
+		return false;
+	}
+
+	std::string_view Tokens = RangeHeader.substr(UnitDelim);
+	while (!Tokens.empty())
+	{
+		// Skip =,
+		Tokens = Tokens.substr(1);
+
+		size_t Delim = Tokens.find_first_of(',');
+		if (Delim == std::string_view::npos)
+		{
+			Delim = Tokens.length();
+		}
+
+		std::string_view Token = Tokens.substr(0, Delim);
+		Tokens				   = Tokens.substr(Delim);
+
+		Delim = Token.find_first_of('-');
+		if (Delim == std::string_view::npos)
+		{
+			return false;
+		}
+
+		const auto Start = ParseInt<uint32_t>(Token.substr(0, Delim));
+		const auto End	 = ParseInt<uint32_t>(Token.substr(Delim + 1));
+
+		if (Start.has_value() && End.has_value() && End.value() > Start.value())
+		{
+			Ranges.push_back({.Start = Start.value(), .End = End.value()});
+		}
+		else if (Start)
+		{
+			Ranges.push_back({.Start = Start.value()});
+		}
+		else if (End)
+		{
+			Ranges.push_back({.End = End.value()});
+		}
+	}
+
+	return Count != Ranges.size();
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 const std::string_view
@@ -641,6 +704,9 @@ enum class HttpServerClass
 	kHttpNull
 };
 
+// Implemented in httpsys.cpp
+Ref<HttpServer> CreateHttpSysServer(int Concurrency, int BackgroundWorkerThreads);
+
 Ref<HttpServer>
 CreateHttpServer(std::string_view ServerClass)
 {
@@ -677,7 +743,7 @@ CreateHttpServer(std::string_view ServerClass)
 #if ZEN_WITH_HTTPSYS
 		case HttpServerClass::kHttpSys:
 			ZEN_INFO("using http.sys server implementation");
-			return new HttpSysServer(std::thread::hardware_concurrency(), /* background worker threads */ 16);
+			return CreateHttpSysServer(std::thread::hardware_concurrency(), /* background worker threads */ 16);
 #endif
 
 		case HttpServerClass::kHttpNull:
diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp
index f6f8024ca..16ec135cd 100644
--- a/zenhttp/httpsys.cpp
+++ b/zenhttp/httpsys.cpp
@@ -188,6 +188,7 @@ public:
 	virtual void	 WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span<IoBuffer> Blobs) override;
 	virtual void	 WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::u8string_view ResponseString) override;
 	virtual void	 WriteResponseAsync(std::function<void(HttpServerRequest&)>&& ContinuationHandler) override;
+	virtual bool	 TryGetRanges(HttpRanges& Ranges) override;
 
 	using HttpServerRequest::WriteResponse;
 
@@ -1408,6 +1409,15 @@ HttpSysServerRequest::WriteResponseAsync(std::function<void(HttpServerRequest&)>
 	}
 }
 
+bool
+HttpSysServerRequest::TryGetRanges(HttpRanges& Ranges)
+{
+	HTTP_REQUEST*			 Req		 = m_HttpTx.HttpRequest();
+	const HTTP_KNOWN_HEADER& RangeHeader = Req->Headers.KnownHeaders[HttpHeaderRange];
+
+	return TryParseHttpRangeHeader({RangeHeader.pRawValue, RangeHeader.RawValueLength}, Ranges);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 InitialRequestHandler::InitialRequestHandler(HttpSysTransaction& InRequest) : HttpSysRequestHandler(InRequest)
@@ -1652,5 +1662,11 @@ HttpSysServer::RegisterService(HttpService& Service)
 	RegisterService(Service.BaseUri(), Service);
 }
 
+Ref<HttpServer>
+CreateHttpSysServer(int Concurrency, int BackgroundWorkerThreads)
+{
+	return Ref<HttpServer>(new HttpSysServer(Concurrency, BackgroundWorkerThreads));
+}
+
 }  // namespace zen
 #endif
diff --git a/zenhttp/include/zenhttp/httpcommon.h b/zenhttp/include/zenhttp/httpcommon.h
index 3e213ece4..19fda8db4 100644
--- a/zenhttp/include/zenhttp/httpcommon.h
+++ b/zenhttp/include/zenhttp/httpcommon.h
@@ -17,9 +17,18 @@ class CbObject;
 class CbPackage;
 class StringBuilderBase;
 
+struct HttpRange
+{
+	uint32_t Start = ~uint32_t(0);
+	uint32_t End   = ~uint32_t(0);
+};
+
+using HttpRanges = std::vector<HttpRange>;
+
 std::string_view MapContentTypeToString(HttpContentType ContentType);
 extern HttpContentType (*ParseContentType)(const std::string_view& ContentTypeString);
 std::string_view ReasonStringForHttpResultCode(int HttpCode);
+bool			 TryParseHttpRangeHeader(std::string_view RangeHeader, HttpRanges& Ranges);
 
 [[nodiscard]] inline bool
 IsHttpSuccessCode(int HttpCode)
diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h
index 5bd51740a..451a47b4a 100644
--- a/zenhttp/include/zenhttp/httpserver.h
+++ b/zenhttp/include/zenhttp/httpserver.h
@@ -59,6 +59,8 @@ public:
 		}
 	};
 
+	virtual bool TryGetRanges(HttpRanges&) { return false; }
+
 	QueryParams GetQueryParams();
 
 	inline HttpVerb		   RequestVerb() const { return m_Verb; }
diff --git a/zenhttp/xmake.lua b/zenhttp/xmake.lua
index 528b72d52..b0dbdbc79 100644
--- a/zenhttp/xmake.lua
+++ b/zenhttp/xmake.lua
@@ -4,6 +4,7 @@ target('zenhttp')
     set_kind("static")
     add_headerfiles("**.h")
     add_files("**.cpp")
+    add_files("httpsys.cpp", {unity_ignored=true})
     add_includedirs("include", {public=true})
     add_deps("zencore")
     add_packages(
diff --git a/zenhttp/zenhttp.cpp b/zenhttp/zenhttp.cpp
index 0194abdcb..4bd6a5697 100644
--- a/zenhttp/zenhttp.cpp
+++ b/zenhttp/zenhttp.cpp
@@ -2,8 +2,11 @@
 
 #include <zenhttp/zenhttp.h>
 
-#include <zenhttp/httpserver.h>
-#include <zenhttp/httpshared.h>
+#if ZEN_WITH_TESTS
+
+#	include <zenhttp/httpclient.h>
+#	include <zenhttp/httpserver.h>
+#	include <zenhttp/httpshared.h>
 
 namespace zen {
 
@@ -15,3 +18,5 @@ zenhttp_forcelinktests()
 }
 
 }  // namespace zen
+
+#endif
diff --git a/zenserver-test/xmake.lua b/zenserver-test/xmake.lua
index 57df8b356..f0b34f6ca 100644
--- a/zenserver-test/xmake.lua
+++ b/zenserver-test/xmake.lua
@@ -4,6 +4,7 @@ target("zenserver-test")
     set_kind("binary")
     add_headerfiles("**.h")
     add_files("*.cpp")
+    add_files("zenserver-test.cpp", {unity_ignored  = true })
     add_deps("zencore", "zenutil", "zenhttp")
     add_deps("zenserver", {inherit=false})
     add_packages("vcpkg::http-parser", "vcpkg::mimalloc")
diff --git a/zenserver/auth/authservice.cpp b/zenserver/auth/authservice.cpp
index 761c087f4..1cc679540 100644
--- a/zenserver/auth/authservice.cpp
+++ b/zenserver/auth/authservice.cpp
@@ -1,8 +1,9 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
-#include <auth/authmgr.h>
 #include <auth/authservice.h>
 
+#include <auth/authmgr.h>
+
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/string.h>
 
diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp
index 1a8b427ca..ca569c467 100644
--- a/zenserver/diag/logging.cpp
+++ b/zenserver/diag/logging.cpp
@@ -326,7 +326,6 @@ private:
 	memory_buf_t		 m_CachedDatetime;
 	std::string			 m_LogId;
 };
-}  // namespace logging
 
 bool
 EnableVTMode()
@@ -355,6 +354,8 @@ EnableVTMode()
 	return true;
 }
 
+}  // namespace logging
+
 #if ZEN_USE_SENTRY
 
 class sentry_sink final : public spdlog::sinks::base_sink<spdlog::details::null_mutex>
@@ -391,8 +392,7 @@ void
 InitializeLogging(const ZenServerOptions& GlobalOptions)
 {
 	zen::logging::InitializeLogging();
-
-	EnableVTMode();
+	logging::EnableVTMode();
 
 	bool					  IsAsync  = true;
 	spdlog::level::level_enum LogLevel = spdlog::level::info;
diff --git a/zenserver/objectstore/objectstore.cpp b/zenserver/objectstore/objectstore.cpp
index 950505bcb..e5739418e 100644
--- a/zenserver/objectstore/objectstore.cpp
+++ b/zenserver/objectstore/objectstore.cpp
@@ -8,6 +8,7 @@
 #include <zencore/string.h>
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
+#include "zenhttp/httpserver.h"
 
 #include <thread>
 
@@ -130,6 +131,13 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
+	zen::HttpRanges Ranges;
+	if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
+	{
+		// Only a single range is supported
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
 	FileContents File = ReadFile(FilePath);
 	if (File.ErrorCode)
 	{
@@ -144,16 +152,45 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 	}
 
 	const IoBuffer& FileBuf = File.Data[0];
-	const uint64_t	Total	= TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
 
-	ZEN_LOG_DEBUG(LogObj,
-				  "GET - '{}/{}' ({}) [OK] (Total: {})",
-				  BucketName,
-				  RelativeBucketPath,
-				  NiceBytes(FileBuf.Size()),
-				  NiceBytes(Total));
+	if (Ranges.empty())
+	{
+		const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
 
-	Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kUnknownContentType, FileBuf);
+		ZEN_LOG_DEBUG(LogObj,
+					  "GET - '{}/{}' ({}) [OK] (Served: {})",
+					  BucketName,
+					  RelativeBucketPath,
+					  NiceBytes(FileBuf.Size()),
+					  NiceBytes(TotalServed));
+
+		Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf);
+	}
+	else
+	{
+		const auto	   Range	   = Ranges[0];
+		const uint64_t RangeSize   = Range.End - Range.Start;
+		const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+
+		ZEN_LOG_DEBUG(LogObj,
+					  "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
+					  BucketName,
+					  RelativeBucketPath,
+					  Range.Start,
+					  Range.End,
+					  NiceBytes(RangeSize),
+					  NiceBytes(FileBuf.Size()),
+					  NiceBytes(TotalServed));
+
+		MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize);
+		if (RangeView.GetSize() != RangeSize)
+		{
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		}
+
+		IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize());
+		Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf);
+	}
 }
 
 void
diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua
index 992923d48..23bfb9535 100644
--- a/zenserver/xmake.lua
+++ b/zenserver/xmake.lua
@@ -2,9 +2,10 @@
 
 target("zenserver")
     set_kind("binary")
+    add_deps("zencore", "zenhttp", "zenstore", "zenutil")
     add_headerfiles("**.h")
     add_files("**.cpp")
-    add_deps("zencore", "zenhttp", "zenstore", "zenutil")
+    add_files("zenserver.cpp", {unity_ignored  = true })
     add_includedirs(".")
     set_symbols("debug")
 
@@ -17,6 +18,7 @@ target("zenserver")
         add_ldflags("/MANIFEST:EMBED")
         add_ldflags("/LTCG")
         add_files("zenserver.rc")
+        add_cxxflags("/bigobj")
     else
         remove_files("windows/**")
     end
diff --git a/zenstore/blockstore.cpp b/zenstore/blockstore.cpp
index 5d81d1120..d743c431f 100644
--- a/zenstore/blockstore.cpp
+++ b/zenstore/blockstore.cpp
@@ -859,7 +859,7 @@ TEST_CASE("blockstore.blockfile")
 	CHECK(!std::filesystem::exists(RootDirectory / "1"));
 }
 
-namespace {
+namespace blockstore::impl {
 	BlockStoreLocation WriteStringAsChunk(BlockStore& Store, std::string_view String, size_t PayloadAlignment)
 	{
 		BlockStoreLocation Location;
@@ -907,10 +907,12 @@ namespace {
 
 		return IoBufferBuilder::MakeCloneFromMemory(Values.data(), Values.size());
 	}
-}  // namespace
+}  // namespace blockstore::impl
 
 TEST_CASE("blockstore.chunks")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
@@ -939,6 +941,8 @@ TEST_CASE("blockstore.chunks")
 
 TEST_CASE("blockstore.clean.stray.blocks")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
@@ -963,6 +967,8 @@ TEST_CASE("blockstore.clean.stray.blocks")
 
 TEST_CASE("blockstore.flush.forces.new.block")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
@@ -984,6 +990,8 @@ TEST_CASE("blockstore.flush.forces.new.block")
 
 TEST_CASE("blockstore.iterate.chunks")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
@@ -1078,6 +1086,8 @@ TEST_CASE("blockstore.iterate.chunks")
 
 TEST_CASE("blockstore.reclaim.space")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
@@ -1193,6 +1203,8 @@ TEST_CASE("blockstore.reclaim.space")
 
 TEST_CASE("blockstore.thread.read.write")
 {
+	using namespace blockstore::impl;
+
 	ScopedTemporaryDirectory TempDir;
 	auto					 RootDirectory = TempDir.Path();
 
diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp
index 70a88ecad..60644847f 100644
--- a/zenstore/compactcas.cpp
+++ b/zenstore/compactcas.cpp
@@ -732,7 +732,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 #if ZEN_WITH_TESTS
 
 namespace {
-	static IoBuffer CreateChunk(uint64_t Size)
+	static IoBuffer CreateRandomChunk(uint64_t Size)
 	{
 		static std::random_device rd;
 		static std::mt19937		  g(rd());
@@ -865,7 +865,7 @@ TEST_CASE("compactcas.compact.totalsize")
 
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
 			{
-				IoBuffer			   Chunk		= CreateChunk(kChunkSize);
+				IoBuffer			   Chunk		= CreateRandomChunk(kChunkSize);
 				const IoHash		   Hash			= HashBuffer(Chunk);
 				CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, Hash);
 				ZEN_ASSERT(InsertResult.New);
@@ -904,7 +904,7 @@ TEST_CASE("compactcas.gc.basic")
 	CasContainerStrategy Cas(Gc);
 	Cas.Initialize(TempDir.Path(), "cb", 65536, 1 << 4, true);
 
-	IoBuffer Chunk	   = CreateChunk(128);
+	IoBuffer Chunk	   = CreateRandomChunk(128);
 	IoHash	 ChunkHash = IoHash::HashBuffer(Chunk);
 
 	const CasStore::InsertResult InsertResult = Cas.InsertChunk(Chunk, ChunkHash);
@@ -923,7 +923,7 @@ TEST_CASE("compactcas.gc.removefile")
 {
 	ScopedTemporaryDirectory TempDir;
 
-	IoBuffer Chunk	   = CreateChunk(128);
+	IoBuffer Chunk	   = CreateRandomChunk(128);
 	IoHash	 ChunkHash = IoHash::HashBuffer(Chunk);
 	{
 		GcManager			 Gc;
@@ -964,7 +964,7 @@ TEST_CASE("compactcas.gc.compact")
 		Chunks.reserve(9);
 		for (uint64_t Size : ChunkSizes)
 		{
-			Chunks.push_back(CreateChunk(Size));
+			Chunks.push_back(CreateRandomChunk(Size));
 		}
 
 		std::vector<IoHash> ChunkHashes;
@@ -1190,7 +1190,7 @@ TEST_CASE("compactcas.gc.deleteblockonopen")
 	Chunks.reserve(20);
 	for (uint64_t Size : ChunkSizes)
 	{
-		Chunks.push_back(CreateChunk(Size));
+		Chunks.push_back(CreateRandomChunk(Size));
 	}
 
 	std::vector<IoHash> ChunkHashes;
@@ -1256,7 +1256,7 @@ TEST_CASE("compactcas.gc.handleopeniobuffer")
 	Chunks.reserve(20);
 	for (const uint64_t& Size : ChunkSizes)
 	{
-		Chunks.push_back(CreateChunk(Size));
+		Chunks.push_back(CreateRandomChunk(Size));
 	}
 
 	std::vector<IoHash> ChunkHashes;
@@ -1308,7 +1308,7 @@ TEST_CASE("compactcas.threadedinsert")
 		{
 			while (true)
 			{
-				IoBuffer Chunk = CreateChunk(kChunkSize);
+				IoBuffer Chunk = CreateRandomChunk(kChunkSize);
 				IoHash	 Hash  = HashBuffer(Chunk);
 				if (Chunks.contains(Hash))
 				{
@@ -1377,7 +1377,7 @@ TEST_CASE("compactcas.threadedinsert")
 
 			for (int32_t Idx = 0; Idx < kChunkCount; ++Idx)
 			{
-				IoBuffer Chunk	= CreateChunk(kChunkSize);
+				IoBuffer Chunk	= CreateRandomChunk(kChunkSize);
 				IoHash	 Hash	= HashBuffer(Chunk);
 				NewChunks[Hash] = Chunk;
 			}
diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp
index 5b4a2aabc..986cc9ae6 100644
--- a/zenstore/filecas.cpp
+++ b/zenstore/filecas.cpp
@@ -39,7 +39,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
-namespace {
+namespace filecas::impl {
 	const char* IndexExtension = ".uidx";
 	const char* LogExtension   = ".ulog";
 
@@ -77,7 +77,7 @@ namespace {
 
 #pragma pack(pop)
 
-}  // namespace
+}  // namespace filecas::impl
 
 FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash)
 {
@@ -125,6 +125,8 @@ FileCasStrategy::~FileCasStrategy()
 void
 FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsNewStore)
 {
+	using namespace filecas::impl;
+
 	m_IsInitialized = true;
 
 	m_RootDirectory = RootDirectory;
@@ -138,7 +140,43 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN
 	{
 		std::filesystem::remove(LogPath);
 		std::filesystem::remove(IndexPath);
-		std::filesystem::remove_all(RootDirectory);
+
+		if (std::filesystem::is_directory(m_RootDirectory))
+		{
+			// We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside
+			// in this folder as well
+			struct Visitor : public FileSystemTraversal::TreeVisitor
+			{
+				virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t) override
+				{
+					// We don't care about files
+				}
+				static bool IsHexChar(std::filesystem::path::value_type C)
+				{
+					return std::find(&HexChars[0], &HexChars[16], C) != &HexChars[16];
+				}
+				virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent,
+											[[maybe_unused]] const path_view&			  DirectoryName) override
+				{
+					if (DirectoryName.length() == 3)
+					{
+						if (IsHexChar(DirectoryName[0]) && IsHexChar(DirectoryName[1]) && IsHexChar(DirectoryName[2]))
+						{
+							ShardedRoots.push_back(Parent / DirectoryName);
+						}
+					}
+					return false;
+				}
+				std::vector<std::filesystem::path> ShardedRoots;
+			} CasVisitor;
+
+			FileSystemTraversal Traversal;
+			Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor);
+			for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots)
+			{
+				std::filesystem::remove_all(SharededRoot);
+			}
+		}
 	}
 
 	m_LogFlushPosition	   = ReadIndexFile();
@@ -977,6 +1015,8 @@ FileCasStrategy::ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutR
 void
 FileCasStrategy::MakeIndexSnapshot()
 {
+	using namespace filecas::impl;
+
 	uint64_t LogCount = m_CasLog.GetLogCount();
 	if (m_LogFlushPosition == LogCount)
 	{
@@ -1026,12 +1066,12 @@ FileCasStrategy::MakeIndexSnapshot()
 
 		BasicFile ObjectIndexFile;
 		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kTruncate);
-		FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = LogCount};
+		filecas::impl::FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = LogCount};
 
-		Header.Checksum = FileCasIndexHeader::ComputeChecksum(Header);
+		Header.Checksum = filecas::impl::FileCasIndexHeader::ComputeChecksum(Header);
 
-		ObjectIndexFile.Write(&Header, sizeof(FileCasIndexHeader), 0);
-		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(FileCasIndexEntry), sizeof(FileCasIndexHeader));
+		ObjectIndexFile.Write(&Header, sizeof(filecas::impl::FileCasIndexHeader), 0);
+		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(FileCasIndexEntry), sizeof(filecas::impl::FileCasIndexHeader));
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.Close();
 		EntryCount		   = Entries.size();
@@ -1057,6 +1097,8 @@ FileCasStrategy::MakeIndexSnapshot()
 uint64_t
 FileCasStrategy::ReadIndexFile()
 {
+	using namespace filecas::impl;
+
 	std::vector<FileCasIndexEntry> Entries;
 	std::filesystem::path		   IndexPath = GetIndexPath(m_RootDirectory);
 	if (std::filesystem::is_regular_file(IndexPath))
@@ -1143,6 +1185,8 @@ FileCasStrategy::ReadIndexFile()
 uint64_t
 FileCasStrategy::ReadLog(uint64_t SkipEntryCount)
 {
+	using namespace filecas::impl;
+
 	std::filesystem::path LogPath = GetLogPath(m_RootDirectory);
 	if (std::filesystem::is_regular_file(LogPath))
 	{
@@ -1194,6 +1238,8 @@ FileCasStrategy::ReadLog(uint64_t SkipEntryCount)
 std::vector<FileCasStrategy::FileCasIndexEntry>
 FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 {
+	using namespace filecas::impl;
+
 	std::vector<FileCasIndexEntry> Entries;
 	struct Visitor : public FileSystemTraversal::TreeVisitor
 	{
diff --git a/zenstore/gc.cpp b/zenstore/gc.cpp
index 8d3b8d018..370c3c965 100644
--- a/zenstore/gc.cpp
+++ b/zenstore/gc.cpp
@@ -888,7 +888,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& ExpireTime, bool Delete, b
 
 #if ZEN_WITH_TESTS
 
-namespace {
+namespace gc::impl {
 	static IoBuffer CreateChunk(uint64_t Size)
 	{
 		static std::random_device rd;
@@ -909,10 +909,12 @@ namespace {
 	{
 		return CompressedBuffer::Compress(SharedBuffer::MakeView(Buffer.GetData(), Buffer.GetSize()));
 	}
-}  // namespace
+}  // namespace gc::impl
 
 TEST_CASE("gc.basic")
 {
+	using namespace gc::impl;
+
 	ScopedTemporaryDirectory TempDir;
 
 	CidStoreConfiguration CasConfig;
@@ -940,6 +942,8 @@ TEST_CASE("gc.basic")
 
 TEST_CASE("gc.full")
 {
+	using namespace gc::impl;
+
 	ScopedTemporaryDirectory TempDir;
 
 	CidStoreConfiguration CasConfig;
@@ -1140,6 +1144,8 @@ TEST_CASE("gc.full")
 
 TEST_CASE("gc.diskusagewindow")
 {
+	using namespace gc::impl;
+
 	DiskUsageWindow Stats;
 	Stats.Append({.SampleTime = 0, .DiskUsage = 0});	// 0	0
 	Stats.Append({.SampleTime = 10, .DiskUsage = 10});	// 1	10
diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp
index d8e05a334..d87652fde 100644
--- a/zenstore/zenstore.cpp
+++ b/zenstore/zenstore.cpp
@@ -2,14 +2,16 @@
 
 #include "zenstore/zenstore.h"
 
-#include <zenstore/blockstore.h>
-#include <zenstore/gc.h>
-#include <zenstore/hashkeyset.h>
-#include <zenutil/basicfile.h>
+#if ZEN_WITH_TESTS
 
-#include "cas.h"
-#include "compactcas.h"
-#include "filecas.h"
+#	include <zenstore/blockstore.h>
+#	include <zenstore/gc.h>
+#	include <zenstore/hashkeyset.h>
+#	include <zenutil/basicfile.h>
+
+#	include "cas.h"
+#	include "compactcas.h"
+#	include "filecas.h"
 
 namespace zen {
 
@@ -26,3 +28,5 @@ zenstore_forcelinktests()
 }
 
 }  // namespace zen
+
+#endif
diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp
index 3d9324af9..b43313b7c 100644
--- a/zenutil/zenserverprocess.cpp
+++ b/zenutil/zenserverprocess.cpp
@@ -376,7 +376,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd)
 
 //////////////////////////////////////////////////////////////////////////
 
-std::atomic<int> TestCounter{0};
+std::atomic<int> ZenServerTestCounter{0};
 
 ZenServerEnvironment::ZenServerEnvironment()
 {
@@ -416,7 +416,7 @@ ZenServerEnvironment::CreateNewTestDir()
 	using namespace std::literals;
 
 	ExtendableWideStringBuilder<256> TestDir;
-	TestDir << "test"sv << int64_t(++TestCounter);
+	TestDir << "test"sv << int64_t(++ZenServerTestCounter);
 
 	std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str();