Merge branch 'main' of https://github.com/EpicGames/zen

117 files changed, 9027 insertions, 5211 deletions

diff --git a/src/transports/transport-sdk/include/transportplugin.h b/src/transports/transport-sdk/include/transportplugin.h
index 2a3b8075f..4347868e6 100644
--- a/src/transports/transport-sdk/include/transportplugin.h
+++ b/src/transports/transport-sdk/include/transportplugin.h
@@ -77,11 +77,12 @@ public:
 class TransportPlugin
 {
 public:
-	virtual uint32_t AddRef() const											   = 0;
-	virtual uint32_t Release() const										   = 0;
-	virtual void	 Configure(const char* OptionTag, const char* OptionValue) = 0;
-	virtual void	 Initialize(TransportServer* ServerInterface)			   = 0;
-	virtual void	 Shutdown()												   = 0;
+	virtual uint32_t	AddRef() const											  = 0;
+	virtual uint32_t	Release() const											  = 0;
+	virtual void		Configure(const char* OptionTag, const char* OptionValue) = 0;
+	virtual void		Initialize(TransportServer* ServerInterface)			  = 0;
+	virtual void		Shutdown()												  = 0;
+	virtual const char* GetDebugName()											  = 0;
 
 	/** Check whether this transport is usable.
 	 */
@@ -99,9 +100,10 @@ public:
 class TransportConnection
 {
 public:
-	virtual int64_t WriteBytes(const void* Buffer, size_t DataSize) = 0;
-	virtual void	Shutdown(bool Receive, bool Transmit)			= 0;
-	virtual void	CloseConnection()								= 0;
+	virtual int64_t		WriteBytes(const void* Buffer, size_t DataSize) = 0;
+	virtual void		Shutdown(bool Receive, bool Transmit)			= 0;
+	virtual void		CloseConnection()								= 0;
+	virtual const char* GetDebugName()									= 0;
 };
 
 }  // namespace zen
diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp
index 28ac10ec1..7ee2b5ed1 100644
--- a/src/transports/winsock/winsock.cpp
+++ b/src/transports/winsock/winsock.cpp
@@ -51,12 +51,13 @@ public:
 
 	// TransportPlugin implementation
 
-	virtual uint32_t AddRef() const override;
-	virtual uint32_t Release() const override;
-	virtual void	 Configure(const char* OptionTag, const char* OptionValue) override;
-	virtual void	 Initialize(TransportServer* ServerInterface) override;
-	virtual void	 Shutdown() override;
-	virtual bool	 IsAvailable() override;
+	virtual uint32_t	AddRef() const override;
+	virtual uint32_t	Release() const override;
+	virtual void		Configure(const char* OptionTag, const char* OptionValue) override;
+	virtual void		Initialize(TransportServer* ServerInterface) override;
+	virtual void		Shutdown() override;
+	virtual const char* GetDebugName() override;
+	virtual bool		IsAvailable() override;
 
 private:
 	TransportServer* m_ServerInterface = nullptr;
@@ -80,9 +81,10 @@ public:
 
 	// TransportConnection implementation
 
-	virtual int64_t WriteBytes(const void* Buffer, size_t DataSize) override;
-	virtual void	Shutdown(bool Receive, bool Transmit) override;
-	virtual void	CloseConnection() override;
+	virtual int64_t		WriteBytes(const void* Buffer, size_t DataSize) override;
+	virtual void		Shutdown(bool Receive, bool Transmit) override;
+	virtual void		CloseConnection() override;
+	virtual const char* GetDebugName() override;
 
 private:
 	zen::Ref<TransportServerConnection> m_ConnectionHandler;
@@ -153,6 +155,12 @@ WinsockTransportConnection::CloseConnection()
 	m_ClientSocket = 0;
 }
 
+const char*
+WinsockTransportConnection::GetDebugName()
+{
+	return nullptr;
+}
+
 int64_t
 WinsockTransportConnection::WriteBytes(const void* Buffer, size_t DataSize)
 {
@@ -342,6 +350,12 @@ WinsockTransportPlugin::Shutdown()
 	}
 }
 
+const char*
+WinsockTransportPlugin::GetDebugName()
+{
+	return nullptr;
+}
+
 bool
 WinsockTransportPlugin::IsAvailable()
 {
diff --git a/src/transports/winsock/xmake.lua b/src/transports/winsock/xmake.lua
index 552a62702..c14283546 100644
--- a/src/transports/winsock/xmake.lua
+++ b/src/transports/winsock/xmake.lua
@@ -6,9 +6,9 @@ target("winsock")
     add_headerfiles("**.h")
     add_files("**.cpp")
     add_links("Ws2_32")
-    add_includedirs(".", "../../zenbase/include")
+    add_includedirs(".")
     set_symbols("debug")
-    add_deps("transport-sdk")
+    add_deps("zenbase", "transport-sdk")
 
     if is_mode("release") then
         set_optimize("fastest")
diff --git a/src/transports/xmake.lua b/src/transports/xmake.lua
index 44800a8af..78d637d85 100644
--- a/src/transports/xmake.lua
+++ b/src/transports/xmake.lua
@@ -5,6 +5,10 @@ set_languages("cxx20")
 
 includes('transport-sdk')
 
+if os.isdir('zenbase') then
+    includes('zenbase')
+end
+
 if is_plat("windows") then
     includes("winsock")
 end
diff --git a/src/zen/bench.cpp b/src/zen/bench.cpp
new file mode 100644
index 000000000..614454ed5
--- /dev/null
+++ b/src/zen/bench.cpp
@@ -0,0 +1,134 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "bench.h"
+
+#include <zenbase/zenbase.h>
+#include <zencore/except.h>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <stdio.h>
+#	include <tchar.h>
+#	include <windows.h>
+#	include <exception>
+#	include <fmt/format.h>
+
+namespace zen::bench::util {
+
+// See https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/set.htm
+
+typedef DWORD NTSTATUS;
+
+#	define NT_SUCCESS(Status)		  (((NTSTATUS)(Status)) >= 0)
+#	define STATUS_PRIVILEGE_NOT_HELD ((NTSTATUS)0xC0000061L)
+
+typedef enum _SYSTEM_INFORMATION_CLASS
+{
+	SystemMemoryListInformation =
+		80,	 // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege)
+} SYSTEM_INFORMATION_CLASS;
+
+// private
+typedef enum _SYSTEM_MEMORY_LIST_COMMAND
+{
+	MemoryCaptureAccessedBits,
+	MemoryCaptureAndResetAccessedBits,
+	MemoryEmptyWorkingSets,
+	MemoryFlushModifiedList,
+	MemoryPurgeStandbyList,
+	MemoryPurgeLowPriorityStandbyList,
+	MemoryCommandMax
+} SYSTEM_MEMORY_LIST_COMMAND;
+
+BOOL
+ObtainPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
+{
+	LUID			 Luid;
+	TOKEN_PRIVILEGES CurrentPriv;
+	TOKEN_PRIVILEGES NewPriv;
+
+	DWORD dwBufferLength = 16;
+	if (LookupPrivilegeValueA(0, lpName, &Luid))
+	{
+		NewPriv.PrivilegeCount			 = 1;
+		NewPriv.Privileges[0].Luid		 = Luid;
+		NewPriv.Privileges[0].Attributes = 0;
+
+		if (AdjustTokenPrivileges(TokenHandle,
+								  0,
+								  &NewPriv,
+								  DWORD((LPBYTE) & (NewPriv.Privileges[1]) - (LPBYTE)&NewPriv),
+								  &CurrentPriv,
+								  &dwBufferLength))
+		{
+			CurrentPriv.PrivilegeCount			 = 1;
+			CurrentPriv.Privileges[0].Luid		 = Luid;
+			CurrentPriv.Privileges[0].Attributes = flags != 0 ? 2 : 0;
+
+			return AdjustTokenPrivileges(TokenHandle, 0, &CurrentPriv, dwBufferLength, 0, 0);
+		}
+	}
+	return FALSE;
+}
+
+typedef NTSTATUS(WINAPI* NtSetSystemInformationFn)(INT, PVOID, ULONG);
+typedef NTSTATUS(WINAPI* NtQuerySystemInformationFn)(INT, PVOID, ULONG, PULONG);
+
+void
+EmptyStandByList()
+{
+	HMODULE NtDll = LoadLibrary(L"ntdll.dll");
+	if (!NtDll)
+	{
+		zen::ThrowLastError("Could not LoadLibrary ntdll");
+	}
+
+	HANDLE hToken;
+
+	if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
+	{
+		zen::ThrowLastError("Could not open current process token");
+	}
+
+	if (!ObtainPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
+	{
+		zen::ThrowLastError("Unable to obtain SeProfileSingleProcessPrivilege");
+	}
+
+	CloseHandle(hToken);
+
+	NtSetSystemInformationFn   NtSetSystemInformation	= (NtSetSystemInformationFn)GetProcAddress(NtDll, "NtSetSystemInformation");
+	NtQuerySystemInformationFn NtQuerySystemInformation = (NtQuerySystemInformationFn)GetProcAddress(NtDll, "NtQuerySystemInformation");
+
+	if (!NtSetSystemInformation || !NtQuerySystemInformation)
+	{
+		throw std::runtime_error("Failed to look up required ntdll functions");
+	}
+
+	SYSTEM_MEMORY_LIST_COMMAND MemoryListCommand = MemoryPurgeStandbyList;
+	NTSTATUS NtStatus = NtSetSystemInformation(SystemMemoryListInformation, &MemoryListCommand, sizeof(MemoryListCommand));
+
+	if (NtStatus == STATUS_PRIVILEGE_NOT_HELD)
+	{
+		throw elevation_required_exception("Insufficient privileges to execute the memory list command");
+	}
+	else if (!NT_SUCCESS(NtStatus))
+	{
+		throw std::runtime_error(fmt::format("Unable to execute the memory list command (status={})", NtStatus));
+	}
+}
+
+}  // namespace zen::bench::util
+
+#else
+
+namespace zen::bench::util {
+
+void
+EmptyStandByList()
+{
+	return;
+}
+
+}  // namespace zen::bench::util
+
+#endif
diff --git a/src/zen/bench.h b/src/zen/bench.h
new file mode 100644
index 000000000..6c03463ef
--- /dev/null
+++ b/src/zen/bench.h
@@ -0,0 +1,16 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <stdexcept>
+
+namespace zen::bench::util {
+
+void EmptyStandByList();
+
+struct elevation_required_exception : public std::runtime_error
+{
+	explicit elevation_required_exception(const std::string& What) : std::runtime_error{What} {}
+};
+
+}  // namespace zen::bench::util
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp
index b041aa46e..1bde785c7 100644
--- a/src/zen/cmds/admin_cmd.cpp
+++ b/src/zen/cmds/admin_cmd.cpp
@@ -20,6 +20,9 @@ ScrubCommand::ScrubCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
 	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+	m_Options.add_option("", "n", "dry", "Dry run (do not delete any data)", cxxopts::value(m_DryRun), "<bool>");
+	m_Options.add_option("", "", "no-gc", "Do not perform GC after scrub pass", cxxopts::value(m_NoGc), "<bool>");
+	m_Options.add_option("", "", "no-cas", "Do not scrub CAS stores", cxxopts::value(m_NoCas), "<bool>");
 }
 
 ScrubCommand::~ScrubCommand() = default;
@@ -41,11 +44,13 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 		throw OptionParseException("unable to resolve server specification");
 	}
 
-	zen::HttpClient Http(m_HostName);
+	HttpClient Http(m_HostName);
+
+	HttpClient::KeyValueMap Params{{"skipdelete", ToString(m_DryRun)}, {"skipgc", ToString(m_NoGc)}, {"skipcid", ToString(m_NoCas)}};
 
-	if (zen::HttpClient::Response Response = Http.Post("/admin/scrub"sv))
+	if (HttpClient::Response Response = Http.Post("/admin/scrub"sv, /* headers */ HttpClient::KeyValueMap{}, Params))
 	{
-		ZEN_CONSOLE("OK: {}", Response.ToText());
+		ZEN_CONSOLE("scrub started OK: {}", Response.ToText());
 
 		return 0;
 	}
@@ -78,7 +83,7 @@ GcCommand::GcCommand()
 						 "<smallobjects>");
 	m_Options.add_option("", "", "skipcid", "Skip collection of CAS data", cxxopts::value(m_SkipCid)->default_value("false"), "<skipcid>");
 	m_Options.add_option("",
-						 "",
+						 "n",
 						 "skipdelete",
 						 "Skip deletion of data (dryrun)",
 						 cxxopts::value(m_SkipDelete)->default_value("false"),
@@ -99,6 +104,15 @@ GcCommand::GcCommand()
 		.add_option("", "", "usegcv1", "Force use of GC version 1", cxxopts::value(m_ForceUseGCV1)->default_value("false"), "<usegcv2>");
 	m_Options
 		.add_option("", "", "usegcv2", "Force use of GC version 2", cxxopts::value(m_ForceUseGCV2)->default_value("false"), "<usegcv2>");
+	m_Options.add_option("",
+						 "",
+						 "compactblockthreshold",
+						 "How much of a compact block should be used to skip compacting the block. 0 - compact only empty eligible blocks, "
+						 "100 - compact all non-full eligible blocks.",
+						 cxxopts::value(m_CompactBlockThreshold)->default_value("60"),
+						 "<compactblockthreshold>");
+	m_Options
+		.add_option("", "", "verbose", "Enable verbose logging for GC", cxxopts::value(m_Verbose)->default_value("false"), "<verbose>");
 }
 
 GcCommand::~GcCommand()
@@ -123,10 +137,7 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	}
 
 	cpr::Parameters Params;
-	if (m_SmallObjects)
-	{
-		Params.Add({"smallobjects", "true"});
-	}
+	Params.Add({"smallobjects", m_SmallObjects ? "true" : "false"});
 	if (m_MaxCacheDuration != 0)
 	{
 		Params.Add({"maxcacheduration", fmt::format("{}", m_MaxCacheDuration)});
@@ -135,14 +146,8 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	{
 		Params.Add({"disksizesoftlimit", fmt::format("{}", m_DiskSizeSoftLimit)});
 	}
-	if (m_SkipCid)
-	{
-		Params.Add({"skipcid", "true"});
-	}
-	if (m_SkipDelete)
-	{
-		Params.Add({"skipdelete", "true"});
-	}
+	Params.Add({"skipcid", m_SkipCid ? "true" : "false"});
+	Params.Add({"skipdelete", m_SkipDelete ? "true" : "false"});
 	if (m_ForceUseGCV1)
 	{
 		if (m_ForceUseGCV2)
@@ -155,6 +160,11 @@ GcCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	{
 		Params.Add({"forceusegcv2", "true"});
 	}
+	if (m_CompactBlockThreshold)
+	{
+		Params.Add({"compactblockthreshold", fmt::format("{}", m_CompactBlockThreshold)});
+	}
+	Params.Add({"verbose", m_Verbose ? "true" : "false"});
 
 	cpr::Session Session;
 	Session.SetHeader(cpr::Header{{"Accept", "application/json"}});
@@ -237,6 +247,60 @@ GcStatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	return 1;
 }
 
+GcStopCommand::GcStopCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>");
+}
+
+GcStopCommand::~GcStopCommand()
+{
+}
+
+int
+GcStopCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	m_HostName = ResolveTargetHostSpec(m_HostName);
+
+	if (m_HostName.empty())
+	{
+		throw OptionParseException("unable to resolve server specification");
+	}
+
+	cpr::Session Session;
+	Session.SetUrl({fmt::format("{}/admin/gc-stop", m_HostName)});
+	cpr::Response Result = Session.Post();
+
+	if (static_cast<HttpResponseCode>(Result.status_code) == HttpResponseCode::Accepted)
+	{
+		ZEN_CONSOLE("OK: {}", "Cancel request accepted");
+		return 0;
+	}
+	else if (zen::IsHttpSuccessCode(Result.status_code))
+	{
+		ZEN_CONSOLE("OK: {}", "No GC running");
+		return 0;
+	}
+
+	if (Result.status_code)
+	{
+		ZEN_ERROR("GC status failed: {}: {} ({})", Result.status_code, Result.reason, Result.text);
+	}
+	else
+	{
+		ZEN_ERROR("GC status failed: {}", Result.error.message);
+	}
+
+	return 1;
+}
+
 ////////////////////////////////////////////
 
 JobCommand::JobCommand()
@@ -479,21 +543,9 @@ static void
 Copy(const std::filesystem::path& Source, const std::filesystem::path& Target)
 {
 	CreateDirectories(Target.parent_path());
-	BasicFile SourceFile;
-	SourceFile.Open(Source, BasicFile::Mode::kRead);
-	BasicFile TargetFile;
-	TargetFile.Open(Target, BasicFile::Mode::kTruncate);
-	uint64_t			 Size	= SourceFile.FileSize();
-	uint64_t			 Offset = 0;
-	std::vector<uint8_t> Buffer(Min(size_t(Size), size_t(65536u)));
-	while (Offset < Size)
-	{
-		uint64_t CopyCount = Min<uint64_t>(Size - Offset, size_t(Buffer.size()));
-		SourceFile.Read(Buffer.data(), CopyCount, Offset);
-		TargetFile.Write(Buffer.data(), CopyCount, Offset);
-		Offset += CopyCount;
-	}
-	TargetFile.Flush();
+
+	CopyFileOptions Options;
+	CopyFile(Source, Target, Options);
 }
 
 static bool
@@ -503,8 +555,11 @@ TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target
 	{
 		return false;
 	}
-	Copy(Source, Target);
-	return true;
+
+	CreateDirectories(Target.parent_path());
+
+	CopyFileOptions Options;
+	return CopyFile(Source, Target, Options);
 }
 
 int
@@ -559,6 +614,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			std::filesystem::path BucketName	   = BucketPath.filename();
 			std::filesystem::path TargetBucketPath = TargetNamespacePath / BucketName;
 
+			// TODO: make these use file naming helpers from cache implementation?
+
 			std::filesystem::path ManifestPath		 = BucketPath / "zen_manifest";
 			std::filesystem::path TargetManifestPath = TargetBucketPath / "zen_manifest";
 			if (TryCopy(ManifestPath, TargetManifestPath))
@@ -575,6 +632,11 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 				std::filesystem::path IndexPath		  = BucketPath / IndexName;
 				std::filesystem::path TargetIndexPath = TargetBucketPath / IndexName;
 				TryCopy(IndexPath, TargetIndexPath);
+
+				std::filesystem::path MetaName		 = fmt::format("{}.{}", BucketName.string(), "meta");
+				std::filesystem::path MetaPath		 = BucketPath / MetaName;
+				std::filesystem::path TargetMetaPath = TargetBucketPath / MetaName;
+				TryCopy(MetaPath, TargetMetaPath);
 			}
 		}
 	}
diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h
index 356f58363..12029d57e 100644
--- a/src/zen/cmds/admin_cmd.h
+++ b/src/zen/cmds/admin_cmd.h
@@ -10,7 +10,7 @@ namespace zen {
 
 /** Scrub storage
  */
-class ScrubCommand : public ZenCmdBase
+class ScrubCommand : public StorageCommand
 {
 public:
 	ScrubCommand();
@@ -22,11 +22,14 @@ public:
 private:
 	cxxopts::Options m_Options{"scrub", "Scrub zen storage"};
 	std::string		 m_HostName;
+	bool			 m_DryRun = false;
+	bool			 m_NoGc	  = false;
+	bool			 m_NoCas  = false;
 };
 
 /** Garbage collect storage
  */
-class GcCommand : public ZenCmdBase
+class GcCommand : public StorageCommand
 {
 public:
 	GcCommand();
@@ -45,9 +48,11 @@ private:
 	uint64_t		 m_DiskSizeSoftLimit{0};
 	bool			 m_ForceUseGCV1{false};
 	bool			 m_ForceUseGCV2{false};
+	uint32_t		 m_CompactBlockThreshold = 90;
+	bool			 m_Verbose{false};
 };
 
-class GcStatusCommand : public ZenCmdBase
+class GcStatusCommand : public StorageCommand
 {
 public:
 	GcStatusCommand();
@@ -62,6 +67,20 @@ private:
 	bool			 m_Details = false;
 };
 
+class GcStopCommand : public StorageCommand
+{
+public:
+	GcStopCommand();
+	~GcStopCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+
+private:
+	cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"};
+	std::string		 m_HostName;
+};
+
 ////////////////////////////////////////////
 
 class JobCommand : public ZenCmdBase
@@ -101,7 +120,7 @@ private:
 
 /** Flush storage
  */
-class FlushCommand : public ZenCmdBase
+class FlushCommand : public StorageCommand
 {
 public:
 	FlushCommand();
@@ -117,7 +136,7 @@ private:
 
 /** Copy state
  */
-class CopyStateCommand : public ZenCmdBase
+class CopyStateCommand : public StorageCommand
 {
 public:
 	CopyStateCommand();
diff --git a/src/zen/cmds/bench_cmd.cpp b/src/zen/cmds/bench_cmd.cpp
index 06b8967a3..5c955e980 100644
--- a/src/zen/cmds/bench_cmd.cpp
+++ b/src/zen/cmds/bench_cmd.cpp
@@ -1,134 +1,18 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "bench_cmd.h"
+#include "bench.h"
 
 #include <zencore/except.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/string.h>
 #include <zencore/thread.h>
 #include <zencore/timer.h>
 
-#if ZEN_PLATFORM_WINDOWS
-#	include <stdio.h>
-#	include <tchar.h>
-#	include <windows.h>
-#	include <exception>
-
-namespace zen::bench::util {
-
-// See https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/set.htm
-
-typedef DWORD NTSTATUS;
-
-#	define NT_SUCCESS(Status)		  (((NTSTATUS)(Status)) >= 0)
-#	define STATUS_PRIVILEGE_NOT_HELD ((NTSTATUS)0xC0000061L)
-
-typedef enum _SYSTEM_INFORMATION_CLASS
-{
-	SystemMemoryListInformation =
-		80,	 // 80, q: SYSTEM_MEMORY_LIST_INFORMATION; s: SYSTEM_MEMORY_LIST_COMMAND (requires SeProfileSingleProcessPrivilege)
-} SYSTEM_INFORMATION_CLASS;
-
-// private
-typedef enum _SYSTEM_MEMORY_LIST_COMMAND
-{
-	MemoryCaptureAccessedBits,
-	MemoryCaptureAndResetAccessedBits,
-	MemoryEmptyWorkingSets,
-	MemoryFlushModifiedList,
-	MemoryPurgeStandbyList,
-	MemoryPurgeLowPriorityStandbyList,
-	MemoryCommandMax
-} SYSTEM_MEMORY_LIST_COMMAND;
-
-BOOL
-ObtainPrivilege(HANDLE TokenHandle, LPCSTR lpName, int flags)
-{
-	LUID			 Luid;
-	TOKEN_PRIVILEGES CurrentPriv;
-	TOKEN_PRIVILEGES NewPriv;
-
-	DWORD dwBufferLength = 16;
-	if (LookupPrivilegeValueA(0, lpName, &Luid))
-	{
-		NewPriv.PrivilegeCount			 = 1;
-		NewPriv.Privileges[0].Luid		 = Luid;
-		NewPriv.Privileges[0].Attributes = 0;
-
-		if (AdjustTokenPrivileges(TokenHandle,
-								  0,
-								  &NewPriv,
-								  DWORD((LPBYTE) & (NewPriv.Privileges[1]) - (LPBYTE)&NewPriv),
-								  &CurrentPriv,
-								  &dwBufferLength))
-		{
-			CurrentPriv.PrivilegeCount			 = 1;
-			CurrentPriv.Privileges[0].Luid		 = Luid;
-			CurrentPriv.Privileges[0].Attributes = flags != 0 ? 2 : 0;
-
-			return AdjustTokenPrivileges(TokenHandle, 0, &CurrentPriv, dwBufferLength, 0, 0);
-		}
-	}
-	return FALSE;
-}
-
-typedef NTSTATUS(WINAPI* NtSetSystemInformationFn)(INT, PVOID, ULONG);
-typedef NTSTATUS(WINAPI* NtQuerySystemInformationFn)(INT, PVOID, ULONG, PULONG);
-
-struct elevation_required_exception : public std::runtime_error
-{
-	explicit elevation_required_exception(const std::string& What) : std::runtime_error{What} {}
-};
-
-void
-EmptyStandByList()
-{
-	HMODULE NtDll = LoadLibrary(L"ntdll.dll");
-	if (!NtDll)
-	{
-		zen::ThrowLastError("Could not LoadLibrary ntdll");
-	}
-
-	HANDLE hToken;
-
-	if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken))
-	{
-		zen::ThrowLastError("Could not open current process token");
-	}
-
-	if (!ObtainPrivilege(hToken, "SeProfileSingleProcessPrivilege", 1))
-	{
-		zen::ThrowLastError("Unable to obtain SeProfileSingleProcessPrivilege");
-	}
-
-	CloseHandle(hToken);
-
-	NtSetSystemInformationFn   NtSetSystemInformation	= (NtSetSystemInformationFn)GetProcAddress(NtDll, "NtSetSystemInformation");
-	NtQuerySystemInformationFn NtQuerySystemInformation = (NtQuerySystemInformationFn)GetProcAddress(NtDll, "NtQuerySystemInformation");
-
-	if (!NtSetSystemInformation || !NtQuerySystemInformation)
-	{
-		throw std::runtime_error("Failed to look up required ntdll functions");
-	}
-
-	SYSTEM_MEMORY_LIST_COMMAND MemoryListCommand = MemoryPurgeStandbyList;
-	NTSTATUS NtStatus = NtSetSystemInformation(SystemMemoryListInformation, &MemoryListCommand, sizeof(MemoryListCommand));
-
-	if (NtStatus == STATUS_PRIVILEGE_NOT_HELD)
-	{
-		throw elevation_required_exception("Insufficient privileges to execute the memory list command");
-	}
-	else if (!NT_SUCCESS(NtStatus))
-	{
-		throw std::runtime_error(fmt::format("Unable to execute the memory list command (status={})", NtStatus));
-	}
-}
-
-}  // namespace zen::bench::util
-
-#endif
+namespace zen {
 
 BenchCommand::BenchCommand()
 {
@@ -215,3 +99,5 @@ BenchCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 	return 0;
 }
+
+}  // namespace zen
diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h
index 8a8bd4a7c..29d7fcc08 100644
--- a/src/zen/cmds/bench_cmd.h
+++ b/src/zen/cmds/bench_cmd.h
@@ -4,6 +4,8 @@
 
 #include "../zen.h"
 
+namespace zen {
+
 class BenchCommand : public ZenCmdBase
 {
 public:
@@ -12,9 +14,12 @@ public:
 
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
 	virtual cxxopts::Options& Options() override { return m_Options; }
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
 	cxxopts::Options m_Options{"bench", "Benchmarking utility command"};
 	bool			 m_PurgeStandbyLists = false;
 	bool			 m_SingleProcess	 = false;
 };
+
+}  // namespace zen
diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp
index 1bf6ee60e..823f10f1c 100644
--- a/src/zen/cmds/cache_cmd.cpp
+++ b/src/zen/cmds/cache_cmd.cpp
@@ -14,6 +14,8 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #include <cpr/cpr.h>
 ZEN_THIRD_PARTY_INCLUDES_END
 
+namespace zen {
+
 DropCommand::DropCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
@@ -302,3 +304,5 @@ CacheDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar
 
 	return 1;
 }
+
+}  // namespace zen
diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h
index 1f368bdec..80079c452 100644
--- a/src/zen/cmds/cache_cmd.h
+++ b/src/zen/cmds/cache_cmd.h
@@ -4,7 +4,9 @@
 
 #include "../zen.h"
 
-class DropCommand : public ZenCmdBase
+namespace zen {
+
+class DropCommand : public CacheStoreCommand
 {
 public:
 	DropCommand();
@@ -20,7 +22,7 @@ private:
 	std::string		 m_BucketName;
 };
 
-class CacheInfoCommand : public ZenCmdBase
+class CacheInfoCommand : public CacheStoreCommand
 {
 public:
 	CacheInfoCommand();
@@ -35,7 +37,7 @@ private:
 	std::string		 m_BucketName;
 };
 
-class CacheStatsCommand : public ZenCmdBase
+class CacheStatsCommand : public CacheStoreCommand
 {
 public:
 	CacheStatsCommand();
@@ -48,7 +50,7 @@ private:
 	std::string		 m_HostName;
 };
 
-class CacheDetailsCommand : public ZenCmdBase
+class CacheDetailsCommand : public CacheStoreCommand
 {
 public:
 	CacheDetailsCommand();
@@ -66,3 +68,5 @@ private:
 	std::string		 m_Bucket;
 	std::string		 m_ValueKey;
 };
+
+}  // namespace zen
diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp
index e5ddbfa85..956d9c9d2 100644
--- a/src/zen/cmds/copy_cmd.cpp
+++ b/src/zen/cmds/copy_cmd.cpp
@@ -14,6 +14,9 @@ CopyCommand::CopyCommand()
 {
 	m_Options.add_options()("h,help", "Print help");
 	m_Options.add_options()("no-clone", "Do not perform block clone", cxxopts::value(m_NoClone)->default_value("false"));
+	m_Options.add_options()("must-clone",
+							"Always perform block clone (fails if clone is not possible)",
+							cxxopts::value(m_MustClone)->default_value("false"));
 	m_Options.add_option("", "s", "source", "Copy source", cxxopts::value(m_CopySource), "<file/directory>");
 	m_Options.add_option("", "t", "target", "Copy target", cxxopts::value(m_CopyTarget), "<file/directory>");
 	m_Options.parse_positional({"source", "target"});
@@ -45,6 +48,22 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 	FromPath = m_CopySource;
 	ToPath	 = m_CopyTarget;
 
+	std::error_code		  Ec;
+	std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
+
+	if (!Ec)
+	{
+		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
+
+		if (!Ec)
+		{
+			if (FromCanonical == ToCanonical)
+			{
+				throw std::runtime_error("Target and source must be distinct files or directories");
+			}
+		}
+	}
+
 	const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource);
 	const bool IsDirCopy  = std::filesystem::is_directory(m_CopySource);
 
@@ -76,6 +95,17 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 			std::filesystem::create_directories(ToPath);
 		}
 
+		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
+
+		if (!Ec)
+		{
+			if (ToCanonical.generic_string().starts_with(FromCanonical.generic_string()) ||
+				FromCanonical.generic_string().starts_with(ToCanonical.generic_string()))
+			{
+				throw std::runtime_error("Invalid parent/child relationship for source/target directories");
+			}
+		}
+
 		// Multi file copy
 
 		ZEN_CONSOLE("copying {} -> {}", FromPath, ToPath);
@@ -139,6 +169,7 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
 
 		zen::CopyFileOptions CopyOptions;
 		CopyOptions.EnableClone = !m_NoClone;
+		CopyOptions.MustClone	= m_MustClone;
 
 		CopyVisitor Visitor{FromPath, CopyOptions};
 		Visitor.TargetPath = ToPath;
diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h
index 549114160..876aff3f5 100644
--- a/src/zen/cmds/copy_cmd.h
+++ b/src/zen/cmds/copy_cmd.h
@@ -16,12 +16,14 @@ public:
 
 	virtual cxxopts::Options& Options() override { return m_Options; }
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
-	cxxopts::Options m_Options{"copy", "Copy files"};
+	cxxopts::Options m_Options{"copy", "Copy files efficiently"};
 	std::string		 m_CopySource;
 	std::string		 m_CopyTarget;
-	bool			 m_NoClone = false;
+	bool			 m_NoClone	 = false;
+	bool			 m_MustClone = false;
 };
 
 }  // namespace zen
diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h
index 6318704f5..c4f0068e4 100644
--- a/src/zen/cmds/dedup_cmd.h
+++ b/src/zen/cmds/dedup_cmd.h
@@ -16,6 +16,7 @@ public:
 
 	virtual cxxopts::Options& Options() override { return m_Options; }
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
 	cxxopts::Options		 m_Options{"dedup", "Deduplicate files"};
diff --git a/src/zen/cmds/hash_cmd.cpp b/src/zen/cmds/hash_cmd.cpp
deleted file mode 100644
index f5541906b..000000000
--- a/src/zen/cmds/hash_cmd.cpp
+++ /dev/null
@@ -1,172 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "hash_cmd.h"
-
-#include <zencore/blake3.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/string.h>
-#include <zencore/timer.h>
-
-#if ZEN_PLATFORM_WINDOWS
-#	include <ppl.h>
-#endif
-
-namespace zen {
-
-////////////////////////////////////////////////////////////////////////////////
-
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-
-namespace Concurrency {
-
-	template<typename IterType, typename LambdaType>
-	void parallel_for_each(IterType Cursor, IterType End, const LambdaType& Lambda)
-	{
-		for (; Cursor < End; ++Cursor)
-		{
-			Lambda(*Cursor);
-		}
-	}
-
-	template<typename T>
-	struct combinable
-	{
-		combinable<T>& local() { return *this; }
-
-		void operator+=(T Rhs) { Value += Rhs; }
-
-		template<typename LambdaType>
-		void combine_each(const LambdaType& Lambda)
-		{
-			Lambda(Value);
-		}
-
-		T Value = 0;
-	};
-
-}  // namespace Concurrency
-
-#endif	// ZEN_PLATFORM_LINUX|MAC
-
-////////////////////////////////////////////////////////////////////////////////
-
-HashCommand::HashCommand()
-{
-	m_Options.add_options()("d,dir", "Directory to scan", cxxopts::value<std::string>(m_ScanDirectory))(
-		"o,output",
-		"Output file",
-		cxxopts::value<std::string>(m_OutputFile));
-}
-
-HashCommand::~HashCommand() = default;
-
-int
-HashCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
-{
-	ZEN_UNUSED(GlobalOptions);
-
-	if (!ParseOptions(argc, argv))
-	{
-		return 0;
-	}
-
-	bool valid = m_ScanDirectory.length();
-
-	if (!valid)
-		throw zen::OptionParseException("Hash command requires a directory to scan");
-
-	// Gather list of files to process
-
-	ZEN_CONSOLE("Gathering files from {}", m_ScanDirectory);
-
-	struct FileEntry
-	{
-		std::filesystem::path FilePath;
-		zen::BLAKE3			  FileHash;
-	};
-
-	std::vector<FileEntry> FileList;
-	uint64_t			   FileBytes = 0;
-
-	std::filesystem::path ScanDirectoryPath{m_ScanDirectory};
-
-	for (const std::filesystem::directory_entry& Entry : std::filesystem::recursive_directory_iterator(ScanDirectoryPath))
-	{
-		if (Entry.is_regular_file())
-		{
-			FileList.push_back({Entry.path()});
-			FileBytes += Entry.file_size();
-		}
-	}
-
-	ZEN_CONSOLE("Gathered {} files, total size {}", FileList.size(), zen::NiceBytes(FileBytes));
-
-	Concurrency::combinable<uint64_t> TotalBytes;
-
-	auto hashFile = [&](FileEntry& File) {
-		InternalFile InputFile;
-		InputFile.OpenRead(File.FilePath);
-		const uint8_t* DataPointer = (const uint8_t*)InputFile.MemoryMapFile();
-		const size_t   DataSize	   = InputFile.GetFileSize();
-
-		File.FileHash = zen::BLAKE3::HashMemory(DataPointer, DataSize);
-
-		TotalBytes.local() += DataSize;
-	};
-
-	// Process them as quickly as possible
-
-	zen::Stopwatch Timer;
-
-#if 1
-	Concurrency::parallel_for_each(begin(FileList), end(FileList), [&](auto& file) { hashFile(file); });
-#else
-	for (const auto& file : FileList)
-	{
-		hashFile(file);
-	}
-#endif
-
-	size_t TotalByteCount = 0;
-
-	TotalBytes.combine_each([&](size_t Total) { TotalByteCount += Total; });
-
-	const uint64_t ElapsedMs = Timer.GetElapsedTimeMs();
-	ZEN_CONSOLE("Scanned {} files in {}", FileList.size(), zen::NiceTimeSpanMs(ElapsedMs));
-	ZEN_CONSOLE("Total bytes {} ({})", zen::NiceBytes(TotalByteCount), zen::NiceByteRate(TotalByteCount, ElapsedMs));
-
-	InternalFile Output;
-
-	if (m_OutputFile.empty())
-	{
-		// TEMPORARY -- should properly open stdout
-		Output.OpenWrite("CONOUT$", false);
-	}
-	else
-	{
-		Output.OpenWrite(m_OutputFile, true);
-	}
-
-	zen::ExtendableStringBuilder<256> Line;
-
-	uint64_t CurrentOffset = 0;
-
-	for (const auto& File : FileList)
-	{
-		Line.Append(File.FilePath.generic_u8string().c_str());
-		Line.Append(',');
-		File.FileHash.ToHexString(Line);
-		Line.Append('\n');
-
-		Output.Write(Line.Data(), Line.Size(), CurrentOffset);
-		CurrentOffset += Line.Size();
-
-		Line.Reset();
-	}
-
-	// TODO: implement snapshot enumeration and display
-	return 0;
-}
-
-}  // namespace zen
diff --git a/src/zen/cmds/hash_cmd.h b/src/zen/cmds/hash_cmd.h
deleted file mode 100644
index e5ee071e9..000000000
--- a/src/zen/cmds/hash_cmd.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include "../internalfile.h"
-#include "../zen.h"
-
-namespace zen {
-
-/** Generate hash list file
- */
-class HashCommand : public ZenCmdBase
-{
-public:
-	HashCommand();
-	~HashCommand();
-
-	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
-	virtual cxxopts::Options& Options() override { return m_Options; }
-
-private:
-	cxxopts::Options m_Options{"hash", "Hash files"};
-	std::string		 m_ScanDirectory;
-	std::string		 m_OutputFile;
-};
-
-}  // namespace zen
diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h
index 09d91830a..4d6a492b7 100644
--- a/src/zen/cmds/print_cmd.h
+++ b/src/zen/cmds/print_cmd.h
@@ -16,6 +16,7 @@ public:
 
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
 	virtual cxxopts::Options& Options() override { return m_Options; }
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
 	cxxopts::Options m_Options{"print", "Print compact binary object"};
@@ -32,6 +33,7 @@ public:
 
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
 	virtual cxxopts::Options& Options() override { return m_Options; }
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
 
 private:
 	cxxopts::Options m_Options{"printpkg", "Print compact binary package"};
diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h
index fd1590423..8891fdaf4 100644
--- a/src/zen/cmds/projectstore_cmd.h
+++ b/src/zen/cmds/projectstore_cmd.h
@@ -6,7 +6,12 @@
 
 namespace zen {
 
-class DropProjectCommand : public ZenCmdBase
+class ProjectStoreCommand : public ZenCmdBase
+{
+	virtual ZenCmdCategory& CommandCategory() const override { return g_ProjectStoreCategory; }
+};
+
+class DropProjectCommand : public ProjectStoreCommand
 {
 public:
 	DropProjectCommand();
@@ -22,7 +27,7 @@ private:
 	std::string		 m_OplogName;
 };
 
-class ProjectInfoCommand : public ZenCmdBase
+class ProjectInfoCommand : public ProjectStoreCommand
 {
 public:
 	ProjectInfoCommand();
@@ -37,7 +42,7 @@ private:
 	std::string		 m_OplogName;
 };
 
-class CreateProjectCommand : public ZenCmdBase
+class CreateProjectCommand : public ProjectStoreCommand
 {
 public:
 	CreateProjectCommand();
@@ -57,7 +62,7 @@ private:
 	bool			 m_ForceUpdate = false;
 };
 
-class DeleteProjectCommand : public ZenCmdBase
+class DeleteProjectCommand : public ProjectStoreCommand
 {
 public:
 	DeleteProjectCommand();
@@ -72,7 +77,7 @@ private:
 	std::string		 m_ProjectId;
 };
 
-class CreateOplogCommand : public ZenCmdBase
+class CreateOplogCommand : public ProjectStoreCommand
 {
 public:
 	CreateOplogCommand();
@@ -90,7 +95,7 @@ private:
 	bool			 m_ForceUpdate = false;
 };
 
-class DeleteOplogCommand : public ZenCmdBase
+class DeleteOplogCommand : public ProjectStoreCommand
 {
 public:
 	DeleteOplogCommand();
@@ -106,7 +111,7 @@ private:
 	std::string		 m_OplogId;
 };
 
-class ExportOplogCommand : public ZenCmdBase
+class ExportOplogCommand : public ProjectStoreCommand
 {
 public:
 	ExportOplogCommand();
@@ -150,7 +155,7 @@ private:
 	bool		m_FileForceEnableTempBlocks = false;
 };
 
-class ImportOplogCommand : public ZenCmdBase
+class ImportOplogCommand : public ProjectStoreCommand
 {
 public:
 	ImportOplogCommand();
@@ -188,7 +193,7 @@ private:
 	std::string m_FileName;
 };
 
-class SnapshotOplogCommand : public ZenCmdBase
+class SnapshotOplogCommand : public ProjectStoreCommand
 {
 public:
 	SnapshotOplogCommand();
@@ -204,7 +209,7 @@ private:
 	std::string		 m_OplogName;
 };
 
-class ProjectStatsCommand : public ZenCmdBase
+class ProjectStatsCommand : public ProjectStoreCommand
 {
 public:
 	ProjectStatsCommand();
@@ -217,7 +222,7 @@ private:
 	std::string		 m_HostName;
 };
 
-class ProjectDetailsCommand : public ZenCmdBase
+class ProjectDetailsCommand : public ProjectStoreCommand
 {
 public:
 	ProjectDetailsCommand();
@@ -237,7 +242,7 @@ private:
 	std::string		 m_OpId;
 };
 
-class OplogMirrorCommand : public ZenCmdBase
+class OplogMirrorCommand : public ProjectStoreCommand
 {
 public:
 	OplogMirrorCommand();
diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp
index 10cd3aebd..202829aa0 100644
--- a/src/zen/cmds/rpcreplay_cmd.cpp
+++ b/src/zen/cmds/rpcreplay_cmd.cpp
@@ -6,6 +6,7 @@
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/scopeguard.h>
 #include <zencore/session.h>
 #include <zencore/stream.h>
diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h
index e1c2831a5..42cdd4ac1 100644
--- a/src/zen/cmds/rpcreplay_cmd.h
+++ b/src/zen/cmds/rpcreplay_cmd.h
@@ -6,7 +6,7 @@
 
 namespace zen {
 
-class RpcStartRecordingCommand : public ZenCmdBase
+class RpcStartRecordingCommand : public CacheStoreCommand
 {
 public:
 	RpcStartRecordingCommand();
@@ -21,7 +21,7 @@ private:
 	std::string		 m_RecordingPath;
 };
 
-class RpcStopRecordingCommand : public ZenCmdBase
+class RpcStopRecordingCommand : public CacheStoreCommand
 {
 public:
 	RpcStopRecordingCommand();
@@ -35,7 +35,7 @@ private:
 	std::string		 m_HostName;
 };
 
-class RpcReplayCommand : public ZenCmdBase
+class RpcReplayCommand : public CacheStoreCommand
 {
 public:
 	RpcReplayCommand();
diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp
new file mode 100644
index 000000000..a99ba9704
--- /dev/null
+++ b/src/zen/cmds/run_cmd.cpp
@@ -0,0 +1,197 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "run_cmd.h"
+
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/process.h>
+#include <zencore/string.h>
+#include <zencore/timer.h>
+
+using namespace std::literals;
+
+#define ZEN_COLOR_BLACK	  "\033[0;30m"
+#define ZEN_COLOR_RED	  "\033[0;31m"
+#define ZEN_COLOR_GREEN	  "\033[0;32m"
+#define ZEN_COLOR_YELLOW  "\033[0;33m"
+#define ZEN_COLOR_BLUE	  "\033[0;34m"
+#define ZEN_COLOR_MAGENTA "\033[0;35m"
+#define ZEN_COLOR_CYAN	  "\033[0;36m"
+#define ZEN_COLOR_WHITE	  "\033[0;37m"
+
+#define ZEN_BRIGHT_COLOR_BLACK	 "\033[1;30m"
+#define ZEN_BRIGHT_COLOR_RED	 "\033[1;31m"
+#define ZEN_BRIGHT_COLOR_GREEN	 "\033[1;32m"
+#define ZEN_BRIGHT_COLOR_YELLOW	 "\033[1;33m"
+#define ZEN_BRIGHT_COLOR_BLUE	 "\033[1;34m"
+#define ZEN_BRIGHT_COLOR_MAGENTA "\033[1;35m"
+#define ZEN_BRIGHT_COLOR_CYAN	 "\033[1;36m"
+#define ZEN_BRIGHT_COLOR_WHITE	 "\033[1;37m"
+
+#define ZEN_COLOR_RESET "\033[0m"
+
+namespace zen {
+
+RunCommand::RunCommand()
+{
+	m_Options.add_options()("h,help", "Print help");
+	m_Options.add_option("", "n", "count", "Number of times to run command", cxxopts::value(m_RunCount), "<count>");
+	m_Options.add_option("", "t", "time", "How long to run command(s) for", cxxopts::value(m_RunTime), "<seconds>");
+	m_Options.add_option("",
+						 "",
+						 "basepath",
+						 "Where to run command. Each run will execute in its own numbered subdirectory below this directory. Additionally, "
+						 "stdout will be redirected to a file in the provided directory",
+						 cxxopts::value(m_BaseDirectory),
+						 "<path>");
+	m_Options.add_option("",
+						 "",
+						 "max-dirs",
+						 "Number of base directories to retain on rotation",
+						 cxxopts::value(m_MaxBaseDirectoryCount),
+						 "<count>");
+}
+
+RunCommand::~RunCommand()
+{
+}
+
+int
+RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
+{
+	ZEN_UNUSED(GlobalOptions);
+
+	if (!ParseOptions(argc, argv))
+	{
+		return 0;
+	}
+
+	// Validate arguments
+
+	if (GlobalOptions.PassthroughArgV.empty() || GlobalOptions.PassthroughArgV[0].empty())
+		throw OptionParseException("No command specified. The command to run is passed in after a double dash ('--') on the command line");
+
+	if (m_RunCount < 0)
+		throw OptionParseException("Invalid count specified");
+
+	if (m_RunTime < -1 || m_RunTime == 0)
+		throw OptionParseException("Invalid run time specified");
+
+	if (m_MaxBaseDirectoryCount < 0)
+		throw OptionParseException("Invalid directory count specified");
+
+	if (m_RunTime > 0 && m_RunCount > 0)
+		throw OptionParseException("Specify either time or count, not both");
+
+	if (m_RunCount == 0)
+		m_RunCount = 1;
+
+	std::filesystem::path BaseDirectory;
+
+	if (!m_BaseDirectory.empty())
+	{
+		BaseDirectory = m_BaseDirectory;
+
+		if (m_MaxBaseDirectoryCount)
+		{
+			RotateDirectories(BaseDirectory, m_MaxBaseDirectoryCount);
+			CreateDirectories(BaseDirectory);
+		}
+		else
+		{
+			CleanDirectory(BaseDirectory);
+		}
+	}
+
+	bool TimedRun			 = false;
+	auto CommandStartTime	 = std::chrono::system_clock::now();
+	auto CommandDeadlineTime = std::chrono::system_clock::now();
+
+	if (m_RunTime > 0)
+	{
+		m_RunCount = 1'000'000;
+		TimedRun   = true;
+
+		CommandDeadlineTime += std::chrono::seconds(m_RunTime);
+	}
+
+	struct RunResults
+	{
+		int										ExitCode = 0;
+		std::chrono::duration<long, std::milli> Duration{};
+	};
+
+	std::vector<RunResults> Results;
+	int						ErrorCount = 0;
+
+	std::filesystem::path ExecutablePath   = SearchPathForExecutable(GlobalOptions.PassthroughArgV[0]);
+	std::string			  CommandArguments = GlobalOptions.PassthroughArgs;
+
+	for (int i = 0; i < m_RunCount; ++i)
+	{
+		std::filesystem::path RunDir;
+		if (!BaseDirectory.empty())
+		{
+			RunDir = BaseDirectory / IntNum(i + 1).c_str();
+			CreateDirectories(RunDir);
+		}
+
+		Stopwatch Timer;
+
+		CreateProcOptions ProcOptions;
+
+		if (!RunDir.empty())
+		{
+			ProcOptions.WorkingDirectory = &RunDir;
+			ProcOptions.StdoutFile		 = RunDir / "stdout.txt";
+		}
+
+		fmt::print(ZEN_BRIGHT_COLOR_WHITE "run #{}" ZEN_COLOR_RESET ": {}\n", i + 1, GlobalOptions.PassthroughCommandLine);
+
+		ProcessHandle Proc;
+		Proc.Initialize(CreateProc(ExecutablePath, GlobalOptions.PassthroughCommandLine, ProcOptions));
+		if (!Proc.IsValid())
+		{
+			throw std::runtime_error(fmt::format("failed to launch '{}'", ExecutablePath));
+		}
+
+		int ExitCode = Proc.WaitExitCode();
+
+		auto EndTime = std::chrono::system_clock::now();
+
+		if (ExitCode)
+			++ErrorCount;
+
+		Results.emplace_back(RunResults{.ExitCode = ExitCode, .Duration = std::chrono::milliseconds(Timer.GetElapsedTimeMs())});
+
+		if (TimedRun)
+		{
+			if (EndTime >= CommandDeadlineTime)
+			{
+				m_RunCount = i + 1;
+				break;
+			}
+		}
+	}
+
+	fmt::print("{:>5} {:>3} {:>6}\n", "run", "rc", "time");
+	int i = 0;
+	for (const auto& Entry : Results)
+	{
+		fmt::print("{:5} {:3} {:>6}\n", ++i, Entry.ExitCode, NiceTimeSpanMs(Entry.Duration.count()));
+	}
+
+	if (ErrorCount)
+	{
+		fmt::print("run complete ({}/{} failed)\n", ErrorCount, m_RunCount);
+	}
+	else
+	{
+		fmt::print("run complete, no error exit code\n", m_RunCount);
+	}
+
+	return 0;
+}
+
+}  // namespace zen
diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h
new file mode 100644
index 000000000..f6512a4e8
--- /dev/null
+++ b/src/zen/cmds/run_cmd.h
@@ -0,0 +1,27 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "../zen.h"
+
+namespace zen {
+
+class RunCommand : public ZenCmdBase
+{
+public:
+	RunCommand();
+	~RunCommand();
+
+	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override;
+	virtual cxxopts::Options& Options() override { return m_Options; }
+	virtual ZenCmdCategory&	  CommandCategory() const override { return g_UtilitiesCategory; }
+
+private:
+	cxxopts::Options m_Options{"run", "Run executable"};
+	int				 m_RunCount = 0;
+	int				 m_RunTime	= -1;
+	std::string		 m_BaseDirectory;
+	int				 m_MaxBaseDirectoryCount = 10;
+};
+
+}  // namespace zen
diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp
index cb8bd8ecf..837cc7edf 100644
--- a/src/zen/cmds/up_cmd.cpp
+++ b/src/zen/cmds/up_cmd.cpp
@@ -4,6 +4,7 @@
 
 #include <zencore/filesystem.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zenutil/zenserverprocess.h>
 
 #include <memory>
diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h
index 35546c9b6..9b2497c0e 100644
--- a/src/zen/cmds/vfs_cmd.h
+++ b/src/zen/cmds/vfs_cmd.h
@@ -6,7 +6,7 @@
 
 namespace zen {
 
-class VfsCommand : public ZenCmdBase
+class VfsCommand : public StorageCommand
 {
 public:
 	VfsCommand();
diff --git a/src/zen/internalfile.cpp b/src/zen/internalfile.cpp
deleted file mode 100644
index 671a2093e..000000000
--- a/src/zen/internalfile.cpp
+++ /dev/null
@@ -1,306 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "internalfile.h"
-
-#include <zencore/except.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/logging.h>
-#include <zencore/memory.h>
-
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-#	include <fcntl.h>
-#	include <sys/file.h>
-#	include <sys/mman.h>
-#	include <sys/stat.h>
-#	include <unistd.h>
-#endif
-
-#include <gsl/gsl-lite.hpp>
-
-namespace zen {
-
-#define ZEN_USE_SLIST ZEN_PLATFORM_WINDOWS
-
-#if ZEN_USE_SLIST == 0
-struct FileBufferManager::Impl
-{
-	RwLock				m_Lock;
-	std::list<IoBuffer> m_FreeBuffers;
-
-	uint64_t m_BufferSize;
-	uint64_t m_MaxBufferCount;
-
-	Impl(uint64_t BufferSize, uint64_t MaxBuffers) : m_BufferSize(BufferSize), m_MaxBufferCount(MaxBuffers) {}
-
-	IoBuffer AllocBuffer()
-	{
-		RwLock::ExclusiveLockScope _(m_Lock);
-
-		if (m_FreeBuffers.empty())
-		{
-			return IoBuffer{m_BufferSize, 64 * 1024};
-		}
-		else
-		{
-			IoBuffer Buffer = std::move(m_FreeBuffers.front());
-			m_FreeBuffers.pop_front();
-			return Buffer;
-		}
-	}
-
-	void ReturnBuffer(IoBuffer Buffer)
-	{
-		RwLock::ExclusiveLockScope _(m_Lock);
-
-		m_FreeBuffers.push_front(std::move(Buffer));
-	}
-};
-#else
-struct FileBufferManager::Impl
-{
-	struct BufferItem
-	{
-		SLIST_ENTRY ItemEntry;
-		IoBuffer	Buffer;
-	};
-
-	SLIST_HEADER m_FreeList;
-	uint64_t	 m_BufferSize;
-	uint64_t	 m_MaxBufferCount;
-
-	Impl(uint64_t BufferSize, uint64_t MaxBuffers) : m_BufferSize(BufferSize), m_MaxBufferCount(MaxBuffers)
-	{
-		InitializeSListHead(&m_FreeList);
-	}
-
-	~Impl()
-	{
-		while (SLIST_ENTRY* Entry = InterlockedPopEntrySList(&m_FreeList))
-		{
-			BufferItem* Item = reinterpret_cast<BufferItem*>(Entry);
-			delete Item;
-		}
-	}
-
-	IoBuffer AllocBuffer()
-	{
-		SLIST_ENTRY* Entry = InterlockedPopEntrySList(&m_FreeList);
-
-		if (Entry == nullptr)
-		{
-			return IoBuffer{m_BufferSize, 64 * 1024};
-		}
-		else
-		{
-			BufferItem* Item   = reinterpret_cast<BufferItem*>(Entry);
-			IoBuffer	Buffer = std::move(Item->Buffer);
-			delete Item;  // Todo: could keep this around in another list
-
-			return Buffer;
-		}
-	}
-
-	void ReturnBuffer(IoBuffer Buffer)
-	{
-		BufferItem* Item = new BufferItem{nullptr, std::move(Buffer)};
-
-		InterlockedPushEntrySList(&m_FreeList, &Item->ItemEntry);
-	}
-};
-#endif
-
-FileBufferManager::FileBufferManager(uint64_t BufferSize, uint64_t MaxBuffers)
-{
-	m_Impl = new Impl{BufferSize, MaxBuffers};
-}
-
-FileBufferManager::~FileBufferManager()
-{
-	delete m_Impl;
-}
-
-IoBuffer
-FileBufferManager::AllocBuffer()
-{
-	return m_Impl->AllocBuffer();
-}
-
-void
-FileBufferManager::ReturnBuffer(IoBuffer Buffer)
-{
-	return m_Impl->ReturnBuffer(Buffer);
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-InternalFile::InternalFile()
-{
-}
-
-InternalFile::~InternalFile()
-{
-	if (m_Memory)
-		Memory::Free(m_Memory);
-
-#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	if (m_Mmap)
-		munmap(m_Mmap, GetFileSize());
-	if (m_File)
-		close(int(intptr_t(m_File)));
-#endif
-}
-
-size_t
-InternalFile::GetFileSize()
-{
-#if ZEN_PLATFORM_WINDOWS
-	ULONGLONG sz;
-	m_File.GetSize(sz);
-	return size_t(sz);
-#else
-	int Fd = int(intptr_t(m_File));
-	static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files");
-	struct stat Stat;
-	fstat(Fd, &Stat);
-	return size_t(Stat.st_size);
-#endif
-}
-
-void
-InternalFile::OpenWrite(std::filesystem::path FileName, bool IsCreate)
-{
-	bool Success = false;
-
-#if ZEN_PLATFORM_WINDOWS
-	const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING;
-
-	HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, dwCreationDisposition);
-	Success		 = SUCCEEDED(hRes);
-#else
-	int OpenFlags = O_RDWR | O_CLOEXEC;
-	OpenFlags |= IsCreate ? O_CREAT | O_TRUNC : 0;
-
-	int Fd = open(FileName.c_str(), OpenFlags, 0666);
-	if (Fd >= 0)
-	{
-		if (IsCreate)
-		{
-			fchmod(Fd, 0666);
-		}
-		Success = true;
-		m_File	= (void*)(intptr_t(Fd));
-	}
-#endif	// ZEN_PLATFORM_WINDOWS
-
-	if (Success)
-	{
-		ThrowLastError(fmt::format("Failed to open file for writing: '{}'", FileName));
-	}
-}
-
-void
-InternalFile::OpenRead(std::filesystem::path FileName)
-{
-	bool Success = false;
-
-#if ZEN_PLATFORM_WINDOWS
-	const DWORD dwCreationDisposition = OPEN_EXISTING;
-
-	HRESULT hRes = m_File.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ, dwCreationDisposition);
-	Success		 = SUCCEEDED(hRes);
-#else
-	int Fd = open(FileName.c_str(), O_RDONLY);
-	if (Fd >= 0)
-	{
-		Success = true;
-		m_File	= (void*)(intptr_t(Fd));
-	}
-#endif
-
-	if (Success)
-	{
-		ThrowLastError(fmt::format("Failed to open file for reading: '{}'", FileName));
-	}
-}
-
-const void*
-InternalFile::MemoryMapFile()
-{
-	auto FileSize = GetFileSize();
-
-	if (FileSize <= 100 * 1024 * 1024)
-	{
-		m_Memory = Memory::Alloc(FileSize, 64);
-		if (!m_Memory)
-		{
-			ThrowOutOfMemory(fmt::format("failed allocating {:#x} bytes aligned to {:#x}", FileSize, 64));
-		}
-		Read(m_Memory, FileSize, 0);
-
-		return m_Memory;
-	}
-
-#if ZEN_PLATFORM_WINDOWS
-	m_Mmap.MapFile(m_File);
-	return m_Mmap.GetData();
-#else
-	int Fd = int(intptr_t(m_File));
-	m_Mmap = mmap(nullptr, FileSize, PROT_READ, MAP_PRIVATE, Fd, 0);
-	return m_Mmap;
-#endif
-}
-
-void
-InternalFile::Read(void* Data, uint64_t Size, uint64_t Offset)
-{
-	bool Success;
-
-#if ZEN_PLATFORM_WINDOWS
-	OVERLAPPED ovl{};
-
-	ovl.Offset	   = DWORD(Offset & 0xffff'ffffu);
-	ovl.OffsetHigh = DWORD(Offset >> 32);
-
-	HRESULT hRes = m_File.Read(Data, gsl::narrow<DWORD>(Size), &ovl);
-	Success		 = SUCCEEDED(hRes);
-#else
-	int Fd			 = int(intptr_t(m_File));
-	int BytesRead	 = pread(Fd, Data, Size, Offset);
-	Success			 = (BytesRead > 0);
-#endif
-
-	if (Success)
-	{
-		std::error_code DummyEc;
-		ThrowLastError(fmt::format("Failed to read from file '{}'", PathFromHandle(m_File, DummyEc)));
-	}
-}
-
-void
-InternalFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
-{
-	bool Success;
-
-#if ZEN_PLATFORM_WINDOWS
-	OVERLAPPED Ovl{};
-
-	Ovl.Offset	   = DWORD(Offset & 0xffff'ffffu);
-	Ovl.OffsetHigh = DWORD(Offset >> 32);
-
-	HRESULT hRes = m_File.Write(Data, gsl::narrow<DWORD>(Size), &Ovl);
-	Success		 = SUCCEEDED(hRes);
-#else
-	int Fd			 = int(intptr_t(m_File));
-	int BytesWritten = pwrite(Fd, Data, Size, Offset);
-	Success			 = (BytesWritten > 0);
-#endif
-
-	if (Success)
-	{
-		std::error_code DummyEc;
-		ThrowLastError(fmt::format("Failed to write to file '{}'", PathFromHandle(m_File, DummyEc)));
-	}
-}
-
-}  // namespace zen
diff --git a/src/zen/internalfile.h b/src/zen/internalfile.h
deleted file mode 100644
index 90d370e28..000000000
--- a/src/zen/internalfile.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/zencore.h>
-
-#include <zenbase/refcount.h>
-#include <zencore/iobuffer.h>
-#include <zencore/thread.h>
-
-#if ZEN_PLATFORM_WINDOWS
-#	include <zencore/windows.h>
-#endif
-
-#include <filesystem>
-#include <list>
-
-namespace zen {
-
-//////////////////////////////////////////////////////////////////////////
-
-class FileBufferManager : public RefCounted
-{
-public:
-	FileBufferManager(uint64_t BufferSize, uint64_t MaxBufferCount);
-	~FileBufferManager();
-
-	IoBuffer AllocBuffer();
-	void	 ReturnBuffer(IoBuffer Buffer);
-
-private:
-	struct Impl;
-
-	Impl* m_Impl;
-};
-
-class InternalFile : public RefCounted
-{
-public:
-	InternalFile();
-	~InternalFile();
-
-	void OpenRead(std::filesystem::path FileName);
-	void Read(void* Data, uint64_t Size, uint64_t Offset);
-
-	void OpenWrite(std::filesystem::path FileName, bool isCreate);
-	void Write(const void* Data, uint64_t Size, uint64_t Offset);
-
-	const void* MemoryMapFile();
-	size_t		GetFileSize();
-
-private:
-#if ZEN_PLATFORM_WINDOWS
-	windows::FileHandle	 m_File;
-	windows::FileMapping m_Mmap;
-#else
-	void* m_File = nullptr;
-	void* m_Mmap = nullptr;
-#endif
-
-	void* m_Memory = nullptr;
-};
-
-}  // namespace zen
diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp
index 65a02633a..c949008ff 100644
--- a/src/zen/zen.cpp
+++ b/src/zen/zen.cpp
@@ -10,10 +10,10 @@
 #include "cmds/cache_cmd.h"
 #include "cmds/copy_cmd.h"
 #include "cmds/dedup_cmd.h"
-#include "cmds/hash_cmd.h"
 #include "cmds/print_cmd.h"
 #include "cmds/projectstore_cmd.h"
 #include "cmds/rpcreplay_cmd.h"
+#include "cmds/run_cmd.h"
 #include "cmds/serve_cmd.h"
 #include "cmds/status_cmd.h"
 #include "cmds/top_cmd.h"
@@ -47,11 +47,34 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 //////////////////////////////////////////////////////////////////////////
 
+namespace zen {
+
+ZenCmdCategory DefaultCategory{.Name = "general commands"};
+ZenCmdCategory g_UtilitiesCategory{.Name = "utility commands"};
+ZenCmdCategory g_ProjectStoreCategory{.Name = "project store commands"};
+ZenCmdCategory g_CacheStoreCategory{.Name = "cache store commands"};
+ZenCmdCategory g_StorageCategory{.Name = "storage management commands"};
+
+ZenCmdCategory&
+ZenCmdBase::CommandCategory() const
+{
+	return DefaultCategory;
+}
+
 bool
 ZenCmdBase::ParseOptions(int argc, char** argv)
 {
 	cxxopts::Options&	 CmdOptions = Options();
-	cxxopts::ParseResult Result		= CmdOptions.parse(argc, argv);
+	cxxopts::ParseResult Result;
+
+	try
+	{
+		Result = CmdOptions.parse(argc, argv);
+	}
+	catch (std::exception& Ex)
+	{
+		throw zen::OptionParseException(Ex.what());
+	}
 
 	CmdOptions.show_positional_help();
 
@@ -140,28 +163,61 @@ ZenCmdBase::MapHttpToCommandReturnCode(const cpr::Response& Response)
 	return 1;
 }
 
-#if ZEN_WITH_TESTS
-
-class RunTestsCommand : public ZenCmdBase
+std::string
+ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort)
 {
-public:
-	virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override
+	if (InHostSpec.empty())
 	{
-		ZEN_UNUSED(GlobalOptions);
+		// If no host is specified then look to see if we have an instance
+		// running on this host and use that as the default to interact with
 
-		// Set output mode to handle virtual terminal sequences
-		zen::logging::EnableVTMode();
+		zen::ZenServerState Servers;
+
+		if (Servers.InitializeReadOnly())
+		{
+			std::string ResolvedSpec;
 
-		return ZEN_RUN_TESTS(argc, argv);
+			Servers.Snapshot([&](const zen::ZenServerState::ZenServerEntry& Entry) {
+				if (ResolvedSpec.empty())
+				{
+					ResolvedSpec	 = fmt::format("http://localhost:{}", Entry.EffectiveListenPort.load());
+					OutEffectivePort = Entry.EffectiveListenPort;
+				}
+			});
+
+			return ResolvedSpec;
+		}
 	}
 
-	virtual cxxopts::Options& Options() override { return m_Options; }
+	// Parse out port from the specification provided, to be consistent with
+	// the auto-discovery logic above.
 
-private:
-	cxxopts::Options m_Options{"runtests", "Run tests"};
-};
+	std::string_view PortSpec(InHostSpec);
+	if (size_t PrefixIndex = PortSpec.find_last_of(":"); PrefixIndex != std::string_view::npos)
+	{
+		PortSpec.remove_prefix(PrefixIndex + 1);
 
-#endif
+		std::optional<uint16_t> EffectivePort = zen::ParseInt<uint16_t>(PortSpec);
+
+		if (EffectivePort)
+		{
+			OutEffectivePort = EffectivePort.value();
+		}
+	}
+
+	// note: We should consider adding validation/normalization of the provided spec here
+
+	return InHostSpec;
+}
+
+std::string
+ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
+{
+	uint16_t Dummy = 0;
+	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
+}
+
+}  // namespace zen
 
 //////////////////////////////////////////////////////////////////////////
 // TODO: should make this Unicode-aware so we can pass anything in on the
@@ -171,6 +227,7 @@ int
 main(int argc, char** argv)
 {
 	using namespace zen;
+	using namespace std::literals;
 
 #if ZEN_USE_MIMALLOC
 	mi_version();
@@ -214,7 +271,7 @@ main(int argc, char** argv)
 	FlushCommand			 FlushCmd;
 	GcCommand				 GcCmd;
 	GcStatusCommand			 GcStatusCmd;
-	HashCommand				 HashCmd;
+	GcStopCommand			 GcStopCmd;
 	ImportOplogCommand		 ImportOplogCmd;
 	JobCommand				 JobCmd;
 	OplogMirrorCommand		 OplogMirrorCmd;
@@ -227,6 +284,7 @@ main(int argc, char** argv)
 	RpcReplayCommand		 RpcReplayCmd;
 	RpcStartRecordingCommand RpcStartRecordingCmd;
 	RpcStopRecordingCommand	 RpcStopRecordingCmd;
+	RunCommand				 RunCmd;
 	ScrubCommand			 ScrubCmd;
 	ServeCommand			 ServeCmd;
 	SnapshotOplogCommand	 SnapshotOplogCmd;
@@ -237,9 +295,6 @@ main(int argc, char** argv)
 	UpCommand				 UpCmd;
 	VersionCommand			 VersionCmd;
 	VfsCommand				 VfsCmd;
-#if ZEN_WITH_TESTS
-	RunTestsCommand RunTestsCmd;
-#endif
 
 	const struct CommandInfo
 	{
@@ -248,8 +303,8 @@ main(int argc, char** argv)
 		const char* CmdSummary;
 	} Commands[] = {
 		// clang-format off
+		{"attach",				&AttachCmd,				"Add a sponsor process to a running zen service"},
 		{"bench",				&BenchCmd,				"Utility command for benchmarking"},
-//		{"chunk",				&ChunkCmd,				"Perform chunking"},
 		{"cache-details",		&CacheDetailsCmd,		"Details on cache"},
 		{"cache-info",			&CacheInfoCmd,			"Info on cache, namespace or bucket"},
 		{"cache-stats",			&CacheStatsCmd,			"Stats on cache"},
@@ -259,8 +314,10 @@ main(int argc, char** argv)
 		{"down",				&DownCmd,				"Bring zen server down"},
 		{"drop",				&DropCmd,				"Drop cache namespace or bucket"},
 		{"gc-status",			&GcStatusCmd,			"Garbage collect zen storage status check"},
+		{"gc-stop",				&GcStopCmd,				"Request cancel of running garbage collection in zen storage"},
 		{"gc",					&GcCmd,					"Garbage collect zen storage"},
-		{"hash",				&HashCmd,				"Compute file hashes"},
+		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
+		{"logs",				&LoggingCmd,			"Show/control zen logging"},
 		{"oplog-create",		&CreateOplogCmd,		"Create a project oplog"},
 		{"oplog-delete",		&DeleteOplogCmd,		"Delete a project oplog"},
 		{"oplog-export",		&ExportOplogCmd,		"Export project store oplog"},
@@ -276,24 +333,19 @@ main(int argc, char** argv)
 		{"project-info",		&ProjectInfoCmd,		"Info on project or project oplog"},
 		{"project-stats",		&ProjectStatsCmd,		"Stats on project store"},
 		{"ps",					&PsCmd,					"Enumerate running zen server instances"},
-		{"rpc-record-replay",	&RpcReplayCmd,			"Stops recording of cache rpc requests on a host"},
-		{"rpc-record-start",	&RpcStartRecordingCmd,	"Replays a previously recorded session of rpc requests"},
-		{"rpc-record-stop",		&RpcStopRecordingCmd,	"Starts recording of cache rpc requests on a host"},
+		{"rpc-record-replay",	&RpcReplayCmd,			"Replays a previously recorded session of rpc requests"},
+		{"rpc-record-start",	&RpcStartRecordingCmd,	"Starts recording of cache rpc requests on a host"},
+		{"rpc-record-stop",		&RpcStopRecordingCmd,	"Stops recording of cache rpc requests on a host"},
+		{"run",					&RunCmd,				"Run command with special options"},
 		{"scrub",				&ScrubCmd,				"Scrub zen storage (verify data integrity)"},
 		{"serve",				&ServeCmd,				"Serve files from a directory"},
 		{"status",				&StatusCmd,				"Show zen status"},
-		{"logs",				&LoggingCmd,			"Show/control zen logging"},
-		{"jobs",				&JobCmd,				"Show/cancel zen background jobs"},
 		{"top",					&TopCmd,				"Monitor zen server activity"},
 		{"trace",				&TraceCmd,				"Control zen realtime tracing"},
 		{"up",					&UpCmd,					"Bring zen server up"},
-		{"attach",				&AttachCmd,				"Add a sponsor process to a running zen service"},
 		{"version",				&VersionCmd,			"Get zen server version"},
 		{"vfs",					&VfsCmd,				"Manage virtual file system"},
 		{"flush",				&FlushCmd,				"Flush storage"},
-#if ZEN_WITH_TESTS
-		{"runtests",			&RunTestsCmd,			"Run zen tests"},
-#endif
 		// clang-format on
 	};
 
@@ -307,46 +359,57 @@ main(int argc, char** argv)
 	// Split command line into options, commands and any pass-through arguments
 
 	std::string				 Passthrough;
-	std::vector<std::string> PassthroughV;
+	std::string				 PassthroughArgs;
+	std::vector<std::string> PassthroughArgV;
 
 	for (int i = 1; i < argc; ++i)
 	{
-		if (strcmp(argv[i], "--") == 0)
+		if ("--"sv == argv[i])
 		{
 			bool							  IsFirst = true;
 			zen::ExtendableStringBuilder<256> Line;
+			zen::ExtendableStringBuilder<256> Arguments;
 
 			for (int j = i + 1; j < argc; ++j)
 			{
+				auto AppendAscii = [&](auto X) {
+					Line.Append(X);
+					if (!IsFirst)
+					{
+						Arguments.Append(X);
+					}
+				};
+
 				if (!IsFirst)
 				{
-					Line.AppendAscii(" ");
+					AppendAscii(" ");
 				}
 
 				std::string_view ThisArg(argv[j]);
-				PassthroughV.push_back(std::string(ThisArg));
+				PassthroughArgV.push_back(std::string(ThisArg));
 
 				const bool NeedsQuotes = (ThisArg.find(' ') != std::string_view::npos);
 
 				if (NeedsQuotes)
 				{
-					Line.AppendAscii("\"");
+					AppendAscii("\"");
 				}
 
-				Line.Append(ThisArg);
+				AppendAscii(ThisArg);
 
 				if (NeedsQuotes)
 				{
-					Line.AppendAscii("\"");
+					AppendAscii("\"");
 				}
 
 				IsFirst = false;
 			}
 
-			Passthrough = Line.c_str();
+			Passthrough		= Line.c_str();
+			PassthroughArgs = Arguments.c_str();
 
 			// This will "truncate" the arg vector and terminate the loop
-			argc = i - 1;
+			argc = i;
 		}
 	}
 
@@ -374,8 +437,9 @@ main(int argc, char** argv)
 
 	ZenCliOptions GlobalOptions;
 
-	GlobalOptions.PassthroughArgs = Passthrough;
-	GlobalOptions.PassthroughV	  = PassthroughV;
+	GlobalOptions.PassthroughCommandLine = Passthrough;
+	GlobalOptions.PassthroughArgs		 = PassthroughArgs;
+	GlobalOptions.PassthroughArgV		 = PassthroughArgV;
 
 	std::string SubCommand = "<None>";
 
@@ -402,9 +466,26 @@ main(int argc, char** argv)
 
 			printf("available commands:\n");
 
+			std::map<std::string, ZenCmdCategory*> Categories;
+
 			for (const CommandInfo& CmdInfo : Commands)
 			{
-				printf("  %-20s %s\n", CmdInfo.CmdName, CmdInfo.CmdSummary);
+				ZenCmdCategory& Category = CmdInfo.Cmd->CommandCategory();
+
+				Categories[Category.Name]			 = &Category;
+				Category.SortedCmds[CmdInfo.CmdName] = CmdInfo.CmdSummary;
+			}
+
+			for (const auto& CategoryKv : Categories)
+			{
+				fmt::print("  {}\n\n", CategoryKv.first);
+
+				for (const auto& Kv : CategoryKv.second->SortedCmds)
+				{
+					printf("    %-20s %s\n", Kv.first.c_str(), Kv.second.c_str());
+				}
+
+				printf("\n");
 			}
 
 			exit(0);
@@ -429,14 +510,6 @@ main(int argc, char** argv)
 				{
 					return CmdInfo.Cmd->Run(GlobalOptions, (int)CommandArgVec.size(), CommandArgVec.data());
 				}
-				catch (cxxopts::OptionParseException& Ex)
-				{
-					std::string help = VerbOptions.help();
-
-					printf("Error parsing arguments for command '%s': %s\n\n%s", SubCommand.c_str(), Ex.what(), help.c_str());
-
-					exit(11);
-				}
 				catch (OptionParseException& Ex)
 				{
 					std::string help = VerbOptions.help();
@@ -450,14 +523,6 @@ main(int argc, char** argv)
 
 		printf("Unknown command specified: '%s', exiting\n", SubCommand.c_str());
 	}
-	catch (cxxopts::OptionParseException& Ex)
-	{
-		std::string HelpMessage = Options.help();
-
-		printf("Error parsing program arguments: %s\n\n%s", Ex.what(), HelpMessage.c_str());
-
-		return 9;
-	}
 	catch (OptionParseException& Ex)
 	{
 		std::string HelpMessage = Options.help();
@@ -475,57 +540,3 @@ main(int argc, char** argv)
 
 	return 0;
 }
-
-std::string
-ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort)
-{
-	if (InHostSpec.empty())
-	{
-		// If no host is specified then look to see if we have an instance
-		// running on this host and use that as the default to interact with
-
-		zen::ZenServerState Servers;
-
-		if (Servers.InitializeReadOnly())
-		{
-			std::string ResolvedSpec;
-
-			Servers.Snapshot([&](const zen::ZenServerState::ZenServerEntry& Entry) {
-				if (ResolvedSpec.empty())
-				{
-					ResolvedSpec	 = fmt::format("http://localhost:{}", Entry.EffectiveListenPort.load());
-					OutEffectivePort = Entry.EffectiveListenPort;
-				}
-			});
-
-			return ResolvedSpec;
-		}
-	}
-
-	// Parse out port from the specification provided, to be consistent with
-	// the auto-discovery logic above.
-
-	std::string_view PortSpec(InHostSpec);
-	if (size_t PrefixIndex = PortSpec.find_last_of(":"); PrefixIndex != std::string_view::npos)
-	{
-		PortSpec.remove_prefix(PrefixIndex + 1);
-
-		std::optional<uint16_t> EffectivePort = zen::ParseInt<uint16_t>(PortSpec);
-
-		if (EffectivePort)
-		{
-			OutEffectivePort = EffectivePort.value();
-		}
-	}
-
-	// note: We should consider adding validation/normalization of the provided spec here
-
-	return InHostSpec;
-}
-
-std::string
-ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec)
-{
-	uint16_t Dummy = 0;
-	return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy);
-}
diff --git a/src/zen/zen.h b/src/zen/zen.h
index 7258c10ce..78f22cad6 100644
--- a/src/zen/zen.h
+++ b/src/zen/zen.h
@@ -13,16 +13,30 @@ namespace cpr {
 class Response;
 }
 
+namespace zen {
+
 struct ZenCliOptions
 {
 	bool IsDebug   = false;
 	bool IsVerbose = false;
 
 	// Arguments after " -- " on command line are passed through and not parsed
+	std::string				 PassthroughCommandLine;
 	std::string				 PassthroughArgs;
-	std::vector<std::string> PassthroughV;
+	std::vector<std::string> PassthroughArgV;
+};
+
+struct ZenCmdCategory
+{
+	std::string						   Name;
+	std::map<std::string, std::string> SortedCmds;
 };
 
+extern ZenCmdCategory g_UtilitiesCategory;
+extern ZenCmdCategory g_ProjectStoreCategory;
+extern ZenCmdCategory g_CacheStoreCategory;
+extern ZenCmdCategory g_StorageCategory;
+
 /** Base class for command implementations
  */
 
@@ -31,6 +45,7 @@ class ZenCmdBase
 public:
 	virtual int				  Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) = 0;
 	virtual cxxopts::Options& Options()														 = 0;
+	virtual ZenCmdCategory&	  CommandCategory() const;
 
 	bool			   ParseOptions(int argc, char** argv);
 	static std::string FormatHttpResponse(const cpr::Response& Response);
@@ -38,3 +53,15 @@ public:
 	static std::string ResolveTargetHostSpec(const std::string& InHostSpec);
 	static std::string ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort);
 };
+
+class StorageCommand : public ZenCmdBase
+{
+	virtual ZenCmdCategory& CommandCategory() const override { return g_StorageCategory; }
+};
+
+class CacheStoreCommand : public ZenCmdBase
+{
+	virtual ZenCmdCategory& CommandCategory() const override { return g_CacheStoreCategory; }
+};
+
+}  // namespace zen
diff --git a/src/zen/zen.rc b/src/zen/zen.rc
index 14a9afb70..661d75011 100644
--- a/src/zen/zen.rc
+++ b/src/zen/zen.rc
@@ -18,11 +18,11 @@ PRODUCTVERSION ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER
         BLOCK "040904b0"
         {
             VALUE "CompanyName",        "Epic Games Inc\0"
-            VALUE "FileDescription",    "CLI utility for Zen Storage Service\0"
+            VALUE "FileDescription",    "CLI utility for Unreal Zen Storage Service\0"
             VALUE "FileVersion",        ZEN_CFG_VERSION "\0"
             VALUE "LegalCopyright",     "Copyright Epic Games Inc. All Rights Reserved\0"
             VALUE "OriginalFilename",   "zen.exe\0"
-            VALUE "ProductName",        "Zen Storage Server\0"
+            VALUE "ProductName",        "Unreal Zen Storage Server\0"
 			VALUE "ProductVersion",		ZEN_CFG_VERSION_BUILD_STRING_FULL "\0"
         }
     }
diff --git a/src/zenbase/include/zenbase/zenbase.h b/src/zenbase/include/zenbase/zenbase.h
index 1df375b28..401bcd088 100644
--- a/src/zenbase/include/zenbase/zenbase.h
+++ b/src/zenbase/include/zenbase/zenbase.h
@@ -197,9 +197,11 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1];
 //////////////////////////////////////////////////////////////////////////
 
 #if ZEN_COMPILER_MSC
-#	define ZEN_NOINLINE __declspec(noinline)
+#	define ZEN_NOINLINE	__declspec(noinline)
+#	define ZEN_FORCEINLINE [[msvc::forceinline]]
 #else
-#	define ZEN_NOINLINE __attribute__((noinline))
+#	define ZEN_NOINLINE	__attribute__((noinline))
+#	define ZEN_FORCEINLINE __attribute__((always_inline))
 #endif
 
 #if ZEN_PLATFORM_WINDOWS
diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp
index 5e8ce22ed..9152a8bfc 100644
--- a/src/zencore/compactbinary.cpp
+++ b/src/zencore/compactbinary.cpp
@@ -2,6 +2,7 @@
 
 #include "zencore/compactbinary.h"
 
+#include <zencore/assertfmt.h>
 #include <zencore/base64.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinaryvalidation.h>
@@ -855,10 +856,10 @@ void
 CbFieldView::CopyTo(MutableMemoryView Buffer) const
 {
 	const MemoryView Source = GetViewNoType();
-	ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize());
-	//	   TEXT("A buffer of %" UINT64_FMT " bytes was provided when %" UINT64_FMT " bytes are required"),
-	//	   Buffer.GetSize(),
-	//	   sizeof(CbFieldType) + Source.GetSize());
+	ZEN_ASSERT_FORMAT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize(),
+					  "A buffer of {} bytes was provided when {} bytes are required",
+					  Buffer.GetSize(),
+					  sizeof(CbFieldType) + Source.GetSize());
 	*static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetSerializedType(Type);
 	Buffer.RightChopInline(sizeof(CbFieldType));
 	memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize());
@@ -963,10 +964,10 @@ void
 CbArrayView::CopyTo(MutableMemoryView Buffer) const
 {
 	const MemoryView Source = GetPayloadView();
-	ZEN_ASSERT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize());
-	// TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
-	// Buffer.GetSize(),
-	// sizeof(CbFieldType) + Source.GetSize());
+	ZEN_ASSERT_FORMAT(Buffer.GetSize() == sizeof(CbFieldType) + Source.GetSize(),
+					  "Buffer is {} bytes but {} is required.",
+					  Buffer.GetSize(),
+					  sizeof(CbFieldType) + Source.GetSize());
 
 	*static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType());
 	Buffer.RightChopInline(sizeof(CbFieldType));
@@ -1077,10 +1078,10 @@ void
 CbObjectView::CopyTo(MutableMemoryView Buffer) const
 {
 	const MemoryView Source = GetPayloadView();
-	ZEN_ASSERT(Buffer.GetSize() == (sizeof(CbFieldType) + Source.GetSize()));
-	// TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
-	// Buffer.GetSize(),
-	// sizeof(CbFieldType) + Source.GetSize());
+	ZEN_ASSERT_FORMAT(Buffer.GetSize() == (sizeof(CbFieldType) + Source.GetSize()),
+					  "Buffer is {} bytes but {} is required.",
+					  Buffer.GetSize(),
+					  sizeof(CbFieldType) + Source.GetSize());
 	*static_cast<CbFieldType*>(Buffer.GetData()) = CbFieldTypeOps::GetType(GetType());
 	Buffer.RightChopInline(sizeof(CbFieldType));
 	memcpy(Buffer.GetData(), Source.GetData(), Source.GetSize());
@@ -1151,10 +1152,10 @@ TCbFieldIterator<FieldType>::CopyRangeTo(MutableMemoryView InBuffer) const
 	MemoryView Source;
 	if (TryGetSerializedRangeView(Source))
 	{
-		ZEN_ASSERT(InBuffer.GetSize() == Source.GetSize());
-		// TEXT("Buffer is %" UINT64_FMT " bytes but %" UINT64_FMT " is required."),
-		// InBuffer.GetSize(),
-		// Source.GetSize());
+		ZEN_ASSERT_FORMAT(InBuffer.GetSize() == Source.GetSize(),
+						  "Buffer is {} bytes but {} is required.",
+						  InBuffer.GetSize(),
+						  Source.GetSize());
 		memcpy(InBuffer.GetData(), Source.GetData(), Source.GetSize());
 	}
 	else
@@ -1654,7 +1655,7 @@ public:
 					break;
 				}
 			default:
-				ZEN_ASSERT(false);
+				ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType()));
 				break;
 		}
 
diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp
index d4ccd434d..5c08d2e6e 100644
--- a/src/zencore/compactbinarybuilder.cpp
+++ b/src/zencore/compactbinarybuilder.cpp
@@ -2,6 +2,7 @@
 
 #include "zencore/compactbinarybuilder.h"
 
+#include <zencore/assertfmt.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryvalidation.h>
 #include <zencore/endian.h>
@@ -128,13 +129,10 @@ CbWriter::Save()
 CbFieldViewIterator
 CbWriter::Save(const MutableMemoryView Buffer)
 {
-	ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None);
-	// TEXT("It is invalid to save while there are incomplete write operations."));
-	ZEN_ASSERT(Data.size() > 0);  // TEXT("It is invalid to save when nothing has been written."));
-	ZEN_ASSERT(Buffer.GetSize() == Data.size());
-	// TEXT("Buffer is %" UINT64_FMT " bytes but %" INT64_FMT " is required."),
-	// Buffer.GetSize(),
-	// Data.Num());
+	ZEN_ASSERT_FORMAT(States.size() == 1 && States.back().Flags == StateFlags::None,
+					  "It is invalid to save while there are incomplete write operations.");
+	ZEN_ASSERT_FORMAT(Data.size() > 0, "It is invalid to save when nothing has been written.");
+	ZEN_ASSERT_FORMAT(Buffer.GetSize() == Data.size(), "Buffer is {} bytes but {} is required.", Buffer.GetSize(), Data.size());
 	memcpy(Buffer.GetData(), Data.data(), Data.size());
 	return CbFieldViewIterator::MakeRange(Buffer);
 }
@@ -142,9 +140,9 @@ CbWriter::Save(const MutableMemoryView Buffer)
 void
 CbWriter::Save(BinaryWriter& Writer)
 {
-	ZEN_ASSERT(States.size() == 1 && States.back().Flags == StateFlags::None);
-	// TEXT("It is invalid to save while there are incomplete write operations."));
-	ZEN_ASSERT(Data.size() > 0);  // TEXT("It is invalid to save when nothing has been written."));
+	ZEN_ASSERT_FORMAT(States.size() == 1 && States.back().Flags == StateFlags::None,
+					  "It is invalid to save while there are incomplete write operations.");
+	ZEN_ASSERT_FORMAT(Data.size() > 0, "It is invalid to save when nothing has been written.");
 	Writer.Write(Data.data(), Data.size());
 }
 
@@ -166,10 +164,9 @@ CbWriter::BeginField()
 	}
 	else
 	{
-		ZEN_ASSERT((State.Flags & StateFlags::Name) == StateFlags::Name);
-		// TEXT("A new field cannot be written until the previous field '%.*hs' is finished."),
-		// GetActiveName().Len(),
-		// GetActiveName().GetData());
+		ZEN_ASSERT_FORMAT((State.Flags & StateFlags::Name) == StateFlags::Name,
+						  "A new field cannot be written until the previous field '{}' is finished.",
+						  GetActiveName());
 	}
 }
 
@@ -184,8 +181,8 @@ CbWriter::EndField(CbFieldType Type)
 	}
 	else
 	{
-		ZEN_ASSERT((State.Flags & StateFlags::Object) == StateFlags::None);
-		// TEXT("It is invalid to write an object field without a unique non-empty name."));
+		ZEN_ASSERT((State.Flags & StateFlags::Object) == StateFlags::None,
+				   "It is invalid to write an object field without a unique non-empty name.");
 	}
 
 	if (State.Count == 0)
@@ -207,21 +204,18 @@ CbWriter&
 CbWriter::SetName(const std::string_view Name)
 {
 	WriterState& State = States.back();
-	ZEN_ASSERT((State.Flags & StateFlags::Array) != StateFlags::Array);
-	// TEXT("It is invalid to write a name for an array field. Name '%.*hs'"),
-	// Name.Len(),
-	// Name.GetData());
-	ZEN_ASSERT(!Name.empty());
-	// TEXT("%s"),
-	//(State.Flags & EStateFlags::Object) == EStateFlags::Object
-	// ? TEXT("It is invalid to write an empty name for an object field. Specify a unique non-empty name.")
-	// : TEXT("It is invalid to write an empty name for a top-level field. Specify a name or avoid this call."));
-	ZEN_ASSERT((State.Flags & (StateFlags::Name | StateFlags::Field)) == StateFlags::None);
-	// TEXT("A new field '%.*hs' cannot be written until the previous field '%.*hs' is finished."),
-	// Name.Len(),
-	// Name.GetData(),
-	// GetActiveName().Len(),
-	// GetActiveName().GetData());
+	ZEN_ASSERT_FORMAT((State.Flags & StateFlags::Array) != StateFlags::Array,
+					  "It is invalid to write a name for an array field. Name '{}'",
+					  Name);
+	ZEN_ASSERT_FORMAT(!Name.empty(),
+					  "{}",
+					  (State.Flags & StateFlags::Object) == StateFlags::Object
+						  ? "It is invalid to write an empty name for an object field. Specify a unique non-empty name."
+						  : "It is invalid to write an empty name for a top-level field. Specify a name or avoid this call.");
+	ZEN_ASSERT_FORMAT((State.Flags & (StateFlags::Name | StateFlags::Field)) == StateFlags::None,
+					  "A new field '{}' cannot be written until the previous field '{}' is finished.",
+					  Name,
+					  GetActiveName());
 
 	BeginField();
 	State.Flags |= StateFlags::Name;
@@ -296,7 +290,7 @@ CbWriter::MakeFieldsUniform(const int64_t FieldBeginOffset, const int64_t FieldE
 void
 CbWriter::AddField(const CbFieldView& Value)
 {
-	ZEN_ASSERT(Value.HasValue());  // , TEXT("It is invalid to write a field with no value."));
+	ZEN_ASSERT_FORMAT(Value.HasValue(), "It is invalid to write a field with no value.");
 	BeginField();
 	EndField(AppendCompactBinary(Value, Data));
 }
@@ -318,11 +312,10 @@ CbWriter::BeginObject()
 void
 CbWriter::EndObject()
 {
-	ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Object) == StateFlags::Object);
-
-	// TEXT("It is invalid to end an object when an object is not at the top of the stack."));
-	ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None);
-	// TEXT("It is invalid to end an object until the previous field is finished."));
+	ZEN_ASSERT_FORMAT(States.size() > 1 && (States.back().Flags & StateFlags::Object) == StateFlags::Object,
+					  "It is invalid to end an object when an object is not at the top of the stack.");
+	ZEN_ASSERT_FORMAT((States.back().Flags & StateFlags::Field) == StateFlags::None,
+					  "It is invalid to end an object until the previous field is finished.");
 
 	const bool	   bUniform = IsUniformType(States.back().UniformType);
 	const uint64_t Count	= States.back().Count;
@@ -378,10 +371,10 @@ CbWriter::BeginArray()
 void
 CbWriter::EndArray()
 {
-	ZEN_ASSERT(States.size() > 1 && (States.back().Flags & StateFlags::Array) == StateFlags::Array);
-	// TEXT("Invalid attempt to end an array when an array is not at the top of the stack."));
-	ZEN_ASSERT((States.back().Flags & StateFlags::Field) == StateFlags::None);
-	// TEXT("It is invalid to end an array until the previous field is finished."));
+	ZEN_ASSERT_FORMAT(States.size() > 1 && (States.back().Flags & StateFlags::Array) == StateFlags::Array,
+					  "Invalid attempt to end an array when an array is not at the top of the stack.");
+	ZEN_ASSERT_FORMAT((States.back().Flags & StateFlags::Field) == StateFlags::None,
+					  "It is invalid to end an array until the previous field is finished.");
 	const bool	   bUniform = IsUniformType(States.back().UniformType);
 	const uint64_t Count	= States.back().Count;
 	States.pop_back();
diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp
index 06cda7382..29ec14e0c 100644
--- a/src/zencore/filesystem.cpp
+++ b/src/zencore/filesystem.cpp
@@ -7,6 +7,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/stream.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
@@ -333,11 +334,17 @@ SupportsBlockRefCounting(std::filesystem::path Path)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
-bool
+static bool
 CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 {
 #if ZEN_PLATFORM_WINDOWS
-	windows::Handle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr));
+	windows::Handle FromFile(CreateFileW(FromPath.c_str(),
+										 GENERIC_READ,
+										 FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+										 nullptr,
+										 OPEN_EXISTING,
+										 0,
+										 nullptr));
 	if (FromFile == INVALID_HANDLE_VALUE)
 	{
 		FromFile.Detach();
@@ -401,8 +408,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 	FILE_DISPOSITION_INFO FileDisposition = {TRUE};
 	if (!SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition))
 	{
+		const DWORD ErrorCode = ::GetLastError();
 		TargetFile.Close();
 		DeleteFileW(ToPath.c_str());
+		SetLastError(ErrorCode);
 		return false;
 	}
 
@@ -531,6 +540,19 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
+void
+CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode)
+{
+	OutErrorCode.clear();
+
+	bool Success = CopyFile(FromPath, ToPath, Options);
+
+	if (!Success)
+	{
+		OutErrorCode = MakeErrorCodeFromLastError();
+	}
+}
+
 bool
 CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
 {
@@ -613,6 +635,133 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop
 }
 
 void
+CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options)
+{
+	// Validate arguments
+
+	if (FromPath.empty() || !std::filesystem::is_directory(FromPath))
+		throw std::runtime_error("invalid CopyTree source directory specified");
+
+	if (ToPath.empty())
+		throw std::runtime_error("no CopyTree target specified");
+
+	if (Options.MustClone && !SupportsBlockRefCounting(FromPath))
+		throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath));
+
+	if (std::filesystem::exists(ToPath))
+	{
+		if (!std::filesystem::is_directory(ToPath))
+		{
+			throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath));
+		}
+	}
+	else
+	{
+		std::filesystem::create_directories(ToPath);
+	}
+
+	if (Options.MustClone && !SupportsBlockRefCounting(ToPath))
+		throw std::runtime_error(fmt::format("cloning not possible from '{}'", ToPath));
+
+	// Verify source/target relationships
+
+	std::error_code		  Ec;
+	std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec);
+
+	if (!Ec)
+	{
+		std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec);
+
+		if (!Ec)
+		{
+			if (FromCanonical == ToCanonical)
+			{
+				throw std::runtime_error("Target and source must be distinct files or directories");
+			}
+
+			if (ToCanonical.generic_string().starts_with(FromCanonical.generic_string()) ||
+				FromCanonical.generic_string().starts_with(ToCanonical.generic_string()))
+			{
+				throw std::runtime_error("Invalid parent/child relationship for source/target directories");
+			}
+		}
+	}
+
+	struct CopyVisitor : public FileSystemTraversal::TreeVisitor
+	{
+		CopyVisitor(std::filesystem::path InBasePath, zen::CopyFileOptions InCopyOptions) : BasePath(InBasePath), CopyOptions(InCopyOptions)
+		{
+		}
+
+		virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			std::error_code				Ec;
+			const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec);
+
+			if (Ec)
+			{
+				FailedFileCount++;
+			}
+			else
+			{
+				const std::filesystem::path FromPath = Parent / File;
+				std::filesystem::path		ToPath;
+
+				if (Relative.compare("."))
+				{
+					zen::CreateDirectories(TargetPath / Relative);
+
+					ToPath = TargetPath / Relative / File;
+				}
+				else
+				{
+					ToPath = TargetPath / File;
+				}
+
+				try
+				{
+					if (zen::CopyFile(FromPath, ToPath, CopyOptions))
+					{
+						++FileCount;
+						ByteCount += FileSize;
+					}
+					else
+					{
+						throw std::runtime_error("CopyFile failed in an unexpected way");
+					}
+				}
+				catch (std::exception& Ex)
+				{
+					++FailedFileCount;
+
+					throw std::runtime_error(fmt::format("failed to copy '{}' to '{}': '{}'", FromPath, ToPath, Ex.what()));
+				}
+			}
+		}
+
+		virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; }
+
+		std::filesystem::path BasePath;
+		std::filesystem::path TargetPath;
+		zen::CopyFileOptions  CopyOptions;
+		int					  FileCount		  = 0;
+		uint64_t			  ByteCount		  = 0;
+		int					  FailedFileCount = 0;
+	};
+
+	CopyVisitor Visitor{FromPath, Options};
+	Visitor.TargetPath = ToPath;
+
+	FileSystemTraversal Traversal;
+	Traversal.TraverseFileSystem(FromPath, Visitor);
+
+	if (Visitor.FailedFileCount)
+	{
+		throw std::runtime_error(fmt::format("{} file copy operations FAILED", Visitor.FailedFileCount));
+	}
+}
+
+void
 WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount)
 {
 #if ZEN_PLATFORM_WINDOWS
@@ -1139,12 +1288,12 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 {
 	if (NativeHandle == nullptr)
 	{
-		return std::filesystem::path();
+		return "<error handle 'nullptr'>";
 	}
 #if ZEN_PLATFORM_WINDOWS
 	if (NativeHandle == INVALID_HANDLE_VALUE)
 	{
-		return std::filesystem::path();
+		return "<error handle 'invalid handle'>";
 	}
 
 	auto GetFinalPathNameByHandleWRetry =
@@ -1181,7 +1330,7 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 	if (Error != ERROR_SUCCESS)
 	{
 		Ec = MakeErrorCodeFromLastError();
-		return std::filesystem::path();
+		return fmt::format("<error handle '{}'>", Ec.message());
 	}
 
 	if (RequiredLengthIncludingNul < PathDataSize)
@@ -1198,7 +1347,7 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 	if (Error != ERROR_SUCCESS)
 	{
 		Ec = MakeErrorCodeFromLastError();
-		return std::filesystem::path();
+		return fmt::format("<error handle '{}'>", Ec.message());
 	}
 	ZEN_UNUSED(FinalLength);
 	return FullPath;
@@ -1212,7 +1361,7 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 	if (BytesRead <= 0)
 	{
 		Ec = MakeErrorCodeFromLastError();
-		return {};
+		return fmt::format("<error handle '{}'>", Ec.message());
 	}
 
 	Link[BytesRead] = '\0';
@@ -1223,7 +1372,7 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec)
 	if (fcntl(Fd, F_GETPATH, Path) < 0)
 	{
 		Ec = MakeErrorCodeFromLastError();
-		return {};
+		return fmt::format("<error handle '{}'>", Ec.message());
 	}
 
 	return Path;
@@ -1460,6 +1609,76 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles)
 	return Result;
 }
 
+std::error_code
+RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDirectories)
+{
+	const std::filesystem::path BasePath(DirectoryName.parent_path());
+	const std::string			Stem(DirectoryName.stem().string());
+
+	auto GetPathForIndex = [&](size_t Index) -> std::filesystem::path {
+		if (Index == 0)
+		{
+			return BasePath / Stem;
+		}
+		return BasePath / fmt::format("{}.{}", Stem, Index);
+	};
+
+	auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool { return std::filesystem::is_empty(Path, Ec); };
+
+	std::error_code Result;
+	const bool		BaseIsEmpty = IsEmpty(GetPathForIndex(0), Result);
+	if (Result)
+	{
+		return Result;
+	}
+
+	if (BaseIsEmpty)
+		return Result;
+
+	for (std::size_t i = MaxDirectories; i > 0; i--)
+	{
+		const std::filesystem::path SourcePath = GetPathForIndex(i - 1);
+
+		if (std::filesystem::exists(SourcePath))
+		{
+			std::filesystem::path TargetPath = GetPathForIndex(i);
+
+			std::error_code DummyEc;
+			if (std::filesystem::exists(TargetPath, DummyEc))
+			{
+				std::filesystem::remove_all(TargetPath, DummyEc);
+			}
+			std::filesystem::rename(SourcePath, TargetPath, DummyEc);
+		}
+	}
+
+	return Result;
+}
+
+std::filesystem::path
+SearchPathForExecutable(std::string_view ExecutableName)
+{
+#if ZEN_PLATFORM_WINDOWS
+	std::wstring Executable(Utf8ToWide(ExecutableName));
+
+	DWORD Result = SearchPathW(nullptr, Executable.c_str(), L".exe", 0, nullptr, nullptr);
+
+	if (!Result)
+		return ExecutableName;
+
+	auto PathBuffer = std::make_unique_for_overwrite<WCHAR[]>(Result);
+
+	Result = SearchPathW(nullptr, Executable.c_str(), L".exe", Result, PathBuffer.get(), nullptr);
+
+	if (!Result)
+		return ExecutableName;
+
+	return PathBuffer.get();
+#else
+	return ExecutableName;
+#endif
+}
+
 //////////////////////////////////////////////////////////////////////////
 //
 // Testing related code follows...
@@ -1619,6 +1838,55 @@ TEST_CASE("PathBuilder")
 #	endif
 }
 
+TEST_CASE("RotateDirectories")
+{
+	std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test";
+	CleanDirectory(TestBaseDir);
+	std::filesystem::path RotateDir		= TestBaseDir / "rotate_dir" / "dir_to_rotate";
+	IoBuffer			  DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5);
+
+	auto NewDir = [&] {
+		CreateDirectories(RotateDir);
+		WriteFile(RotateDir / ".placeholder", DummyFileData);
+	};
+
+	auto DirWithSuffix = [&](int Index) -> std::filesystem::path { return RotateDir.generic_string().append(fmt::format(".{}", Index)); };
+
+	const int RotateMax = 10;
+
+	NewDir();
+	CHECK(std::filesystem::exists(RotateDir));
+	RotateDirectories(RotateDir, RotateMax);
+	CHECK(!std::filesystem::exists(RotateDir));
+	CHECK(std::filesystem::exists(DirWithSuffix(1)));
+	NewDir();
+	CHECK(std::filesystem::exists(RotateDir));
+	RotateDirectories(RotateDir, RotateMax);
+	CHECK(!std::filesystem::exists(RotateDir));
+	CHECK(std::filesystem::exists(DirWithSuffix(1)));
+	CHECK(std::filesystem::exists(DirWithSuffix(2)));
+
+	for (int i = 0; i < RotateMax; ++i)
+	{
+		NewDir();
+		std::error_code Ec		= RotateDirectories(RotateDir, 10);
+		const bool		IsError = !!Ec;
+		CHECK_EQ(IsError, false);
+	}
+
+	CHECK(!std::filesystem::exists(RotateDir));
+
+	for (int i = 0; i < RotateMax; ++i)
+	{
+		CHECK(std::filesystem::exists(DirWithSuffix(i + 1)));
+	}
+
+	for (int i = RotateMax; i < RotateMax + 5; ++i)
+	{
+		CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1)));
+	}
+}
+
 #endif
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/assertfmt.h b/src/zencore/include/zencore/assertfmt.h
new file mode 100644
index 000000000..56383ffd9
--- /dev/null
+++ b/src/zencore/include/zencore/assertfmt.h
@@ -0,0 +1,48 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/zencore.h>
+
+#include <fmt/args.h>
+#include <string_view>
+
+namespace zen {
+
+namespace assert {
+	template<typename... T>
+	auto AssertCaptureArguments(T&&... Args)
+	{
+		return fmt::make_format_args(Args...);
+	}
+
+	void ExecAssertFmt
+		[[noreturn]] (const char* Filename, int LineNumber, const char* FunctionName, std::string_view Format, fmt::format_args Args);
+
+	// MSVC (v19.00.24215.1 at time of writing) ignores no-inline attributes on
+	// lambdas. This can be worked around by calling the lambda from inside this
+	// templated (and correctly non-inlined) function.
+	template<typename RetType = void, class InnerType, typename... ArgTypes>
+	RetType ZEN_FORCENOINLINE ZEN_DEBUG_SECTION CallColdNoInline(InnerType&& Inner, ArgTypes const&... Args)
+	{
+		return Inner(Args...);
+	}
+
+}  // namespace assert
+
+#define ZEN_ASSERT_FORMAT(x, fmt, ...)                                                    \
+	do                                                                                    \
+	{                                                                                     \
+		using namespace std::literals;                                                    \
+		if (x) [[likely]]                                                                 \
+			break;                                                                        \
+		zen::assert::CallColdNoInline([&]() ZEN_FORCEINLINE {                             \
+			zen::assert::ExecAssertFmt(__FILE__,                                          \
+									   __LINE__,                                          \
+									   __FUNCTION__,                                      \
+									   "assert(" #x ") failed: " fmt ""sv,                \
+									   zen::assert::AssertCaptureArguments(__VA_ARGS__)); \
+		});                                                                               \
+	} while (false)
+
+}  // namespace zen
diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h
index f92df5a54..e91fdc659 100644
--- a/src/zencore/include/zencore/blockingqueue.h
+++ b/src/zencore/include/zencore/blockingqueue.h
@@ -22,7 +22,6 @@ public:
 		{
 			std::lock_guard Lock(m_Lock);
 			m_Queue.emplace_back(std::move(Item));
-			m_Size++;
 		}
 
 		m_NewItemSignal.notify_one();
@@ -30,31 +29,33 @@ public:
 
 	bool WaitAndDequeue(T& Item)
 	{
-		if (m_CompleteAdding.load())
-		{
-			return false;
-		}
-
 		std::unique_lock Lock(m_Lock);
-		m_NewItemSignal.wait(Lock, [this]() { return !m_Queue.empty() || m_CompleteAdding.load(); });
-
-		if (!m_Queue.empty())
+		if (m_Queue.empty())
 		{
-			Item = std::move(m_Queue.front());
-			m_Queue.pop_front();
-			m_Size--;
-
-			return true;
+			if (m_CompleteAdding)
+			{
+				return false;
+			}
+			m_NewItemSignal.wait(Lock, [this]() { return !m_Queue.empty() || m_CompleteAdding; });
+			if (m_Queue.empty())
+			{
+				ZEN_ASSERT(m_CompleteAdding);
+				return false;
+			}
 		}
-
-		return false;
+		Item = std::move(m_Queue.front());
+		m_Queue.pop_front();
+		return true;
 	}
 
 	void CompleteAdding()
 	{
-		if (!m_CompleteAdding.load())
+		std::unique_lock Lock(m_Lock);
+		if (!m_CompleteAdding)
 		{
-			m_CompleteAdding.store(true);
+			m_CompleteAdding = true;
+
+			Lock.unlock();
 			m_NewItemSignal.notify_all();
 		}
 	}
@@ -69,8 +70,7 @@ private:
 	mutable std::mutex		m_Lock;
 	std::condition_variable m_NewItemSignal;
 	std::deque<T>			m_Queue;
-	std::atomic_bool		m_CompleteAdding{false};
-	std::atomic_uint32_t	m_Size;
+	bool					m_CompleteAdding = false;
 };
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h
index 89f69c1ab..9c81cf490 100644
--- a/src/zencore/include/zencore/compactbinarybuilder.h
+++ b/src/zencore/include/zencore/compactbinarybuilder.h
@@ -654,6 +654,24 @@ operator<<(CbWriter& Writer, const Oid& Value)
 ZENCORE_API CbWriter& operator<<(CbWriter& Writer, DateTime Value);
 ZENCORE_API CbWriter& operator<<(CbWriter& Writer, TimeSpan Value);
 
+ZENCORE_API inline TimeSpan
+ToTimeSpan(std::chrono::seconds Secs)
+{
+	return TimeSpan(0, 0, gsl::narrow<int>(Secs.count()));
+};
+ZENCORE_API inline TimeSpan
+ToTimeSpan(std::chrono::milliseconds MS)
+{
+	return TimeSpan(MS.count() * TimeSpan::TicksPerMillisecond);
+}
+ZENCORE_API inline DateTime
+ToDateTime(std::chrono::system_clock::time_point TimePoint)
+{
+	time_t Time	   = std::chrono::system_clock::to_time_t(TimePoint);
+	tm	   UTCTime = *gmtime(&Time);
+	return DateTime(1900 + UTCTime.tm_year, UTCTime.tm_mon, UTCTime.tm_mday, UTCTime.tm_hour, UTCTime.tm_min, UTCTime.tm_sec);
+}
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 void usonbuilder_forcelink();  // internal
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index 22eb40e45..233941479 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -1,5 +1,4 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
-// Copyright Epic Games, Inc. All Rights Reserved.
 
 #pragma once
 
@@ -87,6 +86,11 @@ struct CopyFileOptions
 };
 
 ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
+ZENCORE_API void CopyFile(std::filesystem::path	 FromPath,
+						  std::filesystem::path	 ToPath,
+						  const CopyFileOptions& Options,
+						  std::error_code&		 OutError);
+ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
 ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
 
 ZENCORE_API void PathToUtf8(const std::filesystem::path& Path, StringBuilderBase& Out);
@@ -211,7 +215,10 @@ void GetDirectoryContent(const std::filesystem::path& RootDir, uint8_t Flags, Di
 
 std::string GetEnvVariable(std::string_view VariableName);
 
+std::filesystem::path SearchPathForExecutable(std::string_view ExecutableName);
+
 std::error_code RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles);
+std::error_code RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDirectories);
 
 //////////////////////////////////////////////////////////////////////////
 
diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h
index 7accce41c..d891ed55b 100644
--- a/src/zencore/include/zencore/iobuffer.h
+++ b/src/zencore/include/zencore/iobuffer.h
@@ -31,6 +31,7 @@ enum class ZenContentType : uint8_t
 	kCSS				= 11,
 	kPNG				= 12,
 	kIcon				= 13,
+	kXML				= 14,
 	kCOUNT
 };
 
@@ -70,6 +71,8 @@ ToString(ZenContentType ContentType)
 			return "png"sv;
 		case ZenContentType::kIcon:
 			return "icon"sv;
+		case ZenContentType::kXML:
+			return "xml"sv;
 	}
 }
 
diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h
index ad873aa51..00af68b0a 100644
--- a/src/zencore/include/zencore/logbase.h
+++ b/src/zencore/include/zencore/logbase.h
@@ -90,6 +90,9 @@ struct LoggerRef
 	bool   ShouldLog(int Level) const;
 	inline operator bool() const { return SpdLogger != nullptr; }
 
+	void					 SetLogLevel(logging::level::LogLevel NewLogLevel);
+	logging::level::LogLevel GetLogLevel();
+
 	spdlog::logger* SpdLogger = nullptr;
 };
 
diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h
index d14d1ab8d..6d44e31df 100644
--- a/src/zencore/include/zencore/logging.h
+++ b/src/zencore/include/zencore/logging.h
@@ -35,6 +35,10 @@ LoggerRef ErrorLog();
 void	  SetErrorLog(std::string_view LoggerId);
 LoggerRef Get(std::string_view Name);
 
+void ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers);
+void RefreshLogLevels();
+void RefreshLogLevels(level::LogLevel DefaultLevel);
+
 struct LogCategory
 {
 	inline LogCategory(std::string_view InCategory) : CategoryName(InCategory) {}
@@ -235,6 +239,14 @@ std::string_view EmitActivitiesForLogging(StringBuilderBase& OutString);
 
 #define ZEN_LOG_SCOPE(...) ScopedLazyActivity $Activity##__LINE__([&](StringBuilderBase& Out) { Out << fmt::format(__VA_ARGS__); })
 
+#define ZEN_SCOPED_WARN(fmtstr, ...)                                                   \
+	do                                                                                 \
+	{                                                                                  \
+		ExtendableStringBuilder<256> ScopeString;                                      \
+		const std::string_view		 Scopes = EmitActivitiesForLogging(ScopeString);   \
+		ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \
+	} while (false)
+
 #define ZEN_SCOPED_ERROR(fmtstr, ...)                                                               \
 	do                                                                                              \
 	{                                                                                               \
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
new file mode 100644
index 000000000..d90a32301
--- /dev/null
+++ b/src/zencore/include/zencore/process.h
@@ -0,0 +1,98 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/thread.h>
+#include <zencore/zencore.h>
+
+#include <filesystem>
+
+namespace zen {
+
+/** Basic process abstraction
+ */
+class ProcessHandle
+{
+public:
+	ZENCORE_API ProcessHandle();
+
+	ProcessHandle(const ProcessHandle&) = delete;
+	ProcessHandle& operator=(const ProcessHandle&) = delete;
+
+	ZENCORE_API ~ProcessHandle();
+
+	ZENCORE_API void			   Initialize(int Pid);
+	ZENCORE_API void			   Initialize(void* ProcessHandle);	 /// Initialize with an existing handle - takes ownership of the handle
+	ZENCORE_API [[nodiscard]] bool IsRunning() const;
+	ZENCORE_API [[nodiscard]] bool IsValid() const;
+	ZENCORE_API bool			   Wait(int TimeoutMs = -1);
+	ZENCORE_API int				   WaitExitCode();
+	ZENCORE_API void			   Terminate(int ExitCode);
+	ZENCORE_API void			   Reset();
+	[[nodiscard]] inline int	   Pid() const { return m_Pid; }
+
+private:
+	void* m_ProcessHandle = nullptr;
+	int	  m_Pid			  = 0;
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	int m_ExitCode = -1;
+#endif
+};
+
+/** Basic process creation
+ */
+struct CreateProcOptions
+{
+	enum
+	{
+		Flag_NewConsole = 1 << 0,
+		Flag_Elevated	= 1 << 1,
+		Flag_Unelevated = 1 << 2,
+	};
+
+	const std::filesystem::path* WorkingDirectory = nullptr;
+	uint32_t					 Flags			  = 0;
+	std::filesystem::path		 StdoutFile;
+};
+
+#if ZEN_PLATFORM_WINDOWS
+using CreateProcResult = void*;	 // handle to the process
+#else
+using CreateProcResult = int32_t;  // pid
+#endif
+
+ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable,
+										std::string_view			 CommandLine,  // should also include arg[0] (executable name)
+										const CreateProcOptions&	 Options = {});
+
+/** Process monitor - monitors a list of running processes via polling
+
+	Intended to be used to monitor a set of "sponsor" processes, where
+	we need to determine when none of them remain alive
+
+ */
+
+class ProcessMonitor
+{
+public:
+	ProcessMonitor();
+	~ProcessMonitor();
+
+	ZENCORE_API bool IsRunning();
+	ZENCORE_API void AddPid(int Pid);
+	ZENCORE_API bool IsActive() const;
+
+private:
+	using HandleType = void*;
+
+	mutable RwLock			m_Lock;
+	std::vector<HandleType> m_ProcessHandles;
+};
+
+ZENCORE_API bool IsProcessRunning(int pid);
+ZENCORE_API int	 GetCurrentProcessId();
+int				 GetProcessId(CreateProcResult ProcId);
+
+void process_forcelink();  // internal
+
+}  // namespace zen
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index e3de2224c..3aec1647d 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -809,6 +809,19 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func)
 
 //////////////////////////////////////////////////////////////////////////
 
+inline std::string_view
+ToString(bool Value)
+{
+	using namespace std::literals;
+	if (Value)
+	{
+		return "true"sv;
+	}
+	return "false"sv;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 inline int32_t
 StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length = -1)
 {
diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h
index 9f2671610..2d0ef7396 100644
--- a/src/zencore/include/zencore/thread.h
+++ b/src/zencore/include/zencore/thread.h
@@ -10,6 +10,8 @@
 #include <string_view>
 #include <vector>
 
+#define ZEN_USE_WINDOWS_EVENTS ZEN_PLATFORM_WINDOWS
+
 namespace zen {
 
 void SetCurrentThreadName(std::string_view ThreadName);
@@ -107,7 +109,7 @@ public:
 	ZENCORE_API bool Wait(int TimeoutMs = -1);
 	ZENCORE_API void Close();
 
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	inline void* GetWindowsHandle() { return m_EventHandle; }
 #endif
 
@@ -204,87 +206,7 @@ private:
 	Event				  Complete;
 };
 
-/** Basic process abstraction
- */
-class ProcessHandle
-{
-public:
-	ZENCORE_API ProcessHandle();
-
-	ProcessHandle(const ProcessHandle&) = delete;
-	ProcessHandle& operator=(const ProcessHandle&) = delete;
-
-	ZENCORE_API ~ProcessHandle();
-
-	ZENCORE_API void			   Initialize(int Pid);
-	ZENCORE_API void			   Initialize(void* ProcessHandle);	 /// Initialize with an existing handle - takes ownership of the handle
-	ZENCORE_API [[nodiscard]] bool IsRunning() const;
-	ZENCORE_API [[nodiscard]] bool IsValid() const;
-	ZENCORE_API bool			   Wait(int TimeoutMs = -1);
-	ZENCORE_API int				   WaitExitCode();
-	ZENCORE_API void			   Terminate(int ExitCode);
-	ZENCORE_API void			   Reset();
-	[[nodiscard]] inline int	   Pid() const { return m_Pid; }
-
-private:
-	void* m_ProcessHandle = nullptr;
-	int	  m_Pid			  = 0;
-};
-
-/** Basic process creation
- */
-struct CreateProcOptions
-{
-	enum
-	{
-		Flag_NewConsole = 1 << 0,
-		Flag_Elevated	= 1 << 1,
-		Flag_Unelevated = 1 << 2,
-	};
-
-	const std::filesystem::path* WorkingDirectory = nullptr;
-	uint32_t					 Flags			  = 0;
-};
-
-#if ZEN_PLATFORM_WINDOWS
-using CreateProcResult = void*;	 // handle to the process
-#else
-using CreateProcResult = int32_t;  // pid
-#endif
-
-ZENCORE_API CreateProcResult CreateProc(const std::filesystem::path& Executable,
-										std::string_view			 CommandLine,  // should also include arg[0] (executable name)
-										const CreateProcOptions&	 Options = {});
-
-/** Process monitor - monitors a list of running processes via polling
-
-	Intended to be used to monitor a set of "sponsor" processes, where
-	we need to determine when none of them remain alive
-
- */
-
-class ProcessMonitor
-{
-public:
-	ProcessMonitor();
-	~ProcessMonitor();
-
-	ZENCORE_API bool IsRunning();
-	ZENCORE_API void AddPid(int Pid);
-	ZENCORE_API bool IsActive() const;
-
-private:
-	using HandleType = void*;
-
-	mutable RwLock			m_Lock;
-	std::vector<HandleType> m_ProcessHandles;
-};
-
-ZENCORE_API bool IsProcessRunning(int pid);
-ZENCORE_API int	 GetCurrentProcessId();
 ZENCORE_API int	 GetCurrentThreadId();
-int				 GetProcessId(CreateProcResult ProcId);
-
 ZENCORE_API void Sleep(int ms);
 
 void thread_forcelink();  // internal
diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h
index 665df5808..2d4c1e610 100644
--- a/src/zencore/include/zencore/trace.h
+++ b/src/zencore/include/zencore/trace.h
@@ -17,6 +17,7 @@ ZEN_THIRD_PARTY_INCLUDES_START
 ZEN_THIRD_PARTY_INCLUDES_END
 
 #define ZEN_TRACE_CPU(x) TRACE_CPU_SCOPE(x)
+#define ZEN_TRACE_CPU_FLUSH(x) TRACE_CPU_SCOPE(x, trace::CpuScopeFlags::CpuFlush)
 
 enum class TraceType
 {
@@ -25,10 +26,10 @@ enum class TraceType
 	None
 };
 
-void TraceInit();
+void TraceInit(std::string_view ProgramName);
 void TraceShutdown();
 bool IsTracing();
-void TraceStart(const char* HostOrPath, TraceType Type);
+void TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type);
 bool TraceStop();
 
 #else
diff --git a/src/zencore/include/zencore/windows.h b/src/zencore/include/zencore/windows.h
index 0943a85ea..14026fef1 100644
--- a/src/zencore/include/zencore/windows.h
+++ b/src/zencore/include/zencore/windows.h
@@ -24,6 +24,7 @@ struct IUnknown;  // Workaround for "combaseapi.h(229): error C2187: syntax erro
 #	include <windows.h>
 #	undef GetObject
 #	undef SendMessage
+#	undef CopyFile
 
 ZEN_THIRD_PARTY_INCLUDES_END
 
diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp
index 0d34372a9..90f4e2428 100644
--- a/src/zencore/logging.cpp
+++ b/src/zencore/logging.cpp
@@ -4,10 +4,14 @@
 
 #include <zencore/string.h>
 #include <zencore/testing.h>
+#include <zencore/thread.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <spdlog/details/registry.h>
 #include <spdlog/sinks/null_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 #include <spdlog/spdlog.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 #if ZEN_PLATFORM_WINDOWS
 #	pragma section(".zlog$a", read)
@@ -46,6 +50,8 @@ LoggingContext::~LoggingContext()
 {
 }
 
+//////////////////////////////////////////////////////////////////////////
+
 static inline bool
 IsErrorLevel(int LogLevel)
 {
@@ -176,8 +182,77 @@ ToStringView(level::LogLevel Level)
 
 }  // namespace zen::logging::level
 
+//////////////////////////////////////////////////////////////////////////
+
 namespace zen::logging {
 
+RwLock		LogLevelsLock;
+std::string LogLevels[level::LogLevelCount];
+
+void
+ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers)
+{
+	RwLock::ExclusiveLockScope _(LogLevelsLock);
+	LogLevels[Level] = Loggers;
+}
+
+void
+RefreshLogLevels(level::LogLevel* DefaultLevel)
+{
+	spdlog::details::registry::log_levels Levels;
+
+	{
+		RwLock::SharedLockScope _(LogLevelsLock);
+
+		for (int i = 0; i < level::LogLevelCount; ++i)
+		{
+			level::LogLevel CurrentLevel{i};
+
+			std::string_view Spec = LogLevels[i];
+
+			while (!Spec.empty())
+			{
+				std::string LoggerName;
+
+				if (auto CommaPos = Spec.find_first_of(','); CommaPos != std::string_view::npos)
+				{
+					LoggerName = Spec.substr(CommaPos + 1);
+					Spec.remove_prefix(CommaPos + 1);
+				}
+				else
+				{
+					LoggerName = Spec;
+					Spec	   = {};
+				}
+
+				Levels[LoggerName] = to_spdlog_level(CurrentLevel);
+			}
+		}
+	}
+
+	if (DefaultLevel)
+	{
+		spdlog::level::level_enum SpdDefaultLevel = to_spdlog_level(*DefaultLevel);
+		spdlog::details::registry::instance().set_levels(Levels, &SpdDefaultLevel);
+	}
+	else
+	{
+		spdlog::details::registry::instance().set_levels(Levels, nullptr);
+	}
+}
+
+void
+RefreshLogLevels(level::LogLevel DefaultLevel)
+{
+	RefreshLogLevels(&DefaultLevel);
+}
+
+void
+RefreshLogLevels()
+{
+	RefreshLogLevels(nullptr);
+}
+
 void
 SetLogLevel(level::LogLevel NewLogLevel)
 {
@@ -240,6 +315,7 @@ Get(std::string_view Name)
 	if (!Logger)
 	{
 		Logger = Default().SpdLogger->clone(std::string(Name));
+		spdlog::apply_logger_env_levels(Logger);
 		spdlog::register_logger(Logger);
 	}
 
@@ -252,6 +328,11 @@ std::shared_ptr<spdlog::logger> ConLogger;
 void
 SuppressConsoleLog()
 {
+	if (ConLogger)
+	{
+		spdlog::drop("console");
+		ConLogger = {};
+	}
 	ConLogger = spdlog::null_logger_mt("console");
 }
 
@@ -262,6 +343,7 @@ ConsoleLog()
 		if (!ConLogger)
 		{
 			ConLogger = spdlog::stdout_color_mt("console");
+			spdlog::apply_logger_env_levels(ConLogger);
 
 			ConLogger->set_pattern("%v");
 		}
@@ -279,7 +361,6 @@ InitializeLogging()
 void
 ShutdownLogging()
 {
-	spdlog::drop_all();
 	spdlog::shutdown();
 	TheDefaultLogger = {};
 }
@@ -321,6 +402,18 @@ LoggerRef::ShouldLog(int Level) const
 	return SpdLogger->should_log(static_cast<spdlog::level::level_enum>(Level));
 }
 
+void
+LoggerRef::SetLogLevel(logging::level::LogLevel NewLogLevel)
+{
+	SpdLogger->set_level(to_spdlog_level(NewLogLevel));
+}
+
+logging::level::LogLevel
+LoggerRef::GetLogLevel()
+{
+	return logging::level::to_logging_level(SpdLogger->level());
+}
+
 thread_local ScopedActivityBase* t_ScopeStack = nullptr;
 
 ScopedActivityBase*
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
new file mode 100644
index 000000000..2d0ec2de6
--- /dev/null
+++ b/src/zencore/process.cpp
@@ -0,0 +1,765 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zencore/process.h>
+
+#include <zencore/except.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/testing.h>
+
+#include <thread>
+
+#if ZEN_PLATFORM_WINDOWS
+#	include <shellapi.h>
+#	include <Shlobj.h>
+#	include <zencore/windows.h>
+#else
+#	include <fcntl.h>
+#	include <pthread.h>
+#	include <signal.h>
+#	include <sys/file.h>
+#	include <sys/sem.h>
+#	include <sys/stat.h>
+#	include <sys/syscall.h>
+#	include <sys/wait.h>
+#	include <time.h>
+#	include <unistd.h>
+#endif
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+#if ZEN_PLATFORM_LINUX
+const bool bNoZombieChildren = []() {
+	// When a child process exits it is put into a zombie state until the parent
+	// collects its result. This doesn't fit the Windows-like model that Zen uses
+	// where there is a less strict familial model and no zombification. Ignoring
+	// SIGCHLD signals removes the need to call wait() on zombies. Another option
+	// would be for the child to call setsid() but that would detatch the child
+	// from the terminal.
+	struct sigaction Action = {};
+	sigemptyset(&Action.sa_mask);
+	Action.sa_handler = SIG_IGN;
+	sigaction(SIGCHLD, &Action, nullptr);
+	return true;
+}();
+#endif
+
+ProcessHandle::ProcessHandle() = default;
+
+#if ZEN_PLATFORM_WINDOWS
+void
+ProcessHandle::Initialize(void* ProcessHandle)
+{
+	ZEN_ASSERT(m_ProcessHandle == nullptr);
+
+	if (ProcessHandle == INVALID_HANDLE_VALUE)
+	{
+		ProcessHandle = nullptr;
+	}
+
+	// TODO: perform some debug verification here to verify it's a valid handle?
+	m_ProcessHandle = ProcessHandle;
+	m_Pid			= GetProcessId(m_ProcessHandle);
+}
+#endif	// ZEN_PLATFORM_WINDOWS
+
+ProcessHandle::~ProcessHandle()
+{
+	Reset();
+}
+
+void
+ProcessHandle::Initialize(int Pid)
+{
+	ZEN_ASSERT(m_ProcessHandle == nullptr);
+
+#if ZEN_PLATFORM_WINDOWS
+	m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid);
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	if (Pid > 0)
+	{
+		m_ProcessHandle = (void*)(intptr_t(Pid));
+	}
+#endif
+
+	if (!m_ProcessHandle)
+	{
+		ThrowLastError(fmt::format("ProcessHandle::Initialize(pid: {}) failed", Pid));
+	}
+
+	m_Pid = Pid;
+}
+
+bool
+ProcessHandle::IsRunning() const
+{
+	bool bActive = false;
+
+#if ZEN_PLATFORM_WINDOWS
+	DWORD ExitCode = 0;
+	GetExitCodeProcess(m_ProcessHandle, &ExitCode);
+	bActive = (ExitCode == STILL_ACTIVE);
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	bActive = (kill(pid_t(m_Pid), 0) == 0);
+#endif
+
+	return bActive;
+}
+
+bool
+ProcessHandle::IsValid() const
+{
+	return (m_ProcessHandle != nullptr);
+}
+
+void
+ProcessHandle::Terminate(int ExitCode)
+{
+	if (!IsRunning())
+	{
+		return;
+	}
+
+	bool bSuccess = false;
+
+#if ZEN_PLATFORM_WINDOWS
+	TerminateProcess(m_ProcessHandle, ExitCode);
+	DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE);
+	bSuccess		 = (WaitResult != WAIT_OBJECT_0);
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	ZEN_UNUSED(ExitCode);
+	bSuccess			= (kill(m_Pid, SIGKILL) == 0);
+#endif
+
+	if (!bSuccess)
+	{
+		// What might go wrong here, and what is meaningful to act on?
+	}
+}
+
+void
+ProcessHandle::Reset()
+{
+	if (IsValid())
+	{
+#if ZEN_PLATFORM_WINDOWS
+		CloseHandle(m_ProcessHandle);
+#endif
+		m_ProcessHandle = nullptr;
+		m_Pid			= 0;
+	}
+}
+
+bool
+ProcessHandle::Wait(int TimeoutMs)
+{
+	using namespace std::literals;
+
+#if ZEN_PLATFORM_WINDOWS
+	const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs;
+
+	const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout);
+
+	switch (WaitResult)
+	{
+		case WAIT_OBJECT_0:
+			return true;
+
+		case WAIT_TIMEOUT:
+			return false;
+
+		case WAIT_FAILED:
+			break;
+	}
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	const int SleepMs	= 20;
+	timespec  SleepTime = {0, SleepMs * 1000 * 1000};
+	for (int SleepedTimeMS = 0;; SleepedTimeMS += SleepMs)
+	{
+		int WaitState = 0;
+		waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
+
+		if (WIFEXITED(WaitState))
+		{
+			m_ExitCode = WEXITSTATUS(WaitState);
+		}
+
+		if (kill(m_Pid, 0) < 0)
+		{
+			int32_t LastError = zen::GetLastError();
+			if (LastError == ESRCH)
+			{
+				return true;
+			}
+			ThrowSystemError(static_cast<uint32_t>(LastError), "Process::Wait kill failed"sv);
+		}
+
+		if (TimeoutMs >= 0 && SleepedTimeMS >= TimeoutMs)
+		{
+			return false;
+		}
+
+		nanosleep(&SleepTime, nullptr);
+	}
+#endif
+
+	// What might go wrong here, and what is meaningful to act on?
+	ThrowLastError("Process::Wait failed"sv);
+}
+
+int
+ProcessHandle::WaitExitCode()
+{
+	Wait(-1);
+
+#if ZEN_PLATFORM_WINDOWS
+	DWORD ExitCode = 0;
+	GetExitCodeProcess(m_ProcessHandle, &ExitCode);
+
+	ZEN_ASSERT(ExitCode != STILL_ACTIVE);
+
+	return ExitCode;
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	return m_ExitCode;
+#else
+	ZEN_NOT_IMPLEMENTED();
+
+	return 0;
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#if !ZEN_PLATFORM_WINDOWS || ZEN_WITH_TESTS
+static void
+BuildArgV(std::vector<char*>& Out, char* CommandLine)
+{
+	char* Cursor = CommandLine;
+	while (true)
+	{
+		// Skip leading whitespace
+		for (; *Cursor == ' '; ++Cursor)
+			;
+
+		// Check for nullp terminator
+		if (*Cursor == '\0')
+		{
+			break;
+		}
+
+		Out.push_back(Cursor);
+
+		// Extract word
+		int QuoteCount = 0;
+		do
+		{
+			QuoteCount += (*Cursor == '\"');
+			if (*Cursor == ' ' && !(QuoteCount & 1))
+			{
+				break;
+			}
+			++Cursor;
+		} while (*Cursor != '\0');
+
+		if (*Cursor == '\0')
+		{
+			break;
+		}
+
+		*Cursor = '\0';
+		++Cursor;
+	}
+}
+#endif	// !WINDOWS || TESTS
+
+#if ZEN_PLATFORM_WINDOWS
+static CreateProcResult
+CreateProcNormal(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
+{
+	PROCESS_INFORMATION ProcessInfo{};
+	STARTUPINFO			StartupInfo{.cb = sizeof(STARTUPINFO)};
+
+	bool				  InheritHandles	= false;
+	void*				  Environment		= nullptr;
+	LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr;
+	LPSECURITY_ATTRIBUTES ThreadAttributes	= nullptr;
+
+	DWORD CreationFlags = 0;
+	if (Options.Flags & CreateProcOptions::Flag_NewConsole)
+	{
+		CreationFlags |= CREATE_NEW_CONSOLE;
+	}
+
+	const wchar_t* WorkingDir = nullptr;
+	if (Options.WorkingDirectory != nullptr)
+	{
+		WorkingDir = Options.WorkingDirectory->c_str();
+	}
+
+	ExtendableWideStringBuilder<256> CommandLineZ;
+	CommandLineZ << CommandLine;
+
+	if (!Options.StdoutFile.empty())
+	{
+		SECURITY_ATTRIBUTES sa;
+		sa.nLength				= sizeof sa;
+		sa.lpSecurityDescriptor = nullptr;
+		sa.bInheritHandle		= TRUE;
+
+		StartupInfo.hStdInput  = nullptr;
+		StartupInfo.hStdOutput = CreateFileW(Options.StdoutFile.c_str(),
+											 GENERIC_READ | GENERIC_WRITE,
+											 FILE_SHARE_READ,
+											 &sa,
+											 CREATE_ALWAYS,
+											 FILE_ATTRIBUTE_NORMAL,
+											 nullptr);
+
+		const BOOL Success = DuplicateHandle(GetCurrentProcess(),
+											 StartupInfo.hStdOutput,
+											 GetCurrentProcess(),
+											 &StartupInfo.hStdError,
+											 0,
+											 TRUE,
+											 DUPLICATE_SAME_ACCESS);
+
+		if (Success)
+		{
+			StartupInfo.dwFlags |= STARTF_USESTDHANDLES;
+			InheritHandles = true;
+		}
+		else
+		{
+			CloseHandle(StartupInfo.hStdOutput);
+			StartupInfo.hStdOutput = 0;
+		}
+	}
+
+	BOOL Success = CreateProcessW(Executable.c_str(),
+								  CommandLineZ.Data(),
+								  ProcessAttributes,
+								  ThreadAttributes,
+								  InheritHandles,
+								  CreationFlags,
+								  Environment,
+								  WorkingDir,
+								  &StartupInfo,
+								  &ProcessInfo);
+
+	if (StartupInfo.dwFlags & STARTF_USESTDHANDLES)
+	{
+		CloseHandle(StartupInfo.hStdError);
+		CloseHandle(StartupInfo.hStdOutput);
+	}
+
+	if (!Success)
+	{
+		return nullptr;
+	}
+
+	CloseHandle(ProcessInfo.hThread);
+	return ProcessInfo.hProcess;
+}
+
+static CreateProcResult
+CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
+{
+	/* Launches a binary with the shell as its parent. The shell (such as
+	   Explorer) should be an unelevated process. */
+
+	// No sense in using this route if we are not elevated in the first place
+	if (IsUserAnAdmin() == FALSE)
+	{
+		return CreateProcNormal(Executable, CommandLine, Options);
+	}
+
+	// Get the users' shell process and open it for process creation
+	HWND ShellWnd = GetShellWindow();
+	if (ShellWnd == nullptr)
+	{
+		return nullptr;
+	}
+
+	DWORD ShellPid;
+	GetWindowThreadProcessId(ShellWnd, &ShellPid);
+
+	HANDLE Process = OpenProcess(PROCESS_CREATE_PROCESS, FALSE, ShellPid);
+	if (Process == nullptr)
+	{
+		return nullptr;
+	}
+	auto $0 = MakeGuard([&] { CloseHandle(Process); });
+
+	// Creating a process as a child of another process is done by setting a
+	// thread-attribute list on the startup info passed to CreateProcess()
+	SIZE_T AttrListSize;
+	InitializeProcThreadAttributeList(nullptr, 1, 0, &AttrListSize);
+
+	auto AttrList = (PPROC_THREAD_ATTRIBUTE_LIST)malloc(AttrListSize);
+	auto $1		  = MakeGuard([&] { free(AttrList); });
+
+	if (!InitializeProcThreadAttributeList(AttrList, 1, 0, &AttrListSize))
+	{
+		return nullptr;
+	}
+
+	BOOL bOk =
+		UpdateProcThreadAttribute(AttrList, 0, PROC_THREAD_ATTRIBUTE_PARENT_PROCESS, (HANDLE*)&Process, sizeof(Process), nullptr, nullptr);
+	if (!bOk)
+	{
+		return nullptr;
+	}
+
+	// By this point we know we are an elevated process. It is not allowed to
+	// create a process as a child of another unelevated process that share our
+	// elevated console window if we have one. So we'll need to create a new one.
+	uint32_t CreateProcFlags = EXTENDED_STARTUPINFO_PRESENT;
+	if (GetConsoleWindow() != nullptr)
+	{
+		CreateProcFlags |= CREATE_NEW_CONSOLE;
+	}
+	else
+	{
+		CreateProcFlags |= DETACHED_PROCESS;
+	}
+
+	// Everything is set up now so we can proceed and launch the process
+	STARTUPINFOEXW StartupInfo = {
+		.StartupInfo	 = {.cb = sizeof(STARTUPINFOEXW)},
+		.lpAttributeList = AttrList,
+	};
+	PROCESS_INFORMATION ProcessInfo = {};
+
+	if (Options.Flags & CreateProcOptions::Flag_NewConsole)
+	{
+		CreateProcFlags |= CREATE_NEW_CONSOLE;
+	}
+
+	ExtendableWideStringBuilder<256> CommandLineZ;
+	CommandLineZ << CommandLine;
+
+	ExtendableWideStringBuilder<256> CurrentDirZ;
+	LPCWSTR							 WorkingDirectoryPtr = nullptr;
+	if (Options.WorkingDirectory)
+	{
+		CurrentDirZ << Options.WorkingDirectory->native();
+		WorkingDirectoryPtr = CurrentDirZ.c_str();
+	}
+
+	bOk = CreateProcessW(Executable.c_str(),
+						 CommandLineZ.Data(),
+						 nullptr,
+						 nullptr,
+						 FALSE,
+						 CreateProcFlags,
+						 nullptr,
+						 WorkingDirectoryPtr,
+						 &StartupInfo.StartupInfo,
+						 &ProcessInfo);
+	if (bOk == FALSE)
+	{
+		return nullptr;
+	}
+
+	CloseHandle(ProcessInfo.hThread);
+	return ProcessInfo.hProcess;
+}
+
+static CreateProcResult
+CreateProcElevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
+{
+	ExtendableWideStringBuilder<256> CommandLineZ;
+	CommandLineZ << CommandLine;
+
+	SHELLEXECUTEINFO ShellExecuteInfo;
+	ZeroMemory(&ShellExecuteInfo, sizeof(ShellExecuteInfo));
+	ShellExecuteInfo.cbSize		  = sizeof(ShellExecuteInfo);
+	ShellExecuteInfo.fMask		  = SEE_MASK_UNICODE | SEE_MASK_NOCLOSEPROCESS;
+	ShellExecuteInfo.lpFile		  = Executable.c_str();
+	ShellExecuteInfo.lpVerb		  = TEXT("runas");
+	ShellExecuteInfo.nShow		  = SW_SHOW;
+	ShellExecuteInfo.lpParameters = CommandLineZ.c_str();
+
+	if (Options.WorkingDirectory != nullptr)
+	{
+		ShellExecuteInfo.lpDirectory = Options.WorkingDirectory->c_str();
+	}
+
+	if (::ShellExecuteEx(&ShellExecuteInfo))
+	{
+		return ShellExecuteInfo.hProcess;
+	}
+
+	return nullptr;
+}
+#endif	// ZEN_PLATFORM_WINDOWS
+
+CreateProcResult
+CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
+{
+#if ZEN_PLATFORM_WINDOWS
+	if (Options.Flags & CreateProcOptions::Flag_Unelevated)
+	{
+		return CreateProcUnelevated(Executable, CommandLine, Options);
+	}
+
+	if (Options.Flags & CreateProcOptions::Flag_Elevated)
+	{
+		return CreateProcElevated(Executable, CommandLine, Options);
+	}
+
+	return CreateProcNormal(Executable, CommandLine, Options);
+#else
+	std::vector<char*> ArgV;
+	std::string		   CommandLineZ(CommandLine);
+	BuildArgV(ArgV, CommandLineZ.data());
+	ArgV.push_back(nullptr);
+
+	int ChildPid = fork();
+	if (ChildPid < 0)
+	{
+		ThrowLastError("Failed to fork a new child process");
+	}
+	else if (ChildPid == 0)
+	{
+		if (Options.WorkingDirectory != nullptr)
+		{
+			int Result = chdir(Options.WorkingDirectory->c_str());
+			ZEN_UNUSED(Result);
+		}
+
+		if (execv(Executable.c_str(), ArgV.data()) < 0)
+		{
+			ThrowLastError("Failed to exec() a new process image");
+		}
+	}
+
+	return ChildPid;
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+ProcessMonitor::ProcessMonitor()
+{
+}
+
+ProcessMonitor::~ProcessMonitor()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	for (HandleType& Proc : m_ProcessHandles)
+	{
+#if ZEN_PLATFORM_WINDOWS
+		CloseHandle(Proc);
+#endif
+		Proc = 0;
+	}
+}
+
+bool
+ProcessMonitor::IsRunning()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	bool FoundOne = false;
+
+	for (HandleType& Proc : m_ProcessHandles)
+	{
+		bool ProcIsActive;
+
+#if ZEN_PLATFORM_WINDOWS
+		DWORD ExitCode = 0;
+		GetExitCodeProcess(Proc, &ExitCode);
+
+		ProcIsActive = (ExitCode == STILL_ACTIVE);
+		if (!ProcIsActive)
+		{
+			CloseHandle(Proc);
+		}
+#else
+		int Pid		 = int(intptr_t(Proc));
+		ProcIsActive = IsProcessRunning(Pid);
+#endif
+
+		if (!ProcIsActive)
+		{
+			Proc = 0;
+		}
+
+		// Still alive
+		FoundOne |= ProcIsActive;
+	}
+
+	std::erase_if(m_ProcessHandles, [](HandleType Handle) { return Handle == 0; });
+
+	return FoundOne;
+}
+
+void
+ProcessMonitor::AddPid(int Pid)
+{
+	HandleType ProcessHandle;
+
+#if ZEN_PLATFORM_WINDOWS
+	ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid);
+#else
+	ProcessHandle = HandleType(intptr_t(Pid));
+#endif
+
+	if (ProcessHandle)
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		m_ProcessHandles.push_back(ProcessHandle);
+	}
+}
+
+bool
+ProcessMonitor::IsActive() const
+{
+	RwLock::SharedLockScope _(m_Lock);
+	return m_ProcessHandles.empty() == false;
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+bool
+IsProcessRunning(int pid)
+{
+	// This function is arguably not super useful, a pid can be re-used
+	// by the OS so holding on to a pid and polling it over some time
+	// period will not necessarily tell you what you probably want to know.
+
+#if ZEN_PLATFORM_WINDOWS
+	HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
+
+	if (!hProc)
+	{
+		DWORD Error = zen::GetLastError();
+
+		if (Error == ERROR_INVALID_PARAMETER)
+		{
+			return false;
+		}
+
+		ThrowSystemError(Error, fmt::format("failed to open process with pid {}", pid));
+	}
+
+	bool  bStillActive = true;
+	DWORD ExitCode	   = 0;
+	if (0 != GetExitCodeProcess(hProc, &ExitCode))
+	{
+		bStillActive = ExitCode == STILL_ACTIVE;
+	}
+	else
+	{
+		ZEN_WARN("Unable to get exit code from handle for process '{}', treating the process as active", pid);
+	}
+
+	CloseHandle(hProc);
+
+	return bStillActive;
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+	return (kill(pid_t(pid), 0) == 0);
+#endif
+}
+
+int
+GetCurrentProcessId()
+{
+#if ZEN_PLATFORM_WINDOWS
+	return ::GetCurrentProcessId();
+#else
+	return int(getpid());
+#endif
+}
+
+int
+GetProcessId(CreateProcResult ProcId)
+{
+#if ZEN_PLATFORM_WINDOWS
+	return static_cast<int>(::GetProcessId(ProcId));
+#else
+	return ProcId;
+#endif
+}
+
+#if ZEN_WITH_TESTS
+
+void
+process_forcelink()
+{
+}
+
+TEST_SUITE_BEGIN("core.process");
+
+TEST_CASE("Process")
+{
+	int Pid = GetCurrentProcessId();
+	CHECK(Pid > 0);
+	CHECK(IsProcessRunning(Pid));
+}
+
+TEST_CASE("BuildArgV")
+{
+	const char* Words[] = {"one", "two", "three", "four", "five"};
+	struct
+	{
+		int			WordCount;
+		const char* Input;
+	} Cases[] = {
+		{0, ""},
+		{0, "  "},
+		{1, "one"},
+		{1, " one"},
+		{1, "one  "},
+		{2, "one two"},
+		{2, " one two"},
+		{2, "one two "},
+		{2, " one two"},
+		{2, "one two "},
+		{2, "one two "},
+		{3, "one two three"},
+		{3, "\"one\" two \"three\""},
+		{5, "one two three four five"},
+	};
+
+	for (const auto& Case : Cases)
+	{
+		std::vector<char*> OutArgs;
+		StringBuilder<64>  Mutable;
+		Mutable << Case.Input;
+		BuildArgV(OutArgs, Mutable.Data());
+
+		CHECK_EQ(OutArgs.size(), Case.WordCount);
+
+		for (int i = 0, n = int(OutArgs.size()); i < n; ++i)
+		{
+			const char* Truth	 = Words[i];
+			size_t		TruthLen = strlen(Truth);
+
+			const char* Candidate = OutArgs[i];
+			bool		bQuoted	  = (Candidate[0] == '\"');
+			Candidate += bQuoted;
+
+			CHECK(strncmp(Truth, Candidate, TruthLen) == 0);
+
+			if (bQuoted)
+			{
+				CHECK_EQ(Candidate[TruthLen], '\"');
+			}
+		}
+	}
+}
+
+TEST_SUITE_END(/* core.process */);
+
+#endif
+
+}  // namespace zen
diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp
index 54d89ded2..936424e0f 100644
--- a/src/zencore/testing.cpp
+++ b/src/zencore/testing.cpp
@@ -5,12 +5,64 @@
 
 #if ZEN_WITH_TESTS
 
+#	include <doctest/doctest.h>
+
 namespace zen::testing {
 
 using namespace std::literals;
 
+struct TestListener : public doctest::IReporter
+{
+	const std::string_view ColorYellow = "\033[0;33m"sv;
+	const std::string_view ColorNone   = "\033[0m"sv;
+
+	// constructor has to accept the ContextOptions by ref as a single argument
+	TestListener(const doctest::ContextOptions&) {}
+
+	void report_query(const doctest::QueryData& /*in*/) override {}
+
+	void test_run_start() override {}
+
+	void test_run_end(const doctest::TestRunStats& /*in*/) override {}
+
+	void test_case_start(const doctest::TestCaseData& in) override
+	{
+		Current = &in;
+		ZEN_CONSOLE("{}======== TEST_CASE: {:<50} ========{}", ColorYellow, Current->m_name, ColorNone);
+	}
+
+	// called when a test case is reentered because of unfinished subcases
+	void test_case_reenter(const doctest::TestCaseData& /*in*/) override
+	{
+		ZEN_CONSOLE("{}-------------------------------------------------------------------------------{}", ColorYellow, ColorNone);
+	}
+
+	void test_case_end(const doctest::CurrentTestCaseStats& /*in*/) override { Current = nullptr; }
+
+	void test_case_exception(const doctest::TestCaseException& /*in*/) override {}
+
+	void subcase_start(const doctest::SubcaseSignature& in) override
+	{
+		ZEN_CONSOLE("{}-------- SUBCASE:   {:<50} --------{}",
+					ColorYellow,
+					fmt::format("{}/{}", Current->m_name, in.m_name.c_str()),
+					ColorNone);
+	}
+
+	void subcase_end() override {}
+
+	void log_assert(const doctest::AssertData& /*in*/) override {}
+
+	void log_message(const doctest::MessageData& /*in*/) override {}
+
+	void test_case_skipped(const doctest::TestCaseData& /*in*/) override {}
+
+	const doctest::TestCaseData* Current = nullptr;
+};
+
 struct TestRunner::Impl
 {
+	Impl() { REGISTER_LISTENER("ZenTestListener", 1, TestListener); }
 	doctest::Context Session;
 };
 
diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp
index 1f1b1b8f5..149a0d781 100644
--- a/src/zencore/thread.cpp
+++ b/src/zencore/thread.cpp
@@ -8,6 +8,9 @@
 #include <zencore/scopeguard.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
+#include <zencore/trace.h>
+
+#include <thread>
 
 #if ZEN_PLATFORM_LINUX
 #	if !defined(_GNU_SOURCE)
@@ -15,15 +18,15 @@
 #	endif
 #endif
 
-#if ZEN_PLATFORM_WINDOWS
-#	include <shellapi.h>
-#	include <Shlobj.h>
-#	include <zencore/windows.h>
-#else
+#if !ZEN_USE_WINDOWS_EVENTS
 #	include <chrono>
 #	include <condition_variable>
 #	include <mutex>
+#endif
 
+#if ZEN_PLATFORM_WINDOWS
+#	include <zencore/windows.h>
+#else
 #	include <fcntl.h>
 #	include <pthread.h>
 #	include <signal.h>
@@ -36,10 +39,6 @@
 #	include <unistd.h>
 #endif
 
-#include <zencore/trace.h>
-
-#include <thread>
-
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 ZEN_THIRD_PARTY_INCLUDES_END
@@ -78,22 +77,6 @@ SetNameInternal(DWORD thread_id, const char* name)
 }
 #endif
 
-#if ZEN_PLATFORM_LINUX
-const bool bNoZombieChildren = []() {
-	// When a child process exits it is put into a zombie state until the parent
-	// collects its result. This doesn't fit the Windows-like model that Zen uses
-	// where there is a less strict familial model and no zombification. Ignoring
-	// SIGCHLD siganals removes the need to call wait() on zombies. Another option
-	// would be for the child to call setsid() but that would detatch the child
-	// from the terminal.
-	struct sigaction Action = {};
-	sigemptyset(&Action.sa_mask);
-	Action.sa_handler = SIG_IGN;
-	sigaction(SIGCHLD, &Action, nullptr);
-	return true;
-}();
-#endif
-
 void
 SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName)
 {
@@ -152,12 +135,12 @@ RwLock::ReleaseExclusive() noexcept
 
 //////////////////////////////////////////////////////////////////////////
 
-#if !ZEN_PLATFORM_WINDOWS
+#if !ZEN_USE_WINDOWS_EVENTS
 struct EventInner
 {
 	std::mutex				Mutex;
 	std::condition_variable CondVar;
-	bool volatile bSet = false;
+	std::atomic_bool		bSet{false};
 };
 #endif	// !ZEN_PLATFORM_WINDOWS
 
@@ -166,7 +149,7 @@ Event::Event()
 	bool bManualReset  = true;
 	bool bInitialState = false;
 
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	m_EventHandle = CreateEvent(nullptr, bManualReset, bInitialState, nullptr);
 #else
 	ZEN_UNUSED(bManualReset);
@@ -184,13 +167,13 @@ Event::~Event()
 void
 Event::Set()
 {
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	SetEvent(m_EventHandle);
 #else
 	auto* Inner	  = (EventInner*)m_EventHandle;
 	{
 		std::unique_lock Lock(Inner->Mutex);
-		Inner->bSet = true;
+		Inner->bSet.store(true);
 	}
 	Inner->CondVar.notify_all();
 #endif
@@ -199,13 +182,13 @@ Event::Set()
 void
 Event::Reset()
 {
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	ResetEvent(m_EventHandle);
 #else
 	auto* Inner = (EventInner*)m_EventHandle;
 	{
 		std::unique_lock Lock(Inner->Mutex);
-		Inner->bSet = false;
+		Inner->bSet.store(false);
 	}
 #endif
 }
@@ -213,10 +196,14 @@ Event::Reset()
 void
 Event::Close()
 {
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	CloseHandle(m_EventHandle);
 #else
 	auto* Inner = (EventInner*)m_EventHandle;
+	{
+		std::unique_lock Lock(Inner->Mutex);
+		Inner->bSet.store(true);
+	}
 	delete Inner;
 #endif
 	m_EventHandle = nullptr;
@@ -225,7 +212,7 @@ Event::Close()
 bool
 Event::Wait(int TimeoutMs)
 {
-#if ZEN_PLATFORM_WINDOWS
+#if ZEN_USE_WINDOWS_EVENTS
 	using namespace std::literals;
 
 	const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs;
@@ -239,25 +226,34 @@ Event::Wait(int TimeoutMs)
 
 	return (Result == WAIT_OBJECT_0);
 #else
-	auto* Inner = (EventInner*)m_EventHandle;
+	auto* Inner = reinterpret_cast<EventInner*>(m_EventHandle);
+
+	if (Inner->bSet.load())
+	{
+		return true;
+	}
 
 	if (TimeoutMs >= 0)
 	{
 		std::unique_lock Lock(Inner->Mutex);
 
-		if (Inner->bSet)
+		if (Inner->bSet.load())
 		{
 			return true;
 		}
 
-		return Inner->CondVar.wait_for(Lock, std::chrono::milliseconds(TimeoutMs), [&] { return Inner->bSet; });
+		return Inner->CondVar.wait_for(Lock, std::chrono::milliseconds(TimeoutMs), [&] { return Inner->bSet.load(); });
 	}
 
+	// Infinite wait. This does not actually call the wait() function to work around
+	// an apparent issue in the underlying implementation.
+
 	std::unique_lock Lock(Inner->Mutex);
 
-	if (!Inner->bSet)
+	if (!Inner->bSet.load())
 	{
-		Inner->CondVar.wait(Lock, [&] { return Inner->bSet; });
+		while (!Inner->CondVar.wait_for(Lock, std::chrono::milliseconds(1000), [&] { return Inner->bSet.load(); }))
+			;
 	}
 
 	return true;
@@ -398,9 +394,10 @@ NamedEvent::Wait(int TimeoutMs)
 	}
 
 #	if defined(_GNU_SOURCE)
+	const int		TimeoutSec	 = TimeoutMs / 1000;
 	struct timespec TimeoutValue = {
-		.tv_sec	 = TimeoutMs >> 10,
-		.tv_nsec = (TimeoutMs & 0x3ff) << 20,
+		.tv_sec	 = TimeoutSec,
+		.tv_nsec = (TimeoutMs - (TimeoutSec * 1000)) * 1000000,
 	};
 	Result = semtimedop(Sem, &SemOp, 1, &TimeoutValue);
 #	else
@@ -418,7 +415,6 @@ NamedEvent::Wait(int TimeoutMs)
 		TimeoutMs -= SleepTimeMs;
 	} while (TimeoutMs > 0);
 #	endif	// _GNU_SOURCE
-
 	return Result == 0;
 #endif
 }
@@ -520,582 +516,6 @@ NamedMutex::Exists(std::string_view MutexName)
 #endif	// ZEN_PLATFORM_WINDOWS
 }
 
-//////////////////////////////////////////////////////////////////////////
-
-ProcessHandle::ProcessHandle() = default;
-
-#if ZEN_PLATFORM_WINDOWS
-void
-ProcessHandle::Initialize(void* ProcessHandle)
-{
-	ZEN_ASSERT(m_ProcessHandle == nullptr);
-
-	if (ProcessHandle == INVALID_HANDLE_VALUE)
-	{
-		ProcessHandle = nullptr;
-	}
-
-	// TODO: perform some debug verification here to verify it's a valid handle?
-	m_ProcessHandle = ProcessHandle;
-	m_Pid			= GetProcessId(m_ProcessHandle);
-}
-#endif	// ZEN_PLATFORM_WINDOWS
-
-ProcessHandle::~ProcessHandle()
-{
-	Reset();
-}
-
-void
-ProcessHandle::Initialize(int Pid)
-{
-	ZEN_ASSERT(m_ProcessHandle == nullptr);
-
-#if ZEN_PLATFORM_WINDOWS
-	m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid);
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	if (Pid > 0)
-	{
-		m_ProcessHandle = (void*)(intptr_t(Pid));
-	}
-#endif
-
-	if (!m_ProcessHandle)
-	{
-		ThrowLastError(fmt::format("ProcessHandle::Initialize(pid: {}) failed", Pid));
-	}
-
-	m_Pid = Pid;
-}
-
-bool
-ProcessHandle::IsRunning() const
-{
-	bool bActive = false;
-
-#if ZEN_PLATFORM_WINDOWS
-	DWORD ExitCode = 0;
-	GetExitCodeProcess(m_ProcessHandle, &ExitCode);
-	bActive = (ExitCode == STILL_ACTIVE);
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	bActive = (kill(pid_t(m_Pid), 0) == 0);
-#endif
-
-	return bActive;
-}
-
-bool
-ProcessHandle::IsValid() const
-{
-	return (m_ProcessHandle != nullptr);
-}
-
-void
-ProcessHandle::Terminate(int ExitCode)
-{
-	if (!IsRunning())
-	{
-		return;
-	}
-
-	bool bSuccess = false;
-
-#if ZEN_PLATFORM_WINDOWS
-	TerminateProcess(m_ProcessHandle, ExitCode);
-	DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE);
-	bSuccess		 = (WaitResult != WAIT_OBJECT_0);
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	ZEN_UNUSED(ExitCode);
-	bSuccess			= (kill(m_Pid, SIGKILL) == 0);
-#endif
-
-	if (!bSuccess)
-	{
-		// What might go wrong here, and what is meaningful to act on?
-	}
-}
-
-void
-ProcessHandle::Reset()
-{
-	if (IsValid())
-	{
-#if ZEN_PLATFORM_WINDOWS
-		CloseHandle(m_ProcessHandle);
-#endif
-		m_ProcessHandle = nullptr;
-		m_Pid			= 0;
-	}
-}
-
-bool
-ProcessHandle::Wait(int TimeoutMs)
-{
-	using namespace std::literals;
-
-#if ZEN_PLATFORM_WINDOWS
-	const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs;
-
-	const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout);
-
-	switch (WaitResult)
-	{
-		case WAIT_OBJECT_0:
-			return true;
-
-		case WAIT_TIMEOUT:
-			return false;
-
-		case WAIT_FAILED:
-			break;
-	}
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	const int SleepMs	= 20;
-	timespec  SleepTime = {0, SleepMs * 1000 * 1000};
-	for (int i = 0;; i += SleepMs)
-	{
-#	if ZEN_PLATFORM_MAC
-		int WaitState = 0;
-		waitpid(m_Pid, &WaitState, WNOHANG | WCONTINUED | WUNTRACED);
-#	endif
-
-		if (kill(m_Pid, 0) < 0)
-		{
-			if (zen::GetLastError() == ESRCH)
-			{
-				return true;
-			}
-			break;
-		}
-
-		if (TimeoutMs >= 0 && i >= TimeoutMs)
-		{
-			return false;
-		}
-
-		nanosleep(&SleepTime, nullptr);
-	}
-#endif
-
-	// What might go wrong here, and what is meaningful to act on?
-	ThrowLastError("Process::Wait failed"sv);
-}
-
-int
-ProcessHandle::WaitExitCode()
-{
-	Wait(-1);
-
-#if ZEN_PLATFORM_WINDOWS
-	DWORD ExitCode = 0;
-	GetExitCodeProcess(m_ProcessHandle, &ExitCode);
-
-	ZEN_ASSERT(ExitCode != STILL_ACTIVE);
-
-	return ExitCode;
-#else
-	ZEN_NOT_IMPLEMENTED();
-
-	return 0;
-#endif
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-#if !ZEN_PLATFORM_WINDOWS || ZEN_WITH_TESTS
-static void
-BuildArgV(std::vector<char*>& Out, char* CommandLine)
-{
-	char* Cursor = CommandLine;
-	while (true)
-	{
-		// Skip leading whitespace
-		for (; *Cursor == ' '; ++Cursor)
-			;
-
-		// Check for nullp terminator
-		if (*Cursor == '\0')
-		{
-			break;
-		}
-
-		Out.push_back(Cursor);
-
-		// Extract word
-		int QuoteCount = 0;
-		do
-		{
-			QuoteCount += (*Cursor == '\"');
-			if (*Cursor == ' ' && !(QuoteCount & 1))
-			{
-				break;
-			}
-			++Cursor;
-		} while (*Cursor != '\0');
-
-		if (*Cursor == '\0')
-		{
-			break;
-		}
-
-		*Cursor = '\0';
-		++Cursor;
-	}
-}
-#endif	// !WINDOWS || TESTS
-
-#if ZEN_PLATFORM_WINDOWS
-static CreateProcResult
-CreateProcNormal(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
-{
-	PROCESS_INFORMATION ProcessInfo{};
-	STARTUPINFO			StartupInfo{.cb = sizeof(STARTUPINFO)};
-
-	const bool			  InheritHandles	= false;
-	void*				  Environment		= nullptr;
-	LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr;
-	LPSECURITY_ATTRIBUTES ThreadAttributes	= nullptr;
-
-	DWORD CreationFlags = 0;
-	if (Options.Flags & CreateProcOptions::Flag_NewConsole)
-	{
-		CreationFlags |= CREATE_NEW_CONSOLE;
-	}
-
-	const wchar_t* WorkingDir = nullptr;
-	if (Options.WorkingDirectory != nullptr)
-	{
-		WorkingDir = Options.WorkingDirectory->c_str();
-	}
-
-	ExtendableWideStringBuilder<256> CommandLineZ;
-	CommandLineZ << CommandLine;
-
-	BOOL Success = CreateProcessW(Executable.c_str(),
-								  CommandLineZ.Data(),
-								  ProcessAttributes,
-								  ThreadAttributes,
-								  InheritHandles,
-								  CreationFlags,
-								  Environment,
-								  WorkingDir,
-								  &StartupInfo,
-								  &ProcessInfo);
-
-	if (!Success)
-	{
-		return nullptr;
-	}
-
-	CloseHandle(ProcessInfo.hThread);
-	return ProcessInfo.hProcess;
-}
-
-static CreateProcResult
-CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
-{
-	/* Launches a binary with the shell as its parent. The shell (such as
-	   Explorer) should be an unelevated process. */
-
-	// No sense in using this route if we are not elevated in the first place
-	if (IsUserAnAdmin() == FALSE)
-	{
-		return CreateProcNormal(Executable, CommandLine, Options);
-	}
-
-	// Get the users' shell process and open it for process creation
-	HWND ShellWnd = GetShellWindow();
-	if (ShellWnd == nullptr)
-	{
-		return nullptr;
-	}
-
-	DWORD ShellPid;
-	GetWindowThreadProcessId(ShellWnd, &ShellPid);
-
-	HANDLE Process = OpenProcess(PROCESS_CREATE_PROCESS, FALSE, ShellPid);
-	if (Process == nullptr)
-	{
-		return nullptr;
-	}
-	auto $0 = MakeGuard([&] { CloseHandle(Process); });
-
-	// Creating a process as a child of another process is done by setting a
-	// thread-attribute list on the startup info passed to CreateProcess()
-	SIZE_T AttrListSize;
-	InitializeProcThreadAttributeList(nullptr, 1, 0, &AttrListSize);
-
-	auto AttrList = (PPROC_THREAD_ATTRIBUTE_LIST)malloc(AttrListSize);
-	auto $1		  = MakeGuard([&] { free(AttrList); });
-
-	if (!InitializeProcThreadAttributeList(AttrList, 1, 0, &AttrListSize))
-	{
-		return nullptr;
-	}
-
-	BOOL bOk =
-		UpdateProcThreadAttribute(AttrList, 0, PROC_THREAD_ATTRIBUTE_PARENT_PROCESS, (HANDLE*)&Process, sizeof(Process), nullptr, nullptr);
-	if (!bOk)
-	{
-		return nullptr;
-	}
-
-	// By this point we know we are an elevated process. It is not allowed to
-	// create a process as a child of another unelevated process that share our
-	// elevated console window if we have one. So we'll need to create a new one.
-	uint32_t CreateProcFlags = EXTENDED_STARTUPINFO_PRESENT;
-	if (GetConsoleWindow() != nullptr)
-	{
-		CreateProcFlags |= CREATE_NEW_CONSOLE;
-	}
-	else
-	{
-		CreateProcFlags |= DETACHED_PROCESS;
-	}
-
-	// Everything is set up now so we can proceed and launch the process
-	STARTUPINFOEXW StartupInfo = {
-		.StartupInfo	 = {.cb = sizeof(STARTUPINFOEXW)},
-		.lpAttributeList = AttrList,
-	};
-	PROCESS_INFORMATION ProcessInfo = {};
-
-	if (Options.Flags & CreateProcOptions::Flag_NewConsole)
-	{
-		CreateProcFlags |= CREATE_NEW_CONSOLE;
-	}
-
-	ExtendableWideStringBuilder<256> CommandLineZ;
-	CommandLineZ << CommandLine;
-
-	bOk = CreateProcessW(Executable.c_str(),
-						 CommandLineZ.Data(),
-						 nullptr,
-						 nullptr,
-						 FALSE,
-						 CreateProcFlags,
-						 nullptr,
-						 nullptr,
-						 &StartupInfo.StartupInfo,
-						 &ProcessInfo);
-	if (bOk == FALSE)
-	{
-		return nullptr;
-	}
-
-	CloseHandle(ProcessInfo.hThread);
-	return ProcessInfo.hProcess;
-}
-
-static CreateProcResult
-CreateProcElevated(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
-{
-	ExtendableWideStringBuilder<256> CommandLineZ;
-	CommandLineZ << CommandLine;
-
-	SHELLEXECUTEINFO ShellExecuteInfo;
-	ZeroMemory(&ShellExecuteInfo, sizeof(ShellExecuteInfo));
-	ShellExecuteInfo.cbSize		  = sizeof(ShellExecuteInfo);
-	ShellExecuteInfo.fMask		  = SEE_MASK_UNICODE | SEE_MASK_NOCLOSEPROCESS;
-	ShellExecuteInfo.lpFile		  = Executable.c_str();
-	ShellExecuteInfo.lpVerb		  = TEXT("runas");
-	ShellExecuteInfo.nShow		  = SW_SHOW;
-	ShellExecuteInfo.lpParameters = CommandLineZ.c_str();
-
-	if (Options.WorkingDirectory != nullptr)
-	{
-		ShellExecuteInfo.lpDirectory = Options.WorkingDirectory->c_str();
-	}
-
-	if (::ShellExecuteEx(&ShellExecuteInfo))
-	{
-		return ShellExecuteInfo.hProcess;
-	}
-
-	return nullptr;
-}
-#endif	// ZEN_PLATFORM_WINDOWS
-
-CreateProcResult
-CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options)
-{
-#if ZEN_PLATFORM_WINDOWS
-	if (Options.Flags & CreateProcOptions::Flag_Unelevated)
-	{
-		return CreateProcUnelevated(Executable, CommandLine, Options);
-	}
-
-	if (Options.Flags & CreateProcOptions::Flag_Elevated)
-	{
-		return CreateProcElevated(Executable, CommandLine, Options);
-	}
-
-	return CreateProcNormal(Executable, CommandLine, Options);
-#else
-	std::vector<char*> ArgV;
-	std::string		   CommandLineZ(CommandLine);
-	BuildArgV(ArgV, CommandLineZ.data());
-	ArgV.push_back(nullptr);
-
-	int ChildPid = fork();
-	if (ChildPid < 0)
-	{
-		ThrowLastError("Failed to fork a new child process");
-	}
-	else if (ChildPid == 0)
-	{
-		if (Options.WorkingDirectory != nullptr)
-		{
-			int Result = chdir(Options.WorkingDirectory->c_str());
-			ZEN_UNUSED(Result);
-		}
-
-		if (execv(Executable.c_str(), ArgV.data()) < 0)
-		{
-			ThrowLastError("Failed to exec() a new process image");
-		}
-	}
-
-	return ChildPid;
-#endif
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-ProcessMonitor::ProcessMonitor()
-{
-}
-
-ProcessMonitor::~ProcessMonitor()
-{
-	RwLock::ExclusiveLockScope _(m_Lock);
-
-	for (HandleType& Proc : m_ProcessHandles)
-	{
-#if ZEN_PLATFORM_WINDOWS
-		CloseHandle(Proc);
-#endif
-		Proc = 0;
-	}
-}
-
-bool
-ProcessMonitor::IsRunning()
-{
-	RwLock::ExclusiveLockScope _(m_Lock);
-
-	bool FoundOne = false;
-
-	for (HandleType& Proc : m_ProcessHandles)
-	{
-		bool ProcIsActive;
-
-#if ZEN_PLATFORM_WINDOWS
-		DWORD ExitCode = 0;
-		GetExitCodeProcess(Proc, &ExitCode);
-
-		ProcIsActive = (ExitCode == STILL_ACTIVE);
-		if (!ProcIsActive)
-		{
-			CloseHandle(Proc);
-		}
-#else
-		int Pid		 = int(intptr_t(Proc));
-		ProcIsActive = IsProcessRunning(Pid);
-#endif
-
-		if (!ProcIsActive)
-		{
-			Proc = 0;
-		}
-
-		// Still alive
-		FoundOne |= ProcIsActive;
-	}
-
-	std::erase_if(m_ProcessHandles, [](HandleType Handle) { return Handle == 0; });
-
-	return FoundOne;
-}
-
-void
-ProcessMonitor::AddPid(int Pid)
-{
-	HandleType ProcessHandle;
-
-#if ZEN_PLATFORM_WINDOWS
-	ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid);
-#else
-	ProcessHandle = HandleType(intptr_t(Pid));
-#endif
-
-	if (ProcessHandle)
-	{
-		RwLock::ExclusiveLockScope _(m_Lock);
-		m_ProcessHandles.push_back(ProcessHandle);
-	}
-}
-
-bool
-ProcessMonitor::IsActive() const
-{
-	RwLock::SharedLockScope _(m_Lock);
-	return m_ProcessHandles.empty() == false;
-}
-
-//////////////////////////////////////////////////////////////////////////
-
-bool
-IsProcessRunning(int pid)
-{
-	// This function is arguably not super useful, a pid can be re-used
-	// by the OS so holding on to a pid and polling it over some time
-	// period will not necessarily tell you what you probably want to know.
-
-#if ZEN_PLATFORM_WINDOWS
-	HANDLE hProc = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid);
-
-	if (!hProc)
-	{
-		DWORD Error = zen::GetLastError();
-
-		if (Error == ERROR_INVALID_PARAMETER)
-		{
-			return false;
-		}
-
-		ThrowSystemError(Error, fmt::format("failed to open process with pid {}", pid));
-	}
-
-	bool  bStillActive = true;
-	DWORD ExitCode	   = 0;
-	if (0 != GetExitCodeProcess(hProc, &ExitCode))
-	{
-		bStillActive = ExitCode == STILL_ACTIVE;
-	}
-	else
-	{
-		ZEN_WARN("Unable to get exit code from handle for process '{}', treating the process as active", pid);
-	}
-
-	CloseHandle(hProc);
-
-	return bStillActive;
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-	return (kill(pid_t(pid), 0) == 0);
-#endif
-}
-
-int
-GetCurrentProcessId()
-{
-#if ZEN_PLATFORM_WINDOWS
-	return ::GetCurrentProcessId();
-#else
-	return int(getpid());
-#endif
-}
-
 int
 GetCurrentThreadId()
 {
@@ -1108,16 +528,6 @@ GetCurrentThreadId()
 #endif
 }
 
-int
-GetProcessId(CreateProcResult ProcId)
-{
-#if ZEN_PLATFORM_WINDOWS
-	return static_cast<int>(::GetProcessId(ProcId));
-#else
-	return ProcId;
-#endif
-}
-
 void
 Sleep(int ms)
 {
@@ -1140,65 +550,11 @@ thread_forcelink()
 {
 }
 
-TEST_CASE("Thread")
-{
-	int Pid = GetCurrentProcessId();
-	CHECK(Pid > 0);
-	CHECK(IsProcessRunning(Pid));
-
-	CHECK_FALSE(GetCurrentThreadId() == 0);
-}
+TEST_SUITE_BEGIN("core.thread");
 
-TEST_CASE("BuildArgV")
+TEST_CASE("GetCurrentThreadId")
 {
-	const char* Words[] = {"one", "two", "three", "four", "five"};
-	struct
-	{
-		int			WordCount;
-		const char* Input;
-	} Cases[] = {
-		{0, ""},
-		{0, "  "},
-		{1, "one"},
-		{1, " one"},
-		{1, "one  "},
-		{2, "one two"},
-		{2, " one two"},
-		{2, "one two "},
-		{2, " one two"},
-		{2, "one two "},
-		{2, "one two "},
-		{3, "one two three"},
-		{3, "\"one\" two \"three\""},
-		{5, "one two three four five"},
-	};
-
-	for (const auto& Case : Cases)
-	{
-		std::vector<char*> OutArgs;
-		StringBuilder<64>  Mutable;
-		Mutable << Case.Input;
-		BuildArgV(OutArgs, Mutable.Data());
-
-		CHECK_EQ(OutArgs.size(), Case.WordCount);
-
-		for (int i = 0, n = int(OutArgs.size()); i < n; ++i)
-		{
-			const char* Truth	 = Words[i];
-			size_t		TruthLen = strlen(Truth);
-
-			const char* Candidate = OutArgs[i];
-			bool		bQuoted	  = (Candidate[0] == '\"');
-			Candidate += bQuoted;
-
-			CHECK(strncmp(Truth, Candidate, TruthLen) == 0);
-
-			if (bQuoted)
-			{
-				CHECK_EQ(Candidate[TruthLen], '\"');
-			}
-		}
-	}
+	CHECK_FALSE(GetCurrentThreadId() == 0);
 }
 
 TEST_CASE("NamedEvent")
@@ -1213,19 +569,20 @@ TEST_CASE("NamedEvent")
 		CHECK(!bEventSet);
 	}
 
+	NamedEvent ReadyEvent(Name + "_ready");
+
 	// Thread check
 	std::thread Waiter = std::thread([Name]() {
 		NamedEvent ReadyEvent(Name + "_ready");
 		ReadyEvent.Set();
 
 		NamedEvent TestEvent(Name);
-		TestEvent.Wait(100);
+		TestEvent.Wait(1000);
 	});
 
-	NamedEvent ReadyEvent(Name + "_ready");
 	ReadyEvent.Wait();
 
-	zen::Sleep(50);
+	zen::Sleep(100);
 	TestEvent.Set();
 
 	Waiter.join();
@@ -1253,6 +610,8 @@ TEST_CASE("NamedMutex")
 	CHECK(!NamedMutex::Exists(Name));
 }
 
+TEST_SUITE_END();
+
 #endif	// ZEN_WITH_TESTS
 
 }  // namespace zen
diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp
index d71ca0984..f7e4c4b68 100644
--- a/src/zencore/trace.cpp
+++ b/src/zencore/trace.cpp
@@ -9,7 +9,7 @@
 #	include <zencore/trace.h>
 
 void
-TraceInit()
+TraceInit(std::string_view ProgramName)
 {
 	static std::atomic_bool gInited	 = false;
 	bool					Expected = false;
@@ -23,14 +23,20 @@ TraceInit()
 	};
 	trace::Initialize(Desc);
 
+#	if ZEN_PLATFORM_WINDOWS
+	const char* CommandLineString = GetCommandLineA();
+#	else
+	const char* CommandLineString = "";
+#	endif
+
 	trace::ThreadRegister("main", /* system id */ 0, /* sort id */ 0);
-	trace::DescribeSession("zenserver",
+	trace::DescribeSession(ProgramName,
 #	if ZEN_BUILD_DEBUG
 						   trace::Build::Debug,
 #	else
 						   trace::Build::Development,
 #	endif
-						   "",
+						   CommandLineString,
 						   ZEN_CFG_VERSION_BUILD_STRING);
 }
 
@@ -48,9 +54,9 @@ IsTracing()
 }
 
 void
-TraceStart(const char* HostOrPath, TraceType Type)
+TraceStart(std::string_view ProgramName, const char* HostOrPath, TraceType Type)
 {
-	TraceInit();
+	TraceInit(ProgramName);
 	switch (Type)
 	{
 		case TraceType::Network:
diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp
index 3a4b1e6a1..6ff6463dd 100644
--- a/src/zencore/workthreadpool.cpp
+++ b/src/zencore/workthreadpool.cpp
@@ -74,6 +74,7 @@ struct WorkerThreadPool::Impl
 	{
 		WaitForThreadpoolWorkCallbacks(m_Work, /* CancelPendingCallbacks */ TRUE);
 		CloseThreadpoolWork(m_Work);
+		CloseThreadpool(m_ThreadPool);
 	}
 
 	void ScheduleWork(Ref<IWork> Work)
@@ -109,6 +110,7 @@ struct WorkerThreadPool::Impl
 			m_WorkQueue.pop_front();
 		}
 
+		ZEN_TRACE_CPU_FLUSH("AsyncWork");
 		WorkFromQueue->Execute();
 	}
 };
@@ -150,7 +152,10 @@ struct WorkerThreadPool::Impl
 
 		for (std::thread& Thread : m_WorkerThreads)
 		{
-			Thread.join();
+			if (Thread.joinable())
+			{
+				Thread.join();
+			}
 		}
 
 		m_WorkerThreads.clear();
@@ -174,6 +179,7 @@ WorkerThreadPool::Impl::WorkerThreadFunction(ThreadStartInfo Info)
 		{
 			try
 			{
+				ZEN_TRACE_CPU_FLUSH("AsyncWork");
 				Work->Execute();
 			}
 			catch (std::exception& e)
@@ -219,7 +225,17 @@ WorkerThreadPool::ScheduleWork(Ref<IWork> Work)
 	}
 	else
 	{
-		Work->Execute();
+		try
+		{
+			ZEN_TRACE_CPU_FLUSH("SyncWork");
+			Work->Execute();
+		}
+		catch (std::exception& e)
+		{
+			Work->m_Exception = std::current_exception();
+
+			ZEN_WARN("Caught exception when executing worker synchronously: {}", e.what());
+		}
 	}
 }
 
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index 9377a733b..eed903f54 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -10,6 +10,7 @@
 #	include <pthread.h>
 #endif
 
+#include <zencore/assertfmt.h>
 #include <zencore/blake3.h>
 #include <zencore/compactbinary.h>
 #include <zencore/compactbinarybuilder.h>
@@ -24,6 +25,7 @@
 #include <zencore/logging.h>
 #include <zencore/memory.h>
 #include <zencore/mpscqueue.h>
+#include <zencore/process.h>
 #include <zencore/sha1.h>
 #include <zencore/stats.h>
 #include <zencore/stream.h>
@@ -33,6 +35,22 @@
 #include <zencore/uid.h>
 #include <zencore/workthreadpool.h>
 
+#include <fmt/format.h>
+
+namespace zen::assert {
+
+void
+ExecAssertFmt(const char* Filename, int LineNumber, const char* FunctionName, std::string_view Format, fmt::format_args Args)
+{
+	fmt::basic_memory_buffer<char, 1024> Message;
+	fmt::vformat_to(fmt::appender(Message), Format, Args);
+	Message.push_back('\0');
+
+	AssertImpl::ExecAssert(Filename, LineNumber, FunctionName, Message.data());
+}
+
+}  // namespace zen::assert
+
 namespace zen {
 
 void refcount_forcelink();
@@ -123,6 +141,7 @@ zencore_forcelinktests()
 	zen::logging_forcelink();
 	zen::memory_forcelink();
 	zen::mpscqueue_forcelink();
+	zen::process_forcelink();
 	zen::refcount_forcelink();
 	zen::sha1_forcelink();
 	zen::stats_forcelink();
@@ -142,6 +161,8 @@ zencore_forcelinktests()
 
 namespace zen {
 
+TEST_SUITE_BEGIN("core.assert");
+
 TEST_CASE("Assert.Default")
 {
 	bool A = true;
@@ -149,6 +170,13 @@ TEST_CASE("Assert.Default")
 	CHECK_THROWS_WITH(ZEN_ASSERT(A == B), "A == B");
 }
 
+TEST_CASE("Assert.Format")
+{
+	bool A = true;
+	bool B = false;
+	CHECK_THROWS_WITH(ZEN_ASSERT_FORMAT(A == B, "{} == {}", A, B), "assert(A == B) failed: true == false");
+}
+
 TEST_CASE("Assert.Custom")
 {
 	struct MyAssertImpl : AssertImpl
@@ -183,6 +211,7 @@ TEST_CASE("Assert.Custom")
 	CHECK(strcmp(MyAssert.Message, "A == B") == 0);
 }
 
+TEST_SUITE_END();
 #endif
 
 }  // namespace zen
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index fa75060db..97d6a01fe 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -80,6 +80,9 @@ MapContentTypeToString(HttpContentType ContentType)
 
 		case HttpContentType::kIcon:
 			return "image/x-icon"sv;
+
+		case HttpContentType::kXML:
+			return "application/xml"sv;
 	}
 }
 
@@ -111,6 +114,7 @@ static constinit uint32_t HashPng						= HashStringDjb2("png"sv);
 static constinit uint32_t HashImagePng					= HashStringDjb2("image/png"sv);
 static constinit uint32_t HashIcon						= HashStringDjb2("ico"sv);
 static constinit uint32_t HashImageIcon					= HashStringDjb2("image/x-icon"sv);
+static constinit uint32_t HashXml						= HashStringDjb2("application/xml"sv);
 
 std::once_flag InitContentTypeLookup;
 
@@ -143,6 +147,7 @@ struct HashedTypeEntry
 	{HashImagePng,					HttpContentType::kPNG},
 	{HashIcon,						HttpContentType::kIcon},
 	{HashImageIcon,					HttpContentType::kIcon},
+	{HashXml,						HttpContentType::kXML},
 	// clang-format on
 };
 
@@ -593,6 +598,18 @@ HttpServerRequest::ReadPayloadObject()
 {
 	if (IoBuffer Payload = ReadPayload())
 	{
+		if (m_ContentType == HttpContentType::kJSON)
+		{
+			std::string Json(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize());
+			std::string Err;
+
+			CbFieldIterator It = LoadCompactBinaryFromJson(Json, Err);
+			if (Err.empty())
+			{
+				return It.AsObject();
+			}
+			return CbObject();
+		}
 		return LoadCompactBinaryObject(std::move(Payload));
 	}
 
@@ -756,20 +773,20 @@ CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config)
 
 #	if 0
 				Ref<TransportPlugin> WinsockPlugin{CreateSocketTransportPlugin()};
-				WinsockPlugin->Configure("port", "8055");
+				WinsockPlugin->Configure("port", "8558");
 				Server->AddPlugin(WinsockPlugin);
 #	endif
 
 #	if 0
 				Ref<TransportPlugin> AsioPlugin{CreateAsioTransportPlugin()};
-				AsioPlugin->Configure("port", "8055");
+				AsioPlugin->Configure("port", "8558");
 				Server->AddPlugin(AsioPlugin);
 #	endif
 
 #	if 1
 				Ref<DllTransportPlugin> DllPlugin{CreateDllTransportPlugin()};
 				DllPlugin->LoadDll("winsock");
-				DllPlugin->ConfigureDll("winsock", "port", "8055");
+				DllPlugin->ConfigureDll("winsock", "port", "8558");
 				Server->AddPlugin(DllPlugin);
 #	endif
 
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index eabad4728..1089dd221 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -175,11 +175,11 @@ private:
 class HttpServer : public RefCounted
 {
 public:
-	virtual void RegisterService(HttpService& Service) = 0;
-	virtual int	 Initialize(int BasePort)			   = 0;
-	virtual void Run(bool IsInteractiveSession)		   = 0;
-	virtual void RequestExit()						   = 0;
-	virtual void Close()							   = 0;
+	virtual void RegisterService(HttpService& Service)					 = 0;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) = 0;
+	virtual void Run(bool IsInteractiveSession)							 = 0;
+	virtual void RequestExit()											 = 0;
+	virtual void Close()												 = 0;
 };
 
 struct HttpServerConfig
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index c62aca001..9fca314b3 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -941,7 +941,7 @@ public:
 	~HttpAsioServer();
 
 	virtual void RegisterService(HttpService& Service) override;
-	virtual int	 Initialize(int BasePort) override;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) override;
 	virtual void Run(bool IsInteractiveSession) override;
 	virtual void RequestExit() override;
 	virtual void Close() override;
@@ -992,8 +992,9 @@ HttpAsioServer::RegisterService(HttpService& Service)
 }
 
 int
-HttpAsioServer::Initialize(int BasePort)
+HttpAsioServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_UNUSED(DataDir);
 	m_BasePort = m_Impl->Start(gsl::narrow<uint16_t>(BasePort), m_ForceLoopback, m_ThreadCount);
 	return m_BasePort;
 }
diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp
index d8ebdc9c0..2a6a90d2e 100644
--- a/src/zenhttp/servers/httpmulti.cpp
+++ b/src/zenhttp/servers/httpmulti.cpp
@@ -28,15 +28,16 @@ HttpMultiServer::RegisterService(HttpService& Service)
 }
 
 int
-HttpMultiServer::Initialize(int BasePort)
+HttpMultiServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_UNUSED(DataDir);
 	ZEN_ASSERT(!m_IsInitialized);
 
 	int EffectivePort = 0;
 
 	for (auto& Server : m_Servers)
 	{
-		const int InitializeResult = Server->Initialize(BasePort);
+		const int InitializeResult = Server->Initialize(BasePort, DataDir);
 
 		if (!EffectivePort)
 		{
diff --git a/src/zenhttp/servers/httpmulti.h b/src/zenhttp/servers/httpmulti.h
index d5b21d3c3..53cf57568 100644
--- a/src/zenhttp/servers/httpmulti.h
+++ b/src/zenhttp/servers/httpmulti.h
@@ -16,7 +16,7 @@ public:
 	~HttpMultiServer();
 
 	virtual void RegisterService(HttpService& Service) override;
-	virtual int	 Initialize(int BasePort) override;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) override;
 	virtual void Run(bool IsInteractiveSession) override;
 	virtual void RequestExit() override;
 	virtual void Close() override;
diff --git a/src/zenhttp/servers/httpnull.cpp b/src/zenhttp/servers/httpnull.cpp
index 7d3e9079a..9ac1c61ce 100644
--- a/src/zenhttp/servers/httpnull.cpp
+++ b/src/zenhttp/servers/httpnull.cpp
@@ -25,8 +25,9 @@ HttpNullServer::RegisterService(HttpService& Service)
 }
 
 int
-HttpNullServer::Initialize(int BasePort)
+HttpNullServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_UNUSED(DataDir);
 	return BasePort;
 }
 
diff --git a/src/zenhttp/servers/httpnull.h b/src/zenhttp/servers/httpnull.h
index 965e729f7..818020604 100644
--- a/src/zenhttp/servers/httpnull.h
+++ b/src/zenhttp/servers/httpnull.h
@@ -18,7 +18,7 @@ public:
 	~HttpNullServer();
 
 	virtual void RegisterService(HttpService& Service) override;
-	virtual int	 Initialize(int BasePort) override;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) override;
 	virtual void Run(bool IsInteractiveSession) override;
 	virtual void RequestExit() override;
 	virtual void Close() override;
diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp
index 4ae7cd87a..3eed9db8f 100644
--- a/src/zenhttp/servers/httpplugin.cpp
+++ b/src/zenhttp/servers/httpplugin.cpp
@@ -7,7 +7,11 @@
 #	include "httpparser.h"
 
 #	include <zencore/except.h>
+#	include <zencore/filesystem.h>
+#	include <zencore/fmtutils.h>
 #	include <zencore/logging.h>
+#	include <zencore/scopeguard.h>
+#	include <zencore/session.h>
 #	include <zencore/thread.h>
 #	include <zencore/trace.h>
 #	include <zenhttp/httpserver.h>
@@ -15,6 +19,8 @@
 #	include <memory>
 #	include <string_view>
 
+#	include <fmt/format.h>
+
 #	if ZEN_PLATFORM_WINDOWS
 #		include <conio.h>
 #	endif
@@ -38,17 +44,21 @@ using namespace std::literals;
 
 struct HttpPluginConnectionHandler : public TransportServerConnection, public HttpRequestParserCallbacks, RefCounted
 {
+	HttpPluginConnectionHandler();
+	~HttpPluginConnectionHandler();
+
+	// TransportServerConnection
+
 	virtual uint32_t AddRef() const override;
 	virtual uint32_t Release() const override;
-
-	virtual void OnBytesRead(const void* Buffer, size_t DataSize) override;
+	virtual void	 OnBytesRead(const void* Buffer, size_t DataSize) override;
 
 	// HttpRequestParserCallbacks
 
 	virtual void HandleRequest() override;
 	virtual void TerminateConnection() override;
 
-	void Initialize(TransportConnection* Transport, HttpPluginServerImpl& Server);
+	void Initialize(TransportConnection* Transport, HttpPluginServerImpl& Server, uint32_t ConnectionId);
 
 private:
 	enum class RequestState
@@ -65,7 +75,8 @@ private:
 	RequestState	  m_RequestState = RequestState::kInitialState;
 	HttpRequestParser m_RequestParser{*this};
 
-	uint32_t				 m_ConnectionId = 0;
+	uint32_t				 m_ConnectionId	  = 0;
+	std::atomic_uint32_t	 m_RequestCounter = 0;
 	Ref<IHttpPackageHandler> m_PackageHandler;
 
 	TransportConnection*  m_TransportConnection = nullptr;
@@ -82,7 +93,7 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 	// HttpPluginServer
 
 	virtual void RegisterService(HttpService& Service) override;
-	virtual int	 Initialize(int BasePort) override;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) override;
 	virtual void Run(bool IsInteractiveSession) override;
 	virtual void RequestExit() override;
 	virtual void Close() override;
@@ -92,6 +103,8 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 
 	HttpService* RouteRequest(std::string_view Url);
 
+	void WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload);
+
 	struct ServiceEntry
 	{
 		std::string	 ServiceUrlPath;
@@ -103,6 +116,11 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer
 	std::vector<ServiceEntry>		  m_UriHandlers;
 	std::vector<Ref<TransportPlugin>> m_Plugins;
 	Event							  m_ShutdownEvent;
+	bool							  m_IsRequestLoggingEnabled = false;
+	LoggerRef						  m_RequestLog;
+	std::atomic_uint32_t			  m_ConnectionIdCounter{0};
+	std::filesystem::path			  m_DataDir;	 // Application data directory
+	std::filesystem::path			  m_PayloadDir;	 // Request debugging payload directory
 
 	// TransportServer
 
@@ -147,14 +165,20 @@ public:
 	HttpPluginResponse() = default;
 	explicit HttpPluginResponse(HttpContentType ContentType) : m_ContentType(ContentType) {}
 
+	HttpPluginResponse(const HttpPluginResponse&) = delete;
+	HttpPluginResponse& operator=(const HttpPluginResponse&) = delete;
+
 	void InitializeForPayload(uint16_t ResponseCode, std::span<IoBuffer> BlobList);
 
-	inline uint16_t ResponseCode() const { return m_ResponseCode; }
-	inline uint64_t ContentLength() const { return m_ContentLength; }
+	inline uint16_t		   ResponseCode() const { return m_ResponseCode; }
+	inline uint64_t		   ContentLength() const { return m_ContentLength; }
+	inline HttpContentType ContentType() const { return m_ContentType; }
 
 	const std::vector<IoBuffer>& ResponseBuffers() const { return m_ResponseBuffers; }
 	void						 SuppressPayload() { m_ResponseBuffers.resize(1); }
 
+	std::string_view GetHeaders();
+
 private:
 	uint16_t					 m_ResponseCode	 = 0;
 	bool						 m_IsKeepAlive	 = true;
@@ -162,8 +186,6 @@ private:
 	uint64_t					 m_ContentLength = 0;
 	std::vector<IoBuffer>		 m_ResponseBuffers;
 	ExtendableStringBuilder<160> m_Headers;
-
-	std::string_view GetHeaders();
 };
 
 void
@@ -210,27 +232,55 @@ HttpPluginResponse::InitializeForPayload(uint16_t ResponseCode, std::span<IoBuff
 std::string_view
 HttpPluginResponse::GetHeaders()
 {
-	m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
-			  << "Content-Type: " << MapContentTypeToString(m_ContentType) << "\r\n"
-			  << "Content-Length: " << ContentLength() << "\r\n"sv;
-
-	if (!m_IsKeepAlive)
+	if (m_Headers.Size() == 0)
 	{
-		m_Headers << "Connection: close\r\n"sv;
-	}
+		m_Headers << "HTTP/1.1 " << ResponseCode() << " " << ReasonStringForHttpResultCode(ResponseCode()) << "\r\n"
+				  << "Content-Type: " << MapContentTypeToString(m_ContentType) << "\r\n"
+				  << "Content-Length: " << ContentLength() << "\r\n"sv;
+
+		if (!m_IsKeepAlive)
+		{
+			m_Headers << "Connection: close\r\n"sv;
+		}
 
-	m_Headers << "\r\n"sv;
+		m_Headers << "\r\n"sv;
+	}
 
 	return m_Headers;
 }
 
 //////////////////////////////////////////////////////////////////////////
 
+HttpPluginConnectionHandler::HttpPluginConnectionHandler()
+{
+}
+
+HttpPluginConnectionHandler::~HttpPluginConnectionHandler()
+{
+	if (m_Server)
+	{
+		ZEN_LOG_TRACE(m_Server->m_RequestLog, "END connection #{}", m_ConnectionId);
+	}
+}
+
 void
-HttpPluginConnectionHandler::Initialize(TransportConnection* Transport, HttpPluginServerImpl& Server)
+HttpPluginConnectionHandler::Initialize(TransportConnection* Transport, HttpPluginServerImpl& Server, uint32_t ConnectionId)
 {
 	m_TransportConnection = Transport;
 	m_Server			  = &Server;
+	m_ConnectionId		  = ConnectionId;
+
+	std::string_view ConnectionName;
+	if (const char* Name = Transport->GetDebugName())
+	{
+		ConnectionName = Name;
+	}
+	else
+	{
+		ConnectionName = "anonymous";
+	}
+
+	ZEN_LOG_TRACE(m_Server->m_RequestLog, "NEW connection #{} ('')", m_ConnectionId, ConnectionName);
 }
 
 uint32_t
@@ -248,13 +298,19 @@ HttpPluginConnectionHandler::Release() const
 void
 HttpPluginConnectionHandler::OnBytesRead(const void* Buffer, size_t AvailableBytes)
 {
+	ZEN_ASSERT(m_Server);
+
+	ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} OnBytesRead: {}", m_ConnectionId, AvailableBytes);
+
 	while (AvailableBytes)
 	{
 		const size_t ConsumedBytes = m_RequestParser.ConsumeData((const char*)Buffer, AvailableBytes);
 
 		if (ConsumedBytes == ~0ull)
 		{
-			// terminate connection
+			// request parser error -- terminate connection
+
+			ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} terminating due to request parsing error", m_ConnectionId);
 
 			return TerminateConnection();
 		}
@@ -269,15 +325,21 @@ HttpPluginConnectionHandler::OnBytesRead(const void* Buffer, size_t AvailableByt
 void
 HttpPluginConnectionHandler::HandleRequest()
 {
+	ZEN_ASSERT(m_Server);
+
+	const uint32_t RequestNumber = m_RequestCounter.fetch_add(1);
+
+	ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} ENTER HandleRequest #{}", m_ConnectionId, RequestNumber);
+	auto $Exit =
+		MakeGuard([&] { ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} EXIT HandleRequest #{}", m_ConnectionId, RequestNumber); });
+
 	if (!m_RequestParser.IsKeepAlive())
 	{
 		// Once response has been written, connection is done
 		m_RequestState = RequestState::kWritingFinal;
 
-		// We're not going to read any more data from this socket
-
-		const bool Receive	= true;
-		const bool Transmit = false;
+		const bool Receive	= true;	  // We're not going to read any more data from this socket
+		const bool Transmit = false;  // We will write more data however
 		m_TransportConnection->Shutdown(Receive, Transmit);
 	}
 	else
@@ -300,6 +362,24 @@ HttpPluginConnectionHandler::HandleRequest()
 
 		HttpPluginServerRequest Request(m_RequestParser, *Service, m_RequestParser.Body());
 
+		const HttpVerb		   RequestVerb = Request.RequestVerb();
+		const std::string_view Uri		   = Request.RelativeUri();
+
+		if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace))
+		{
+			ZEN_LOG_TRACE(m_Server->m_RequestLog,
+						  "connection #{} Handling Request: {} {} ({} bytes ({}), accept: {})",
+						  m_ConnectionId,
+						  ToString(RequestVerb),
+						  Uri,
+						  Request.ContentLength(),
+						  ToString(Request.RequestContentType()),
+						  ToString(Request.AcceptContentType()));
+
+			m_Server->WriteDebugPayload(fmt::format("request_{}_{}.bin", m_ConnectionId, RequestNumber),
+										std::vector<IoBuffer>{Request.ReadPayload()});
+		}
+
 		if (!HandlePackageOffers(*Service, Request, m_PackageHandler))
 		{
 			try
@@ -340,6 +420,17 @@ HttpPluginConnectionHandler::HandleRequest()
 
 		if (std::unique_ptr<HttpPluginResponse> Response = std::move(Request.m_Response))
 		{
+			{
+				const uint16_t ResponseCode = Response->ResponseCode();
+				ZEN_LOG_TRACE(m_Server->m_RequestLog,
+							  "connection #{} Response: {} {} ({} bytes, {})",
+							  m_ConnectionId,
+							  ResponseCode,
+							  ToString(HttpResponseCode(ResponseCode)),
+							  Response->ContentLength(),
+							  ToString(Response->ContentType()));
+			}
+
 			// Transmit the response
 
 			if (m_RequestParser.RequestVerb() == HttpVerb::kHead)
@@ -349,10 +440,19 @@ HttpPluginConnectionHandler::HandleRequest()
 
 			const std::vector<IoBuffer>& ResponseBuffers = Response->ResponseBuffers();
 
-			//// TODO: should cork/uncork for Linux?
+			if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace))
+			{
+				m_Server->WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber), ResponseBuffers);
+			}
 
 			for (const IoBuffer& Buffer : ResponseBuffers)
 			{
+				ZEN_LOG_TRACE(m_Server->m_RequestLog,
+							  "connection #{} SEND: {} bytes, {}",
+							  m_ConnectionId,
+							  Buffer.GetSize(),
+							  ToString(Buffer.GetContentType()));
+
 				int64_t SentBytes = SendBuffer(Buffer);
 
 				if (SentBytes < 0)
@@ -558,7 +658,7 @@ HttpPluginServerRequest::TryGetRanges(HttpRanges& Ranges)
 
 //////////////////////////////////////////////////////////////////////////
 
-HttpPluginServerImpl::HttpPluginServerImpl()
+HttpPluginServerImpl::HttpPluginServerImpl() : m_RequestLog(logging::Get("http_requests"))
 {
 }
 
@@ -570,13 +670,19 @@ TransportServerConnection*
 HttpPluginServerImpl::CreateConnectionHandler(TransportConnection* Connection)
 {
 	HttpPluginConnectionHandler* Handler{new HttpPluginConnectionHandler()};
-	Handler->Initialize(Connection, *this);
+	const uint32_t				 ConnectionId = m_ConnectionIdCounter.fetch_add(1);
+	Handler->Initialize(Connection, *this, ConnectionId);
 	return Handler;
 }
 
 int
-HttpPluginServerImpl::Initialize(int BasePort)
+HttpPluginServerImpl::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	m_DataDir	 = DataDir;
+	m_PayloadDir = DataDir / "debug" / GetSessionIdString();
+
+	ZEN_INFO("any debug payloads will be written to '{}'", m_PayloadDir);
+
 	try
 	{
 		RwLock::ExclusiveLockScope _(m_Lock);
@@ -742,6 +848,23 @@ HttpPluginServerImpl::RouteRequest(std::string_view Url)
 	return CandidateService;
 }
 
+void
+HttpPluginServerImpl::WriteDebugPayload(std::string_view Filename, const std::span<const IoBuffer> Payload)
+{
+	uint64_t					 PayloadSize = 0;
+	std::vector<const IoBuffer*> Buffers;
+	for (auto& Io : Payload)
+	{
+		Buffers.push_back(&Io);
+		PayloadSize += Io.GetSize();
+	}
+
+	if (PayloadSize)
+	{
+		WriteFile(m_PayloadDir / Filename, Buffers.data(), Buffers.size());
+	}
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 struct HttpPluginServerImpl;
diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp
index 0b11d396b..d2cb63cd7 100644
--- a/src/zenhttp/servers/httpsys.cpp
+++ b/src/zenhttp/servers/httpsys.cpp
@@ -43,7 +43,7 @@ public:
 
 	// HttpServer interface implementation
 
-	virtual int	 Initialize(int BasePort) override;
+	virtual int	 Initialize(int BasePort, std::filesystem::path DataDir) override;
 	virtual void Run(bool TestMode) override;
 	virtual void RequestExit() override;
 	virtual void RegisterService(HttpService& Service) override;
@@ -2012,8 +2012,9 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT
 //
 
 int
-HttpSysServer::Initialize(int BasePort)
+HttpSysServer::Initialize(int BasePort, std::filesystem::path DataDir)
 {
+	ZEN_UNUSED(DataDir);
 	if (int EffectivePort = InitializeServer(BasePort))
 	{
 		StartServer();
diff --git a/src/zenhttp/servers/iothreadpool.cpp b/src/zenhttp/servers/iothreadpool.cpp
index da4b42e28..e941606e2 100644
--- a/src/zenhttp/servers/iothreadpool.cpp
+++ b/src/zenhttp/servers/iothreadpool.cpp
@@ -33,7 +33,12 @@ WinIoThreadPool::WinIoThreadPool(int InThreadCount, int InMaxThreadCount)
 
 WinIoThreadPool::~WinIoThreadPool()
 {
+	// this will wait for all callbacks to complete and tear down the `CreateThreadpoolIo`
+	// object and release all related objects
+	CloseThreadpoolCleanupGroupMembers(m_CleanupGroup, /* cancel pending callbacks */ TRUE, nullptr);
+	CloseThreadpoolCleanupGroup(m_CleanupGroup);
 	CloseThreadpool(m_ThreadPool);
+	DestroyThreadpoolEnvironment(&m_CallbackEnvironment);
 }
 
 void
diff --git a/src/zenhttp/transports/asiotransport.cpp b/src/zenhttp/transports/asiotransport.cpp
index ab053a748..a9a782821 100644
--- a/src/zenhttp/transports/asiotransport.cpp
+++ b/src/zenhttp/transports/asiotransport.cpp
@@ -34,12 +34,13 @@ public:
 	AsioTransportPlugin();
 	~AsioTransportPlugin();
 
-	virtual uint32_t AddRef() const override;
-	virtual uint32_t Release() const override;
-	virtual void	 Configure(const char* OptionTag, const char* OptionValue) override;
-	virtual void	 Initialize(TransportServer* ServerInterface) override;
-	virtual void	 Shutdown() override;
-	virtual bool	 IsAvailable() override;
+	virtual uint32_t	AddRef() const override;
+	virtual uint32_t	Release() const override;
+	virtual void		Configure(const char* OptionTag, const char* OptionValue) override;
+	virtual void		Initialize(TransportServer* ServerInterface) override;
+	virtual void		Shutdown() override;
+	virtual const char* GetDebugName() override { return nullptr; }
+	virtual bool		IsAvailable() override;
 
 private:
 	bool	 m_IsOk		   = true;
@@ -63,9 +64,10 @@ struct AsioTransportConnection : public TransportConnection, std::enable_shared_
 
 	// TransportConnectionInterface
 
-	virtual int64_t WriteBytes(const void* Buffer, size_t DataSize) override;
-	virtual void	Shutdown(bool Receive, bool Transmit) override;
-	virtual void	CloseConnection() override;
+	virtual int64_t		WriteBytes(const void* Buffer, size_t DataSize) override;
+	virtual void		Shutdown(bool Receive, bool Transmit) override;
+	virtual void		CloseConnection() override;
+	virtual const char* GetDebugName() override { return nullptr; }
 
 private:
 	void EnqueueRead();
diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp
index dd4479e39..e09e62ec5 100644
--- a/src/zenhttp/transports/dlltransport.cpp
+++ b/src/zenhttp/transports/dlltransport.cpp
@@ -19,69 +19,6 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
-struct DllTransportConnection : public TransportConnection
-{
-public:
-	DllTransportConnection();
-	~DllTransportConnection();
-
-	void Initialize(TransportServerConnection& ServerConnection);
-	void HandleConnection();
-
-	// TransportConnection
-
-	virtual int64_t WriteBytes(const void* Buffer, size_t DataSize) override;
-	virtual void	Shutdown(bool Receive, bool Transmit) override;
-	virtual void	CloseConnection() override;
-
-private:
-	Ref<TransportServerConnection> m_ConnectionHandler;
-	bool						   m_IsTerminated = false;
-};
-
-DllTransportConnection::DllTransportConnection()
-{
-}
-
-DllTransportConnection::~DllTransportConnection()
-{
-}
-
-void
-DllTransportConnection::Initialize(TransportServerConnection& ServerConnection)
-{
-	m_ConnectionHandler = &ServerConnection;  // TODO: this is awkward
-}
-
-void
-DllTransportConnection::HandleConnection()
-{
-}
-
-void
-DllTransportConnection::CloseConnection()
-{
-	if (m_IsTerminated)
-	{
-		return;
-	}
-
-	m_IsTerminated = true;
-}
-
-int64_t
-DllTransportConnection::WriteBytes(const void* Buffer, size_t DataSize)
-{
-	ZEN_UNUSED(Buffer, DataSize);
-	return DataSize;
-}
-
-void
-DllTransportConnection::Shutdown(bool Receive, bool Transmit)
-{
-	ZEN_UNUSED(Receive, Transmit);
-}
-
 //////////////////////////////////////////////////////////////////////////
 
 struct LoadedDll
@@ -97,12 +34,13 @@ public:
 	DllTransportPluginImpl();
 	~DllTransportPluginImpl();
 
-	virtual uint32_t AddRef() const override;
-	virtual uint32_t Release() const override;
-	virtual void	 Configure(const char* OptionTag, const char* OptionValue) override;
-	virtual void	 Initialize(TransportServer* ServerInterface) override;
-	virtual void	 Shutdown() override;
-	virtual bool	 IsAvailable() override;
+	virtual uint32_t	AddRef() const override;
+	virtual uint32_t	Release() const override;
+	virtual void		Configure(const char* OptionTag, const char* OptionValue) override;
+	virtual void		Initialize(TransportServer* ServerInterface) override;
+	virtual void		Shutdown() override;
+	virtual const char* GetDebugName() override;
+	virtual bool		IsAvailable() override;
 
 	virtual void LoadDll(std::string_view Name) override;
 	virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override;
@@ -179,6 +117,12 @@ DllTransportPluginImpl::Shutdown()
 	}
 }
 
+const char*
+DllTransportPluginImpl::GetDebugName()
+{
+	return nullptr;
+}
+
 bool
 DllTransportPluginImpl::IsAvailable()
 {
diff --git a/src/zenhttp/transports/winsocktransport.cpp b/src/zenhttp/transports/winsocktransport.cpp
index 2397dd7cf..7407c55dd 100644
--- a/src/zenhttp/transports/winsocktransport.cpp
+++ b/src/zenhttp/transports/winsocktransport.cpp
@@ -31,9 +31,10 @@ public:
 
 	// TransportConnection
 
-	virtual int64_t WriteBytes(const void* Buffer, size_t DataSize) override;
-	virtual void	Shutdown(bool Receive, bool Transmit) override;
-	virtual void	CloseConnection() override;
+	virtual int64_t		WriteBytes(const void* Buffer, size_t DataSize) override;
+	virtual void		Shutdown(bool Receive, bool Transmit) override;
+	virtual void		CloseConnection() override;
+	virtual const char* GetDebugName() override;
 
 private:
 	Ref<TransportServerConnection> m_ConnectionHandler;
@@ -103,6 +104,12 @@ SocketTransportConnection::CloseConnection()
 	m_ClientSocket = 0;
 }
 
+const char*
+SocketTransportConnection::GetDebugName()
+{
+	return nullptr;
+}
+
 int64_t
 SocketTransportConnection::WriteBytes(const void* Buffer, size_t DataSize)
 {
@@ -157,12 +164,13 @@ public:
 	SocketTransportPluginImpl();
 	~SocketTransportPluginImpl();
 
-	virtual uint32_t AddRef() const override;
-	virtual uint32_t Release() const override;
-	virtual void	 Configure(const char* OptionTag, const char* OptionValue) override;
-	virtual void	 Initialize(TransportServer* ServerInterface) override;
-	virtual void	 Shutdown() override;
-	virtual bool	 IsAvailable() override;
+	virtual uint32_t	AddRef() const override;
+	virtual uint32_t	Release() const override;
+	virtual void		Configure(const char* OptionTag, const char* OptionValue) override;
+	virtual void		Initialize(TransportServer* ServerInterface) override;
+	virtual void		Shutdown() override;
+	virtual const char* GetDebugName() override;
+	virtual bool		IsAvailable() override;
 
 private:
 	TransportServer* m_ServerInterface = nullptr;
@@ -337,6 +345,12 @@ SocketTransportPluginImpl::Shutdown()
 	}
 }
 
+const char*
+SocketTransportPluginImpl::GetDebugName()
+{
+	return nullptr;
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 TransportPlugin*
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 6aa58ee14..3efa57fdb 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -101,22 +101,23 @@ int
 main(int argc, char** argv)
 {
 	using namespace std::literals;
+	using namespace zen;
 
 #	if ZEN_USE_MIMALLOC
 	mi_version();
 #	endif
 
-	zen::zencore_forcelinktests();
-	zen::zenhttp_forcelinktests();
-	zen::cacherequests_forcelink();
+	zencore_forcelinktests();
+	zenhttp_forcelinktests();
+	cacherequests_forcelink();
 
 	zen::logging::InitializeLogging();
 
 	zen::logging::SetLogLevel(zen::logging::level::Debug);
 	spdlog::set_formatter(std::make_unique<zen::logging::full_test_formatter>("test", std::chrono::system_clock::now()));
 
-	std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path();
-	std::filesystem::path TestBaseDir	 = ProgramBaseDir.parent_path().parent_path() / ".test";
+	std::filesystem::path ProgramBaseDir = GetRunningExecutablePath().parent_path();
+	std::filesystem::path TestBaseDir	 = std::filesystem::current_path() / ".test";
 
 	// This is pretty janky because we're passing most of the options through to the test
 	// framework, so we can't just use cxxopts (I think). This should ideally be cleaned up
@@ -153,11 +154,7 @@ TEST_CASE("default.single")
 
 	ZenServerInstance Instance(TestEnv);
 	Instance.SetTestDir(TestDir);
-	Instance.SpawnServer(13337);
-
-	ZEN_INFO("Waiting...");
-
-	Instance.WaitUntilReady();
+	const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 	std::atomic<uint64_t> BatchCounter{0};
@@ -170,7 +167,7 @@ TEST_CASE("default.single")
 
 		ZEN_INFO("query batch {} started (thread {})", BatchNo, ThreadId);
 		cpr::Session cli;
-		cli.SetUrl(cpr::Url{"http://localhost:13337/test/hello"});
+		cli.SetUrl(cpr::Url{fmt::format("http://localhost:{}/test/hello", PortNumber)});
 
 		for (int i = 0; i < 10000; ++i)
 		{
@@ -206,17 +203,17 @@ TEST_CASE("multi.basic")
 	ZenServerInstance	  Instance1(TestEnv);
 	std::filesystem::path TestDir1 = TestEnv.CreateNewTestDir();
 	Instance1.SetTestDir(TestDir1);
-	Instance1.SpawnServer(13337);
+	Instance1.SpawnServer();
 
 	ZenServerInstance	  Instance2(TestEnv);
 	std::filesystem::path TestDir2 = TestEnv.CreateNewTestDir();
 	Instance2.SetTestDir(TestDir2);
-	Instance2.SpawnServer(13338);
+	Instance2.SpawnServer();
 
 	ZEN_INFO("Waiting...");
 
-	Instance1.WaitUntilReady();
-	Instance2.WaitUntilReady();
+	const uint16_t PortNum1 = Instance1.WaitUntilReady();
+	const uint16_t PortNum2 = Instance2.WaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 	std::atomic<uint64_t> BatchCounter{0};
@@ -242,10 +239,10 @@ TEST_CASE("multi.basic")
 
 	ZEN_INFO("Running multi-server test...");
 
-	Concurrency::parallel_invoke([&] { IssueTestRequests(13337); },
-								 [&] { IssueTestRequests(13338); },
-								 [&] { IssueTestRequests(13337); },
-								 [&] { IssueTestRequests(13338); });
+	Concurrency::parallel_invoke([&] { IssueTestRequests(PortNum1); },
+								 [&] { IssueTestRequests(PortNum2); },
+								 [&] { IssueTestRequests(PortNum1); },
+								 [&] { IssueTestRequests(PortNum2); });
 
 	uint64_t Elapsed = timer.GetElapsedTimeMs();
 
@@ -261,12 +258,10 @@ TEST_CASE("project.basic")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
-	const uint16_t PortNumber = 13337;
-
 	ZenServerInstance Instance1(TestEnv);
 	Instance1.SetTestDir(TestDir);
-	Instance1.SpawnServer(PortNumber);
-	Instance1.WaitUntilReady();
+
+	const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
 
 	std::atomic<uint64_t> RequestCount{0};
 
@@ -423,29 +418,6 @@ TEST_CASE("project.basic")
 			 zen::NiceRate(RequestCount, (uint32_t)Elapsed, "req"));
 }
 
-#	if 0  // this is extremely WIP
-TEST_CASE("project.pipe")
-{
-	using namespace std::literals;
-
-	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
-
-	const uint16_t PortNumber = 13337;
-
-	ZenServerInstance Instance1(TestEnv);
-	Instance1.SetTestDir(TestDir);
-	Instance1.SpawnServer(PortNumber);
-	Instance1.WaitUntilReady();
-
-	zen::LocalProjectClient LocalClient(PortNumber);
-
-	zen::CbObjectWriter Cbow;
-	Cbow << "hey" << 42;
-
-	zen::CbObject Response = LocalClient.MessageTransaction(Cbow.Save());
-}
-#	endif
-
 namespace utils {
 
 	struct ZenConfig
@@ -455,43 +427,46 @@ namespace utils {
 		std::string			  BaseUri;
 		std::string			  Args;
 
-		static ZenConfig New(uint16_t Port = 13337, std::string Args = "")
+		static ZenConfig New(std::string Args = "")
+		{
+			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = TestEnv.GetNewPortNumber(), .Args = std::move(Args)};
+		}
+
+		static ZenConfig New(uint16_t Port, std::string Args = "")
 		{
-			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(),
-							 .Port	  = Port,
-							 .BaseUri = fmt::format("http://localhost:{}/z$", Port),
-							 .Args	  = std::move(Args)};
+			return ZenConfig{.DataDir = TestEnv.CreateNewTestDir(), .Port = Port, .Args = std::move(Args)};
 		}
 
-		static ZenConfig NewWithUpstream(uint16_t UpstreamPort)
+		static ZenConfig NewWithUpstream(uint16_t Port, uint16_t UpstreamPort)
 		{
-			return New(13337, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort));
+			return New(Port, fmt::format("--debug --upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", UpstreamPort));
 		}
 
-		static ZenConfig NewWithThreadedUpstreams(std::span<uint16_t> UpstreamPorts, bool Debug)
+		static ZenConfig NewWithThreadedUpstreams(uint16_t NewPort, std::span<uint16_t> UpstreamPorts, bool Debug)
 		{
 			std::string Args = Debug ? "--debug" : "";
 			for (uint16_t Port : UpstreamPorts)
 			{
 				Args = fmt::format("{}{}--upstream-zen-url=http://localhost:{}", Args, Args.length() > 0 ? " " : "", Port);
 			}
-			return New(13337, Args);
+			return New(NewPort, Args);
 		}
 
 		void Spawn(ZenServerInstance& Inst)
 		{
 			Inst.SetTestDir(DataDir);
 			Inst.SpawnServer(Port, Args);
-			Inst.WaitUntilReady();
+			const uint16_t InstancePort = Inst.WaitUntilReady();
+
+			if (Port != InstancePort)
+				ZEN_DEBUG("relocation detected from {} to {}", Port, InstancePort);
+
+			Port	= InstancePort;
+			BaseUri = fmt::format("http://localhost:{}/z$", Port);
 		}
 	};
 
-	void SpawnServer(ZenServerInstance& Server, ZenConfig& Cfg)
-	{
-		Server.SetTestDir(Cfg.DataDir);
-		Server.SpawnServer(Cfg.Port, Cfg.Args);
-		Server.WaitUntilReady();
-	}
+	void SpawnServer(ZenServerInstance& Server, ZenConfig& Cfg) { Cfg.Spawn(Server); }
 
 }  // namespace utils
 
@@ -501,18 +476,16 @@ TEST_CASE("zcache.basic")
 
 	std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
-	const uint16_t PortNumber = 13337;
-
-	const int  kIterationCount = 100;
-	const auto BaseUri		   = fmt::format("http://localhost:{}/z$", PortNumber);
+	const int kIterationCount = 100;
 
 	auto HashKey = [](int i) -> zen::IoHash { return zen::IoHash::HashBuffer(&i, sizeof i); };
 
 	{
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+
+		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		// Populate with some simple data
 
@@ -569,8 +542,9 @@ TEST_CASE("zcache.basic")
 	{
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+		const uint16_t PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+
+		const std::string BaseUri = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		// Retrieve data again
 
@@ -647,14 +621,12 @@ TEST_CASE("zcache.cbpackage")
 
 	SUBCASE("PUT/GET returns correct package")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Instance1(TestEnv);
 		Instance1.SetTestDir(TestDir);
-		Instance1.SpawnServer(PortNumber);
-		Instance1.WaitUntilReady();
+		const uint16_t	  PortNumber = Instance1.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -687,24 +659,21 @@ TEST_CASE("zcache.cbpackage")
 	SUBCASE("PUT propagates upstream")
 	{
 		// Setup local and remote server
-		std::filesystem::path LocalDataDir	   = TestEnv.CreateNewTestDir();
-		std::filesystem::path RemoteDataDir	   = TestEnv.CreateNewTestDir();
-		const uint16_t		  LocalPortNumber  = 13337;
-		const uint16_t		  RemotePortNumber = 13338;
-
-		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
+		std::filesystem::path LocalDataDir	= TestEnv.CreateNewTestDir();
+		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
 		RemoteInstance.SetTestDir(RemoteDataDir);
-		RemoteInstance.SpawnServer(RemotePortNumber);
-		RemoteInstance.WaitUntilReady();
+		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
 		LocalInstance.SetTestDir(LocalDataDir);
-		LocalInstance.SpawnServer(LocalPortNumber,
+		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
-		LocalInstance.WaitUntilReady();
+		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
+
+		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -750,24 +719,21 @@ TEST_CASE("zcache.cbpackage")
 	SUBCASE("GET finds upstream when missing in local")
 	{
 		// Setup local and remote server
-		std::filesystem::path LocalDataDir	   = TestEnv.CreateNewTestDir();
-		std::filesystem::path RemoteDataDir	   = TestEnv.CreateNewTestDir();
-		const uint16_t		  LocalPortNumber  = 13337;
-		const uint16_t		  RemotePortNumber = 13338;
-
-		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
+		std::filesystem::path LocalDataDir	= TestEnv.CreateNewTestDir();
+		std::filesystem::path RemoteDataDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance RemoteInstance(TestEnv);
 		RemoteInstance.SetTestDir(RemoteDataDir);
-		RemoteInstance.SpawnServer(RemotePortNumber);
-		RemoteInstance.WaitUntilReady();
+		const uint16_t RemotePortNumber = RemoteInstance.SpawnServerAndWaitUntilReady();
 
 		ZenServerInstance LocalInstance(TestEnv);
 		LocalInstance.SetTestDir(LocalDataDir);
-		LocalInstance.SpawnServer(LocalPortNumber,
+		LocalInstance.SpawnServer(TestEnv.GetNewPortNumber(),
 								  fmt::format("--upstream-thread-count=0 --upstream-zen-url=http://localhost:{}", RemotePortNumber));
-		LocalInstance.WaitUntilReady();
+		const uint16_t LocalPortNumber = LocalInstance.WaitUntilReady();
+
+		const auto LocalBaseUri	 = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+		const auto RemoteBaseUri = fmt::format("http://localhost:{}/z$", RemotePortNumber);
 
 		const std::string_view Bucket = "mosdef"sv;
 		zen::IoHash			   Key;
@@ -843,15 +809,17 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("query - 'local' does not query upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+		const uint16_t UpstreamPort = UpstreamCfg.Port;
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamPort);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const std::string_view Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -878,15 +846,17 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local' does not store upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+		const uint16_t UpstreamPort = UpstreamCfg.Port;
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamPort);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -913,15 +883,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local/remote' stores local and upstream (binary)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -948,15 +919,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("query - 'local' does not query upstream (cppackage)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -985,15 +957,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local' does not store upstream (cbpackge)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1022,15 +995,16 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("store - 'local/remote' stores local and upstream (cbpackage)")
 	{
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamInst(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalInst(TestEnv);
-		const auto		  Bucket = "legacy"sv;
-
 		UpstreamCfg.Spawn(UpstreamInst);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalInst(TestEnv);
 		LocalCfg.Spawn(LocalInst);
 
+		const auto Bucket = "legacy"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1059,12 +1033,12 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("skip - 'data' returns cache record without attachments/empty payload")
 	{
-		ZenConfig		  Cfg = ZenConfig::New();
+		ZenConfig		  Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance Instance(TestEnv);
-		const auto		  Bucket = "test"sv;
-
 		Cfg.Spawn(Instance);
 
+		const auto Bucket = "test"sv;
+
 		zen::IoHash	   Key;
 		zen::IoHash	   PayloadId;
 		zen::CbPackage Package = GeneratePackage(Key, PayloadId);
@@ -1111,12 +1085,12 @@ TEST_CASE("zcache.policy")
 
 	SUBCASE("skip - 'data' returns empty binary value")
 	{
-		ZenConfig		  Cfg = ZenConfig::New();
+		ZenConfig		  Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance Instance(TestEnv);
-		const auto		  Bucket = "test"sv;
-
 		Cfg.Spawn(Instance);
 
+		const auto Bucket = "test"sv;
+
 		zen::IoHash Key;
 		auto		BinaryValue = GenerateData(1024, Key);
 
@@ -1245,14 +1219,13 @@ TEST_CASE("zcache.rpc")
 
 	SUBCASE("get cache records")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		CachePolicy				   Policy = CachePolicy::Default;
 		std::vector<zen::CacheKey> Keys	  = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128);
@@ -1276,14 +1249,12 @@ TEST_CASE("zcache.rpc")
 
 	SUBCASE("get missing cache records")
 	{
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		CachePolicy				   Policy		= CachePolicy::Default;
 		std::vector<zen::CacheKey> ExistingKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 128);
@@ -1324,12 +1295,12 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamServer(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalServer(TestEnv);
-
 		SpawnServer(UpstreamServer, UpstreamCfg);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalServer(TestEnv);
 		SpawnServer(LocalServer, LocalCfg);
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4);
@@ -1349,12 +1320,12 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+		ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 		ZenServerInstance UpstreamServer(TestEnv);
-		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-		ZenServerInstance LocalServer(TestEnv);
-
 		SpawnServer(UpstreamServer, UpstreamCfg);
+
+		ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+		ZenServerInstance LocalServer(TestEnv);
 		SpawnServer(LocalServer, LocalCfg);
 
 		std::vector<zen::CacheKey> Keys = PutCacheRecords(UpstreamCfg.BaseUri, "ue4.ddc"sv, "mastodon"sv, 4);
@@ -1376,14 +1347,13 @@ TEST_CASE("zcache.rpc")
 	{
 		using namespace utils;
 
-		std::filesystem::path TestDir	 = TestEnv.CreateNewTestDir();
-		const uint16_t		  PortNumber = 13337;
-		const auto			  BaseUri	 = fmt::format("http://localhost:{}/z$", PortNumber);
+		std::filesystem::path TestDir = TestEnv.CreateNewTestDir();
 
 		ZenServerInstance Inst(TestEnv);
 		Inst.SetTestDir(TestDir);
-		Inst.SpawnServer(PortNumber);
-		Inst.WaitUntilReady();
+
+		const uint16_t	  BasePort = Inst.SpawnServerAndWaitUntilReady();
+		const std::string BaseUri  = fmt::format("http://localhost:{}/z$", BasePort);
 
 		std::vector<zen::CacheKey> SmallKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024);
 		std::vector<zen::CacheKey> LargeKeys = PutCacheRecords(BaseUri, "ue4.ddc"sv, "mastodon"sv, 4, 1024 * 1024 * 16, SmallKeys.size());
@@ -1525,26 +1495,25 @@ TEST_CASE("zcache.failing.upstream")
 	using namespace std::literals;
 	using namespace utils;
 
-	const uint16_t	  Upstream1PortNumber = 13338;
-	ZenConfig		  Upstream1Cfg		  = ZenConfig::New(Upstream1PortNumber);
+	ZenConfig		  Upstream1Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance Upstream1Server(TestEnv);
+	SpawnServer(Upstream1Server, Upstream1Cfg);
 
-	const uint16_t	  Upstream2PortNumber = 13339;
-	ZenConfig		  Upstream2Cfg		  = ZenConfig::New(Upstream2PortNumber);
+	ZenConfig		  Upstream2Cfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance Upstream2Server(TestEnv);
+	SpawnServer(Upstream2Server, Upstream2Cfg);
 
-	std::vector<std::uint16_t> UpstreamPorts = {Upstream1PortNumber, Upstream2PortNumber};
-	ZenConfig				   LocalCfg		 = ZenConfig::NewWithThreadedUpstreams(UpstreamPorts, false);
+	std::vector<std::uint16_t> UpstreamPorts = {Upstream1Cfg.Port, Upstream2Cfg.Port};
+	ZenConfig				   LocalCfg		 = ZenConfig::NewWithThreadedUpstreams(TestEnv.GetNewPortNumber(), UpstreamPorts, false);
 	LocalCfg.Args += (" --upstream-thread-count 2");
 	ZenServerInstance LocalServer(TestEnv);
-	const uint16_t	  LocalPortNumber = 13337;
-	const auto		  LocalUri		  = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-	const auto		  Upstream1Uri	  = fmt::format("http://localhost:{}/z$", Upstream1PortNumber);
-	const auto		  Upstream2Uri	  = fmt::format("http://localhost:{}/z$", Upstream2PortNumber);
-
-	SpawnServer(Upstream1Server, Upstream1Cfg);
-	SpawnServer(Upstream2Server, Upstream2Cfg);
 	SpawnServer(LocalServer, LocalCfg);
+
+	const uint16_t LocalPortNumber = LocalCfg.Port;
+	const auto	   LocalUri		   = fmt::format("http://localhost:{}/z$", LocalPortNumber);
+	const auto	   Upstream1Uri	   = fmt::format("http://localhost:{}/z$", Upstream1Cfg.Port);
+	const auto	   Upstream2Uri	   = fmt::format("http://localhost:{}/z$", Upstream2Cfg.Port);
+
 	bool Upstream1Running = true;
 	bool Upstream2Running = true;
 
@@ -1785,16 +1754,16 @@ TEST_CASE("zcache.rpc.allpolicies")
 	using namespace std::literals;
 	using namespace utils;
 
-	ZenConfig		  UpstreamCfg = ZenConfig::New(13338);
+	ZenConfig		  UpstreamCfg = ZenConfig::New(TestEnv.GetNewPortNumber());
 	ZenServerInstance UpstreamServer(TestEnv);
-	ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(13338);
-	ZenServerInstance LocalServer(TestEnv);
-	const uint16_t	  LocalPortNumber = 13337;
-	const auto		  BaseUri		  = fmt::format("http://localhost:{}/z$", LocalPortNumber);
-
 	SpawnServer(UpstreamServer, UpstreamCfg);
+
+	ZenConfig		  LocalCfg = ZenConfig::NewWithUpstream(TestEnv.GetNewPortNumber(), UpstreamCfg.Port);
+	ZenServerInstance LocalServer(TestEnv);
 	SpawnServer(LocalServer, LocalCfg);
 
+	const auto BaseUri = fmt::format("http://localhost:{}/z$", LocalServer.GetBasePort());
+
 	std::string_view TestVersion   = "F72150A02AE34B57A9EC91D36BA1CE08"sv;
 	std::string_view TestBucket	   = "allpoliciestest"sv;
 	std::string_view TestNamespace = "ue4.ddc"sv;
@@ -2321,7 +2290,7 @@ public:
 
 			Callback(*Instance);
 
-			Instance->SpawnServer(13337 + i, AdditionalServerArgs);
+			Instance->SpawnServer(TestEnv.GetNewPortNumber(), AdditionalServerArgs);
 		}
 
 		for (int i = 0; i < m_ServerCount; ++i)
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp
index d4c69f41b..c2df847ad 100644
--- a/src/zenserver/admin/admin.cpp
+++ b/src/zenserver/admin/admin.cpp
@@ -204,25 +204,24 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 				Details = true;
 			}
 
-			auto SecondsToString = [](std::chrono::seconds Secs) {
-				return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count()));
-			};
-
 			CbObjectWriter Response;
 			Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv);
 			Response.BeginObject("Config");
 			{
 				Response << "RootDirectory" << State.Config.RootDirectory.string();
-				Response << "MonitorInterval" << SecondsToString(State.Config.MonitorInterval);
-				Response << "Interval" << SecondsToString(State.Config.Interval);
-				Response << "MaxCacheDuration" << SecondsToString(State.Config.MaxCacheDuration);
-				Response << "MaxProjectStoreDuration" << SecondsToString(State.Config.MaxProjectStoreDuration);
+				Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval);
+				Response << "Interval" << ToTimeSpan(State.Config.Interval);
+				Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration);
+				Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration);
 				Response << "CollectSmallObjects" << State.Config.CollectSmallObjects;
 				Response << "Enabled" << State.Config.Enabled;
 				Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize);
 				Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit);
 				Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites);
-				Response << "LightweightInterval" << SecondsToString(State.Config.LightweightInterval);
+				Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval);
+				Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1) ? "1" : "2");
+				Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent;
+				Response << "Verbose" << State.Config.Verbose;
 			}
 			Response.EndObject();
 			Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked;
@@ -233,8 +232,8 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 
 			Response.BeginObject("FullGC");
 			{
-				Response << "LastTime" << fmt::format("{}", State.LastFullGcTime);
-				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilFullGc);
+				Response << "LastTime" << ToDateTime(State.LastFullGcTime);
+				Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc);
 				if (State.Config.DiskSizeSoftLimit != 0)
 				{
 					Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC);
@@ -246,7 +245,7 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 				}
 				else
 				{
-					Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count());
+					Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration);
 					Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize);
 					Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize);
 				}
@@ -254,8 +253,8 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 			Response.EndObject();
 			Response.BeginObject("LightweightGC");
 			{
-				Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime);
-				Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc);
+				Response << "LastTime" << ToDateTime(State.LastLightweightGcTime);
+				Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc);
 
 				if (State.LastLightweightGCV2Result)
 				{
@@ -264,7 +263,7 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 				}
 				else
 				{
-					Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count());
+					Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration);
 					Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize);
 					Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize);
 				}
@@ -330,11 +329,36 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 				GcParams.ForceGCVersion = GcVersion::kV2;
 			}
 
+			if (auto Param = Params.GetValue("compactblockthreshold"); Param.empty() == false)
+			{
+				if (auto Value = ParseInt<uint32_t>(Param))
+				{
+					GcParams.CompactBlockUsageThresholdPercent = Value.value();
+				}
+			}
+
+			if (auto Param = Params.GetValue("verbose"); Param.empty() == false)
+			{
+				GcParams.Verbose = Param == "true"sv;
+			}
+
 			const bool Started = m_GcScheduler.TriggerGc(GcParams);
 
 			CbObjectWriter Response;
 			Response << "Status"sv << (Started ? "Started"sv : "Running"sv);
-			HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
+			HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save());
+		},
+		HttpVerb::kPost);
+
+	m_Router.RegisterRoute(
+		"gc-stop",
+		[this](HttpRouterRequest& Req) {
+			HttpServerRequest& HttpReq = Req.ServerRequest();
+			if (m_GcScheduler.CancelGC())
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::Accepted);
+			}
+			HttpReq.WriteResponse(HttpResponseCode::OK);
 		},
 		HttpVerb::kPost);
 
@@ -381,10 +405,30 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 
 			GcScheduler::TriggerScrubParams ScrubParams;
 			ScrubParams.MaxTimeslice = std::chrono::seconds(100);
+
+			if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false)
+			{
+				ScrubParams.SkipDelete = (Param == "true"sv);
+			}
+
+			if (auto Param = Params.GetValue("skipgc"); Param.empty() == false)
+			{
+				ScrubParams.SkipGc = (Param == "true"sv);
+			}
+
+			if (auto Param = Params.GetValue("skipcid"); Param.empty() == false)
+			{
+				ScrubParams.SkipCas = (Param == "true"sv);
+			}
+
 			m_GcScheduler.TriggerScrub(ScrubParams);
 
 			CbObjectWriter Response;
 			Response << "ok"sv << true;
+			Response << "skip_delete" << ScrubParams.SkipDelete;
+			Response << "skip_gc" << ScrubParams.SkipGc;
+			Response << "skip_cas" << ScrubParams.SkipCas;
+			Response << "max_time" << TimeSpan(0, 0, gsl::narrow<int>(ScrubParams.MaxTimeslice.count()));
 			HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save());
 		},
 		HttpVerb::kPost);
@@ -438,7 +482,7 @@ HttpAdminService::HttpAdminService(GcScheduler&	   Scheduler,
 														 HttpContentType::kText,
 														 "Tracing is already enabled"sv);
 			}
-			TraceStart(HostOrPath.c_str(), Type);
+			TraceStart("zenserver", HostOrPath.c_str(), Type);
 			return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started");
 		},
 		HttpVerb::kPost);
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 9bb75480e..0987cd0f1 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -14,6 +14,7 @@
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
 #include <zencore/xxhash.h>
+#include <zenutil/workerpools.h>
 
 #include <future>
 
@@ -25,12 +26,6 @@ namespace {
 #pragma pack(push)
 #pragma pack(1)
 
-	// We use this to indicate if a on disk bucket needs wiping
-	// In version 0.2.5 -> 0.2.11 there was a GC corruption bug that would scrable the references
-	// to block items.
-	// See: https://github.com/EpicGames/zen/pull/299
-	static const uint32_t CurrentDiskBucketVersion = 1;
-
 	struct CacheBucketIndexHeader
 	{
 		static constexpr uint32_t ExpectedMagic	 = 0x75696478;	// 'uidx';
@@ -48,23 +43,94 @@ namespace {
 		{
 			return XXH32(&Header.Magic, sizeof(CacheBucketIndexHeader) - sizeof(uint32_t), 0xC0C0'BABA);
 		}
+
+		bool IsValid() const
+		{
+			if (Magic != ExpectedMagic)
+			{
+				return false;
+			}
+
+			if (Checksum != ComputeChecksum(*this))
+			{
+				return false;
+			}
+
+			if (PayloadAlignment == 0)
+			{
+				return false;
+			}
+
+			return true;
+		}
 	};
 
 	static_assert(sizeof(CacheBucketIndexHeader) == 32);
 
+	struct BucketMetaHeader
+	{
+		static constexpr uint32_t ExpectedMagic	 = 0x61'74'65'6d;  // 'meta';
+		static constexpr uint32_t Version1		 = 1;
+		static constexpr uint32_t CurrentVersion = Version1;
+
+		uint32_t Magic		 = ExpectedMagic;
+		uint32_t Version	 = CurrentVersion;
+		uint64_t EntryCount	 = 0;
+		uint64_t LogPosition = 0;
+		uint32_t Padding	 = 0;
+		uint32_t Checksum	 = 0;
+
+		static uint32_t ComputeChecksum(const BucketMetaHeader& Header)
+		{
+			return XXH32(&Header.Magic, sizeof(BucketMetaHeader) - sizeof(uint32_t), 0xC0C0'BABA);
+		}
+
+		bool IsValid() const
+		{
+			if (Magic != ExpectedMagic)
+			{
+				return false;
+			}
+
+			if (Checksum != ComputeChecksum(*this))
+			{
+				return false;
+			}
+
+			if (Padding != 0)
+			{
+				return false;
+			}
+
+			return true;
+		}
+	};
+
+	static_assert(sizeof(BucketMetaHeader) == 32);
+
 #pragma pack(pop)
 
+	//////////////////////////////////////////////////////////////////////////
+
+	template<typename T>
+	void Reset(T& V)
+	{
+		T Tmp;
+		V.swap(Tmp);
+	}
+
 	const char* IndexExtension = ".uidx";
 	const char* LogExtension   = ".slog";
+	const char* MetaExtension  = ".meta";
 
 	std::filesystem::path GetIndexPath(const std::filesystem::path& BucketDir, const std::string& BucketName)
 	{
 		return BucketDir / (BucketName + IndexExtension);
 	}
 
-	std::filesystem::path GetTempIndexPath(const std::filesystem::path& BucketDir, const std::string& BucketName)
+	std::filesystem::path GetMetaPath(const std::filesystem::path& BucketDir, const std::string& BucketName)
 	{
-		return BucketDir / (BucketName + ".tmp");
+		return BucketDir / (BucketName + MetaExtension);
 	}
 
 	std::filesystem::path GetLogPath(const std::filesystem::path& BucketDir, const std::string& BucketName)
@@ -72,6 +138,12 @@ namespace {
 		return BucketDir / (BucketName + LogExtension);
 	}
 
+	std::filesystem::path GetManifestPath(const std::filesystem::path& BucketDir, const std::string& BucketName)
+	{
+		ZEN_UNUSED(BucketName);
+		return BucketDir / "zen_manifest";
+	}
+
 	bool ValidateCacheBucketIndexEntry(const DiskIndexEntry& Entry, std::string& OutReason)
 	{
 		if (Entry.Key == IoHash::Zero)
@@ -140,26 +212,458 @@ namespace {
 }  // namespace
 
 namespace fs = std::filesystem;
+using namespace std::literals;
 
-static CbObject
-LoadCompactBinaryObject(const fs::path& Path)
+class BucketManifestSerializer
 {
-	FileContents Result = ReadFile(Path);
+	using MetaDataIndex	 = ZenCacheDiskLayer::CacheBucket::MetaDataIndex;
+	using BucketMetaData = ZenCacheDiskLayer::CacheBucket::BucketMetaData;
+
+	using PayloadIndex	= ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+
+public:
+	// We use this to indicate if a on disk bucket needs wiping
+	// In version 0.2.5 -> 0.2.11 there was a GC corruption bug that would scramble the references
+	// to block items.
+	// See: https://github.com/EpicGames/zen/pull/299
+	static inline const uint32_t CurrentDiskBucketVersion = 1;
 
-	if (!Result.ErrorCode)
+	bool Open(std::filesystem::path ManifestPath)
 	{
-		IoBuffer Buffer = Result.Flatten();
-		if (CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All); Error == CbValidateError::None)
+		Manifest = LoadCompactBinaryObject(ManifestPath);
+		return !!Manifest;
+	}
+
+	Oid GetBucketId() const { return Manifest["BucketId"sv].AsObjectId(); }
+
+	bool IsCurrentVersion(uint32_t& OutVersion) const
+	{
+		OutVersion = Manifest["Version"sv].AsUInt32(0);
+		return OutVersion == CurrentDiskBucketVersion;
+	}
+
+	void ParseManifest(RwLock::ExclusiveLockScope&								   BucketLock,
+					   ZenCacheDiskLayer::CacheBucket&							   Bucket,
+					   std::filesystem::path									   ManifestPath,
+					   ZenCacheDiskLayer::CacheBucket::IndexMap&				   Index,
+					   std::vector<AccessTime>&									   AccessTimes,
+					   std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
+
+	Oid GenerateNewManifest(std::filesystem::path ManifestPath);
+
+	IoBuffer MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount);
+	uint64_t GetSidecarSize() const { return m_ManifestEntryCount * sizeof(ManifestData); }
+	void	 WriteSidecarFile(RwLock::SharedLockScope&											 BucketLock,
+							  const std::filesystem::path&										 SidecarPath,
+							  uint64_t															 SnapshotLogPosition,
+							  const ZenCacheDiskLayer::CacheBucket::IndexMap&					 Index,
+							  const std::vector<AccessTime>&									 AccessTimes,
+							  const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&	 Payloads,
+							  const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
+	bool	 ReadSidecarFile(RwLock::ExclusiveLockScope&								 BucketLock,
+							 ZenCacheDiskLayer::CacheBucket&							 Bucket,
+							 std::filesystem::path										 SidecarPath,
+							 ZenCacheDiskLayer::CacheBucket::IndexMap&					 Index,
+							 std::vector<AccessTime>&									 AccessTimes,
+							 std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads);
+
+	IoBuffer MakeManifest(const Oid&													BucketId,
+						  ZenCacheDiskLayer::CacheBucket::IndexMap&&					Index,
+						  std::vector<AccessTime>&&										AccessTimes,
+						  std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&&	Payloads,
+						  std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas);
+
+	CbObject Manifest;
+
+private:
+	CbObject LoadCompactBinaryObject(const fs::path& Path)
+	{
+		FileContents Result = ReadFile(Path);
+
+		if (!Result.ErrorCode)
 		{
-			return LoadCompactBinaryObject(Buffer);
+			IoBuffer Buffer = Result.Flatten();
+			if (CbValidateError Error = ValidateCompactBinary(Buffer, CbValidateMode::All); Error == CbValidateError::None)
+			{
+				return zen::LoadCompactBinaryObject(Buffer);
+			}
 		}
+
+		return CbObject();
 	}
 
-	return CbObject();
+	uint64_t m_ManifestEntryCount = 0;
+
+	struct ManifestData
+	{
+		IoHash	   Key;		   // 20
+		AccessTime Timestamp;  // 4
+		IoHash	   RawHash;	   // 20
+		uint32_t   Padding_0;  // 4
+		size_t	   RawSize;	   // 8
+		uint64_t   Padding_1;  // 8
+	};
+
+	static_assert(sizeof(ManifestData) == 64);
+};
+
+void
+BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope&									BucketLock,
+										ZenCacheDiskLayer::CacheBucket&								Bucket,
+										std::filesystem::path										ManifestPath,
+										ZenCacheDiskLayer::CacheBucket::IndexMap&					Index,
+										std::vector<AccessTime>&									AccessTimes,
+										std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
+{
+	if (Manifest["UsingMetaFile"sv].AsBool())
+	{
+		ReadSidecarFile(BucketLock, Bucket, GetMetaPath(Bucket.m_BucketDir, Bucket.m_BucketName), Index, AccessTimes, Payloads);
+
+		return;
+	}
+
+	ZEN_TRACE_CPU("Z$::ParseManifest");
+
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] { ZEN_INFO("parsed store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
+	const uint64_t			  Count = Manifest["Count"sv].AsUInt64(0);
+	std::vector<PayloadIndex> KeysIndexes;
+	KeysIndexes.reserve(Count);
+
+	CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView();
+	for (CbFieldView& KeyView : KeyArray)
+	{
+		if (auto It = Index.find(KeyView.AsHash()); It != Index.end())
+		{
+			KeysIndexes.push_back(It.value());
+		}
+		else
+		{
+			KeysIndexes.push_back(PayloadIndex());
+		}
+	}
+
+	size_t		KeyIndexOffset = 0;
+	CbArrayView TimeStampArray = Manifest["Timestamps"].AsArrayView();
+	for (CbFieldView& TimeStampView : TimeStampArray)
+	{
+		const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++];
+		if (KeyIndex)
+		{
+			AccessTimes[KeyIndex] = TimeStampView.AsInt64();
+		}
+	}
+
+	KeyIndexOffset			 = 0;
+	CbArrayView RawHashArray = Manifest["RawHash"].AsArrayView();
+	CbArrayView RawSizeArray = Manifest["RawSize"].AsArrayView();
+	if (RawHashArray.Num() == RawSizeArray.Num())
+	{
+		auto RawHashIt = RawHashArray.CreateViewIterator();
+		auto RawSizeIt = RawSizeArray.CreateViewIterator();
+		while (RawHashIt != CbFieldViewIterator())
+		{
+			const PayloadIndex KeyIndex = KeysIndexes[KeyIndexOffset++];
+
+			if (KeyIndex)
+			{
+				uint64_t RawSize = RawSizeIt.AsUInt64();
+				IoHash	 RawHash = RawHashIt.AsHash();
+				if (RawSize != 0 || RawHash != IoHash::Zero)
+				{
+					BucketPayload& Payload = Payloads[KeyIndex];
+					Bucket.SetMetaData(BucketLock, Payload, BucketMetaData{.RawSize = RawSize, .RawHash = RawHash});
+				}
+			}
+
+			RawHashIt++;
+			RawSizeIt++;
+		}
+	}
+	else
+	{
+		ZEN_WARN("Mismatch in size between 'RawHash' and 'RawSize' arrays in {}, skipping meta data", ManifestPath);
+	}
+}
+
+Oid
+BucketManifestSerializer::GenerateNewManifest(std::filesystem::path ManifestPath)
+{
+	const Oid BucketId = Oid::NewOid();
+
+	CbObjectWriter Writer;
+	Writer << "BucketId"sv << BucketId;
+	Writer << "Version"sv << CurrentDiskBucketVersion;
+	Manifest = Writer.Save();
+	WriteFile(ManifestPath, Manifest.GetBuffer().AsIoBuffer());
+
+	return BucketId;
+}
+
+IoBuffer
+BucketManifestSerializer::MakeManifest(const Oid&													 BucketId,
+									   ZenCacheDiskLayer::CacheBucket::IndexMap&&					 Index,
+									   std::vector<AccessTime>&&									 AccessTimes,
+									   std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&&	 Payloads,
+									   std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>&& MetaDatas)
+{
+	using namespace std::literals;
+
+	ZEN_TRACE_CPU("Z$::MakeManifest");
+
+	size_t ItemCount = Index.size();
+
+	// This tends to overestimate a little bit but it is still way more accurate than what we get with exponential growth
+	// And we don't need to reallocate the underlying buffer in almost every case
+	const size_t   EstimatedSizePerItem = 54u;
+	const size_t   ReserveSize			= ItemCount == 0 ? 48u : RoundUp(32u + (ItemCount * EstimatedSizePerItem), 128);
+	CbObjectWriter Writer(ReserveSize);
+
+	Writer << "BucketId"sv << BucketId;
+	Writer << "Version"sv << CurrentDiskBucketVersion;
+
+	if (!Index.empty())
+	{
+		Writer.AddInteger("Count"sv, gsl::narrow<std::uint64_t>(Index.size()));
+		Writer.BeginArray("Keys"sv);
+		for (auto& Kv : Index)
+		{
+			const IoHash& Key = Kv.first;
+			Writer.AddHash(Key);
+		}
+		Writer.EndArray();
+
+		Writer.BeginArray("Timestamps"sv);
+		for (auto& Kv : Index)
+		{
+			GcClock::Tick AccessTime = AccessTimes[Kv.second];
+			Writer.AddInteger(AccessTime);
+		}
+		Writer.EndArray();
+
+		if (!MetaDatas.empty())
+		{
+			Writer.BeginArray("RawHash"sv);
+			for (auto& Kv : Index)
+			{
+				const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = Payloads[Kv.second];
+				if (Payload.MetaData)
+				{
+					Writer.AddHash(MetaDatas[Payload.MetaData].RawHash);
+				}
+				else
+				{
+					Writer.AddHash(IoHash::Zero);
+				}
+			}
+			Writer.EndArray();
+
+			Writer.BeginArray("RawSize"sv);
+			for (auto& Kv : Index)
+			{
+				const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = Payloads[Kv.second];
+				if (Payload.MetaData)
+				{
+					Writer.AddInteger(MetaDatas[Payload.MetaData].RawSize);
+				}
+				else
+				{
+					Writer.AddInteger(0);
+				}
+			}
+			Writer.EndArray();
+		}
+	}
+
+	Manifest = Writer.Save();
+	return Manifest.GetBuffer().AsIoBuffer();
+}
+
+IoBuffer
+BucketManifestSerializer::MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount)
+{
+	m_ManifestEntryCount = EntryCount;
+
+	CbObjectWriter Writer;
+	Writer << "BucketId"sv << BucketId;
+	Writer << "Version"sv << CurrentDiskBucketVersion;
+	Writer << "Count"sv << EntryCount;
+	Writer << "UsingMetaFile"sv << true;
+	Manifest = Writer.Save();
+
+	return Manifest.GetBuffer().AsIoBuffer();
+}
+
+bool
+BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope&								  BucketLock,
+										  ZenCacheDiskLayer::CacheBucket&							  Bucket,
+										  std::filesystem::path										  SidecarPath,
+										  ZenCacheDiskLayer::CacheBucket::IndexMap&					  Index,
+										  std::vector<AccessTime>&									  AccessTimes,
+										  std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>& Payloads)
+{
+	ZEN_ASSERT(AccessTimes.size() == Payloads.size());
+
+	std::error_code Ec;
+
+	BasicFile SidecarFile;
+	SidecarFile.Open(SidecarPath, BasicFile::Mode::kRead, Ec);
+
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to open sidecar file '{}'", SidecarPath));
+	}
+
+	uint64_t FileSize = SidecarFile.FileSize();
+
+	auto InvalidGuard = MakeGuard([&] { ZEN_WARN("skipping invalid sidecar file '{}'", SidecarPath); });
+
+	if (FileSize < sizeof(BucketMetaHeader))
+	{
+		return false;
+	}
+
+	BasicFileBuffer Sidecar(SidecarFile, 128 * 1024);
+
+	BucketMetaHeader Header;
+	Sidecar.Read(&Header, sizeof Header, 0);
+
+	if (!Header.IsValid())
+	{
+		return false;
+	}
+
+	if (Header.Version != BucketMetaHeader::Version1)
+	{
+		return false;
+	}
+
+	const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData);
+	if (Header.EntryCount > ExpectedEntryCount)
+	{
+		return false;
+	}
+
+	InvalidGuard.Dismiss();
+
+	uint64_t RemainingEntryCount = ExpectedEntryCount;
+	uint64_t EntryCount			 = 0;
+	uint64_t CurrentReadOffset	 = sizeof(Header);
+
+	while (RemainingEntryCount--)
+	{
+		const ManifestData* Entry = Sidecar.MakeView<ManifestData>(CurrentReadOffset);
+		CurrentReadOffset += sizeof(ManifestData);
+
+		if (auto It = Index.find(Entry->Key); It != Index.end())
+		{
+			PayloadIndex PlIndex = It.value();
+
+			ZEN_ASSERT(size_t(PlIndex) <= Payloads.size());
+
+			ZenCacheDiskLayer::CacheBucket::BucketPayload& PayloadEntry = Payloads[PlIndex];
+
+			AccessTimes[PlIndex] = Entry->Timestamp;
+
+			if (Entry->RawSize && Entry->RawHash != IoHash::Zero)
+			{
+				Bucket.SetMetaData(BucketLock, PayloadEntry, BucketMetaData{.RawSize = Entry->RawSize, .RawHash = Entry->RawHash});
+			}
+		}
+
+		EntryCount++;
+	}
+
+	ZEN_ASSERT(EntryCount == ExpectedEntryCount);
+
+	return true;
+}
+
+void
+BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
+										   const std::filesystem::path&										  SidecarPath,
+										   uint64_t															  SnapshotLogPosition,
+										   const ZenCacheDiskLayer::CacheBucket::IndexMap&					  Index,
+										   const std::vector<AccessTime>&									  AccessTimes,
+										   const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&  Payloads,
+										   const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas)
+{
+	BucketMetaHeader Header;
+	Header.EntryCount  = m_ManifestEntryCount;
+	Header.LogPosition = SnapshotLogPosition;
+	Header.Checksum	   = Header.ComputeChecksum(Header);
+
+	std::error_code Ec;
+
+	TemporaryFile SidecarFile;
+	SidecarFile.CreateTemporary(SidecarPath.parent_path(), Ec);
+
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed creating '{}'", SidecarFile.GetPath()));
+	}
+
+	SidecarFile.Write(&Header, sizeof Header, 0);
+
+	// TODO: make this batching for better performance
+	{
+		uint64_t WriteOffset = sizeof Header;
+
+		//		BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024);
+
+		std::vector<ManifestData> ManifestDataBuffer;
+		const size_t			  MaxManifestDataBufferCount = Min(Index.size(), 4096u);  // 256 Kb
+		ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
+		for (auto& Kv : Index)
+		{
+			const IoHash&	   Key	   = Kv.first;
+			const PayloadIndex PlIndex = Kv.second;
+
+			IoHash	 RawHash = IoHash::Zero;
+			uint64_t RawSize = 0;
+
+			if (const MetaDataIndex MetaIndex = Payloads[PlIndex].MetaData)
+			{
+				RawHash = MetaDatas[MetaIndex].RawHash;
+				RawSize = MetaDatas[MetaIndex].RawSize;
+			}
+
+			ManifestDataBuffer.emplace_back(ManifestData{.Key		= Key,
+														 .Timestamp = AccessTimes[PlIndex],
+														 .RawHash	= RawHash,
+														 .Padding_0 = 0,
+														 .RawSize	= RawSize,
+														 .Padding_1 = 0});
+			if (ManifestDataBuffer.size() == MaxManifestDataBufferCount)
+			{
+				const uint64_t WriteSize = sizeof(ManifestData) * ManifestDataBuffer.size();
+				SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset);
+				WriteOffset += WriteSize;
+				ManifestDataBuffer.clear();
+				ManifestDataBuffer.reserve(MaxManifestDataBufferCount);
+			}
+		}
+		if (ManifestDataBuffer.size() > 0)
+		{
+			SidecarFile.Write(ManifestDataBuffer.data(), sizeof(ManifestData) * ManifestDataBuffer.size(), WriteOffset);
+		}
+	}
+
+	SidecarFile.MoveTemporaryIntoPlace(SidecarPath, Ec);
+
+	if (Ec)
+	{
+		throw std::system_error(Ec, fmt::format("failed to move '{}' into '{}'", SidecarFile.GetPath(), SidecarPath));
+	}
 }
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
 											std::atomic_uint64_t&	   OuterCacheMemoryUsage,
 											std::string				   BucketName,
@@ -170,6 +674,9 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
 , m_Configuration(Config)
 , m_BucketId(Oid::Zero)
 {
+	m_Index.min_load_factor(IndexMinLoadFactor);
+	m_Index.max_load_factor(IndexMaxLoadFactor);
+
 	if (m_BucketName.starts_with(std::string_view("legacy")) || m_BucketName.ends_with(std::string_view("shadermap")))
 	{
 		const uint64_t LegacyOverrideSize = 16 * 1024 * 1024;
@@ -192,6 +699,10 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 	using namespace std::literals;
 
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::OpenOrCreate");
+	ZEN_ASSERT(m_IsFlushing.load());
+
+	// We want to take the lock here since we register as a GC referencer a construction
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 
 	ZEN_LOG_SCOPE("opening cache bucket '{}'", BucketDir);
 
@@ -200,169 +711,72 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 
 	CreateDirectories(m_BucketDir);
 
-	std::filesystem::path ManifestPath{m_BucketDir / "zen_manifest"};
+	std::filesystem::path ManifestPath = GetManifestPath(m_BucketDir, m_BucketName);
 
 	bool IsNew = false;
 
-	CbObject Manifest = LoadCompactBinaryObject(ManifestPath);
+	BucketManifestSerializer ManifestReader;
 
-	if (Manifest)
+	if (ManifestReader.Open(ManifestPath))
 	{
-		m_BucketId = Manifest["BucketId"sv].AsObjectId();
+		m_BucketId = ManifestReader.GetBucketId();
 		if (m_BucketId == Oid::Zero)
 		{
 			return false;
 		}
-		const uint32_t Version = Manifest["Version"sv].AsUInt32(0);
-		if (Version != CurrentDiskBucketVersion)
+
+		uint32_t Version = 0;
+		if (ManifestReader.IsCurrentVersion(/* out */ Version) == false)
 		{
-			ZEN_INFO("Wiping bucket '{}', found version {}, required version {}", BucketDir, Version, CurrentDiskBucketVersion);
+			ZEN_INFO("Wiping bucket '{}', found version {}, required version {}",
+					 BucketDir,
+					 Version,
+					 BucketManifestSerializer::CurrentDiskBucketVersion);
 			IsNew = true;
 		}
 	}
 	else if (AllowCreate)
 	{
-		m_BucketId.Generate();
-
-		CbObjectWriter Writer;
-		Writer << "BucketId"sv << m_BucketId;
-		Writer << "Version"sv << CurrentDiskBucketVersion;
-		Manifest = Writer.Save();
-		WriteFile(m_BucketDir / "zen_manifest", Manifest.GetBuffer().AsIoBuffer());
-		IsNew = true;
+		m_BucketId = ManifestReader.GenerateNewManifest(ManifestPath);
+		IsNew	   = true;
 	}
 	else
 	{
 		return false;
 	}
 
-	OpenLog(IsNew);
-
-	if (!IsNew)
-	{
-		ZEN_TRACE_CPU("Z$::Disk::Bucket::OpenOrCreate::Manifest");
-
-		Stopwatch  Timer;
-		const auto _ =
-			MakeGuard([&] { ZEN_INFO("read store manifest '{}' in {}", ManifestPath, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
-
-		const uint64_t kInvalidIndex = ~(0ull);
+	InitializeIndexFromDisk(IndexLock, IsNew);
 
-		const uint64_t Count = Manifest["Count"sv].AsUInt64(0);
-		if (Count != 0)
-		{
-			std::vector<size_t> KeysIndexes;
-			KeysIndexes.reserve(Count);
-			CbArrayView KeyArray = Manifest["Keys"sv].AsArrayView();
-			for (CbFieldView& KeyView : KeyArray)
-			{
-				if (auto It = m_Index.find(KeyView.AsHash()); It != m_Index.end())
-				{
-					KeysIndexes.push_back(It.value());
-				}
-				else
-				{
-					KeysIndexes.push_back(kInvalidIndex);
-				}
-			}
-			size_t		KeyIndexOffset = 0;
-			CbArrayView TimeStampArray = Manifest["Timestamps"].AsArrayView();
-			for (CbFieldView& TimeStampView : TimeStampArray)
-			{
-				const size_t KeyIndex = KeysIndexes[KeyIndexOffset++];
-				if (KeyIndex != kInvalidIndex)
-				{
-					m_AccessTimes[KeyIndex] = TimeStampView.AsInt64();
-				}
-			}
-			KeyIndexOffset			 = 0;
-			CbArrayView RawHashArray = Manifest["RawHash"].AsArrayView();
-			CbArrayView RawSizeArray = Manifest["RawSize"].AsArrayView();
-			if (RawHashArray.Num() == RawSizeArray.Num())
-			{
-				auto RawHashIt = RawHashArray.CreateViewIterator();
-				auto RawSizeIt = RawSizeArray.CreateViewIterator();
-				while (RawHashIt != CbFieldViewIterator())
-				{
-					const size_t KeyIndex = KeysIndexes[KeyIndexOffset++];
-
-					if (KeyIndex != kInvalidIndex)
-					{
-						uint64_t RawSize = RawSizeIt.AsUInt64();
-						IoHash	 RawHash = RawHashIt.AsHash();
-						if (RawSize != 0 || RawHash != IoHash::Zero)
-						{
-							BucketPayload& Payload = m_Payloads[KeyIndex];
-							SetMetaData(Payload, BucketMetaData{.RawSize = RawSize, .RawHash = RawHash});
-						}
-					}
-
-					RawHashIt++;
-					RawSizeIt++;
-				}
-			}
-			else
-			{
-				ZEN_WARN("Mismatch in size between 'RawHash' and 'RawSize' arrays in {}, skipping meta data", ManifestPath);
-			}
-		}
-
-		////// Legacy format read
-		{
-			for (CbFieldView Entry : Manifest["Timestamps"sv])
-			{
-				const CbObjectView Obj = Entry.AsObjectView();
-				const IoHash	   Key = Obj["Key"sv].AsHash();
-
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					size_t EntryIndex = It.value();
-					ZEN_ASSERT_SLOW(EntryIndex < m_AccessTimes.size());
-					m_AccessTimes[EntryIndex] = Obj["LastAccess"sv].AsInt64();
-				}
-			}
-			for (CbFieldView Entry : Manifest["RawInfo"sv])
-			{
-				const CbObjectView Obj = Entry.AsObjectView();
-				const IoHash	   Key = Obj["Key"sv].AsHash();
-				if (auto It = m_Index.find(Key); It != m_Index.end())
-				{
-					size_t EntryIndex = It.value();
-					ZEN_ASSERT_SLOW(EntryIndex < m_Payloads.size());
-
-					const IoHash   RawHash = Obj["RawHash"sv].AsHash();
-					const uint64_t RawSize = Obj["RawSize"sv].AsUInt64();
-
-					if (RawHash == IoHash::Zero || RawSize == 0)
-					{
-						ZEN_SCOPED_ERROR("detected bad index entry in index - {}", EntryIndex);
-					}
+	auto _ = MakeGuard([&]() {
+		// We are now initialized, allow flushing when we exit
+		m_IsFlushing.store(false);
+	});
 
-					BucketPayload& Payload = m_Payloads[EntryIndex];
-					SetMetaData(Payload, BucketMetaData{.RawSize = RawSize, .RawHash = RawHash});
-				}
-			}
-		}
+	if (IsNew)
+	{
+		return true;
 	}
 
+	ManifestReader.ParseManifest(IndexLock, *this, ManifestPath, m_Index, m_AccessTimes, m_Payloads);
+
 	return true;
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::MakeIndexSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc)
+ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(const std::function<uint64_t()>& ClaimDiskReserveFunc)
 {
-	ZEN_TRACE_CPU("Z$::Disk::Bucket::MakeIndexSnapshot");
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::WriteIndexSnapshot");
 
-	uint64_t LogCount = m_SlogFile.GetLogCount();
+	const uint64_t LogCount = m_SlogFile.GetLogCount();
 	if (m_LogFlushPosition == LogCount)
 	{
 		return;
 	}
 
 	ZEN_DEBUG("writing store snapshot for '{}'", m_BucketDir);
-	uint64_t   EntryCount = 0;
-	Stopwatch  Timer;
-	const auto _ = MakeGuard([&] {
+	const uint64_t EntryCount = m_Index.size();
+	Stopwatch	   Timer;
+	const auto	   _ = MakeGuard([&] {
 		ZEN_INFO("wrote store snapshot for '{}' containing {} entries in {}",
 				 m_BucketDir,
 				 EntryCount,
@@ -371,42 +785,11 @@ ZenCacheDiskLayer::CacheBucket::MakeIndexSnapshot(const std::function<uint64_t()
 
 	namespace fs = std::filesystem;
 
-	fs::path IndexPath	   = GetIndexPath(m_BucketDir, m_BucketName);
-	fs::path STmpIndexPath = GetTempIndexPath(m_BucketDir, m_BucketName);
-
-	// Move index away, we keep it if something goes wrong
-	if (fs::is_regular_file(STmpIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(STmpIndexPath, Ec) || Ec)
-		{
-			ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message());
-			return;
-		}
-	}
+	fs::path IndexPath = GetIndexPath(m_BucketDir, m_BucketName);
 
 	try
 	{
-		if (fs::is_regular_file(IndexPath))
-		{
-			fs::rename(IndexPath, STmpIndexPath);
-		}
-
-		// Write the current state of the location map to a new index state
-		std::vector<DiskIndexEntry> Entries;
-		Entries.resize(m_Index.size());
-
-		{
-			uint64_t EntryIndex = 0;
-			for (auto& Entry : m_Index)
-			{
-				DiskIndexEntry& IndexEntry = Entries[EntryIndex++];
-				IndexEntry.Key			   = Entry.first;
-				IndexEntry.Location		   = m_Payloads[Entry.second].Location;
-			}
-		}
-
-		uint64_t		IndexSize = sizeof(CacheBucketIndexHeader) + Entries.size() * sizeof(DiskIndexEntry);
+		const uint64_t	IndexSize = sizeof(CacheBucketIndexHeader) + EntryCount * sizeof(DiskIndexEntry);
 		std::error_code Error;
 		DiskSpace		Space = DiskSpaceInfo(m_BucketDir, Error);
 		if (Error)
@@ -426,185 +809,230 @@ ZenCacheDiskLayer::CacheBucket::MakeIndexSnapshot(const std::function<uint64_t()
 				fmt::format("not enough free disk space in '{}' to save index of size {}", m_BucketDir, NiceBytes(IndexSize)));
 		}
 
-		BasicFile ObjectIndexFile;
-		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kTruncate);
-		CacheBucketIndexHeader Header = {.EntryCount	   = Entries.size(),
-										 .LogPosition	   = LogCount,
-										 .PayloadAlignment = gsl::narrow<uint32_t>(m_Configuration.PayloadAlignment)};
+		TemporaryFile	ObjectIndexFile;
+		std::error_code Ec;
+		ObjectIndexFile.CreateTemporary(m_BucketDir, Ec);
+		if (Ec)
+		{
+			throw std::system_error(Ec, fmt::format("failed to create new snapshot file in '{}'", m_BucketDir));
+		}
+
+		{
+			// This is in a separate scope just to ensure IndexWriter goes out
+			// of scope before the file is flushed/closed, in order to ensure
+			// all data is written to the file
+			BasicFileWriter IndexWriter(ObjectIndexFile, 128 * 1024);
 
-		Header.Checksum = CacheBucketIndexHeader::ComputeChecksum(Header);
-		ObjectIndexFile.Write(&Header, sizeof(CacheBucketIndexHeader), 0);
-		ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(DiskIndexEntry), sizeof(CacheBucketIndexHeader));
-		ObjectIndexFile.Flush();
-		ObjectIndexFile.Close();
-		EntryCount		   = Entries.size();
-		m_LogFlushPosition = LogCount;
-	}
-	catch (std::exception& Err)
-	{
-		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
+			CacheBucketIndexHeader Header = {.EntryCount	   = EntryCount,
+											 .LogPosition	   = LogCount,
+											 .PayloadAlignment = gsl::narrow<uint32_t>(m_Configuration.PayloadAlignment)};
+
+			Header.Checksum = CacheBucketIndexHeader::ComputeChecksum(Header);
+			IndexWriter.Write(&Header, sizeof(CacheBucketIndexHeader), 0);
+
+			uint64_t IndexWriteOffset = sizeof(CacheBucketIndexHeader);
 
-		// Restore any previous snapshot
+			for (auto& Entry : m_Index)
+			{
+				DiskIndexEntry IndexEntry;
+				IndexEntry.Key		= Entry.first;
+				IndexEntry.Location = m_Payloads[Entry.second].Location;
+				IndexWriter.Write(&IndexEntry, sizeof(DiskIndexEntry), IndexWriteOffset);
+
+				IndexWriteOffset += sizeof(DiskIndexEntry);
+			}
+
+			IndexWriter.Flush();
+		}
 
-		if (fs::is_regular_file(STmpIndexPath))
+		ObjectIndexFile.Flush();
+		ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec);
+		if (Ec)
 		{
-			std::error_code Ec;
-			fs::remove(IndexPath, Ec);	// We don't care if this fails, we try to move the old temp file regardless
-			fs::rename(STmpIndexPath, IndexPath, Ec);
-			if (Ec)
+			std::filesystem::path TempFilePath = ObjectIndexFile.GetPath();
+			ZEN_WARN("snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", TempFilePath, IndexPath, Ec.message());
+
+			if (std::filesystem::is_regular_file(TempFilePath))
 			{
-				ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message());
+				if (!std::filesystem::remove(TempFilePath, Ec) || Ec)
+				{
+					ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempFilePath, Ec.message());
+				}
 			}
 		}
-	}
-	if (fs::is_regular_file(STmpIndexPath))
-	{
-		std::error_code Ec;
-		if (!fs::remove(STmpIndexPath, Ec) || Ec)
+		else
 		{
-			ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message());
+			// We must only update the log flush position once the snapshot write succeeds
+			m_LogFlushPosition = LogCount;
 		}
 	}
+	catch (std::exception& Err)
+	{
+		ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what());
+	}
 }
 
 uint64_t
-ZenCacheDiskLayer::CacheBucket::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion)
+ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const std::filesystem::path& IndexPath, uint32_t& OutVersion)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::ReadIndexFile");
 
-	if (std::filesystem::is_regular_file(IndexPath))
+	if (!std::filesystem::is_regular_file(IndexPath))
 	{
-		BasicFile ObjectIndexFile;
-		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kRead);
-		uint64_t Size = ObjectIndexFile.FileSize();
-		if (Size >= sizeof(CacheBucketIndexHeader))
-		{
-			CacheBucketIndexHeader Header;
-			ObjectIndexFile.Read(&Header, sizeof(Header), 0);
-			if ((Header.Magic == CacheBucketIndexHeader::ExpectedMagic) &&
-				(Header.Checksum == CacheBucketIndexHeader::ComputeChecksum(Header)) && (Header.PayloadAlignment > 0))
-			{
-				switch (Header.Version)
-				{
-					case CacheBucketIndexHeader::Version2:
-						{
-							uint64_t ExpectedEntryCount = (Size - sizeof(sizeof(CacheBucketIndexHeader))) / sizeof(DiskIndexEntry);
-							if (Header.EntryCount > ExpectedEntryCount)
-							{
-								break;
-							}
-							size_t	   EntryCount = 0;
-							Stopwatch  Timer;
-							const auto _ = MakeGuard([&] {
-								ZEN_INFO("read store '{}' index containing {} entries in {}",
-										 IndexPath,
-										 EntryCount,
-										 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
-							});
+		return 0;
+	}
 
-							m_Configuration.PayloadAlignment = Header.PayloadAlignment;
+	auto InvalidGuard = MakeGuard([&] { ZEN_WARN("skipping invalid index file '{}'", IndexPath); });
 
-							std::vector<DiskIndexEntry> Entries;
-							Entries.resize(Header.EntryCount);
-							ObjectIndexFile.Read(Entries.data(),
-												 Header.EntryCount * sizeof(DiskIndexEntry),
-												 sizeof(CacheBucketIndexHeader));
+	BasicFile ObjectIndexFile;
+	ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kRead);
+	uint64_t FileSize = ObjectIndexFile.FileSize();
+	if (FileSize < sizeof(CacheBucketIndexHeader))
+	{
+		return 0;
+	}
 
-							m_Payloads.reserve(Header.EntryCount);
-							m_Index.reserve(Header.EntryCount);
+	CacheBucketIndexHeader Header;
+	ObjectIndexFile.Read(&Header, sizeof(Header), 0);
 
-							std::string InvalidEntryReason;
-							for (const DiskIndexEntry& Entry : Entries)
-							{
-								if (!ValidateCacheBucketIndexEntry(Entry, InvalidEntryReason))
-								{
-									ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
-									continue;
-								}
-								PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
-								m_Payloads.emplace_back(BucketPayload{.Location = Entry.Location});
-								m_Index.insert_or_assign(Entry.Key, EntryIndex);
-								EntryCount++;
-							}
-							m_AccessTimes.resize(m_Payloads.size(), AccessTime(GcClock::TickCount()));
-							if (m_Configuration.EnableReferenceCaching)
-							{
-								m_FirstReferenceIndex.resize(m_Payloads.size());
-							}
-							OutVersion = CacheBucketIndexHeader::Version2;
-							return Header.LogPosition;
-						}
-						break;
-					default:
-						break;
-				}
-			}
+	if (!Header.IsValid())
+	{
+		return 0;
+	}
+
+	if (Header.Version != CacheBucketIndexHeader::Version2)
+	{
+		return 0;
+	}
+
+	const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(CacheBucketIndexHeader))) / sizeof(DiskIndexEntry);
+	if (Header.EntryCount > ExpectedEntryCount)
+	{
+		return 0;
+	}
+
+	InvalidGuard.Dismiss();
+
+	size_t	   EntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read store '{}' index containing {} entries in {}", IndexPath, EntryCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	m_Configuration.PayloadAlignment = Header.PayloadAlignment;
+
+	m_Payloads.reserve(Header.EntryCount);
+	m_Index.reserve(Header.EntryCount);
+
+	BasicFileBuffer FileBuffer(ObjectIndexFile, 128 * 1024);
+
+	uint64_t CurrentReadOffset	 = sizeof(CacheBucketIndexHeader);
+	uint64_t RemainingEntryCount = Header.EntryCount;
+
+	std::string InvalidEntryReason;
+	while (RemainingEntryCount--)
+	{
+		const DiskIndexEntry* Entry = FileBuffer.MakeView<DiskIndexEntry>(CurrentReadOffset);
+		CurrentReadOffset += sizeof(DiskIndexEntry);
+
+		if (!ValidateCacheBucketIndexEntry(*Entry, InvalidEntryReason))
+		{
+			ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
+			continue;
 		}
-		ZEN_WARN("skipping invalid index file '{}'", IndexPath);
+
+		const PayloadIndex EntryIndex = PayloadIndex(EntryCount);
+		m_Payloads.emplace_back(BucketPayload{.Location = Entry->Location});
+		m_Index.insert_or_assign(Entry->Key, EntryIndex);
+
+		EntryCount++;
 	}
-	return 0;
+
+	ZEN_ASSERT(EntryCount == m_Payloads.size());
+
+	m_AccessTimes.resize(EntryCount, AccessTime(GcClock::TickCount()));
+
+	if (m_Configuration.EnableReferenceCaching)
+	{
+		m_FirstReferenceIndex.resize(EntryCount);
+	}
+
+	OutVersion = CacheBucketIndexHeader::Version2;
+	return Header.LogPosition;
 }
 
 uint64_t
-ZenCacheDiskLayer::CacheBucket::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
+ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::ReadLog");
 
-	if (std::filesystem::is_regular_file(LogPath))
+	if (!std::filesystem::is_regular_file(LogPath))
 	{
-		uint64_t					LogEntryCount = 0;
-		Stopwatch					Timer;
-		const auto					_ = MakeGuard([&] {
-			 ZEN_INFO("read store '{}' log containing {} entries in {}", LogPath, LogEntryCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
-		 });
-		TCasLogFile<DiskIndexEntry> CasLog;
-		CasLog.Open(LogPath, CasLogFile::Mode::kRead);
-		if (CasLog.Initialize())
-		{
-			uint64_t EntryCount = CasLog.GetLogCount();
-			if (EntryCount < SkipEntryCount)
-			{
-				ZEN_WARN("reading full log at '{}', reason: Log position from index snapshot is out of range", LogPath);
-				SkipEntryCount = 0;
-			}
-			LogEntryCount			   = EntryCount - SkipEntryCount;
-			uint64_t InvalidEntryCount = 0;
-			CasLog.Replay(
-				[&](const DiskIndexEntry& Record) {
-					std::string InvalidEntryReason;
-					if (Record.Location.Flags & DiskLocation::kTombStone)
-					{
-						m_Index.erase(Record.Key);
-						return;
-					}
-					if (!ValidateCacheBucketIndexEntry(Record, InvalidEntryReason))
-					{
-						ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", LogPath, InvalidEntryReason);
-						++InvalidEntryCount;
-						return;
-					}
-					PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
-					m_Payloads.emplace_back(BucketPayload{.Location = Record.Location});
-					m_Index.insert_or_assign(Record.Key, EntryIndex);
-				},
-				SkipEntryCount);
-			m_AccessTimes.resize(m_Payloads.size(), AccessTime(GcClock::TickCount()));
-			if (m_Configuration.EnableReferenceCaching)
+		return 0;
+	}
+
+	uint64_t   LogEntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read store '{}' log containing {} entries in {}", LogPath, LogEntryCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	});
+
+	TCasLogFile<DiskIndexEntry> CasLog;
+	CasLog.Open(LogPath, CasLogFile::Mode::kRead);
+	if (!CasLog.Initialize())
+	{
+		return 0;
+	}
+
+	const uint64_t EntryCount = CasLog.GetLogCount();
+	if (EntryCount < SkipEntryCount)
+	{
+		ZEN_WARN("reading full log at '{}', reason: Log position from index snapshot is out of range", LogPath);
+		SkipEntryCount = 0;
+	}
+
+	LogEntryCount			   = EntryCount - SkipEntryCount;
+	uint64_t InvalidEntryCount = 0;
+
+	CasLog.Replay(
+		[&](const DiskIndexEntry& Record) {
+			std::string InvalidEntryReason;
+			if (Record.Location.Flags & DiskLocation::kTombStone)
 			{
-				m_FirstReferenceIndex.resize(m_Payloads.size());
+				// Note: this leaves m_Payloads and other arrays with 'holes' in them
+				m_Index.erase(Record.Key);
+				return;
 			}
-			if (InvalidEntryCount)
+
+			if (!ValidateCacheBucketIndexEntry(Record, InvalidEntryReason))
 			{
-				ZEN_WARN("found {} invalid entries in '{}'", InvalidEntryCount, m_BucketDir);
+				ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", LogPath, InvalidEntryReason);
+				++InvalidEntryCount;
+				return;
 			}
-			return LogEntryCount;
-		}
+			PayloadIndex EntryIndex = PayloadIndex(m_Payloads.size());
+			m_Payloads.emplace_back(BucketPayload{.Location = Record.Location});
+			m_Index.insert_or_assign(Record.Key, EntryIndex);
+		},
+		SkipEntryCount);
+
+	m_AccessTimes.resize(m_Payloads.size(), AccessTime(GcClock::TickCount()));
+
+	if (m_Configuration.EnableReferenceCaching)
+	{
+		m_FirstReferenceIndex.resize(m_Payloads.size());
 	}
-	return 0;
+
+	if (InvalidEntryCount)
+	{
+		ZEN_WARN("found {} invalid entries in '{}'", InvalidEntryCount, m_BucketDir);
+	}
+
+	return LogEntryCount;
 };
 
 void
-ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
+ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockScope& IndexLock, const bool IsNew)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::OpenLog");
 
@@ -639,7 +1067,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 	if (std::filesystem::is_regular_file(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
-		m_LogFlushPosition	  = ReadIndexFile(IndexPath, IndexVersion);
+		m_LogFlushPosition	  = ReadIndexFile(IndexLock, IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
@@ -652,19 +1080,18 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 	{
 		if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath))
 		{
-			LogEntryCount = ReadLog(LogPath, m_LogFlushPosition);
+			LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition);
 		}
 		else if (fs::is_regular_file(LogPath))
 		{
-			ZEN_WARN("removing invalid cas log at '{}'", LogPath);
+			ZEN_WARN("removing invalid log at '{}'", LogPath);
 			std::filesystem::remove(LogPath);
 		}
 	}
 
 	m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite);
 
-	std::vector<BlockStoreLocation> KnownLocations;
-	KnownLocations.reserve(m_Index.size());
+	BlockStore::BlockIndexSet KnownBlocks;
 	for (const auto& Entry : m_Index)
 	{
 		size_t				 EntryIndex = Entry.second;
@@ -674,19 +1101,19 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const bool IsNew)
 		if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
 		{
 			m_StandaloneSize.fetch_add(Location.Size(), std::memory_order::relaxed);
-			continue;
 		}
-		const BlockStoreLocation& BlockLocation = Location.GetBlockLocation(m_Configuration.PayloadAlignment);
-		KnownLocations.push_back(BlockLocation);
+		else
+		{
+			const BlockStoreLocation& BlockLocation = Location.GetBlockLocation(m_Configuration.PayloadAlignment);
+			KnownBlocks.Add(BlockLocation.BlockIndex);
+		}
 	}
-
-	m_BlockStore.SyncExistingBlocksOnDisk(KnownLocations);
+	m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
 
 	if (IsNew || LogEntryCount > 0)
 	{
-		MakeIndexSnapshot();
+		WriteIndexSnapshot(IndexLock);
 	}
-	// TODO: should validate integrity of container files here
 }
 
 void
@@ -759,7 +1186,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 		return false;
 	}
 
-	size_t EntryIndex		  = It.value();
+	PayloadIndex EntryIndex	  = It.value();
 	m_AccessTimes[EntryIndex] = GcClock::TickCount();
 	DiskLocation Location	  = m_Payloads[EntryIndex].Location;
 
@@ -776,7 +1203,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 
 	if (Payload->MemCached)
 	{
-		OutValue.Value = m_MemCachedPayloads[Payload->MemCached];
+		OutValue.Value = m_MemCachedPayloads[Payload->MemCached].Payload;
 		Payload		   = nullptr;
 		IndexLock.ReleaseNow();
 		m_MemoryHitCount++;
@@ -803,14 +1230,14 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 				{
 					ZEN_TRACE_CPU("Z$::Disk::Bucket::Get::MemCache");
 					OutValue.Value = IoBufferBuilder::ReadFromFileMaybe(OutValue.Value);
-					RwLock::ExclusiveLockScope _(m_IndexLock);
+					RwLock::ExclusiveLockScope UpdateIndexLock(m_IndexLock);
 					if (auto UpdateIt = m_Index.find(HashKey); UpdateIt != m_Index.end())
 					{
-						BucketPayload& WritePayload = m_Payloads[EntryIndex];
+						BucketPayload& WritePayload = m_Payloads[UpdateIt->second];
 						// Only update if it has not already been updated by other thread
 						if (!WritePayload.MemCached)
 						{
-							SetMemCachedData(WritePayload, OutValue.Value);
+							SetMemCachedData(UpdateIndexLock, UpdateIt->second, OutValue.Value);
 						}
 					}
 				}
@@ -835,7 +1262,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			OutValue.RawHash = IoHash::HashBuffer(OutValue.Value);
 			OutValue.RawSize = OutValue.Value.GetSize();
 		}
-		RwLock::ExclusiveLockScope __(m_IndexLock);
+		RwLock::ExclusiveLockScope UpdateIndexLock(m_IndexLock);
 		if (auto WriteIt = m_Index.find(HashKey); WriteIt != m_Index.end())
 		{
 			BucketPayload& WritePayload = m_Payloads[WriteIt.value()];
@@ -843,7 +1270,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			// Only set if no other path has already updated the meta data
 			if (!WritePayload.MetaData)
 			{
-				SetMetaData(WritePayload, {.RawSize = OutValue.RawSize, .RawHash = OutValue.RawHash});
+				SetMetaData(UpdateIndexLock, WritePayload, {.RawSize = OutValue.RawSize, .RawHash = OutValue.RawHash});
 			}
 		}
 	}
@@ -877,48 +1304,84 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue&
 	m_DiskWriteCount++;
 }
 
-void
+uint64_t
 ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
 {
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::MemCacheTrim");
+
+	uint64_t	  Trimmed	  = 0;
 	GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
-	RwLock::ExclusiveLockScope _(m_IndexLock);
-	for (const auto& Kv : m_Index)
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+	uint32_t				   MemCachedCount = gsl::narrow<uint32_t>(m_MemCachedPayloads.size());
+	if (MemCachedCount == 0)
 	{
-		if (m_AccessTimes[Kv.second] < ExpireTicks)
+		return 0;
+	}
+
+	uint32_t WriteIndex = 0;
+	for (uint32_t ReadIndex = 0; ReadIndex < MemCachedCount; ++ReadIndex)
+	{
+		MemCacheData& Data = m_MemCachedPayloads[ReadIndex];
+		if (!Data.Payload)
 		{
-			BucketPayload& Payload = m_Payloads[Kv.second];
-			RemoveMemCachedData(Payload);
+			continue;
 		}
+		PayloadIndex Index = Data.OwnerIndex;
+		ZEN_ASSERT_SLOW(m_Payloads[Index].MemCached == MemCachedIndex(ReadIndex));
+		GcClock::Tick AccessTime = m_AccessTimes[Index];
+		if (AccessTime < ExpireTicks)
+		{
+			size_t PayloadSize = Data.Payload.GetSize();
+			RemoveMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
+			Data						= {};
+			m_Payloads[Index].MemCached = {};
+			Trimmed += PayloadSize;
+			continue;
+		}
+		if (ReadIndex > WriteIndex)
+		{
+			m_MemCachedPayloads[WriteIndex] = MemCacheData{.Payload = std::move(Data.Payload), .OwnerIndex = Index};
+			m_Payloads[Index].MemCached		= MemCachedIndex(WriteIndex);
+		}
+		WriteIndex++;
 	}
+	m_MemCachedPayloads.resize(WriteIndex);
+	m_MemCachedPayloads.shrink_to_fit();
+	zen::Reset(m_FreeMemCachedPayloads);
+	return Trimmed;
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint		TickStart,
-												 GcClock::Duration		SectionLength,
-												 std::vector<uint64_t>& InOutUsageSlots)
+ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots)
 {
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::GetUsageByAccess");
+
+	size_t					SlotCount = InOutUsageSlots.capacity();
 	RwLock::SharedLockScope _(m_IndexLock);
-	for (const auto& It : m_Index)
+	uint32_t				MemCachedCount = gsl::narrow<uint32_t>(m_MemCachedPayloads.size());
+	if (MemCachedCount == 0)
 	{
-		size_t		   Index   = It.second;
-		BucketPayload& Payload = m_Payloads[Index];
-		if (!Payload.MemCached)
+		return;
+	}
+	for (uint32_t ReadIndex = 0; ReadIndex < MemCachedCount; ++ReadIndex)
+	{
+		MemCacheData& Data = m_MemCachedPayloads[ReadIndex];
+		if (!Data.Payload)
 		{
 			continue;
 		}
+		PayloadIndex Index = Data.OwnerIndex;
+		ZEN_ASSERT_SLOW(m_Payloads[Index].MemCached == MemCachedIndex(ReadIndex));
 		GcClock::TimePoint ItemAccessTime = GcClock::TimePointFromTick(GcClock::Tick(m_AccessTimes[Index]));
-		GcClock::Duration  Age			  = TickStart.time_since_epoch() - ItemAccessTime.time_since_epoch();
-		uint64_t		   Slot			  = gsl::narrow<uint64_t>(Age.count() > 0 ? Age.count() / SectionLength.count() : 0);
-		if (Slot >= InOutUsageSlots.capacity())
+		GcClock::Duration  Age			  = Now > ItemAccessTime ? Now - ItemAccessTime : GcClock::Duration(0);
+		size_t			   Slot = Age < MaxAge ? gsl::narrow<size_t>((Age.count() * SlotCount) / MaxAge.count()) : (SlotCount - 1);
+		ZEN_ASSERT_SLOW(Slot < SlotCount);
+		if (Slot >= InOutUsageSlots.size())
 		{
-			Slot = InOutUsageSlots.capacity() - 1;
+			InOutUsageSlots.resize(Slot + 1, 0);
 		}
-		if (Slot > InOutUsageSlots.size())
-		{
-			InOutUsageSlots.resize(uint64_t(Slot + 1), 0);
-		}
-		InOutUsageSlots[Slot] += m_MemCachedPayloads[Payload.MemCached].GetSize();
+		InOutUsageSlots[Slot] += EstimateMemCachePayloadMemory(Data.Payload.GetSize());
 	}
 }
 
@@ -976,20 +1439,7 @@ ZenCacheDiskLayer::CacheBucket::Flush()
 		m_BlockStore.Flush(/*ForceNewBlock*/ false);
 		m_SlogFile.Flush();
 
-		std::vector<AccessTime>		AccessTimes;
-		std::vector<BucketPayload>	Payloads;
-		std::vector<BucketMetaData> MetaDatas;
-		IndexMap					Index;
-
-		{
-			RwLock::SharedLockScope IndexLock(m_IndexLock);
-			MakeIndexSnapshot();
-			Index		= m_Index;
-			Payloads	= m_Payloads;
-			AccessTimes = m_AccessTimes;
-			MetaDatas	= m_MetaDatas;
-		}
-		SaveManifest(MakeManifest(std::move(Index), std::move(AccessTimes), Payloads, MetaDatas));
+		SaveSnapshot();
 	}
 	catch (std::exception& Ex)
 	{
@@ -998,113 +1448,108 @@ ZenCacheDiskLayer::CacheBucket::Flush()
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::SaveManifest(CbObject&& Manifest, const std::function<uint64_t()>& ClaimDiskReserveFunc)
+ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc)
 {
-	ZEN_TRACE_CPU("Z$::Disk::Bucket::SaveManifest");
 	try
 	{
-		IoBuffer Buffer = Manifest.GetBuffer().AsIoBuffer();
-
-		std::error_code Error;
-		DiskSpace		Space = DiskSpaceInfo(m_BucketDir, Error);
-		if (Error)
-		{
-			ZEN_WARN("get disk space in '{}' FAILED, reason: '{}'", m_BucketDir, Error.message());
-			return;
-		}
-		bool EnoughSpace = Space.Free >= Buffer.GetSize() + 1024 * 512;
-		if (!EnoughSpace)
-		{
-			uint64_t ReclaimedSpace = ClaimDiskReserveFunc();
-			EnoughSpace				= (Space.Free + ReclaimedSpace) >= Buffer.GetSize() + 1024 * 512;
-		}
-		if (!EnoughSpace)
-		{
-			ZEN_WARN("not enough free disk space in '{}'. FAILED to save manifest of size {}", m_BucketDir, NiceBytes(Buffer.GetSize()));
-			return;
-		}
-		WriteFile(m_BucketDir / "zen_manifest", Buffer);
-	}
-	catch (std::exception& Err)
-	{
-		ZEN_WARN("writing manifest in '{}' FAILED, reason: '{}'", m_BucketDir, Err.what());
-	}
-}
+		bool UseLegacyScheme = false;
 
-CbObject
-ZenCacheDiskLayer::CacheBucket::MakeManifest(IndexMap&&							Index,
-											 std::vector<AccessTime>&&			AccessTimes,
-											 const std::vector<BucketPayload>&	Payloads,
-											 const std::vector<BucketMetaData>& MetaDatas)
-{
-	using namespace std::literals;
+		IoBuffer				 Buffer;
+		BucketManifestSerializer ManifestWriter;
 
-	ZEN_TRACE_CPU("Z$::Disk::Bucket::MakeManifest");
-
-	size_t ItemCount = Index.size();
+		if (UseLegacyScheme)
+		{
+			std::vector<AccessTime>		AccessTimes;
+			std::vector<BucketPayload>	Payloads;
+			std::vector<BucketMetaData> MetaDatas;
+			IndexMap					Index;
 
-	// This tends to overestimate a little bit but it is still way more accurate than what we get with exponential growth
-	// And we don't need to reallocate theunderying buffer in almost every case
-	const size_t   EstimatedSizePerItem = 54u;
-	const size_t   ReserveSize			= ItemCount == 0 ? 48u : RoundUp(32u + (ItemCount * EstimatedSizePerItem), 128);
-	CbObjectWriter Writer(ReserveSize);
+			{
+				RwLock::SharedLockScope IndexLock(m_IndexLock);
+				WriteIndexSnapshot(IndexLock);
+				// Note: this copy could be eliminated on shutdown to
+				// reduce memory usage and execution time
+				Index		= m_Index;
+				Payloads	= m_Payloads;
+				AccessTimes = m_AccessTimes;
+				MetaDatas	= m_MetaDatas;
+			}
 
-	Writer << "BucketId"sv << m_BucketId;
-	Writer << "Version"sv << CurrentDiskBucketVersion;
+			Buffer						 = ManifestWriter.MakeManifest(m_BucketId,
+												   std::move(Index),
+												   std::move(AccessTimes),
+												   std::move(Payloads),
+												   std::move(MetaDatas));
+			const uint64_t RequiredSpace = Buffer.GetSize() + 1024 * 512;
 
-	if (!Index.empty())
-	{
-		Writer.AddInteger("Count"sv, gsl::narrow<std::uint64_t>(Index.size()));
-		Writer.BeginArray("Keys"sv);
-		for (auto& Kv : Index)
-		{
-			const IoHash& Key = Kv.first;
-			Writer.AddHash(Key);
+			std::error_code Error;
+			DiskSpace		Space = DiskSpaceInfo(m_BucketDir, Error);
+			if (Error)
+			{
+				ZEN_WARN("get disk space in '{}' FAILED, reason: '{}'", m_BucketDir, Error.message());
+				return;
+			}
+			bool EnoughSpace = Space.Free >= RequiredSpace;
+			if (!EnoughSpace)
+			{
+				uint64_t ReclaimedSpace = ClaimDiskReserveFunc();
+				EnoughSpace				= (Space.Free + ReclaimedSpace) >= RequiredSpace;
+			}
+			if (!EnoughSpace)
+			{
+				ZEN_WARN("not enough free disk space in '{}'. FAILED to save manifest of size {}",
+						 m_BucketDir,
+						 NiceBytes(Buffer.GetSize()));
+				return;
+			}
 		}
-		Writer.EndArray();
-
-		Writer.BeginArray("Timestamps"sv);
-		for (auto& Kv : Index)
+		else
 		{
-			GcClock::Tick AccessTime = AccessTimes[Kv.second];
-			Writer.AddInteger(AccessTime);
-		}
-		Writer.EndArray();
+			RwLock::SharedLockScope IndexLock(m_IndexLock);
+			WriteIndexSnapshot(IndexLock);
+			const uint64_t EntryCount = m_Index.size();
+			Buffer					  = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount);
+			uint64_t SidecarSize	  = ManifestWriter.GetSidecarSize();
 
-		if (!MetaDatas.empty())
-		{
-			Writer.BeginArray("RawHash"sv);
-			for (auto& Kv : Index)
+			const uint64_t RequiredSpace = SidecarSize + Buffer.GetSize() + 1024 * 512;
+
+			std::error_code Error;
+			DiskSpace		Space = DiskSpaceInfo(m_BucketDir, Error);
+			if (Error)
 			{
-				const BucketPayload& Payload = Payloads[Kv.second];
-				if (Payload.MetaData)
-				{
-					Writer.AddHash(MetaDatas[Payload.MetaData].RawHash);
-				}
-				else
-				{
-					Writer.AddHash(IoHash::Zero);
-				}
+				ZEN_WARN("get disk space in '{}' FAILED, reason: '{}'", m_BucketDir, Error.message());
+				return;
 			}
-			Writer.EndArray();
-
-			Writer.BeginArray("RawSize"sv);
-			for (auto& Kv : Index)
+			bool EnoughSpace = Space.Free >= RequiredSpace;
+			if (!EnoughSpace)
 			{
-				const BucketPayload& Payload = Payloads[Kv.second];
-				if (Payload.MetaData)
-				{
-					Writer.AddInteger(MetaDatas[Payload.MetaData].RawSize);
-				}
-				else
-				{
-					Writer.AddInteger(0);
-				}
+				uint64_t ReclaimedSpace = ClaimDiskReserveFunc();
+				EnoughSpace				= (Space.Free + ReclaimedSpace) >= RequiredSpace;
 			}
-			Writer.EndArray();
+			if (!EnoughSpace)
+			{
+				ZEN_WARN("not enough free disk space in '{}'. FAILED to save manifest of size {}",
+						 m_BucketDir,
+						 NiceBytes(Buffer.GetSize()));
+				return;
+			}
+
+			ManifestWriter.WriteSidecarFile(IndexLock,
+											GetMetaPath(m_BucketDir, m_BucketName),
+											m_LogFlushPosition,
+											m_Index,
+											m_AccessTimes,
+											m_Payloads,
+											m_MetaDatas);
 		}
+
+		std::filesystem::path ManifestPath = GetManifestPath(m_BucketDir, m_BucketName);
+		WriteFile(ManifestPath, Buffer);
+	}
+	catch (std::exception& Err)
+	{
+		ZEN_WARN("writing manifest in '{}' FAILED, reason: '{}'", m_BucketDir, Err.what());
 	}
-	return Writer.Save();
 }
 
 IoHash
@@ -1364,8 +1809,8 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 						m_StandaloneSize.fetch_sub(Location.Size(), std::memory_order::relaxed);
 					}
 
-					RemoveMemCachedData(Payload);
-					RemoveMetaData(Payload);
+					RemoveMemCachedData(IndexLock, Payload);
+					RemoveMetaData(IndexLock, Payload);
 
 					Location.Flags |= DiskLocation::kTombStone;
 					LogEntries.push_back(DiskIndexEntry{.Key = BadKey, .Location = Location});
@@ -1395,13 +1840,13 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 				std::vector<BucketPayload>	Payloads;
 				std::vector<AccessTime>		AccessTimes;
 				std::vector<BucketMetaData> MetaDatas;
-				std::vector<IoBuffer>		MemCachedPayloads;
+				std::vector<MemCacheData>	MemCachedPayloads;
 				std::vector<ReferenceIndex> FirstReferenceIndex;
 				IndexMap					Index;
 
 				{
 					RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-					CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
 				}
 			}
 		}
@@ -1463,6 +1908,10 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 		});
 #endif	// CALCULATE_BLOCKING_TIME
+		if (m_Index.empty())
+		{
+			return;
+		}
 		Index				= m_Index;
 		AccessTimes			= m_AccessTimes;
 		Payloads			= m_Payloads;
@@ -1542,10 +1991,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	for (const auto& Entry : StructuredItemsWithUnknownAttachments)
 	{
-		const IoHash&		Key			 = Entry.first;
-		size_t				PayloadIndex = Entry.second;
-		BucketPayload&		Payload		 = Payloads[PayloadIndex];
-		const DiskLocation& Loc			 = Payload.Location;
+		const IoHash&		Key		= Entry.first;
+		BucketPayload&		Payload = Payloads[Entry.second];
+		const DiskLocation& Loc		= Payload.Location;
 		{
 			IoBuffer Buffer;
 			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
@@ -1568,10 +2016,10 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 #endif	// CALCULATE_BLOCKING_TIME
 				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
-					const BucketPayload& CachedPayload = Payloads[PayloadIndex];
+					const BucketPayload& CachedPayload = Payloads[It->second];
 					if (CachedPayload.MemCached)
 					{
-						Buffer = m_MemCachedPayloads[CachedPayload.MemCached];
+						Buffer = m_MemCachedPayloads[CachedPayload.MemCached].Payload;
 						ZEN_ASSERT_SLOW(Buffer);
 					}
 					else
@@ -1678,20 +2126,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 
 		try
 		{
-			std::vector<AccessTime>		AccessTimes;
-			std::vector<BucketPayload>	Payloads;
-			std::vector<BucketMetaData> MetaDatas;
-			IndexMap					Index;
-			{
-				RwLock::SharedLockScope IndexLock(m_IndexLock);
-				MakeIndexSnapshot([&]() { return GcCtx.ClaimGCReserve(); });
-				Index		= m_Index;
-				Payloads	= m_Payloads;
-				AccessTimes = m_AccessTimes;
-				MetaDatas	= m_MetaDatas;
-			}
-			SaveManifest(MakeManifest(std::move(Index), std::move(AccessTimes), Payloads, MetaDatas),
-						 [&]() { return GcCtx.ClaimGCReserve(); });
+			SaveSnapshot([&]() { return GcCtx.ClaimGCReserve(); });
 		}
 		catch (std::exception& Ex)
 		{
@@ -1699,8 +2134,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		}
 	});
 
-	m_SlogFile.Flush();
-
 	auto __ = MakeGuard([&]() {
 		if (!DeletedChunks.empty())
 		{
@@ -1708,7 +2141,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			std::vector<BucketPayload>	Payloads;
 			std::vector<AccessTime>		AccessTimes;
 			std::vector<BucketMetaData> MetaDatas;
-			std::vector<IoBuffer>		MemCachedPayloads;
+			std::vector<MemCacheData>	MemCachedPayloads;
 			std::vector<ReferenceIndex> FirstReferenceIndex;
 			IndexMap					Index;
 			{
@@ -1719,18 +2152,25 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					WriteBlockTimeUs += ElapsedUs;
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
-				CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
 			}
 			GcCtx.AddDeletedCids(std::vector<IoHash>(DeletedChunks.begin(), DeletedChunks.end()));
 		}
 	});
 
-	std::span<const IoHash>					   ExpiredCacheKeySpan = GcCtx.ExpiredCacheKeys(m_BucketDir.string());
+	std::span<const IoHash> ExpiredCacheKeySpan = GcCtx.ExpiredCacheKeys(m_BucketDir.string());
+	if (ExpiredCacheKeySpan.empty())
+	{
+		return;
+	}
+
+	m_SlogFile.Flush();
+
 	std::unordered_set<IoHash, IoHash::Hasher> ExpiredCacheKeys(ExpiredCacheKeySpan.begin(), ExpiredCacheKeySpan.end());
 
 	std::vector<DiskIndexEntry>		 ExpiredStandaloneEntries;
-	IndexMap						 Index;
-	std::vector<BucketPayload>		 Payloads;
+	IndexMap						 IndexSnapshot;
+	std::vector<BucketPayload>		 PayloadsSnapshot;
 	BlockStore::ReclaimSnapshotState BlockStoreState;
 	{
 		bool Expected = false;
@@ -1741,7 +2181,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		}
 		auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); });
 
-		std::vector<AccessTime> AccessTimes;
 		{
 			ZEN_TRACE_CPU("Z$::Disk::Bucket::CollectGarbage::State");
 			RwLock::SharedLockScope IndexLock(m_IndexLock);
@@ -1755,23 +2194,23 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 
 			BlockStoreState = m_BlockStore.GetReclaimSnapshotState();
 
-			Payloads	= m_Payloads;
-			AccessTimes = m_AccessTimes;
-			Index		= m_Index;
-
 			for (const IoHash& Key : ExpiredCacheKeys)
 			{
-				if (auto It = Index.find(Key); It != Index.end())
+				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
-					const BucketPayload& Payload = Payloads[It->second];
-					DiskIndexEntry		 Entry	 = {.Key = It->first, .Location = Payload.Location};
-					if (Entry.Location.Flags & DiskLocation::kStandaloneFile)
+					const BucketPayload& Payload = m_Payloads[It->second];
+					if (Payload.Location.Flags & DiskLocation::kStandaloneFile)
 					{
+						DiskIndexEntry Entry = {.Key = Key, .Location = Payload.Location};
 						Entry.Location.Flags |= DiskLocation::kTombStone;
 						ExpiredStandaloneEntries.push_back(Entry);
 					}
 				}
 			}
+
+			PayloadsSnapshot = m_Payloads;
+			IndexSnapshot	 = m_Index;
+
 			if (GcCtx.IsDeletionMode())
 			{
 				IndexLock.ReleaseNow();
@@ -1836,7 +2275,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		}
 	}
 
-	TotalChunkCount = Index.size();
+	TotalChunkCount = IndexSnapshot.size();
 
 	std::vector<BlockStoreLocation> ChunkLocations;
 	BlockStore::ChunkIndexArray		KeepChunkIndexes;
@@ -1846,10 +2285,10 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 	ChunkIndexToChunkHash.reserve(TotalChunkCount);
 	{
 		TotalChunkCount = 0;
-		for (const auto& Entry : Index)
+		for (const auto& Entry : IndexSnapshot)
 		{
 			size_t				EntryIndex	 = Entry.second;
-			const DiskLocation& DiskLocation = Payloads[EntryIndex].Location;
+			const DiskLocation& DiskLocation = PayloadsSnapshot[EntryIndex].Location;
 
 			if (DiskLocation.Flags & DiskLocation::kStandaloneFile)
 			{
@@ -1894,7 +2333,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			std::vector<DiskIndexEntry> LogEntries;
 			LogEntries.reserve(MovedChunks.size() + RemovedChunks.size());
 			{
-				RwLock::ExclusiveLockScope __(m_IndexLock);
+				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 				Stopwatch				   Timer;
 				const auto				   ____ = MakeGuard([&] {
 					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
@@ -1908,7 +2347,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					const IoHash&			  ChunkHash	  = ChunkIndexToChunkHash[ChunkIndex];
 					size_t					  EntryIndex  = m_Index[ChunkHash];
 					BucketPayload&			  Payload	  = m_Payloads[EntryIndex];
-					if (Payloads[Index[ChunkHash]].Location != m_Payloads[EntryIndex].Location)
+					if (PayloadsSnapshot[IndexSnapshot[ChunkHash]].Location != m_Payloads[EntryIndex].Location)
 					{
 						// Entry has been updated while GC was running, ignore the move
 						continue;
@@ -1921,7 +2360,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					const IoHash&  ChunkHash  = ChunkIndexToChunkHash[ChunkIndex];
 					size_t		   EntryIndex = m_Index[ChunkHash];
 					BucketPayload& Payload	  = m_Payloads[EntryIndex];
-					if (Payloads[Index[ChunkHash]].Location != Payload.Location)
+					if (PayloadsSnapshot[IndexSnapshot[ChunkHash]].Location != Payload.Location)
 					{
 						// Entry has been updated while GC was running, ignore the delete
 						continue;
@@ -1932,8 +2371,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 																   m_Configuration.PayloadAlignment,
 																   OldDiskLocation.GetFlags() | DiskLocation::kTombStone)});
 
-					RemoveMemCachedData(Payload);
-					RemoveMetaData(Payload);
+					RemoveMemCachedData(IndexLock, Payload);
+					RemoveMetaData(IndexLock, Payload);
 
 					m_Index.erase(ChunkHash);
 					DeletedChunks.insert(ChunkHash);
@@ -1970,7 +2409,7 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
 }
 
 CacheValueDetails::ValueDetails
-ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex Index) const
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const IoHash& Key, PayloadIndex Index) const
 {
 	std::vector<IoHash>	 Attachments;
 	const BucketPayload& Payload = m_Payloads[Index];
@@ -1982,7 +2421,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex
 		CbObjectView Obj(Value.GetData());
 		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
 	}
-	BucketMetaData MetaData = GetMetaData(Payload);
+	BucketMetaData MetaData = GetMetaData(IndexLock, Payload);
 	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
 										   .RawSize		= MetaData.RawSize,
 										   .RawHash		= MetaData.RawHash,
@@ -1992,7 +2431,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex
 }
 
 CacheValueDetails::BucketDetails
-ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilter) const
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const
 {
 	CacheValueDetails::BucketDetails Details;
 	RwLock::SharedLockScope			 _(m_IndexLock);
@@ -2001,7 +2440,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
 		Details.Values.reserve(m_Index.size());
 		for (const auto& It : m_Index)
 		{
-			Details.Values.insert_or_assign(It.first, GetValueDetails(It.first, It.second));
+			Details.Values.insert_or_assign(It.first, GetValueDetails(IndexLock, It.first, It.second));
 		}
 	}
 	else
@@ -2009,7 +2448,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
 		IoHash Key = IoHash::FromHexString(ValueFilter);
 		if (auto It = m_Index.find(Key); It != m_Index.end())
 		{
-			Details.Values.insert_or_assign(It->first, GetValueDetails(It->first, It->second));
+			Details.Values.insert_or_assign(It->first, GetValueDetails(IndexLock, It->first, It->second));
 		}
 	}
 	return Details;
@@ -2019,10 +2458,10 @@ void
 ZenCacheDiskLayer::CacheBucket::EnumerateBucketContents(
 	std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
 {
-	RwLock::SharedLockScope _(m_IndexLock);
+	RwLock::SharedLockScope IndexLock(m_IndexLock);
 	for (const auto& It : m_Index)
 	{
-		CacheValueDetails::ValueDetails Vd = GetValueDetails(It.first, It.second);
+		CacheValueDetails::ValueDetails Vd = GetValueDetails(IndexLock, It.first, It.second);
 
 		Fn(It.first, Vd);
 	}
@@ -2046,7 +2485,10 @@ ZenCacheDiskLayer::CollectGarbage(GcContext& GcCtx)
 	{
 		Bucket->CollectGarbage(GcCtx);
 	}
-	MemCacheTrim(Buckets, GcCtx.CacheExpireTime());
+	if (!m_IsMemCacheTrimming)
+	{
+		MemCacheTrim(Buckets, GcCtx.CacheExpireTime());
+	}
 }
 
 void
@@ -2166,6 +2608,10 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 	ValueLock.ReleaseNow();
+	if (m_UpdatedKeys)
+	{
+		m_UpdatedKeys->insert(HashKey);
+	}
 
 	PayloadIndex EntryIndex = {};
 	if (auto It = m_Index.find(HashKey); It == m_Index.end())
@@ -2193,16 +2639,16 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 			SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
 		}
 		m_AccessTimes[EntryIndex] = GcClock::TickCount();
-		RemoveMemCachedData(Payload);
+		RemoveMemCachedData(IndexLock, Payload);
 		m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed);
 	}
 	if (Value.RawSize != 0 || Value.RawHash != IoHash::Zero)
 	{
-		SetMetaData(m_Payloads[EntryIndex], {.RawSize = Value.RawSize, .RawHash = Value.RawHash});
+		SetMetaData(IndexLock, m_Payloads[EntryIndex], {.RawSize = Value.RawSize, .RawHash = Value.RawHash});
 	}
 	else
 	{
-		RemoveMetaData(m_Payloads[EntryIndex]);
+		RemoveMetaData(IndexLock, m_Payloads[EntryIndex]);
 	}
 
 	m_SlogFile.Append({.Key = HashKey, .Location = Loc});
@@ -2210,7 +2656,9 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::SetMetaData(BucketPayload& Payload, const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData)
+ZenCacheDiskLayer::CacheBucket::SetMetaData(RwLock::ExclusiveLockScope&,
+											BucketPayload&										  Payload,
+											const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData)
 {
 	if (Payload.MetaData)
 	{
@@ -2233,7 +2681,7 @@ ZenCacheDiskLayer::CacheBucket::SetMetaData(BucketPayload& Payload, const ZenCac
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::RemoveMetaData(BucketPayload& Payload)
+ZenCacheDiskLayer::CacheBucket::RemoveMetaData(RwLock::ExclusiveLockScope&, BucketPayload& Payload)
 {
 	if (Payload.MetaData)
 	{
@@ -2243,17 +2691,18 @@ ZenCacheDiskLayer::CacheBucket::RemoveMetaData(BucketPayload& Payload)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffer& MemCachedData)
+ZenCacheDiskLayer::CacheBucket::SetMemCachedData(RwLock::ExclusiveLockScope&, PayloadIndex PayloadIndex, IoBuffer& MemCachedData)
 {
-	uint64_t PayloadSize = MemCachedData.GetSize();
+	BucketPayload& Payload	   = m_Payloads[PayloadIndex];
+	uint64_t	   PayloadSize = MemCachedData.GetSize();
 	ZEN_ASSERT(PayloadSize != 0);
 	if (m_FreeMemCachedPayloads.empty())
 	{
 		if (m_MemCachedPayloads.size() != std::numeric_limits<uint32_t>::max())
 		{
 			Payload.MemCached = MemCachedIndex(gsl::narrow<uint32_t>(m_MemCachedPayloads.size()));
-			m_MemCachedPayloads.push_back(MemCachedData);
-			AddMemCacheUsage(PayloadSize);
+			m_MemCachedPayloads.emplace_back(MemCacheData{.Payload = MemCachedData, .OwnerIndex = PayloadIndex});
+			AddMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
 			m_MemoryWriteCount++;
 		}
 	}
@@ -2261,20 +2710,20 @@ ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffe
 	{
 		Payload.MemCached = m_FreeMemCachedPayloads.back();
 		m_FreeMemCachedPayloads.pop_back();
-		m_MemCachedPayloads[Payload.MemCached] = MemCachedData;
-		AddMemCacheUsage(PayloadSize);
+		m_MemCachedPayloads[Payload.MemCached] = MemCacheData{.Payload = MemCachedData, .OwnerIndex = PayloadIndex};
+		AddMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
 		m_MemoryWriteCount++;
 	}
 }
 
 size_t
-ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(BucketPayload& Payload)
+ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload)
 {
 	if (Payload.MemCached)
 	{
-		size_t PayloadSize = m_MemCachedPayloads[Payload.MemCached].GetSize();
-		RemoveMemCacheUsage(PayloadSize);
-		m_MemCachedPayloads[Payload.MemCached] = IoBuffer{};
+		size_t PayloadSize = m_MemCachedPayloads[Payload.MemCached].Payload.GetSize();
+		RemoveMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
+		m_MemCachedPayloads[Payload.MemCached] = {};
 		m_FreeMemCachedPayloads.push_back(Payload.MemCached);
 		Payload.MemCached = {};
 		return PayloadSize;
@@ -2283,7 +2732,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(BucketPayload& Payload)
 }
 
 ZenCacheDiskLayer::CacheBucket::BucketMetaData
-ZenCacheDiskLayer::CacheBucket::GetMetaData(const BucketPayload& Payload) const
+ZenCacheDiskLayer::CacheBucket::GetMetaData(RwLock::SharedLockScope&, const BucketPayload& Payload) const
 {
 	if (Payload.MetaData)
 	{
@@ -2316,14 +2765,18 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 								m_SlogFile.Append({.Key = HashKey, .Location = Location});
 
 								RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+								if (m_UpdatedKeys)
+								{
+									m_UpdatedKeys->insert(HashKey);
+								}
 								if (auto It = m_Index.find(HashKey); It != m_Index.end())
 								{
 									PayloadIndex EntryIndex = It.value();
 									ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size()));
 									BucketPayload& Payload = m_Payloads[EntryIndex];
 
-									RemoveMemCachedData(Payload);
-									RemoveMetaData(Payload);
+									RemoveMemCachedData(IndexLock, Payload);
+									RemoveMetaData(IndexLock, Payload);
 
 									Payload					  = (BucketPayload{.Location = Location});
 									m_AccessTimes[EntryIndex] = GcClock::TickCount();
@@ -2354,12 +2807,246 @@ ZenCacheDiskLayer::CacheBucket::GetGcName(GcCtx&)
 	return fmt::format("cachebucket:'{}'", m_BucketDir.string());
 }
 
-void
-ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
+class DiskBucketStoreCompactor : public GcStoreCompactor
 {
-	size_t									 TotalEntries = 0;
-	tsl::robin_set<IoHash, IoHash::Hasher>	 ExpiredInlineKeys;
-	std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+public:
+	DiskBucketStoreCompactor(ZenCacheDiskLayer::CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys)
+	: m_Bucket(Bucket)
+	, m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys))
+	{
+		m_ExpiredStandaloneKeys.shrink_to_fit();
+	}
+
+	virtual ~DiskBucketStoreCompactor() {}
+
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
+	{
+		ZEN_TRACE_CPU("Z$::Disk::Bucket::CompactStore");
+
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			Reset(m_ExpiredStandaloneKeys);
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: cachebucket [COMPACT] '{}': RemovedDisk: {} in {}",
+					 m_Bucket.m_BucketDir,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		if (!m_ExpiredStandaloneKeys.empty())
+		{
+			// Compact standalone items
+			size_t					   Skipped = 0;
+			ExtendablePathBuilder<256> Path;
+			for (const std::pair<IoHash, uint64_t>& ExpiredKey : m_ExpiredStandaloneKeys)
+			{
+				if (Ctx.IsCancelledFlag.load())
+				{
+					return;
+				}
+				Path.Reset();
+				m_Bucket.BuildPath(Path, ExpiredKey.first);
+				fs::path FilePath = Path.ToPath();
+
+				RwLock::SharedLockScope IndexLock(m_Bucket.m_IndexLock);
+				if (m_Bucket.m_Index.contains(ExpiredKey.first))
+				{
+					// Someone added it back, let the file on disk be
+					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': skipping z$ delete standalone of file '{}' FAILED, it has been added back",
+							  m_Bucket.m_BucketDir,
+							  Path.ToUtf8());
+					continue;
+				}
+
+				if (Ctx.Settings.IsDeleteMode)
+				{
+					RwLock::ExclusiveLockScope ValueLock(m_Bucket.LockForHash(ExpiredKey.first));
+					IndexLock.ReleaseNow();
+					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
+
+					std::error_code Ec;
+					if (!fs::remove(FilePath, Ec))
+					{
+						continue;
+					}
+					if (Ec)
+					{
+						ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': delete expired z$ standalone file '{}' FAILED, reason: '{}'",
+								 m_Bucket.m_BucketDir,
+								 Path.ToUtf8(),
+								 Ec.message());
+						continue;
+					}
+					Stats.RemovedDisk += ExpiredKey.second;
+				}
+				else
+				{
+					RwLock::SharedLockScope ValueLock(m_Bucket.LockForHash(ExpiredKey.first));
+					IndexLock.ReleaseNow();
+					ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8());
+
+					std::error_code Ec;
+					bool			Existed = std::filesystem::is_regular_file(FilePath, Ec);
+					if (Ec)
+					{
+						ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'",
+								 m_Bucket.m_BucketDir,
+								 FilePath,
+								 Ec.message());
+						continue;
+					}
+					if (!Existed)
+					{
+						continue;
+					}
+					Skipped++;
+				}
+			}
+			if (Skipped > 0)
+			{
+				ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': skipped deleting of {} eligible files", m_Bucket.m_BucketDir, Skipped);
+			}
+		}
+
+		if (Ctx.Settings.CollectSmallObjects)
+		{
+			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
+			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys.reset(); }); });
+
+			size_t					  InlineEntryCount = 0;
+			BlockStore::BlockUsageMap BlockUsage;
+			{
+				RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
+				for (const auto& Entry : m_Bucket.m_Index)
+				{
+					ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
+					const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+					const DiskLocation&									 Loc	 = Payload.Location;
+
+					if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+					{
+						continue;
+					}
+					InlineEntryCount++;
+					uint32_t BlockIndex = Loc.Location.BlockLocation.GetBlockIndex();
+					uint64_t ChunkSize	= RoundUp(Loc.Size(), m_Bucket.m_Configuration.PayloadAlignment);
+					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
+					{
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
+					}
+					else
+					{
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
+					}
+				}
+			}
+
+			{
+				BlockStoreCompactState BlockCompactState;
+				std::vector<IoHash>	   BlockCompactStateKeys;
+				BlockCompactStateKeys.reserve(InlineEntryCount);
+
+				BlockStore::BlockEntryCountMap BlocksToCompact =
+					m_Bucket.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
+				BlockCompactState.IncludeBlocks(BlocksToCompact);
+
+				if (BlocksToCompact.size() > 0)
+				{
+					{
+						RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
+						for (const auto& Entry : m_Bucket.m_Index)
+						{
+							ZenCacheDiskLayer::CacheBucket::PayloadIndex		 Index	 = Entry.second;
+							const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload = m_Bucket.m_Payloads[Index];
+							const DiskLocation&									 Loc	 = Payload.Location;
+
+							if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+							{
+								continue;
+							}
+							if (!BlockCompactState.AddKeepLocation(Loc.GetBlockLocation(m_Bucket.m_Configuration.PayloadAlignment)))
+							{
+								continue;
+							}
+							BlockCompactStateKeys.push_back(Entry.first);
+						}
+					}
+
+					if (Ctx.Settings.IsDeleteMode)
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: cachebucket [COMPACT] '{}': compacting {} blocks",
+									 m_Bucket.m_BucketDir,
+									 BlocksToCompact.size());
+						}
+
+						m_Bucket.m_BlockStore.CompactBlocks(
+							BlockCompactState,
+							m_Bucket.m_Configuration.PayloadAlignment,
+							[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+								std::vector<DiskIndexEntry> MovedEntries;
+								MovedEntries.reserve(MovedArray.size());
+								RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock);
+								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+								{
+									size_t		  ChunkIndex = Moved.first;
+									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+
+									if (m_Bucket.m_UpdatedKeys->contains(Key))
+									{
+										continue;
+									}
+
+									if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end())
+									{
+										ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	   = m_Bucket.m_Payloads[It->second];
+										const BlockStoreLocation&					   NewLocation = Moved.second;
+										Payload.Location										   = DiskLocation(NewLocation,
+																		  m_Bucket.m_Configuration.PayloadAlignment,
+																		  Payload.Location.GetFlags());
+										MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
+									}
+								}
+								m_Bucket.m_SlogFile.Append(MovedEntries);
+								Stats.RemovedDisk += FreedDiskSpace;
+								if (Ctx.IsCancelledFlag.load())
+								{
+									return false;
+								}
+								return true;
+							},
+							ClaimDiskReserveCallback);
+					}
+					else
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: cachebucket [COMPACT] '{}': skipped compacting of {} eligible blocks",
+									 m_Bucket.m_BucketDir,
+									 BlocksToCompact.size());
+						}
+					}
+				}
+			}
+		}
+	}
+
+private:
+	ZenCacheDiskLayer::CacheBucket&			 m_Bucket;
+	std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys;
+};
+
+GcStoreCompactor*
+ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
+{
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::RemoveExpiredData");
+
+	size_t TotalEntries = 0;
 
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
@@ -2367,36 +3054,38 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+		ZEN_INFO("GCV2: cachebucket [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}",
 				 m_BucketDir,
-				 Stats.Count,
-				 Stats.Expired,
-				 Stats.Deleted,
-				 NiceBytes(Stats.RemovedDisk),
-				 NiceBytes(Stats.RemovedMemory),
+				 Stats.CheckedCount,
+				 Stats.FoundCount,
+				 Stats.DeletedCount,
+				 NiceBytes(Stats.FreedMemory),
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count();
 
-	BlockStoreCompactState			 BlockCompactState;
-	BlockStore::ReclaimSnapshotState BlockSnapshotState;
-	std::vector<IoHash>				 BlockCompactStateKeys;
-	std::vector<DiskIndexEntry>		 ExpiredEntries;
-	uint64_t						 RemovedStandaloneSize = 0;
+	std::vector<DiskIndexEntry>				 ExpiredEntries;
+	std::vector<std::pair<IoHash, uint64_t>> ExpiredStandaloneKeys;
+	uint64_t								 RemovedStandaloneSize = 0;
 	{
 		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-		if (Ctx.Settings.CollectSmallObjects)
+		if (Ctx.IsCancelledFlag.load())
+		{
+			return nullptr;
+		}
+		if (m_Index.empty())
 		{
-			BlockSnapshotState = m_BlockStore.GetReclaimSnapshotState();
+			return nullptr;
 		}
+
 		TotalEntries = m_Index.size();
 
-		// Find out expired keys and affected blocks
+		// Find out expired keys
 		for (const auto& Entry : m_Index)
 		{
 			const IoHash& Key		 = Entry.first;
-			size_t		  EntryIndex = Entry.second;
+			PayloadIndex  EntryIndex = Entry.second;
 			GcClock::Tick AccessTime = m_AccessTimes[EntryIndex];
 			if (AccessTime >= ExpireTicks)
 			{
@@ -2415,40 +3104,16 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
 			}
 			else if (Ctx.Settings.CollectSmallObjects)
 			{
-				ExpiredInlineKeys.insert(Key);
-				uint32_t BlockIndex			= Payload.Location.Location.BlockLocation.GetBlockIndex();
-				bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
-				if (!IsActiveWriteBlock)
-				{
-					BlockCompactState.IncludeBlock(BlockIndex);
-				}
 				ExpiredEntries.push_back(ExpiredEntry);
 			}
 		}
 
-		Stats.Expired += ExpiredStandaloneKeys.size() + ExpiredInlineKeys.size();
+		Stats.CheckedCount += TotalEntries;
+		Stats.FoundCount += ExpiredEntries.size();
 
-		// Get all locations we need to keep for affected blocks
-		if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
+		if (Ctx.IsCancelledFlag.load())
 		{
-			for (const auto& Entry : m_Index)
-			{
-				const IoHash& Key = Entry.first;
-				if (ExpiredInlineKeys.contains(Key))
-				{
-					continue;
-				}
-				size_t				 EntryIndex = Entry.second;
-				const BucketPayload& Payload	= m_Payloads[EntryIndex];
-				if (Payload.Location.Flags & DiskLocation::kStandaloneFile)
-				{
-					continue;
-				}
-				if (BlockCompactState.AddKeepLocation(Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment)))
-				{
-					BlockCompactStateKeys.push_back(Key);
-				}
-			}
+			return nullptr;
 		}
 
 		if (Ctx.Settings.IsDeleteMode)
@@ -2458,132 +3123,291 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats&
 				auto It = m_Index.find(Entry.Key);
 				ZEN_ASSERT(It != m_Index.end());
 				BucketPayload& Payload = m_Payloads[It->second];
-				RemoveMetaData(Payload);
-				Stats.RemovedMemory += RemoveMemCachedData(Payload);
+				RemoveMetaData(IndexLock, Payload);
+				Stats.FreedMemory += RemoveMemCachedData(IndexLock, Payload);
 				m_Index.erase(It);
+				Stats.DeletedCount++;
 			}
 			m_SlogFile.Append(ExpiredEntries);
 			m_StandaloneSize.fetch_sub(RemovedStandaloneSize, std::memory_order::relaxed);
 		}
 	}
-	Stats.Count += TotalEntries;
 
-	if (ExpiredEntries.empty())
+	if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty())
 	{
-		return;
+		std::vector<BucketPayload>	Payloads;
+		std::vector<AccessTime>		AccessTimes;
+		std::vector<BucketMetaData> MetaDatas;
+		std::vector<MemCacheData>	MemCachedPayloads;
+		std::vector<ReferenceIndex> FirstReferenceIndex;
+		IndexMap					Index;
+		{
+			RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
+			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+		}
 	}
 
-	if (!Ctx.Settings.IsDeleteMode)
+	if (Ctx.IsCancelledFlag.load())
 	{
-		return;
+		return nullptr;
 	}
 
-	Stats.Deleted += ExpiredEntries.size();
-
-	// Compact standalone items
-	ExtendablePathBuilder<256> Path;
-	for (const std::pair<IoHash, uint64_t>& ExpiredKey : ExpiredStandaloneKeys)
-	{
-		Path.Reset();
-		BuildPath(Path, ExpiredKey.first);
-		fs::path FilePath = Path.ToPath();
+	return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys));
+}
 
-		RwLock::SharedLockScope IndexLock(m_IndexLock);
-		if (m_Index.contains(ExpiredKey.first))
-		{
-			// Someone added it back, let the file on disk be
-			ZEN_DEBUG("gc cache bucket '{}': skipping z$ delete standalone of file '{}' FAILED, it has been added back",
-					  m_BucketDir,
-					  Path.ToUtf8());
-			continue;
-		}
+class DiskBucketReferenceChecker : public GcReferenceChecker
+{
+	using PayloadIndex	 = ZenCacheDiskLayer::CacheBucket::PayloadIndex;
+	using BucketPayload	 = ZenCacheDiskLayer::CacheBucket::BucketPayload;
+	using CacheBucket	 = ZenCacheDiskLayer::CacheBucket;
+	using ReferenceIndex = ZenCacheDiskLayer::CacheBucket::ReferenceIndex;
 
-		RwLock::ExclusiveLockScope ValueLock(LockForHash(ExpiredKey.first));
-		IndexLock.ReleaseNow();
-		ZEN_DEBUG("gc cache bucket '{}': deleting standalone cache file '{}'", m_BucketDir, Path.ToUtf8());
+public:
+	DiskBucketReferenceChecker(CacheBucket& Owner) : m_CacheBucket(Owner) {}
 
-		std::error_code Ec;
-		if (!fs::remove(FilePath, Ec))
+	virtual ~DiskBucketReferenceChecker()
+	{
+		try
 		{
-			continue;
+			m_IndexLock.reset();
+			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
+			{
+				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+				// If reference caching is not enabled, we temporarily used the data structure for reference caching, lets reset it
+				m_CacheBucket.ClearReferenceCache();
+			}
 		}
-		if (Ec)
+		catch (std::exception& Ex)
 		{
-			ZEN_WARN("gc cache bucket '{}': delete expired z$ standalone file '{}' FAILED, reason: '{}'",
-					 m_BucketDir,
-					 Path.ToUtf8(),
-					 Ec.message());
-			continue;
+			ZEN_ERROR("~DiskBucketReferenceChecker threw exception: '{}'", Ex.what());
 		}
-		Stats.RemovedDisk += ExpiredKey.second;
 	}
 
-	if (Ctx.Settings.CollectSmallObjects && !ExpiredInlineKeys.empty())
+	virtual void PreCache(GcCtx& Ctx) override
 	{
-		// Compact block store
-		m_BlockStore.CompactBlocks(
-			BlockCompactState,
-			m_Configuration.PayloadAlignment,
-			[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
-				std::vector<DiskIndexEntry> MovedEntries;
-				RwLock::ExclusiveLockScope	_(m_IndexLock);
-				for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
-				{
-					size_t		  ChunkIndex = Moved.first;
-					const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+		ZEN_TRACE_CPU("Z$::Disk::Bucket::PreCache");
 
-					if (auto It = m_Index.find(Key); It != m_Index.end())
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: cachebucket [PRECACHE] '{}': found {} references in {}",
+					 m_CacheBucket.m_BucketDir,
+					 m_CacheBucket.m_ReferenceCount,
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		std::vector<IoHash> UpdateKeys;
+		std::vector<size_t> ReferenceCounts;
+		std::vector<IoHash> References;
+
+		auto GetAttachments = [&References, &ReferenceCounts](const void* CbObjectData) {
+			size_t		 CurrentReferenceCount = References.size();
+			CbObjectView Obj(CbObjectData);
+			Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
+			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
+		};
+
+		// Refresh cache
+		{
+			// If reference caching is enabled the references will be updated at modification for us so we don't need to track modifications
+			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
+			{
+				m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
+			}
+
+			std::vector<IoHash> StandaloneKeys;
+			{
+				std::vector<IoHash>						  InlineKeys;
+				std::unordered_map<uint32_t, std::size_t> BlockIndexToEntriesPerBlockIndex;
+				struct InlineEntry
+				{
+					uint32_t InlineKeyIndex;
+					uint32_t Offset;
+					uint32_t Size;
+				};
+				std::vector<std::vector<InlineEntry>> EntriesPerBlock;
+				size_t								  UpdateCount = 0;
+				{
+					RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
+					for (const auto& Entry : m_CacheBucket.m_Index)
 					{
-						BucketPayload&			  Payload	  = m_Payloads[It->second];
-						const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
-						if (Payload.Location.GetBlockLocation(m_Configuration.PayloadAlignment) != OldLocation)
+						if (Ctx.IsCancelledFlag.load())
+						{
+							IndexLock.ReleaseNow();
+							m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+							return;
+						}
+
+						PayloadIndex		 EntryIndex = Entry.second;
+						const BucketPayload& Payload	= m_CacheBucket.m_Payloads[EntryIndex];
+						const DiskLocation&	 Loc		= Payload.Location;
+
+						if (!Loc.IsFlagSet(DiskLocation::kStructured))
+						{
+							continue;
+						}
+						if (m_CacheBucket.m_Configuration.EnableReferenceCaching &&
+							m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
+						{
+							continue;
+						}
+						UpdateCount++;
+						const IoHash& Key = Entry.first;
+						if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 						{
-							// Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d at a later
-							// time
+							StandaloneKeys.push_back(Key);
 							continue;
 						}
 
-						const BlockStoreLocation& NewLocation = Moved.second;
+						BlockStoreLocation ChunkLocation = Loc.GetBlockLocation(m_CacheBucket.m_Configuration.PayloadAlignment);
+						InlineEntry		   UpdateEntry	 = {.InlineKeyIndex = gsl::narrow<uint32_t>(InlineKeys.size()),
+															.Offset			= gsl::narrow<uint32_t>(ChunkLocation.Offset),
+															.Size			= gsl::narrow<uint32_t>(ChunkLocation.Size)};
+						InlineKeys.push_back(Key);
 
-						Payload.Location = DiskLocation(NewLocation, m_Configuration.PayloadAlignment, Payload.Location.GetFlags());
-						MovedEntries.push_back({.Key = Key, .Location = Payload.Location});
+						if (auto It = BlockIndexToEntriesPerBlockIndex.find(ChunkLocation.BlockIndex);
+							It != BlockIndexToEntriesPerBlockIndex.end())
+						{
+							EntriesPerBlock[It->second].emplace_back(UpdateEntry);
+						}
+						else
+						{
+							BlockIndexToEntriesPerBlockIndex.insert_or_assign(ChunkLocation.BlockIndex, EntriesPerBlock.size());
+							EntriesPerBlock.emplace_back(std::vector<InlineEntry>{UpdateEntry});
+						}
 					}
 				}
-				m_SlogFile.Append(MovedEntries);
-				Stats.RemovedDisk += FreedDiskSpace;
-			},
-			[&]() { return 0; });
-	}
 
-	std::vector<BucketPayload>	Payloads;
-	std::vector<AccessTime>		AccessTimes;
-	std::vector<BucketMetaData> MetaDatas;
-	std::vector<IoBuffer>		MemCachedPayloads;
-	std::vector<ReferenceIndex> FirstReferenceIndex;
-	IndexMap					Index;
-	{
-		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-		CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
-	}
-}
+				UpdateKeys.reserve(UpdateCount);
 
-class DiskBucketReferenceChecker : public GcReferenceChecker
-{
-public:
-	DiskBucketReferenceChecker(ZenCacheDiskLayer::CacheBucket& Owner) : m_CacheBucket(Owner) {}
+				for (auto It : BlockIndexToEntriesPerBlockIndex)
+				{
+					uint32_t BlockIndex = It.first;
+
+					Ref<BlockStoreFile> BlockFile = m_CacheBucket.m_BlockStore.GetBlockFile(BlockIndex);
+					if (BlockFile)
+					{
+						size_t					  EntriesPerBlockIndex = It.second;
+						std::vector<InlineEntry>& InlineEntries		   = EntriesPerBlock[EntriesPerBlockIndex];
+
+						std::sort(InlineEntries.begin(), InlineEntries.end(), [&](const InlineEntry& Lhs, const InlineEntry& Rhs) -> bool {
+							return Lhs.Offset < Rhs.Offset;
+						});
+
+						uint64_t		BlockFileSize = BlockFile->FileSize();
+						BasicFileBuffer BlockBuffer(BlockFile->GetBasicFile(), 32768);
+						for (const InlineEntry& InlineEntry : InlineEntries)
+						{
+							if ((InlineEntry.Offset + InlineEntry.Size) > BlockFileSize)
+							{
+								ReferenceCounts.push_back(0);
+							}
+							else
+							{
+								MemoryView ChunkView = BlockBuffer.MakeView(InlineEntry.Size, InlineEntry.Offset);
+								if (ChunkView.GetSize() == InlineEntry.Size)
+								{
+									GetAttachments(ChunkView.GetData());
+								}
+								else
+								{
+									std::vector<uint8_t> Buffer(InlineEntry.Size);
+									BlockBuffer.Read(Buffer.data(), InlineEntry.Size, InlineEntry.Offset);
+									GetAttachments(Buffer.data());
+								}
+							}
+							const IoHash& Key = InlineKeys[InlineEntry.InlineKeyIndex];
+							UpdateKeys.push_back(Key);
+						}
+					}
+				}
+			}
+			{
+				for (const IoHash& Key : StandaloneKeys)
+				{
+					if (Ctx.IsCancelledFlag.load())
+					{
+						m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+						return;
+					}
+
+					IoBuffer Buffer = m_CacheBucket.GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
+					if (!Buffer)
+					{
+						continue;
+					}
+
+					GetAttachments(Buffer.GetData());
+					UpdateKeys.push_back(Key);
+				}
+			}
+		}
 
-	virtual ~DiskBucketReferenceChecker()
-	{
-		m_IndexLock.reset();
-		if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
 		{
-			// If reference caching is not enabled, we temporarily used the data structure for reference caching, lets reset it
-			m_CacheBucket.ClearReferenceCache();
+			size_t					   ReferenceOffset = 0;
+			RwLock::ExclusiveLockScope IndexLock(m_CacheBucket.m_IndexLock);
+
+			if (!m_CacheBucket.m_Configuration.EnableReferenceCaching)
+			{
+				ZEN_ASSERT(m_CacheBucket.m_FirstReferenceIndex.empty());
+				ZEN_ASSERT(m_CacheBucket.m_ReferenceHashes.empty());
+				ZEN_ASSERT(m_CacheBucket.m_NextReferenceHashesIndexes.empty());
+				ZEN_ASSERT(m_CacheBucket.m_ReferenceCount == 0);
+				ZEN_ASSERT(m_CacheBucket.m_UpdatedKeys);
+
+				// If reference caching is not enabled, we will resize and use the data structure in place for reference caching when
+				// we figure out what this bucket references. This will be reset once the DiskBucketReferenceChecker is deleted.
+				m_CacheBucket.m_FirstReferenceIndex.resize(m_CacheBucket.m_Payloads.size(), ReferenceIndex::Unknown());
+				m_CacheBucket.m_ReferenceHashes.reserve(References.size());
+				m_CacheBucket.m_NextReferenceHashesIndexes.reserve(References.size());
+			}
+			else
+			{
+				ZEN_ASSERT(!m_CacheBucket.m_UpdatedKeys);
+			}
+
+			for (size_t Index = 0; Index < UpdateKeys.size(); Index++)
+			{
+				const IoHash& Key			 = UpdateKeys[Index];
+				size_t		  ReferenceCount = ReferenceCounts[Index];
+				if (auto It = m_CacheBucket.m_Index.find(Key); It != m_CacheBucket.m_Index.end())
+				{
+					PayloadIndex EntryIndex = It->second;
+					if (m_CacheBucket.m_Configuration.EnableReferenceCaching)
+					{
+						if (m_CacheBucket.m_FirstReferenceIndex[EntryIndex] != ReferenceIndex::Unknown())
+						{
+							// The reference data is valid and what we have is old/redundant
+							continue;
+						}
+					}
+					else if (m_CacheBucket.m_UpdatedKeys->contains(Key))
+					{
+						// Our pre-cache data is invalid
+						continue;
+					}
+
+					m_CacheBucket.SetReferences(IndexLock,
+												m_CacheBucket.m_FirstReferenceIndex[EntryIndex],
+												std::span<IoHash>{References.data() + ReferenceOffset, ReferenceCount});
+				}
+				ReferenceOffset += ReferenceCount;
+			}
+
+			if (m_CacheBucket.m_Configuration.EnableReferenceCaching && !UpdateKeys.empty())
+			{
+				m_CacheBucket.CompactReferences(IndexLock);
+			}
 		}
 	}
 
 	virtual void LockState(GcCtx& Ctx) override
 	{
+		ZEN_TRACE_CPU("Z$::Disk::Bucket::LockState");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
@@ -2597,22 +3421,42 @@ public:
 		});
 
 		m_IndexLock = std::make_unique<RwLock::SharedLockScope>(m_CacheBucket.m_IndexLock);
-
-		// Rescan to see if any cache items needs refreshing since last pass when we had the lock
-		for (const auto& Entry : m_CacheBucket.m_Index)
+		if (Ctx.IsCancelledFlag.load())
 		{
-			size_t												 PayloadIndex = Entry.second;
-			const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	  = m_CacheBucket.m_Payloads[PayloadIndex];
-			const DiskLocation&									 Loc		  = Payload.Location;
+			m_UncachedReferences.clear();
+			m_IndexLock.reset();
+			m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+			return;
+		}
 
-			if (!Loc.IsFlagSet(DiskLocation::kStructured))
-			{
-				continue;
-			}
-			ZEN_ASSERT(!m_CacheBucket.m_FirstReferenceIndex.empty());
-			const IoHash& Key = Entry.first;
-			if (m_CacheBucket.m_FirstReferenceIndex[PayloadIndex] == ZenCacheDiskLayer::CacheBucket::ReferenceIndex::Unknown())
+		if (m_CacheBucket.m_UpdatedKeys)
+		{
+			const HashSet& UpdatedKeys(*m_CacheBucket.m_UpdatedKeys);
+			for (const IoHash& Key : UpdatedKeys)
 			{
+				if (Ctx.IsCancelledFlag.load())
+				{
+					m_UncachedReferences.clear();
+					m_IndexLock.reset();
+					m_CacheBucket.m_IndexLock.WithExclusiveLock([&]() { m_CacheBucket.m_UpdatedKeys.reset(); });
+					return;
+				}
+
+				auto It = m_CacheBucket.m_Index.find(Key);
+				if (It == m_CacheBucket.m_Index.end())
+				{
+					continue;
+				}
+
+				PayloadIndex		 EntryIndex = It->second;
+				const BucketPayload& Payload	= m_CacheBucket.m_Payloads[EntryIndex];
+				const DiskLocation&	 Loc		= Payload.Location;
+
+				if (!Loc.IsFlagSet(DiskLocation::kStructured))
+				{
+					continue;
+				}
+
 				IoBuffer Buffer;
 				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 				{
@@ -2633,21 +3477,48 @@ public:
 		}
 	}
 
-	virtual void RemoveUsedReferencesFromSet(GcCtx&, HashSet& IoCids) override
+	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
 	{
+		ZEN_TRACE_CPU("Z$::Disk::Bucket::RemoveUsedReferencesFromSet");
+
 		ZEN_ASSERT(m_IndexLock);
+		size_t	   InitialCount = IoCids.size();
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: cachebucket [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}",
+					 m_CacheBucket.m_BucketDir,
+					 InitialCount - IoCids.size(),
+					 InitialCount,
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
 
 		for (const IoHash& ReferenceHash : m_CacheBucket.m_ReferenceHashes)
 		{
-			IoCids.erase(ReferenceHash);
+			if (IoCids.erase(ReferenceHash) == 1)
+			{
+				if (IoCids.empty())
+				{
+					return;
+				}
+			}
 		}
 
 		for (const IoHash& ReferenceHash : m_UncachedReferences)
 		{
-			IoCids.erase(ReferenceHash);
+			if (IoCids.erase(ReferenceHash) == 1)
+			{
+				if (IoCids.empty())
+				{
+					return;
+				}
+			}
 		}
 	}
-	ZenCacheDiskLayer::CacheBucket&			 m_CacheBucket;
+	CacheBucket&							 m_CacheBucket;
 	std::unique_ptr<RwLock::SharedLockScope> m_IndexLock;
 	HashSet									 m_UncachedReferences;
 };
@@ -2655,119 +3526,22 @@ public:
 std::vector<GcReferenceChecker*>
 ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 {
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::CreateReferenceCheckers");
+
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
 		if (!Ctx.Settings.Verbose)
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': found {} references in {}",
-				 m_BucketDir,
-				 m_ReferenceCount,
-				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': completed in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
-	std::vector<IoHash> UpdateKeys;
-	std::vector<IoHash> StandaloneKeys;
-	std::vector<size_t> ReferenceCounts;
-	std::vector<IoHash> References;
-
-	// Refresh cache
-	{
-		RwLock::SharedLockScope IndexLock(m_IndexLock);
-		for (const auto& Entry : m_Index)
-		{
-			size_t												 PayloadIndex = Entry.second;
-			const ZenCacheDiskLayer::CacheBucket::BucketPayload& Payload	  = m_Payloads[PayloadIndex];
-			const DiskLocation&									 Loc		  = Payload.Location;
-
-			if (!Loc.IsFlagSet(DiskLocation::kStructured))
-			{
-				continue;
-			}
-			if (m_Configuration.EnableReferenceCaching &&
-				m_FirstReferenceIndex[PayloadIndex] != ZenCacheDiskLayer::CacheBucket::ReferenceIndex::Unknown())
-			{
-				continue;
-			}
-			const IoHash& Key = Entry.first;
-			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
-			{
-				StandaloneKeys.push_back(Key);
-				continue;
-			}
-			IoBuffer Buffer = GetInlineCacheValue(Loc);
-			if (!Buffer)
-			{
-				UpdateKeys.push_back(Key);
-				ReferenceCounts.push_back(0);
-				continue;
-			}
-			size_t CurrentReferenceCount = References.size();
-			{
-				CbObjectView Obj(Buffer.GetData());
-				Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
-				Buffer = {};
-			}
-			UpdateKeys.push_back(Key);
-			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
-		}
-	}
-	{
-		for (const IoHash& Key : StandaloneKeys)
-		{
-			IoBuffer Buffer = GetStandaloneCacheValue(ZenContentType::kCbObject, Key);
-			if (!Buffer)
-			{
-				continue;
-			}
-
-			size_t CurrentReferenceCount = References.size();
-			{
-				CbObjectView Obj(Buffer.GetData());
-				Obj.IterateAttachments([&References](CbFieldView Field) { References.emplace_back(Field.AsAttachment()); });
-				Buffer = {};
-			}
-			UpdateKeys.push_back(Key);
-			ReferenceCounts.push_back(References.size() - CurrentReferenceCount);
-		}
-	}
-
 	{
-		size_t					   ReferenceOffset = 0;
-		RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-		if (!m_Configuration.EnableReferenceCaching)
-		{
-			ZEN_ASSERT(m_FirstReferenceIndex.empty());
-			ZEN_ASSERT(m_ReferenceHashes.empty());
-			ZEN_ASSERT(m_NextReferenceHashesIndexes.empty());
-			ZEN_ASSERT(m_ReferenceCount == 0);
-			// If reference caching is not enabled, we will resize and use the data structure in place for reference caching when
-			// we figure out what this bucket references. This will be reset once the DiskBucketReferenceChecker is deleted.
-			m_FirstReferenceIndex.resize(m_Payloads.size());
-		}
-		for (size_t Index = 0; Index < UpdateKeys.size(); Index++)
-		{
-			const IoHash& Key			 = UpdateKeys[Index];
-			size_t		  ReferenceCount = ReferenceCounts[Index];
-			auto		  It			 = m_Index.find(Key);
-			if (It == m_Index.end())
-			{
-				ReferenceOffset += ReferenceCount;
-				continue;
-			}
-			if (m_FirstReferenceIndex[It->second] != ReferenceIndex::Unknown())
-			{
-				continue;
-			}
-			SetReferences(IndexLock,
-						  m_FirstReferenceIndex[It->second],
-						  std::span<IoHash>{References.data() + ReferenceOffset, ReferenceCount});
-			ReferenceOffset += ReferenceCount;
-		}
-		if (m_Configuration.EnableReferenceCaching)
+		RwLock::SharedLockScope __(m_IndexLock);
+		if (m_Index.empty())
 		{
-			CompactReferences(IndexLock);
+			return {};
 		}
 	}
 
@@ -2777,6 +3551,8 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 void
 ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
 {
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::CompactReferences");
+
 	std::vector<ReferenceIndex> FirstReferenceIndex;
 	std::vector<IoHash>			NewReferenceHashes;
 	std::vector<ReferenceIndex> NewNextReferenceHashesIndexes;
@@ -2813,7 +3589,9 @@ ZenCacheDiskLayer::CacheBucket::CompactReferences(RwLock::ExclusiveLockScope&)
 	}
 	m_FirstReferenceIndex.swap(FirstReferenceIndex);
 	m_ReferenceHashes.swap(NewReferenceHashes);
+	m_ReferenceHashes.shrink_to_fit();
 	m_NextReferenceHashesIndexes.swap(NewNextReferenceHashesIndexes);
+	m_NextReferenceHashesIndexes.shrink_to_fit();
 	m_ReferenceCount = m_ReferenceHashes.size();
 }
 
@@ -2940,24 +3718,24 @@ void
 ZenCacheDiskLayer::CacheBucket::ClearReferenceCache()
 {
 	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-	m_FirstReferenceIndex.clear();
-	m_FirstReferenceIndex.shrink_to_fit();
-	m_ReferenceHashes.clear();
-	m_ReferenceHashes.shrink_to_fit();
-	m_NextReferenceHashesIndexes.clear();
-	m_NextReferenceHashesIndexes.shrink_to_fit();
+	Reset(m_FirstReferenceIndex);
+	Reset(m_ReferenceHashes);
+	Reset(m_NextReferenceHashesIndexes);
 	m_ReferenceCount = 0;
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloads,
+ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
+											 std::vector<BucketPayload>&  Payloads,
 											 std::vector<AccessTime>&	  AccessTimes,
 											 std::vector<BucketMetaData>& MetaDatas,
-											 std::vector<IoBuffer>&		  MemCachedPayloads,
+											 std::vector<MemCacheData>&	  MemCachedPayloads,
 											 std::vector<ReferenceIndex>& FirstReferenceIndex,
 											 IndexMap&					  Index,
 											 RwLock::ExclusiveLockScope&  IndexLock)
 {
+	ZEN_TRACE_CPU("Z$::Disk::Bucket::CompactState");
+
 	size_t EntryCount = m_Index.size();
 	Payloads.reserve(EntryCount);
 	AccessTimes.reserve(EntryCount);
@@ -2966,6 +3744,8 @@ ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloa
 		FirstReferenceIndex.reserve(EntryCount);
 	}
 	Index.reserve(EntryCount);
+	Index.min_load_factor(IndexMinLoadFactor);
+	Index.max_load_factor(IndexMaxLoadFactor);
 	for (auto It : m_Index)
 	{
 		PayloadIndex EntryIndex = PayloadIndex(Payloads.size());
@@ -2975,11 +3755,12 @@ ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloa
 		if (Payload.MetaData)
 		{
 			MetaDatas.push_back(m_MetaDatas[Payload.MetaData]);
-			Payload.MetaData = MetaDataIndex(m_MetaDatas.size() - 1);
+			Payload.MetaData = MetaDataIndex(MetaDatas.size() - 1);
 		}
 		if (Payload.MemCached)
 		{
-			MemCachedPayloads.push_back(std::move(m_MemCachedPayloads[Payload.MemCached]));
+			MemCachedPayloads.emplace_back(
+				MemCacheData{.Payload = std::move(m_MemCachedPayloads[Payload.MemCached].Payload), .OwnerIndex = EntryIndex});
 			Payload.MemCached = MemCachedIndex(gsl::narrow<uint32_t>(MemCachedPayloads.size() - 1));
 		}
 		if (m_Configuration.EnableReferenceCaching)
@@ -2992,11 +3773,9 @@ ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloa
 	m_Payloads.swap(Payloads);
 	m_AccessTimes.swap(AccessTimes);
 	m_MetaDatas.swap(MetaDatas);
-	m_FreeMetaDatas.clear();
-	m_FreeMetaDatas.shrink_to_fit();
+	Reset(m_FreeMetaDatas);
 	m_MemCachedPayloads.swap(MemCachedPayloads);
-	m_FreeMemCachedPayloads.clear();
-	m_FreeMetaDatas.shrink_to_fit();
+	Reset(m_FreeMemCachedPayloads);
 	if (m_Configuration.EnableReferenceCaching)
 	{
 		m_FirstReferenceIndex.swap(FirstReferenceIndex);
@@ -3031,124 +3810,99 @@ ZenCacheDiskLayer::ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const st
 
 ZenCacheDiskLayer::~ZenCacheDiskLayer()
 {
-}
-
-bool
-ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	ZEN_TRACE_CPU("Z$::Disk::Get");
-
-	const auto	 BucketName = std::string(InBucket);
-	CacheBucket* Bucket		= nullptr;
-
+	try
 	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto It = m_Buckets.find(BucketName);
-
-		if (It != m_Buckets.end())
 		{
-			Bucket = It->second.get();
+			RwLock::ExclusiveLockScope _(m_Lock);
+			for (auto& It : m_Buckets)
+			{
+				m_DroppedBuckets.emplace_back(std::move(It.second));
+			}
+			m_Buckets.clear();
 		}
+		// We destroy the buckets without holding a lock since destructor calls GcManager::RemoveGcReferencer which takes an exclusive lock.
+		// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
+		m_DroppedBuckets.clear();
 	}
-
-	if (Bucket == nullptr)
+	catch (std::exception& Ex)
 	{
-		// Bucket needs to be opened/created
+		ZEN_ERROR("~ZenCacheDiskLayer() failed. Reason: '{}'", Ex.what());
+	}
+}
 
-		RwLock::ExclusiveLockScope _(m_Lock);
+ZenCacheDiskLayer::CacheBucket*
+ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
+{
+	ZEN_TRACE_CPU("Z$::Disk::GetOrCreateBucket");
+	const auto BucketName = std::string(InBucket);
 
+	{
+		RwLock::SharedLockScope SharedLock(m_Lock);
 		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
-			Bucket = It->second.get();
-		}
-		else
-		{
-			auto InsertResult =
-				m_Buckets.emplace(BucketName,
-								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
-			Bucket = InsertResult.first->second.get();
-
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName;
-
-			if (!Bucket->OpenOrCreate(BucketPath))
-			{
-				m_Buckets.erase(InsertResult.first);
-				return false;
-			}
+			return It->second.get();
 		}
 	}
 
-	ZEN_ASSERT(Bucket != nullptr);
-	if (Bucket->Get(HashKey, OutValue))
+	// We create the bucket without holding a lock since contructor calls GcManager::AddGcReferencer which takes an exclusive lock.
+	// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
+	std::unique_ptr<CacheBucket> Bucket(
+		std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
+
+	RwLock::ExclusiveLockScope Lock(m_Lock);
+	if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 	{
-		TryMemCacheTrim();
-		return true;
+		return It->second.get();
 	}
-	return false;
-}
-
-void
-ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
-{
-	ZEN_TRACE_CPU("Z$::Disk::Put");
-
-	const auto	 BucketName = std::string(InBucket);
-	CacheBucket* Bucket		= nullptr;
 
+	std::filesystem::path BucketPath = m_RootDir;
+	BucketPath /= BucketName;
+	try
 	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto It = m_Buckets.find(BucketName);
-
-		if (It != m_Buckets.end())
+		if (!Bucket->OpenOrCreate(BucketPath))
 		{
-			Bucket = It->second.get();
+			ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+			return nullptr;
 		}
 	}
-
-	if (Bucket == nullptr)
+	catch (const std::exception& Err)
 	{
-		// New bucket needs to be created
+		ZEN_WARN("Creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
+		throw;
+	}
 
-		RwLock::ExclusiveLockScope _(m_Lock);
+	CacheBucket* Result = Bucket.get();
+	m_Buckets.emplace(BucketName, std::move(Bucket));
 
-		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
-		{
-			Bucket = It->second.get();
-		}
-		else
-		{
-			auto InsertResult =
-				m_Buckets.emplace(BucketName,
-								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
-			Bucket = InsertResult.first->second.get();
+	return Result;
+}
 
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName;
+bool
+ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	ZEN_TRACE_CPU("Z$::Disk::Get");
 
-			try
-			{
-				if (!Bucket->OpenOrCreate(BucketPath))
-				{
-					ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
-					m_Buckets.erase(InsertResult.first);
-					return;
-				}
-			}
-			catch (const std::exception& Err)
-			{
-				ZEN_WARN("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
-				throw;
-			}
+	if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr)
+	{
+		if (Bucket->Get(HashKey, OutValue))
+		{
+			TryMemCacheTrim();
+			return true;
 		}
 	}
+	return false;
+}
 
-	ZEN_ASSERT(Bucket != nullptr);
+void
+ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
+{
+	ZEN_TRACE_CPU("Z$::Disk::Put");
 
-	Bucket->Put(HashKey, Value, References);
-	TryMemCacheTrim();
+	if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr)
+	{
+		Bucket->Put(HashKey, Value, References);
+		TryMemCacheTrim();
+	}
 }
 
 void
@@ -3208,11 +3962,8 @@ ZenCacheDiskLayer::DiscoverBuckets()
 
 	RwLock SyncLock;
 
-	const size_t MaxHwTreadUse		   = std::thread::hardware_concurrency();
-	const int	 WorkerThreadPoolCount = gsl::narrow<int>(Min(MaxHwTreadUse, FoundBucketDirectories.size()));
-
-	WorkerThreadPool Pool(WorkerThreadPoolCount);
-	Latch			 WorkLatch(1);
+	WorkerThreadPool& Pool = GetLargeWorkerPool();
+	Latch			  WorkLatch(1);
 	for (auto& BucketPath : FoundBucketDirectories)
 	{
 		WorkLatch.AddCount(1);
@@ -3301,13 +4052,17 @@ void
 ZenCacheDiskLayer::Flush()
 {
 	std::vector<CacheBucket*> Buckets;
+	Stopwatch				  Timer;
+	const auto				  _ = MakeGuard([&] {
+		   if (Buckets.empty())
+		   {
+			   return;
+		   }
+		   ZEN_INFO("Flushed {} buckets at '{}' in {}", Buckets.size(), m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	   });
 
 	{
-		RwLock::SharedLockScope _(m_Lock);
-		if (m_Buckets.empty())
-		{
-			return;
-		}
+		RwLock::SharedLockScope __(m_Lock);
 		Buckets.reserve(m_Buckets.size());
 		for (auto& Kv : m_Buckets)
 		{
@@ -3315,28 +4070,29 @@ ZenCacheDiskLayer::Flush()
 			Buckets.push_back(Bucket);
 		}
 	}
-	const size_t MaxHwTreadUse		   = Max((std::thread::hardware_concurrency() / 4u), 1u);
-	const int	 WorkerThreadPoolCount = gsl::narrow<int>(Min(MaxHwTreadUse, Buckets.size()));
-
-	WorkerThreadPool Pool(WorkerThreadPoolCount);
-	Latch			 WorkLatch(1);
-	for (auto& Bucket : Buckets)
 	{
-		WorkLatch.AddCount(1);
-		Pool.ScheduleWork([&]() {
-			auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
-			Bucket->Flush();
-		});
+		WorkerThreadPool& Pool = GetSmallWorkerPool();
+		Latch			  WorkLatch(1);
+		for (auto& Bucket : Buckets)
+		{
+			WorkLatch.AddCount(1);
+			Pool.ScheduleWork([&]() {
+				auto _ = MakeGuard([&]() { WorkLatch.CountDown(); });
+				Bucket->Flush();
+			});
+		}
+		WorkLatch.CountDown();
+		while (!WorkLatch.Wait(1000))
+		{
+			ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", WorkLatch.Remaining(), m_RootDir);
+		}
 	}
-	WorkLatch.CountDown();
-	WorkLatch.Wait();
 }
 
 void
 ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
 {
 	RwLock::SharedLockScope _(m_Lock);
-
 	{
 		std::vector<std::future<void>> Results;
 		Results.reserve(m_Buckets.size());
@@ -3457,19 +4213,21 @@ ZenCacheDiskLayer::EnumerateBucketContents(std::string_view
 CacheValueDetails::NamespaceDetails
 ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
 {
-	RwLock::SharedLockScope				_(m_Lock);
 	CacheValueDetails::NamespaceDetails Details;
-	if (BucketFilter.empty())
 	{
-		Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
-		for (auto& Kv : m_Buckets)
+		RwLock::SharedLockScope IndexLock(m_Lock);
+		if (BucketFilter.empty())
 		{
-			Details.Buckets[Kv.first] = Kv.second->GetValueDetails(ValueFilter);
+			Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
+			for (auto& Kv : m_Buckets)
+			{
+				Details.Buckets[Kv.first] = Kv.second->GetValueDetails(IndexLock, ValueFilter);
+			}
+		}
+		else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
+		{
+			Details.Buckets[It->first] = It->second->GetValueDetails(IndexLock, ValueFilter);
 		}
-	}
-	else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
-	{
-		Details.Buckets[It->first] = It->second->GetValueDetails(ValueFilter);
 	}
 	return Details;
 }
@@ -3480,17 +4238,8 @@ ZenCacheDiskLayer::MemCacheTrim()
 	ZEN_TRACE_CPU("Z$::Disk::MemCacheTrim");
 
 	ZEN_ASSERT(m_Configuration.MemCacheTargetFootprintBytes != 0);
-
-	const GcClock::TimePoint Now = GcClock::Now();
-
-	const GcClock::Tick		   NowTick			   = Now.time_since_epoch().count();
-	const std::chrono::seconds TrimInterval		   = std::chrono::seconds(m_Configuration.MemCacheTrimIntervalSeconds);
-	GcClock::Tick			   LastTrimTick		   = m_LastTickMemCacheTrim;
-	const GcClock::Tick		   NextAllowedTrimTick = LastTrimTick + GcClock::Duration(TrimInterval).count();
-	if (NowTick < NextAllowedTrimTick)
-	{
-		return;
-	}
+	ZEN_ASSERT(m_Configuration.MemCacheMaxAgeSeconds != 0);
+	ZEN_ASSERT(m_Configuration.MemCacheTrimIntervalSeconds != 0);
 
 	bool Expected = false;
 	if (!m_IsMemCacheTrimming.compare_exchange_strong(Expected, true))
@@ -3498,75 +4247,90 @@ ZenCacheDiskLayer::MemCacheTrim()
 		return;
 	}
 
-	// Bump time forward so we don't keep trying to do m_IsTrimming.compare_exchange_strong
-	const GcClock::Tick NextTrimTick = NowTick + GcClock::Duration(TrimInterval).count();
-	m_LastTickMemCacheTrim.store(NextTrimTick);
+	try
+	{
+		m_JobQueue.QueueJob("ZenCacheDiskLayer::MemCacheTrim", [this](JobContext&) {
+			ZEN_TRACE_CPU("Z$::ZenCacheDiskLayer::MemCacheTrim [Async]");
+
+			const std::chrono::seconds TrimInterval = std::chrono::seconds(m_Configuration.MemCacheTrimIntervalSeconds);
+			uint64_t				   TrimmedSize	= 0;
+			Stopwatch				   Timer;
+			const auto				   Guard = MakeGuard([&] {
+				ZEN_INFO("trimmed {} (remaining {}), from memory cache in {}",
+						 NiceBytes(TrimmedSize),
+						 NiceBytes(m_TotalMemCachedSize),
+						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+
+				const GcClock::Tick NowTick		 = GcClock::TickCount();
+				const GcClock::Tick NextTrimTick = NowTick + GcClock::Duration(TrimInterval).count();
+				m_NextAllowedTrimTick.store(NextTrimTick);
+				m_IsMemCacheTrimming.store(false);
+			});
 
-	m_JobQueue.QueueJob("ZenCacheDiskLayer::MemCacheTrim", [this, Now, TrimInterval](JobContext&) {
-		ZEN_TRACE_CPU("Z$::ZenCacheDiskLayer::MemCacheTrim [Async]");
+			const std::chrono::seconds MaxAge = std::chrono::seconds(m_Configuration.MemCacheMaxAgeSeconds);
 
-		uint64_t   StartSize = m_TotalMemCachedSize.load();
-		Stopwatch  Timer;
-		const auto Guard = MakeGuard([&] {
-			uint64_t EndSize = m_TotalMemCachedSize.load();
-			ZEN_INFO("trimmed {} (remaining {}), from memory cache in {}",
-					 NiceBytes(StartSize > EndSize ? StartSize - EndSize : 0),
-					 NiceBytes(m_TotalMemCachedSize),
-					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
-			m_IsMemCacheTrimming.store(false);
-		});
-
-		const std::chrono::seconds MaxAge = std::chrono::seconds(m_Configuration.MemCacheMaxAgeSeconds);
+			static const size_t	  UsageSlotCount = 2048;
+			std::vector<uint64_t> UsageSlots;
+			UsageSlots.reserve(UsageSlotCount);
 
-		std::vector<uint64_t> UsageSlots;
-		UsageSlots.reserve(std::chrono::seconds(MaxAge / TrimInterval).count());
+			std::vector<CacheBucket*> Buckets;
+			{
+				RwLock::SharedLockScope __(m_Lock);
+				Buckets.reserve(m_Buckets.size());
+				for (auto& Kv : m_Buckets)
+				{
+					Buckets.push_back(Kv.second.get());
+				}
+			}
 
-		std::vector<CacheBucket*> Buckets;
-		{
-			RwLock::SharedLockScope __(m_Lock);
-			Buckets.reserve(m_Buckets.size());
-			for (auto& Kv : m_Buckets)
+			const GcClock::TimePoint Now = GcClock::Now();
 			{
-				Buckets.push_back(Kv.second.get());
+				ZEN_TRACE_CPU("Z$::ZenCacheDiskLayer::MemCacheTrim GetUsageByAccess");
+				for (CacheBucket* Bucket : Buckets)
+				{
+					Bucket->GetUsageByAccess(Now, MaxAge, UsageSlots);
+				}
 			}
-		}
-		for (CacheBucket* Bucket : Buckets)
-		{
-			Bucket->GetUsageByAccess(Now, GcClock::Duration(TrimInterval), UsageSlots);
-		}
 
-		uint64_t TotalSize = 0;
-		for (size_t Index = 0; Index < UsageSlots.size(); ++Index)
-		{
-			TotalSize += UsageSlots[Index];
-			if (TotalSize >= m_Configuration.MemCacheTargetFootprintBytes)
+			uint64_t TotalSize = 0;
+			for (size_t Index = 0; Index < UsageSlots.size(); ++Index)
 			{
-				GcClock::TimePoint ExpireTime = Now - (TrimInterval * Index);
-				MemCacheTrim(Buckets, ExpireTime);
-				break;
+				TotalSize += UsageSlots[Index];
+				if (TotalSize >= m_Configuration.MemCacheTargetFootprintBytes)
+				{
+					GcClock::TimePoint ExpireTime = Now - ((GcClock::Duration(MaxAge) * Index) / UsageSlotCount);
+					TrimmedSize					  = MemCacheTrim(Buckets, ExpireTime);
+					break;
+				}
 			}
-		}
-	});
+		});
+	}
+	catch (std::exception& Ex)
+	{
+		ZEN_ERROR("Failed scheduling ZenCacheDiskLayer::MemCacheTrim. Reason: '{}'", Ex.what());
+		m_IsMemCacheTrimming.store(false);
+	}
 }
 
-void
+uint64_t
 ZenCacheDiskLayer::MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime)
 {
 	if (m_Configuration.MemCacheTargetFootprintBytes == 0)
 	{
-		return;
+		return 0;
 	}
-	RwLock::SharedLockScope __(m_Lock);
+	uint64_t TrimmedSize = 0;
 	for (CacheBucket* Bucket : Buckets)
 	{
-		Bucket->MemCacheTrim(ExpireTime);
+		TrimmedSize += Bucket->MemCacheTrim(ExpireTime);
 	}
 	const GcClock::TimePoint   Now				   = GcClock::Now();
 	const GcClock::Tick		   NowTick			   = Now.time_since_epoch().count();
 	const std::chrono::seconds TrimInterval		   = std::chrono::seconds(m_Configuration.MemCacheTrimIntervalSeconds);
-	GcClock::Tick			   LastTrimTick		   = m_LastTickMemCacheTrim;
+	GcClock::Tick			   LastTrimTick		   = m_NextAllowedTrimTick;
 	const GcClock::Tick		   NextAllowedTrimTick = NowTick + GcClock::Duration(TrimInterval).count();
-	m_LastTickMemCacheTrim.compare_exchange_strong(LastTrimTick, NextAllowedTrimTick);
+	m_NextAllowedTrimTick.compare_exchange_strong(LastTrimTick, NextAllowedTrimTick);
+	return TrimmedSize;
 }
 
 #if ZEN_WITH_TESTS
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index d46d629e4..6997a12e4 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -29,14 +29,25 @@ struct DiskLocation
 
 	inline DiskLocation(uint64_t ValueSize, uint8_t Flags) : Flags(Flags | kStandaloneFile) { Location.StandaloneSize = ValueSize; }
 
-	inline DiskLocation(const BlockStoreLocation& Location, uint64_t PayloadAlignment, uint8_t Flags) : Flags(Flags & ~kStandaloneFile)
+	inline DiskLocation(const BlockStoreLocation& Location, uint32_t PayloadAlignment, uint8_t Flags) : Flags(Flags & ~kStandaloneFile)
 	{
 		this->Location.BlockLocation = BlockStoreDiskLocation(Location, PayloadAlignment);
 	}
 
-	inline bool operator!=(const DiskLocation& Rhs) const { return memcmp(&Location, &Rhs.Location, sizeof(Location)) != 0; }
+	inline bool operator!=(const DiskLocation& Rhs) const
+	{
+		if (Flags != Rhs.Flags)
+		{
+			return true;
+		}
+		if (Flags & kStandaloneFile)
+		{
+			return Location.StandaloneSize != Rhs.Location.StandaloneSize;
+		}
+		return Location.BlockLocation != Rhs.Location.BlockLocation;
+	}
 
-	inline BlockStoreLocation GetBlockLocation(uint64_t PayloadAlignment) const
+	inline BlockStoreLocation GetBlockLocation(uint32_t PayloadAlignment) const
 	{
 		ZEN_ASSERT(!(Flags & kStandaloneFile));
 		return Location.BlockLocation.Get(PayloadAlignment);
@@ -95,7 +106,7 @@ public:
 	struct BucketConfiguration
 	{
 		uint64_t MaxBlockSize			= 1ull << 30;
-		uint64_t PayloadAlignment		= 1ull << 4;
+		uint32_t PayloadAlignment		= 1u << 4;
 		uint64_t MemCacheSizeThreshold	= 1 * 1024;
 		uint64_t LargeObjectThreshold	= 128 * 1024;
 		bool	 EnableReferenceCaching = false;
@@ -178,7 +189,6 @@ public:
 	void SetAccessTime(std::string_view Bucket, const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
 
-private:
 	/** A cache bucket manages a single directory containing
 		metadata and data for that bucket
 	  */
@@ -187,15 +197,15 @@ private:
 		CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, std::string BucketName, const BucketConfiguration& Config);
 		~CacheBucket();
 
-		bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
-		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
-		void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
-		void MemCacheTrim(GcClock::TimePoint ExpireTime);
-		bool Drop();
-		void Flush();
-		void ScrubStorage(ScrubContext& Ctx);
-		void GatherReferences(GcContext& GcCtx);
-		void CollectGarbage(GcContext& GcCtx);
+		bool	 OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
+		bool	 Get(const IoHash& HashKey, ZenCacheValue& OutValue);
+		void	 Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
+		uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime);
+		bool	 Drop();
+		void	 Flush();
+		void	 ScrubStorage(ScrubContext& Ctx);
+		void	 GatherReferences(GcContext& GcCtx);
+		void	 CollectGarbage(GcContext& GcCtx);
 
 		inline GcStorageSize StorageSize() const
 		{
@@ -205,30 +215,15 @@ private:
 		uint64_t	EntryCount() const;
 		BucketStats Stats();
 
-		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+		CacheValueDetails::BucketDetails GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const;
 		void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
 
-		void GetUsageByAccess(GcClock::TimePoint TickStart, GcClock::Duration SectionLength, std::vector<uint64_t>& InOutUsageSlots);
+		void GetUsageByAccess(GcClock::TimePoint Now, GcClock::Duration MaxAge, std::vector<uint64_t>& InOutUsageSlots);
 #if ZEN_WITH_TESTS
 		void SetAccessTime(const IoHash& HashKey, GcClock::TimePoint Time);
 #endif	// ZEN_WITH_TESTS
 
 	private:
-		GcManager&			  m_Gc;
-		std::atomic_uint64_t& m_OuterCacheMemoryUsage;
-		std::string			  m_BucketName;
-		std::filesystem::path m_BucketDir;
-		std::filesystem::path m_BlocksBasePath;
-		BucketConfiguration	  m_Configuration;
-		BlockStore			  m_BlockStore;
-		Oid					  m_BucketId;
-		std::atomic_bool	  m_IsFlushing{};
-
-		// These files are used to manage storage of small objects for this bucket
-
-		TCasLogFile<DiskIndexEntry> m_SlogFile;
-		uint64_t					m_LogFlushPosition = 0;
-
 #pragma pack(push)
 #pragma pack(1)
 		struct MetaDataIndex
@@ -291,6 +286,11 @@ private:
 
 			operator bool() const { return RawSize != 0 || RawHash != IoHash::Zero; };
 		};
+		struct MemCacheData
+		{
+			IoBuffer	 Payload;
+			PayloadIndex OwnerIndex;
+		};
 #pragma pack(pop)
 		static_assert(sizeof(BucketPayload) == 20u);
 		static_assert(sizeof(BucketMetaData) == 28u);
@@ -298,6 +298,21 @@ private:
 
 		using IndexMap = tsl::robin_map<IoHash, PayloadIndex, IoHash::Hasher>;
 
+		GcManager&			  m_Gc;
+		std::atomic_uint64_t& m_OuterCacheMemoryUsage;
+		std::string			  m_BucketName;
+		std::filesystem::path m_BucketDir;
+		std::filesystem::path m_BlocksBasePath;
+		BucketConfiguration	  m_Configuration;
+		BlockStore			  m_BlockStore;
+		Oid					  m_BucketId;
+		std::atomic_bool	  m_IsFlushing{true};  // Don't allow flush until we are properly initialized
+
+		// These files are used to manage storage of small objects for this bucket
+
+		TCasLogFile<DiskIndexEntry> m_SlogFile;
+		uint64_t					m_LogFlushPosition = 0;
+
 		std::atomic<uint64_t> m_DiskHitCount;
 		std::atomic<uint64_t> m_DiskMissCount;
 		std::atomic<uint64_t> m_DiskWriteCount;
@@ -313,17 +328,18 @@ private:
 		std::vector<BucketPayload>	m_Payloads;
 		std::vector<BucketMetaData> m_MetaDatas;
 		std::vector<MetaDataIndex>	m_FreeMetaDatas;
-		std::vector<IoBuffer>		m_MemCachedPayloads;
+		std::vector<MemCacheData>	m_MemCachedPayloads;
 		std::vector<MemCachedIndex> m_FreeMemCachedPayloads;
 		std::vector<ReferenceIndex> m_FirstReferenceIndex;
 		std::vector<IoHash>			m_ReferenceHashes;
 		std::vector<ReferenceIndex> m_NextReferenceHashesIndexes;
+		std::unique_ptr<HashSet>	m_UpdatedKeys;
 		size_t						m_ReferenceCount = 0;
 		std::atomic_uint64_t		m_StandaloneSize{};
 		std::atomic_uint64_t		m_MemCachedSize{};
 
 		virtual std::string						 GetGcName(GcCtx& Ctx) override;
-		virtual void							 RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
+		virtual GcStoreCompactor*				 RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
 		virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
 
 		void	 BuildPath(PathBuilderBase& Path, const IoHash& HashKey) const;
@@ -331,19 +347,9 @@ private:
 		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
 		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
-		void	 MakeIndexSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
-		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
-		uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
-		void	 OpenLog(const bool IsNew);
-		CbObject MakeManifest(IndexMap&&						 Index,
-							  std::vector<AccessTime>&&			 AccessTimes,
-							  const std::vector<BucketPayload>&	 Payloads,
-							  const std::vector<BucketMetaData>& MetaDatas);
-		void	 SaveManifest(
-				CbObject&&						 Manifest,
-				const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
-		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, PayloadIndex Index) const;
-		void							CompactReferences(RwLock::ExclusiveLockScope&);
+		CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const;
+
+		void		CompactReferences(RwLock::ExclusiveLockScope&);
 		void		SetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex, std::span<IoHash> References);
 		void		RemoveReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex);
 		inline bool GetReferences(RwLock::SharedLockScope&, ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const
@@ -357,16 +363,39 @@ private:
 		ReferenceIndex AllocateReferenceEntry(RwLock::ExclusiveLockScope&, const IoHash& Key);
 		bool		   LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
 		void		   ClearReferenceCache();
-		void		   SetMetaData(BucketPayload& Payload, const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
-		void		   RemoveMetaData(BucketPayload& Payload);
-		BucketMetaData GetMetaData(const BucketPayload& Payload) const;
-		void		   SetMemCachedData(BucketPayload& Payload, IoBuffer& MemCachedData);
-		size_t		   RemoveMemCachedData(BucketPayload& Payload);
 
-		void CompactState(std::vector<BucketPayload>&  Payloads,
+		void		   SetMetaData(RwLock::ExclusiveLockScope&,
+								   BucketPayload&										 Payload,
+								   const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
+		void		   RemoveMetaData(RwLock::ExclusiveLockScope&, BucketPayload& Payload);
+		BucketMetaData GetMetaData(RwLock::SharedLockScope&, const BucketPayload& Payload) const;
+		void		   SetMemCachedData(RwLock::ExclusiveLockScope&, PayloadIndex PayloadIndex, IoBuffer& MemCachedData);
+		size_t		   RemoveMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload);
+
+		void	 InitializeIndexFromDisk(RwLock::ExclusiveLockScope&, bool IsNew);
+		uint64_t ReadIndexFile(RwLock::ExclusiveLockScope&, const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+		uint64_t ReadLog(RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t LogPosition);
+
+		void SaveSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
+		void WriteIndexSnapshot(
+			RwLock::ExclusiveLockScope&,
+			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
+		{
+			WriteIndexSnapshotLocked(ClaimDiskReserveFunc);
+		}
+		void WriteIndexSnapshot(
+			RwLock::SharedLockScope&,
+			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
+		{
+			WriteIndexSnapshotLocked(ClaimDiskReserveFunc);
+		}
+		void WriteIndexSnapshotLocked(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
+
+		void CompactState(RwLock::ExclusiveLockScope&,
+						  std::vector<BucketPayload>&  Payloads,
 						  std::vector<AccessTime>&	   AccessTimes,
 						  std::vector<BucketMetaData>& MetaDatas,
-						  std::vector<IoBuffer>&	   MemCachedPayloads,
+						  std::vector<MemCacheData>&   MemCachedPayloads,
 						  std::vector<ReferenceIndex>& FirstReferenceIndex,
 						  IndexMap&					   Index,
 						  RwLock::ExclusiveLockScope&  IndexLock);
@@ -381,6 +410,10 @@ private:
 			m_MemCachedSize.fetch_sub(ValueSize, std::memory_order::relaxed);
 			m_OuterCacheMemoryUsage.fetch_sub(ValueSize, std::memory_order::relaxed);
 		}
+		static inline uint64_t EstimateMemCachePayloadMemory(uint64_t PayloadSize)
+		{
+			return sizeof(MemCacheData) + sizeof(IoBufferCore) + RoundUp(PayloadSize, 8u);
+		}
 
 		// These locks are here to avoid contention on file creation, therefore it's sufficient
 		// that we take the same lock for the same hash
@@ -392,9 +425,13 @@ private:
 		inline RwLock& LockForHash(const IoHash& Hash) const { return m_ShardedLocks[Hash.Hash[19]]; }
 
 		friend class DiskBucketReferenceChecker;
+		friend class DiskBucketStoreCompactor;
+		friend class BucketManifestSerializer;
 	};
 
-	inline void TryMemCacheTrim()
+private:
+	CacheBucket* GetOrCreateBucket(std::string_view InBucket);
+	inline void	 TryMemCacheTrim()
 	{
 		if (m_Configuration.MemCacheTargetFootprintBytes == 0)
 		{
@@ -408,10 +445,21 @@ private:
 		{
 			return;
 		}
+		if (m_IsMemCacheTrimming)
+		{
+			return;
+		}
+
+		const GcClock::Tick NowTick = GcClock::TickCount();
+		if (NowTick < m_NextAllowedTrimTick)
+		{
+			return;
+		}
+
 		MemCacheTrim();
 	}
-	void MemCacheTrim();
-	void MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime);
+	void	 MemCacheTrim();
+	uint64_t MemCacheTrim(std::vector<CacheBucket*>& Buckets, GcClock::TimePoint ExpireTime);
 
 	GcManager&													  m_Gc;
 	JobQueue&													  m_JobQueue;
@@ -419,7 +467,7 @@ private:
 	Configuration												  m_Configuration;
 	std::atomic_uint64_t										  m_TotalMemCachedSize{};
 	std::atomic_bool											  m_IsMemCacheTrimming = false;
-	std::atomic<GcClock::Tick>									  m_LastTickMemCacheTrim;
+	std::atomic<GcClock::Tick>									  m_NextAllowedTrimTick;
 	mutable RwLock												  m_Lock;
 	std::unordered_map<std::string, std::unique_ptr<CacheBucket>> m_Buckets;
 	std::vector<std::unique_ptr<CacheBucket>>					  m_DroppedBuckets;
@@ -427,6 +475,7 @@ private:
 	ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete;
 	ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete;
 
+	friend class DiskBucketStoreCompactor;
 	friend class DiskBucketReferenceChecker;
 };
 
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index cc6fefc76..9155e209c 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -816,16 +816,28 @@ namespace testutils {
 		return {Key, Buffer};
 	}
 
+	struct FalseType
+	{
+		static const bool Enabled = false;
+	};
+	struct TrueType
+	{
+		static const bool Enabled = true;
+	};
+
 }  // namespace testutils
 
-TEST_CASE("z$.store")
+TEST_CASE_TEMPLATE("z$.store", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
 	GcManager Gc;
 
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 	const int kIterationCount = 100;
 
@@ -859,7 +871,7 @@ TEST_CASE("z$.store")
 	}
 }
 
-TEST_CASE("z$.size")
+TEST_CASE_TEMPLATE("z$.size", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	auto JobQueue = MakeJobQueue(1, "testqueue");
 
@@ -881,7 +893,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold - 256);
 
@@ -915,7 +930,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -939,7 +957,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			CbObject CacheValue = CreateCacheValue(Zcs.GetConfig().DiskLayerConfig.BucketConfig.MemCacheSizeThreshold + 64);
 
@@ -959,7 +980,10 @@ TEST_CASE("z$.size")
 
 		{
 			GcManager		  Gc;
-			ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+			ZenCacheNamespace Zcs(Gc,
+								  *JobQueue,
+								  TempDir.Path() / "cache",
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
@@ -974,7 +998,7 @@ TEST_CASE("z$.size")
 	}
 }
 
-TEST_CASE("z$.gc")
+TEST_CASE_TEMPLATE("z$.gc", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	using namespace testutils;
 
@@ -1001,7 +1025,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			const auto		  Bucket = "teardrinker"sv;
 
 			// Create a cache record
@@ -1041,7 +1065,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace	Zcs(Gc,
 									*JobQueue,
 									TempDir.Path() / "cache",
-									{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+									{.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			std::vector<IoHash> Keep;
 
 			// Collect garbage with 1 hour max cache duration
@@ -1065,7 +1089,7 @@ TEST_CASE("z$.gc")
 		ZenCacheNamespace		 Zcs(Gc,
 								 *JobQueue,
 								 TempDir.Path() / "cache",
-								 {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								 {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		const auto				 Bucket		 = "fortysixandtwo"sv;
 		const GcClock::TimePoint CurrentTime = GcClock::Now();
 
@@ -1114,7 +1138,7 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			const auto		  Bucket = "rightintwo"sv;
 
 			std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
@@ -1162,13 +1186,13 @@ TEST_CASE("z$.gc")
 			ZenCacheNamespace Zcs(Gc,
 								  *JobQueue,
 								  TempDir.Path() / "cache",
-								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+								  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 			CHECK_EQ(0, Zcs.StorageSize().DiskSize);
 		}
 	}
 }
 
-TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
+TEST_CASE_TEMPLATE("z$.threadedinsert", ReferenceCaching, testutils::FalseType, testutils::TrueType)  // * doctest::skip(true))
 {
 	//	for (uint32_t i = 0; i < 100; ++i)
 	{
@@ -1219,7 +1243,10 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 		WorkerThreadPool  ThreadPool(4);
 		GcManager		  Gc;
 		auto			  JobQueue = MakeJobQueue(1, "testqueue");
-		ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path(), {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+		ZenCacheNamespace Zcs(Gc,
+							  *JobQueue,
+							  TempDir.Path(),
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;
@@ -1648,7 +1675,7 @@ TEST_CASE("z$.drop.namespace")
 	}
 }
 
-TEST_CASE("z$.blocked.disklayer.put")
+TEST_CASE_TEMPLATE("z$.blocked.disklayer.put", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1665,7 +1692,10 @@ TEST_CASE("z$.blocked.disklayer.put")
 
 	GcManager		  Gc;
 	auto			  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 	CbObject CacheValue = CreateCacheValue(64 * 1024 + 64);
 
@@ -1701,7 +1731,7 @@ TEST_CASE("z$.blocked.disklayer.put")
 	CHECK(memcmp(NewView.GetData(), Buffer2.GetData(), NewView.GetSize()) == 0);
 }
 
-TEST_CASE("z$.scrub")
+TEST_CASE_TEMPLATE("z$.scrub", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	ScopedTemporaryDirectory TempDir;
 
@@ -1760,7 +1790,10 @@ TEST_CASE("z$.scrub")
 	GcManager			  Gc;
 	CidStore			  CidStore(Gc);
 	auto				  JobQueue = MakeJobQueue(1, "testqueue");
-	ZenCacheNamespace	  Zcs(Gc, *JobQueue, TempDir.Path() / "cache", {});
+	ZenCacheNamespace	  Zcs(Gc,
+						  *JobQueue,
+						  TempDir.Path() / "cache",
+						  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
 	CidStore.Initialize(CidConfig);
 
@@ -1795,7 +1828,7 @@ TEST_CASE("z$.scrub")
 	CHECK(ScrubCtx.BadCids().GetSize() == 0);
 }
 
-TEST_CASE("z$.newgc.basics")
+TEST_CASE_TEMPLATE("z$.newgc.basics", ReferenceCaching, testutils::FalseType, testutils::TrueType)
 {
 	using namespace testutils;
 
@@ -1915,7 +1948,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 
 		// Create some basic data
 		{
@@ -1949,7 +1982,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(1),
@@ -1957,14 +1990,14 @@ TEST_CASE("z$.newgc.basics")
 													   .CollectSmallObjects	   = false,
 													   .IsDeleteMode		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(0u, Result.ReferencerStat.Expired);
-		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(0u, Result.RemovedDisk);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -1983,7 +2016,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -1991,14 +2024,14 @@ TEST_CASE("z$.newgc.basics")
 													   .CollectSmallObjects	   = false,
 													   .IsDeleteMode		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(1u, Result.ReferencerStat.Expired);
-		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(0u, Result.RemovedDisk);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2017,7 +2050,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2025,14 +2058,14 @@ TEST_CASE("z$.newgc.basics")
 													   .CollectSmallObjects	   = true,
 													   .IsDeleteMode		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(7u, Result.ReferencerStat.Expired);
-		CHECK_EQ(0u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(0u, Result.RemovedDisk);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2051,7 +2084,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2060,14 +2093,14 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = true,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(1u, Result.ReferencerStat.Expired);
-		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2086,7 +2119,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2095,14 +2128,14 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = true,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(7u, Result.ReferencerStat.Expired);
-		CHECK_EQ(7u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GE(Result.RemovedDisk, 0);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_GE(Result.CompactStoresStatSum.RemovedDisk, 0);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2121,7 +2154,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2130,17 +2163,20 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(1u, Result.ReferencerStat.Expired);  // Only one cache value is pruned/deleted as that is the only large item in the cache
-													  // (all other large items as in cas)
-		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(1u,
+				 Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);  // Only one cache value is pruned/deleted as that is the only
+																			   // large item in the cache (all other large items as in cas)
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u,
+				 Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats
+					 .FoundCount);	// We won't remove any references since all referencers are small which retains all references
 		CHECK_EQ(0u,
-				 Result.ReferenceStoreStat
-					 .Pruned);	// We won't remove any references since all referencers are small which retains all references
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.RemovedDisk);
-		CHECK_EQ(0u, Result.RemovedMemory);
+				 Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats
+					 .DeletedCount);  // We won't remove any references since all referencers are small which retains all references
+		CHECK_EQ(CacheEntries[UnstructuredCacheValues[2]].Data.GetSize(), Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2159,7 +2195,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
@@ -2168,14 +2204,14 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(7u, Result.ReferencerStat.Expired);
-		CHECK_EQ(7u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GT(Result.RemovedDisk, 0);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
@@ -2195,25 +2231,27 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
-													   .CollectSmallObjects	   = true,
-													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true,
-													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(6u, Result.ReferencerStat.Expired);
-		CHECK_EQ(6u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GT(Result.RemovedDisk, 0);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime					  = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime			  = GcClock::Now() + std::chrono::hours(1),
+													   .CollectSmallObjects				  = true,
+													   .IsDeleteMode					  = true,
+													   .SkipCidDelete					  = true,
+													   .Verbose							  = true,
+													   .CompactBlockUsageThresholdPercent = 100});
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(6u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(6u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		uint64_t MinExpectedRemoveSize = CacheEntries[UnstructuredCacheValues[2]].Data.GetSize();
+		CHECK_LT(MinExpectedRemoveSize, Result.CompactStoresStatSum.RemovedDisk);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2233,7 +2271,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, CacheRecords[0], GcClock::Now() + std::chrono::hours(2));
@@ -2245,14 +2283,14 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(5u, Result.ReferencerStat.Expired);
-		CHECK_EQ(5u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GT(Result.RemovedDisk, 0);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], true, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], true, true));
@@ -2272,7 +2310,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[1], GcClock::Now() + std::chrono::hours(2));
@@ -2285,14 +2323,14 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(4u, Result.ReferencerStat.Expired);
-		CHECK_EQ(4u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(5u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GT(Result.RemovedDisk, 0);
-		CHECK_EQ(0u, Result.RemovedMemory);
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(5u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
+		CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, false));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, false));
@@ -2312,7 +2350,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		// Prime so we can check GC of memory layer
@@ -2329,22 +2367,23 @@ TEST_CASE("z$.newgc.basics")
 		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[2], GcClock::Now() + std::chrono::hours(2));
 		Zcs.SetAccessTime(TearDrinkerBucket, UnstructuredCacheValues[3], GcClock::Now() + std::chrono::hours(2));
 
-		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(1),
-													   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(1),
-													   .CollectSmallObjects	   = true,
-													   .IsDeleteMode		   = true,
-													   .SkipCidDelete		   = true,
-													   .Verbose				   = true});
-		CHECK_EQ(7u, Result.ReferencerStat.Count);
-		CHECK_EQ(4u, Result.ReferencerStat.Expired);
-		CHECK_EQ(4u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(0u, Result.ReferenceStoreStat.Compacted);
-		CHECK_GT(Result.RemovedDisk, 0);
+		GcResult Result = Gc.CollectGarbage(GcSettings{.CacheExpireTime					  = GcClock::Now() + std::chrono::hours(1),
+													   .ProjectStoreExpireTime			  = GcClock::Now() + std::chrono::hours(1),
+													   .CollectSmallObjects				  = true,
+													   .IsDeleteMode					  = true,
+													   .SkipCidDelete					  = true,
+													   .Verbose							  = true,
+													   .CompactBlockUsageThresholdPercent = 100});
+		CHECK_EQ(7u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_GT(Result.CompactStoresStatSum.RemovedDisk, 0);
 		uint64_t MemoryClean = CacheEntries[CacheRecords[0]].Data.GetSize() + CacheEntries[CacheRecords[1]].Data.GetSize() +
 							   CacheEntries[CacheRecords[2]].Data.GetSize() + CacheEntries[UnstructuredCacheValues[0]].Data.GetSize();
-		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+		CHECK_EQ(MemoryClean, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[0], false, true));
 		CHECK(ValidateCacheEntry(Zcs, CidStore, TearDrinkerBucket, CacheRecords[1], false, true));
@@ -2364,7 +2403,7 @@ TEST_CASE("z$.newgc.basics")
 		ZenCacheNamespace Zcs(Gc,
 							  *JobQueue,
 							  TempDir.Path() / "cache",
-							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = true}}});
+							  {.DiskLayerConfig = {.BucketConfig = {.EnableReferenceCaching = ReferenceCaching::Enabled}}});
 		CHECK_EQ(7, Zcs.GetBucketInfo(TearDrinkerBucket).value().DiskLayerInfo.EntryCount);
 
 		auto Attachments =
@@ -2393,15 +2432,17 @@ TEST_CASE("z$.newgc.basics")
 													   .IsDeleteMode		   = true,
 													   .SkipCidDelete		   = false,
 													   .Verbose				   = true});
-		CHECK_EQ(8u, Result.ReferencerStat.Count);
-		CHECK_EQ(1u, Result.ReferencerStat.Expired);
-		CHECK_EQ(1u, Result.ReferencerStat.Deleted);
-		CHECK_EQ(9u, Result.ReferenceStoreStat.Count);
-		CHECK_EQ(4u, Result.ReferenceStoreStat.Pruned);
-		CHECK_EQ(4u, Result.ReferenceStoreStat.Compacted);
-		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(), Result.RemovedDisk);
+		// Write block can't be compacted so Compacted will be less than Deleted
+		CHECK_EQ(8u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount);
+		CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+		CHECK_EQ(9u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+		CHECK_EQ(4u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount);
+		CHECK_EQ(4u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+		CHECK_EQ(Attachments[1].second.GetCompressed().GetSize() + Attachments[3].second.GetCompressed().GetSize(),
+				 Result.CompactStoresStatSum.RemovedDisk);
 		uint64_t MemoryClean = CacheEntries[CacheRecord].Data.GetSize();
-		CHECK_EQ(MemoryClean, Result.RemovedMemory);
+		CHECK_EQ(MemoryClean, Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
 	}
 }
 
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 08ba6dc95..5f2c3351e 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -2,12 +2,14 @@
 
 #include "config.h"
 
+#include "config/luaconfig.h"
 #include "diag/logging.h"
 
 #include <zencore/crypto.h>
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
 #include <zenutil/basicfile.h>
@@ -175,592 +177,156 @@ MakeSafePath(const std::string_view Path)
 #endif
 };
 
-namespace LuaConfig {
-
-	void EscapeBackslash(std::string& InOutString)
+class CachePolicyOption : public LuaConfig::OptionValue
+{
+public:
+	CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {}
+	virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
 	{
-		std::size_t BackslashPos = InOutString.find('\\');
-		if (BackslashPos != std::string::npos)
+		switch (Value)
 		{
-			std::size_t						  Offset = 0;
-			zen::ExtendableStringBuilder<512> PathBuilder;
-			while (BackslashPos != std::string::npos)
-			{
-				PathBuilder.Append(InOutString.substr(Offset, BackslashPos + 1 - Offset));
-				PathBuilder.Append('\\');
-				Offset		 = BackslashPos + 1;
-				BackslashPos = InOutString.find('\\', Offset);
-			}
-			PathBuilder.Append(InOutString.substr(Offset, BackslashPos));
-			InOutString = PathBuilder.ToString();
+			case UpstreamCachePolicy::Read:
+				StringBuilder.Append("readonly");
+				break;
+			case UpstreamCachePolicy::Write:
+				StringBuilder.Append("writeonly");
+				break;
+			case UpstreamCachePolicy::Disabled:
+				StringBuilder.Append("disabled");
+				break;
+			case UpstreamCachePolicy::ReadWrite:
+				StringBuilder.Append("readwrite");
+				break;
+			default:
+				ZEN_ASSERT(false);
 		}
 	}
-
-	class OptionValue
-	{
-	public:
-		virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) = 0;
-		virtual void Parse(sol::object Object)											   = 0;
-
-		virtual ~OptionValue() {}
-	};
-
-	typedef std::shared_ptr<OptionValue> TOptionValue;
-
-	class StringOption : public OptionValue
-	{
-	public:
-		StringOption(std::string& Value) : Value(Value) {}
-		virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
-		{
-			StringBuilder.Append(fmt::format("\"{}\"", Value));
-		}
-		virtual void Parse(sol::object Object) override { Value = Object.as<std::string>(); }
-		std::string& Value;
-	};
-
-	class FilePathOption : public OptionValue
+	virtual void Parse(sol::object Object) override
 	{
-	public:
-		FilePathOption(std::filesystem::path& Value) : Value(Value) {}
-		virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
+		std::string PolicyString = Object.as<std::string>();
+		if (PolicyString == "readonly")
 		{
-			std::string Path = Value.string();
-			EscapeBackslash(Path);
-			StringBuilder.Append(fmt::format("\"{}\"", Path));
+			Value = UpstreamCachePolicy::Read;
 		}
-		virtual void Parse(sol::object Object) override
+		else if (PolicyString == "writeonly")
 		{
-			std::string Str = Object.as<std::string>();
-			if (!Str.empty())
-			{
-				Value = MakeSafePath(Str);
-			}
+			Value = UpstreamCachePolicy::Write;
 		}
-		std::filesystem::path& Value;
-	};
-
-	class BoolOption : public OptionValue
-	{
-	public:
-		BoolOption(bool& Value) : Value(Value) {}
-		virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
+		else if (PolicyString == "disabled")
 		{
-			StringBuilder.Append(Value ? "true" : "false");
+			Value = UpstreamCachePolicy::Disabled;
 		}
-		virtual void Parse(sol::object Object) override { Value = Object.as<bool>(); }
-		bool&		 Value;
-	};
-
-	class CachePolicyOption : public OptionValue
-	{
-	public:
-		CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {}
-		virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
+		else if (PolicyString == "readwrite")
 		{
-			switch (Value)
-			{
-				case UpstreamCachePolicy::Read:
-					StringBuilder.Append("readonly");
-					break;
-				case UpstreamCachePolicy::Write:
-					StringBuilder.Append("writeonly");
-					break;
-				case UpstreamCachePolicy::Disabled:
-					StringBuilder.Append("disabled");
-					break;
-				case UpstreamCachePolicy::ReadWrite:
-					StringBuilder.Append("readwrite");
-					break;
-				default:
-					ZEN_ASSERT(false);
-			}
+			Value = UpstreamCachePolicy::ReadWrite;
 		}
-		virtual void Parse(sol::object Object) override
-		{
-			std::string PolicyString = Object.as<std::string>();
-			if (PolicyString == "readonly")
-			{
-				Value = UpstreamCachePolicy::Read;
-			}
-			else if (PolicyString == "writeonly")
-			{
-				Value = UpstreamCachePolicy::Write;
-			}
-			else if (PolicyString == "disabled")
-			{
-				Value = UpstreamCachePolicy::Disabled;
-			}
-			else if (PolicyString == "readwrite")
-			{
-				Value = UpstreamCachePolicy::ReadWrite;
-			}
-		}
-		UpstreamCachePolicy& Value;
-	};
-
-	template<Integral T>
-	class NumberOption : public OptionValue
-	{
-	public:
-		NumberOption(T& Value) : Value(Value) {}
-		virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override
-		{
-			StringBuilder.Append(fmt::format("{}", Value));
-		}
-		virtual void Parse(sol::object Object) override { Value = Object.as<T>(); }
-		T&			 Value;
-	};
+	}
+	UpstreamCachePolicy& Value;
+};
 
-	class LuaContainerWriter
+class ZenAuthConfigOption : public LuaConfig::OptionValue
+{
+public:
+	ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {}
+	virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
 	{
-	public:
-		LuaContainerWriter(zen::StringBuilderBase& StringBuilder, std::string_view Indent)
-		: StringBuilder(StringBuilder)
-		, InitialIndent(Indent.length())
-		, LocalIndent(Indent)
+		if (Value.OpenIdProviders.empty())
 		{
-			StringBuilder.Append("{\n");
-			LocalIndent.push_back('\t');
-		}
-		~LuaContainerWriter()
-		{
-			LocalIndent.pop_back();
-			StringBuilder.Append(LocalIndent);
-			StringBuilder.Append("}");
-		}
-
-		void BeginContainer(std::string_view Name)
-		{
-			StringBuilder.Append(LocalIndent);
-			if (!Name.empty())
-			{
-				StringBuilder.Append(Name);
-				StringBuilder.Append(" = {\n");
-			}
-			else
-			{
-				StringBuilder.Append("{\n");
-			}
-			LocalIndent.push_back('\t');
+			StringBuilder.Append("{}");
+			return;
 		}
-		void WriteValue(std::string_view Name, std::string_view Value)
+		LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent);
+		for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders)
 		{
-			if (Name.empty())
-			{
-				StringBuilder.Append(fmt::format("{}\"{}\",\n", LocalIndent, Value));
-			}
-			else
+			Writer.BeginContainer("");
 			{
-				StringBuilder.Append(fmt::format("{}{} = \"{}\",\n", LocalIndent, Name, Value));
+				Writer.WriteValue("name", Config.Name);
+				Writer.WriteValue("url", Config.Url);
+				Writer.WriteValue("clientid", Config.ClientId);
 			}
+			Writer.EndContainer();
 		}
-		void EndContainer()
-		{
-			LocalIndent.pop_back();
-			StringBuilder.Append(LocalIndent);
-			StringBuilder.Append("}");
-			StringBuilder.Append(",\n");
-		}
-
-	private:
-		zen::StringBuilderBase& StringBuilder;
-		const std::size_t		InitialIndent;
-		std::string				LocalIndent;
-	};
-
-	class StringArrayOption : public OptionValue
+	}
+	virtual void Parse(sol::object Object) override
 	{
-	public:
-		StringArrayOption(std::vector<std::string>& Value) : Value(Value) {}
-		virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
-		{
-			if (Value.empty())
-			{
-				StringBuilder.Append("{}");
-			}
-			if (Value.size() == 1)
-			{
-				StringBuilder.Append(fmt::format("\"{}\"", Value[0]));
-			}
-			else
-			{
-				LuaContainerWriter Writer(StringBuilder, Indent);
-				for (std::string String : Value)
-				{
-					Writer.WriteValue("", String);
-				}
-			}
-		}
-		virtual void Parse(sol::object Object) override
+		if (sol::optional<sol::table> OpenIdProviders = Object.as<sol::table>())
 		{
-			if (Object.get_type() == sol::type::string)
+			for (const auto& Kv : OpenIdProviders.value())
 			{
-				Value.push_back(Object.as<std::string>());
-			}
-			else if (Object.get_type() == sol::type::table)
-			{
-				for (const auto& Kv : Object.as<sol::table>())
+				if (sol::optional<sol::table> OpenIdProvider = Kv.second.as<sol::table>())
 				{
-					Value.push_back(Kv.second.as<std::string>());
-				}
-			}
-		}
-
-	private:
-		std::vector<std::string>& Value;
-	};
+					std::string Name	 = OpenIdProvider.value().get_or("name", std::string("Default"));
+					std::string Url		 = OpenIdProvider.value().get_or("url", std::string());
+					std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string());
 
-	class ZenAuthConfigOption : public OptionValue
-	{
-	public:
-		ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {}
-		virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
-		{
-			if (Value.OpenIdProviders.empty())
-			{
-				StringBuilder.Append("{}");
-				return;
-			}
-			LuaContainerWriter Writer(StringBuilder, Indent);
-			for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders)
-			{
-				Writer.BeginContainer("");
-				{
-					Writer.WriteValue("name", Config.Name);
-					Writer.WriteValue("url", Config.Url);
-					Writer.WriteValue("clientid", Config.ClientId);
-				}
-				Writer.EndContainer();
-			}
-		}
-		virtual void Parse(sol::object Object) override
-		{
-			if (sol::optional<sol::table> OpenIdProviders = Object.as<sol::table>())
-			{
-				for (const auto& Kv : OpenIdProviders.value())
-				{
-					if (sol::optional<sol::table> OpenIdProvider = Kv.second.as<sol::table>())
-					{
-						std::string Name	 = OpenIdProvider.value().get_or("name", std::string("Default"));
-						std::string Url		 = OpenIdProvider.value().get_or("url", std::string());
-						std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string());
-
-						Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)});
-					}
+					Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)});
 				}
 			}
 		}
-		ZenAuthConfig& Value;
-	};
+	}
+	ZenAuthConfig& Value;
+};
 
-	class ZenObjectStoreConfigOption : public OptionValue
+class ZenObjectStoreConfigOption : public LuaConfig::OptionValue
+{
+public:
+	ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {}
+	virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
 	{
-	public:
-		ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {}
-		virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override
+		if (Value.Buckets.empty())
 		{
-			if (Value.Buckets.empty())
-			{
-				StringBuilder.Append("{}");
-				return;
-			}
-			LuaContainerWriter Writer(StringBuilder, Indent);
-			for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets)
-			{
-				Writer.BeginContainer("");
-				{
-					Writer.WriteValue("name", Config.Name);
-					std::string Directory = Config.Directory.string();
-					EscapeBackslash(Directory);
-					Writer.WriteValue("directory", Directory);
-				}
-				Writer.EndContainer();
-			}
+			StringBuilder.Append("{}");
+			return;
 		}
-		virtual void Parse(sol::object Object) override
+		LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent);
+		for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets)
 		{
-			if (sol::optional<sol::table> Buckets = Object.as<sol::table>())
+			Writer.BeginContainer("");
 			{
-				for (const auto& Kv : Buckets.value())
-				{
-					if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>())
-					{
-						std::string Name	  = Bucket.value().get_or("name", std::string("Default"));
-						std::string Directory = Bucket.value().get_or("directory", std::string());
-
-						Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafePath(Directory)});
-					}
-				}
+				Writer.WriteValue("name", Config.Name);
+				std::string Directory = Config.Directory.string();
+				LuaConfig::EscapeBackslash(Directory);
+				Writer.WriteValue("directory", Directory);
 			}
+			Writer.EndContainer();
 		}
-		ZenObjectStoreConfig& Value;
-	};
-
-	std::shared_ptr<OptionValue> MakeOption(std::string& Value) { return std::make_shared<StringOption>(Value); };
-
-	std::shared_ptr<OptionValue> MakeOption(std::filesystem::path& Value) { return std::make_shared<FilePathOption>(Value); };
-
-	template<Integral T>
-	std::shared_ptr<OptionValue> MakeOption(T& Value)
-	{
-		return std::make_shared<NumberOption<T>>(Value);
-	};
-
-	std::shared_ptr<OptionValue> MakeOption(bool& Value) { return std::make_shared<BoolOption>(Value); };
-
-	std::shared_ptr<OptionValue> MakeOption(UpstreamCachePolicy& Value) { return std::make_shared<CachePolicyOption>(Value); };
-
-	std::shared_ptr<OptionValue> MakeOption(std::vector<std::string>& Value) { return std::make_shared<StringArrayOption>(Value); };
-
-	std::shared_ptr<OptionValue> MakeOption(ZenAuthConfig& Value) { return std::make_shared<ZenAuthConfigOption>(Value); };
-
-	std::shared_ptr<OptionValue> MakeOption(ZenObjectStoreConfig& Value) { return std::make_shared<ZenObjectStoreConfigOption>(Value); };
-
-	struct Option
-	{
-		std::string	 CommandLineOptionName;
-		TOptionValue Value;
-	};
-
-	struct Options
+	}
+	virtual void Parse(sol::object Object) override
 	{
-	public:
-		template<typename T>
-		void AddOption(std::string_view Key, T& Value, std::string_view CommandLineOptionName = "")
-		{
-			OptionMap.insert_or_assign(std::string(Key),
-									   Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)});
-		};
-
-		void Parse(const std::filesystem::path& Path, const cxxopts::ParseResult& CmdLineResult)
-		{
-			zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path);
-
-			if (LuaScript)
-			{
-				sol::state lua;
-
-				lua.open_libraries(sol::lib::base);
-
-				lua.set_function("getenv", [&](const std::string env) -> sol::object {
-#if ZEN_PLATFORM_WINDOWS
-					std::wstring EnvVarValue;
-					size_t		 RequiredSize = 0;
-					std::wstring EnvWide	  = zen::Utf8ToWide(env);
-					_wgetenv_s(&RequiredSize, nullptr, 0, EnvWide.c_str());
-
-					if (RequiredSize == 0)
-						return sol::make_object(lua, sol::lua_nil);
-
-					EnvVarValue.resize(RequiredSize);
-					_wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str());
-					return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str()));
-#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
-			char* EnvVariable = getenv(env.c_str());
-			if (EnvVariable == nullptr)
-			{
-				return sol::make_object(lua, sol::lua_nil);
-			}
-			return sol::make_object(lua, EnvVariable);
-#else
-			ZEN_UNUSED(env);
-			return sol::make_object(lua, sol::lua_nil);
-#endif
-				});
-
-				try
-				{
-					sol::load_result config = lua.load(std::string_view((const char*)LuaScript.Data(), LuaScript.Size()), "zen_cfg");
-
-					if (!config.valid())
-					{
-						sol::error err = config;
-
-						std::string ErrorString = sol::to_string(config.status());
-
-						throw std::runtime_error(fmt::format("{} error: {}", ErrorString, err.what()));
-					}
-
-					config();
-				}
-				catch (std::exception& e)
-				{
-					throw std::runtime_error(fmt::format("failed to load config script ('{}'): {}", Path, e.what()).c_str());
-				}
-
-				Parse(lua, CmdLineResult);
-			}
-		}
-
-		void Parse(const sol::state& LuaState, const cxxopts::ParseResult& CmdLineResult)
-		{
-			for (auto It : LuaState)
-			{
-				sol::object Key		= It.first;
-				sol::type	KeyType = Key.get_type();
-				if (KeyType == sol::type::string)
-				{
-					sol::type ValueType = It.second.get_type();
-					switch (ValueType)
-					{
-						case sol::type::table:
-							{
-								std::string Name = Key.as<std::string>();
-								if (Name.starts_with("_"))
-								{
-									continue;
-								}
-								if (Name == "base")
-								{
-									continue;
-								}
-								Traverse(It.second.as<sol::table>(), Name, CmdLineResult);
-							}
-							break;
-						default:
-							break;
-					}
-				}
-			}
-		}
-
-		void Touch(std::string_view Key) { UsedKeys.insert(std::string(Key)); }
-
-		void Print(zen::StringBuilderBase& SB, const cxxopts::ParseResult& CmdLineResult)
+		if (sol::optional<sol::table> Buckets = Object.as<sol::table>())
 		{
-			for (auto It : OptionMap)
+			for (const auto& Kv : Buckets.value())
 			{
-				if (CmdLineResult.count(It.second.CommandLineOptionName) != 0)
+				if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>())
 				{
-					UsedKeys.insert(It.first);
-				}
-			}
+					std::string Name	  = Bucket.value().get_or("name", std::string("Default"));
+					std::string Directory = Bucket.value().get_or("directory", std::string());
 
-			std::vector<std::string> SortedKeys(UsedKeys.begin(), UsedKeys.end());
-			std::sort(SortedKeys.begin(), SortedKeys.end());
-			auto GetTablePath = [](const std::string& Key) -> std::vector<std::string> {
-				std::vector<std::string> Path;
-				zen::ForEachStrTok(Key, '.', [&Path](std::string_view Part) {
-					Path.push_back(std::string(Part));
-					return true;
-				});
-				return Path;
-			};
-			std::vector<std::string> CurrentTablePath;
-			std::string				 Indent;
-			auto					 It = SortedKeys.begin();
-			for (const std::string& Key : SortedKeys)
-			{
-				std::vector<std::string> KeyPath = GetTablePath(Key);
-				std::string				 Name	 = KeyPath.back();
-				KeyPath.pop_back();
-				if (CurrentTablePath != KeyPath)
-				{
-					size_t EqualCount = 0;
-					while (EqualCount < CurrentTablePath.size() && EqualCount < KeyPath.size() &&
-						   CurrentTablePath[EqualCount] == KeyPath[EqualCount])
-					{
-						EqualCount++;
-					}
-					while (CurrentTablePath.size() > EqualCount)
-					{
-						CurrentTablePath.pop_back();
-						Indent.pop_back();
-						SB.Append(Indent);
-						SB.Append("}");
-						if (CurrentTablePath.size() == EqualCount && !Indent.empty() && KeyPath.size() >= EqualCount)
-						{
-							SB.Append(",");
-						}
-						SB.Append("\n");
-						if (Indent.empty())
-						{
-							SB.Append("\n");
-						}
-					}
-					while (EqualCount < KeyPath.size())
-					{
-						SB.Append(Indent);
-						SB.Append(KeyPath[EqualCount]);
-						SB.Append(" = {\n");
-						Indent.push_back('\t');
-						CurrentTablePath.push_back(KeyPath[EqualCount]);
-						EqualCount++;
-					}
+					Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)});
 				}
-
-				SB.Append(Indent);
-				SB.Append(Name);
-				SB.Append(" = ");
-				OptionMap[Key].Value->Print(Indent, SB);
-				SB.Append(",\n");
-			}
-			while (!CurrentTablePath.empty())
-			{
-				Indent.pop_back();
-				SB.Append(Indent);
-				SB.Append("}\n");
-				CurrentTablePath.pop_back();
 			}
 		}
+	}
+	ZenObjectStoreConfig& Value;
+};
 
-	private:
-		void Traverse(sol::table Table, std::string_view PathPrefix, const cxxopts::ParseResult& CmdLineResult)
-		{
-			for (auto It : Table)
-			{
-				sol::object Key		= It.first;
-				sol::type	KeyType = Key.get_type();
-				if (KeyType == sol::type::string || KeyType == sol::type::number)
-				{
-					sol::type ValueType = It.second.get_type();
-					switch (ValueType)
-					{
-						case sol::type::table:
-						case sol::type::string:
-						case sol::type::number:
-						case sol::type::boolean:
-							{
-								std::string Name = Key.as<std::string>();
-								if (Name.starts_with("_"))
-								{
-									continue;
-								}
-								Name		  = std::string(PathPrefix) + "." + Key.as<std::string>();
-								auto OptionIt = OptionMap.find(Name);
-								if (OptionIt != OptionMap.end())
-								{
-									UsedKeys.insert(Name);
-									if (CmdLineResult.count(OptionIt->second.CommandLineOptionName) != 0)
-									{
-										continue;
-									}
-									OptionIt->second.Value->Parse(It.second);
-									continue;
-								}
-								if (ValueType == sol::type::table)
-								{
-									if (Name == "base")
-									{
-										continue;
-									}
-									Traverse(It.second.as<sol::table>(), Name, CmdLineResult);
-								}
-							}
-							break;
-						default:
-							break;
-					}
-				}
-			}
-		}
+std::shared_ptr<LuaConfig::OptionValue>
+MakeOption(zen::UpstreamCachePolicy& Value)
+{
+	return std::make_shared<CachePolicyOption>(Value);
+};
 
-		std::unordered_map<std::string, Option> OptionMap;
-		std::unordered_set<std::string>			UsedKeys;
-	};
+std::shared_ptr<LuaConfig::OptionValue>
+MakeOption(zen::ZenAuthConfig& Value)
+{
+	return std::make_shared<ZenAuthConfigOption>(Value);
+};
 
-}  // namespace LuaConfig
+std::shared_ptr<LuaConfig::OptionValue>
+MakeOption(zen::ZenObjectStoreConfig& Value)
+{
+	return std::make_shared<ZenObjectStoreConfigOption>(Value);
+};
 
 void
 ParseConfigFile(const std::filesystem::path& Path,
@@ -887,6 +453,10 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("gc.lightweightntervalseconds"sv,
 						 ServerOptions.GcConfig.LightweightIntervalSeconds,
 						 "gc-lightweight-interval-seconds"sv);
+	LuaOptions.AddOption("gc.compactblockthreshold"sv,
+						 ServerOptions.GcConfig.CompactBlockUsageThresholdPercent,
+						 "gc-compactblock-threshold"sv);
+	LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv);
 
 	////// gc
 	LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv);
@@ -938,6 +508,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	std::string AbsLogFile;
 	std::string ConfigFile;
 	std::string OutputConfigFile;
+	std::string BaseSnapshotDir;
 
 	cxxopts::Options options("zenserver", "Zen Server");
 	options.add_options()("dedicated",
@@ -947,12 +518,20 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("clean",
 						  "Clean out all state at startup",
 						  cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false"));
+	options.add_options()("scrub",
+						  "Validate state at startup",
+						  cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"),
+						  "(nocas,nogc,nodelete,yes,no)*");
 	options.add_options()("help", "Show command line help");
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
-	options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
+	options.add_options()("snapshot-dir",
+						  "Specify a snapshot of server state to mirror into the persistence root at startup",
+						  cxxopts::value<std::string>(BaseSnapshotDir));
 	options.add_options()("content-dir", "Frontend content directory", cxxopts::value<std::string>(ContentDir));
-	options.add_options()("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile));
+	options.add_options()("powercycle",
+						  "Exit immediately after initialization is complete",
+						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
@@ -961,7 +540,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("sentry-allow-personal-info",
 						  "Allow personally identifiable information in sentry crash reports",
 						  cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false"));
-	options.add_options()("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"));
+
+	// clang-format off
+	options.add_options("logging")
+		("abslog",			"Path to log file",								cxxopts::value<std::string>(AbsLogFile))
+		("log-id",			"Specify id for adding context to log output",	cxxopts::value<std::string>(ServerOptions.LogId))
+		("quiet", 			"Disable console logging", 						cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+		("log-trace", 		"Change selected loggers to level TRACE", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+		("log-debug", 		"Change selected loggers to level DEBUG", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+		("log-info", 		"Change selected loggers to level INFO", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+		("log-warn", 		"Change selected loggers to level WARN", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+		("log-error", 		"Change selected loggers to level ERROR", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+		("log-critical", 	"Change selected loggers to level CRITICAL", 	cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+		("log-off", 		"Change selected loggers to level OFF", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+	;
+	// clang-format on
 
 	options.add_option("security",
 					   "",
@@ -1313,6 +906,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"),
 					   "");
 
+	options.add_option("gc",
+					   "",
+					   "gc-compactblock-threshold",
+					   "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only "
+					   "empty eligible blocks, 100 - compact all non-full eligible blocks.",
+					   cxxopts::value<uint32_t>(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"),
+					   "");
+
+	options.add_option("gc",
+					   "",
+					   "gc-verbose",
+					   "Enable verbose logging for GC.",
+					   cxxopts::value<bool>(ServerOptions.GcConfig.Verbose)->default_value("false"),
+					   "");
+
 	options.add_option("objectstore",
 					   "",
 					   "objectstore-enabled",
@@ -1337,9 +945,18 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 	try
 	{
-		auto result = options.parse(argc, argv);
+		cxxopts::ParseResult Result;
 
-		if (result.count("help"))
+		try
+		{
+			Result = options.parse(argc, argv);
+		}
+		catch (std::exception& Ex)
+		{
+			throw zen::OptionParseException(Ex.what());
+		}
+
+		if (Result.count("help"))
 		{
 			ZEN_CONSOLE("{}", options.help());
 #if ZEN_PLATFORM_WINDOWS
@@ -1352,12 +969,28 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			exit(0);
 		}
 
+		for (int i = 0; i < logging::level::LogLevelCount; ++i)
+		{
+			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
+		}
+		logging::RefreshLogLevels();
+
 		ServerOptions.DataDir						  = MakeSafePath(DataDir);
+		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
 		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
 		ServerOptions.AbsLogFile					  = MakeSafePath(AbsLogFile);
 		ServerOptions.ConfigFile					  = MakeSafePath(ConfigFile);
 		ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions);
 
+		if (!BaseSnapshotDir.empty())
+		{
+			if (DataDir.empty())
+				throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot");
+
+			if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir))
+				throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir));
+		}
+
 		if (OpenIdProviderUrl.empty() == false)
 		{
 			if (OpenIdClientId.empty())
@@ -1373,21 +1006,15 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 		if (!ServerOptions.ConfigFile.empty())
 		{
-			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, result, OutputConfigFile);
+			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile);
 		}
 		else
 		{
-			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, result, OutputConfigFile);
+			ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile);
 		}
 
 		ValidateOptions(ServerOptions);
 	}
-	catch (cxxopts::OptionParseException& e)
-	{
-		ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help());
-
-		throw;
-	}
 	catch (zen::OptionParseException& e)
 	{
 		ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help());
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index d55f0d5a1..cd2d92523 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/logbase.h>
 #include <zencore/zencore.h>
 #include <zenhttp/httpserver.h>
 #include <filesystem>
@@ -72,6 +73,8 @@ struct ZenGcConfig
 	int32_t						  LightweightIntervalSeconds		= 0;
 	uint64_t					  MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
 	bool						  UseGCV2							= false;
+	uint32_t					  CompactBlockUsageThresholdPercent = 90;
+	bool						  Verbose							= false;
 };
 
 struct ZenOpenIdProviderConfig
@@ -129,6 +132,7 @@ struct ZenServerOptions
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
 	std::filesystem::path	 ConfigFile;				  // Path to Lua config file
+	std::filesystem::path	 BaseSnapshotDir;			  // Path to server state snapshot (will be copied into data dir on start)
 	std::string				 ChildId;					  // Id assigned by parent process (used for lifetime management)
 	std::string				 LogId;						  // Id for tagging log output
 	std::string				 EncryptionKey;				  // 256 bit AES encryption key
@@ -139,6 +143,7 @@ struct ZenServerOptions
 	bool					 UninstallService	= false;  // Flag used to initiate service uninstall (temporary)
 	bool					 IsDebug			= false;
 	bool					 IsCleanStart		= false;  // Indicates whether all state should be wiped on startup or not
+	bool					 IsPowerCycle		= false;  // When true, the process shuts down immediately after initialization
 	bool					 IsTest				= false;
 	bool					 IsDedicated		= false;  // Indicates a dedicated/shared instance, with larger resource requirements
 	bool					 ShouldCrash		= false;  // Option for testing crash handling
@@ -147,6 +152,8 @@ struct ZenServerOptions
 	bool					 SentryAllowPII		= false;  // Allow personally identifiable information in sentry crash reports
 	bool					 ObjectStoreEnabled = false;
 	bool					 NoConsoleOutput	= false;  // Control default use of stdout for diagnostics
+	std::string				 Loggers[zen::logging::level::LogLevelCount];
+	std::string				 ScrubOptions;
 #if ZEN_WITH_TRACE
 	std::string TraceHost;	// Host name or IP address to send trace data to
 	std::string TraceFile;	// Path of a file to write a trace
diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp
new file mode 100644
index 000000000..cdc808cf6
--- /dev/null
+++ b/src/zenserver/config/luaconfig.cpp
@@ -0,0 +1,461 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "luaconfig.h"
+
+namespace zen::LuaConfig {
+
+std::string
+MakeSafePath(const std::string_view Path)
+{
+#if ZEN_PLATFORM_WINDOWS
+	if (Path.empty())
+	{
+		return std::string(Path);
+	}
+
+	std::string FixedPath(Path);
+	std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\');
+	if (!FixedPath.starts_with("\\\\?\\"))
+	{
+		FixedPath.insert(0, "\\\\?\\");
+	}
+	return FixedPath;
+#else
+	return std::string(Path);
+#endif
+};
+
+void
+EscapeBackslash(std::string& InOutString)
+{
+	std::size_t BackslashPos = InOutString.find('\\');
+	if (BackslashPos != std::string::npos)
+	{
+		std::size_t						  Offset = 0;
+		zen::ExtendableStringBuilder<512> PathBuilder;
+		while (BackslashPos != std::string::npos)
+		{
+			PathBuilder.Append(InOutString.substr(Offset, BackslashPos + 1 - Offset));
+			PathBuilder.Append('\\');
+			Offset		 = BackslashPos + 1;
+			BackslashPos = InOutString.find('\\', Offset);
+		}
+		PathBuilder.Append(InOutString.substr(Offset, BackslashPos));
+		InOutString = PathBuilder.ToString();
+	}
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+BoolOption::BoolOption(bool& Value) : Value(Value)
+{
+}
+
+void
+BoolOption::Print(std::string_view, zen::StringBuilderBase& StringBuilder)
+{
+	StringBuilder.Append(Value ? "true" : "false");
+}
+
+void
+BoolOption::Parse(sol::object Object)
+{
+	Value = Object.as<bool>();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+StringOption::StringOption(std::string& Value) : Value(Value)
+{
+}
+
+void
+StringOption::Print(std::string_view, zen::StringBuilderBase& StringBuilder)
+{
+	StringBuilder.Append(fmt::format("\"{}\"", Value));
+}
+
+void
+StringOption::Parse(sol::object Object)
+{
+	Value = Object.as<std::string>();
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+FilePathOption::FilePathOption(std::filesystem::path& Value) : Value(Value)
+{
+}
+
+void
+FilePathOption::Print(std::string_view, zen::StringBuilderBase& StringBuilder)
+{
+	std::string Path = Value.string();
+	EscapeBackslash(Path);
+	StringBuilder.Append(fmt::format("\"{}\"", Path));
+}
+
+void
+FilePathOption::Parse(sol::object Object)
+{
+	std::string Str = Object.as<std::string>();
+	if (!Str.empty())
+	{
+		Value = MakeSafePath(Str);
+	}
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+LuaContainerWriter::LuaContainerWriter(zen::StringBuilderBase& StringBuilder, std::string_view Indent)
+: StringBuilder(StringBuilder)
+, InitialIndent(Indent.length())
+, LocalIndent(Indent)
+{
+	StringBuilder.Append("{\n");
+	LocalIndent.push_back('\t');
+}
+
+LuaContainerWriter::~LuaContainerWriter()
+{
+	LocalIndent.pop_back();
+	StringBuilder.Append(LocalIndent);
+	StringBuilder.Append("}");
+}
+
+void
+LuaContainerWriter::BeginContainer(std::string_view Name)
+{
+	StringBuilder.Append(LocalIndent);
+	if (!Name.empty())
+	{
+		StringBuilder.Append(Name);
+		StringBuilder.Append(" = {\n");
+	}
+	else
+	{
+		StringBuilder.Append("{\n");
+	}
+	LocalIndent.push_back('\t');
+}
+
+void
+LuaContainerWriter::WriteValue(std::string_view Name, std::string_view Value)
+{
+	if (Name.empty())
+	{
+		StringBuilder.Append(fmt::format("{}\"{}\",\n", LocalIndent, Value));
+	}
+	else
+	{
+		StringBuilder.Append(fmt::format("{}{} = \"{}\",\n", LocalIndent, Name, Value));
+	}
+}
+
+void
+LuaContainerWriter::EndContainer()
+{
+	LocalIndent.pop_back();
+	StringBuilder.Append(LocalIndent);
+	StringBuilder.Append("}");
+	StringBuilder.Append(",\n");
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+StringArrayOption::StringArrayOption(std::vector<std::string>& Value) : Value(Value)
+{
+}
+
+void
+StringArrayOption::Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder)
+{
+	if (Value.empty())
+	{
+		StringBuilder.Append("{}");
+	}
+	if (Value.size() == 1)
+	{
+		StringBuilder.Append(fmt::format("\"{}\"", Value[0]));
+	}
+	else
+	{
+		LuaContainerWriter Writer(StringBuilder, Indent);
+		for (std::string String : Value)
+		{
+			Writer.WriteValue("", String);
+		}
+	}
+}
+
+void
+StringArrayOption::Parse(sol::object Object)
+{
+	if (Object.get_type() == sol::type::string)
+	{
+		Value.push_back(Object.as<std::string>());
+	}
+	else if (Object.get_type() == sol::type::table)
+	{
+		for (const auto& Kv : Object.as<sol::table>())
+		{
+			Value.push_back(Kv.second.as<std::string>());
+		}
+	}
+}
+
+std::shared_ptr<OptionValue>
+MakeOption(std::string& Value)
+{
+	return std::make_shared<StringOption>(Value);
+}
+
+std::shared_ptr<OptionValue>
+MakeOption(std::filesystem::path& Value)
+{
+	return std::make_shared<FilePathOption>(Value);
+}
+
+std::shared_ptr<OptionValue>
+MakeOption(bool& Value)
+{
+	return std::make_shared<BoolOption>(Value);
+}
+
+std::shared_ptr<OptionValue>
+MakeOption(std::vector<std::string>& Value)
+{
+	return std::make_shared<StringArrayOption>(Value);
+}
+
+void
+Options::Parse(const std::filesystem::path& Path, const cxxopts::ParseResult& CmdLineResult)
+{
+	zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(Path);
+
+	if (LuaScript)
+	{
+		sol::state lua;
+
+		lua.open_libraries(sol::lib::base);
+
+		lua.set_function("getenv", [&](const std::string env) -> sol::object {
+#if ZEN_PLATFORM_WINDOWS
+			std::wstring EnvVarValue;
+			size_t		 RequiredSize = 0;
+			std::wstring EnvWide	  = zen::Utf8ToWide(env);
+			_wgetenv_s(&RequiredSize, nullptr, 0, EnvWide.c_str());
+
+			if (RequiredSize == 0)
+				return sol::make_object(lua, sol::lua_nil);
+
+			EnvVarValue.resize(RequiredSize);
+			_wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str());
+			return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str()));
+#elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+			char* EnvVariable = getenv(env.c_str());
+			if (EnvVariable == nullptr)
+			{
+				return sol::make_object(lua, sol::lua_nil);
+			}
+			return sol::make_object(lua, EnvVariable);
+#else
+			ZEN_UNUSED(env);
+			return sol::make_object(lua, sol::lua_nil);
+#endif
+		});
+
+		try
+		{
+			sol::load_result config = lua.load(std::string_view((const char*)LuaScript.Data(), LuaScript.Size()), "zen_cfg");
+
+			if (!config.valid())
+			{
+				sol::error err = config;
+
+				std::string ErrorString = sol::to_string(config.status());
+
+				throw std::runtime_error(fmt::format("{} error: {}", ErrorString, err.what()));
+			}
+
+			config();
+		}
+		catch (std::exception& e)
+		{
+			throw std::runtime_error(fmt::format("failed to load config script ('{}'): {}", Path, e.what()).c_str());
+		}
+
+		Parse(lua, CmdLineResult);
+	}
+}
+
+void
+Options::Parse(const sol::state& LuaState, const cxxopts::ParseResult& CmdLineResult)
+{
+	for (auto It : LuaState)
+	{
+		sol::object Key		= It.first;
+		sol::type	KeyType = Key.get_type();
+		if (KeyType == sol::type::string)
+		{
+			sol::type ValueType = It.second.get_type();
+			switch (ValueType)
+			{
+				case sol::type::table:
+					{
+						std::string Name = Key.as<std::string>();
+						if (Name.starts_with("_"))
+						{
+							continue;
+						}
+						if (Name == "base")
+						{
+							continue;
+						}
+						Traverse(It.second.as<sol::table>(), Name, CmdLineResult);
+					}
+					break;
+				default:
+					break;
+			}
+		}
+	}
+}
+
+void
+Options::Touch(std::string_view Key)
+{
+	UsedKeys.insert(std::string(Key));
+}
+
+void
+Options::Print(zen::StringBuilderBase& SB, const cxxopts::ParseResult& CmdLineResult)
+{
+	for (auto It : OptionMap)
+	{
+		if (CmdLineResult.count(It.second.CommandLineOptionName) != 0)
+		{
+			UsedKeys.insert(It.first);
+		}
+	}
+
+	std::vector<std::string> SortedKeys(UsedKeys.begin(), UsedKeys.end());
+	std::sort(SortedKeys.begin(), SortedKeys.end());
+	auto GetTablePath = [](const std::string& Key) -> std::vector<std::string> {
+		std::vector<std::string> Path;
+		zen::ForEachStrTok(Key, '.', [&Path](std::string_view Part) {
+			Path.push_back(std::string(Part));
+			return true;
+		});
+		return Path;
+	};
+	std::vector<std::string> CurrentTablePath;
+	std::string				 Indent;
+	auto					 It = SortedKeys.begin();
+	for (const std::string& Key : SortedKeys)
+	{
+		std::vector<std::string> KeyPath = GetTablePath(Key);
+		std::string				 Name	 = KeyPath.back();
+		KeyPath.pop_back();
+		if (CurrentTablePath != KeyPath)
+		{
+			size_t EqualCount = 0;
+			while (EqualCount < CurrentTablePath.size() && EqualCount < KeyPath.size() &&
+				   CurrentTablePath[EqualCount] == KeyPath[EqualCount])
+			{
+				EqualCount++;
+			}
+			while (CurrentTablePath.size() > EqualCount)
+			{
+				CurrentTablePath.pop_back();
+				Indent.pop_back();
+				SB.Append(Indent);
+				SB.Append("}");
+				if (CurrentTablePath.size() == EqualCount && !Indent.empty() && KeyPath.size() >= EqualCount)
+				{
+					SB.Append(",");
+				}
+				SB.Append("\n");
+				if (Indent.empty())
+				{
+					SB.Append("\n");
+				}
+			}
+			while (EqualCount < KeyPath.size())
+			{
+				SB.Append(Indent);
+				SB.Append(KeyPath[EqualCount]);
+				SB.Append(" = {\n");
+				Indent.push_back('\t');
+				CurrentTablePath.push_back(KeyPath[EqualCount]);
+				EqualCount++;
+			}
+		}
+
+		SB.Append(Indent);
+		SB.Append(Name);
+		SB.Append(" = ");
+		OptionMap[Key].Value->Print(Indent, SB);
+		SB.Append(",\n");
+	}
+	while (!CurrentTablePath.empty())
+	{
+		Indent.pop_back();
+		SB.Append(Indent);
+		SB.Append("}\n");
+		CurrentTablePath.pop_back();
+	}
+}
+
+void
+Options::Traverse(sol::table Table, std::string_view PathPrefix, const cxxopts::ParseResult& CmdLineResult)
+{
+	for (auto It : Table)
+	{
+		sol::object Key		= It.first;
+		sol::type	KeyType = Key.get_type();
+		if (KeyType == sol::type::string || KeyType == sol::type::number)
+		{
+			sol::type ValueType = It.second.get_type();
+			switch (ValueType)
+			{
+				case sol::type::table:
+				case sol::type::string:
+				case sol::type::number:
+				case sol::type::boolean:
+					{
+						std::string Name = Key.as<std::string>();
+						if (Name.starts_with("_"))
+						{
+							continue;
+						}
+						Name		  = std::string(PathPrefix) + "." + Key.as<std::string>();
+						auto OptionIt = OptionMap.find(Name);
+						if (OptionIt != OptionMap.end())
+						{
+							UsedKeys.insert(Name);
+							if (CmdLineResult.count(OptionIt->second.CommandLineOptionName) != 0)
+							{
+								continue;
+							}
+							OptionIt->second.Value->Parse(It.second);
+							continue;
+						}
+						if (ValueType == sol::type::table)
+						{
+							if (Name == "base")
+							{
+								continue;
+							}
+							Traverse(It.second.as<sol::table>(), Name, CmdLineResult);
+						}
+					}
+					break;
+				default:
+					break;
+			}
+		}
+	}
+}
+
+}  // namespace zen::LuaConfig
diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h
new file mode 100644
index 000000000..76b3088a3
--- /dev/null
+++ b/src/zenserver/config/luaconfig.h
@@ -0,0 +1,139 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zenbase/concepts.h>
+#include <zencore/fmtutils.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+#include <cxxopts.hpp>
+#include <sol/sol.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <filesystem>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+
+namespace zen::LuaConfig {
+
+std::string MakeSafePath(const std::string_view Path);
+void		EscapeBackslash(std::string& InOutString);
+
+class OptionValue
+{
+public:
+	virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) = 0;
+	virtual void Parse(sol::object Object)											   = 0;
+
+	virtual ~OptionValue() {}
+};
+
+class StringOption : public OptionValue
+{
+public:
+	explicit StringOption(std::string& Value);
+	virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override;
+	virtual void Parse(sol::object Object) override;
+	std::string& Value;
+};
+
+class FilePathOption : public OptionValue
+{
+public:
+	explicit FilePathOption(std::filesystem::path& Value);
+	virtual void		   Print(std::string_view, zen::StringBuilderBase& StringBuilder) override;
+	virtual void		   Parse(sol::object Object) override;
+	std::filesystem::path& Value;
+};
+
+class BoolOption : public OptionValue
+{
+public:
+	explicit BoolOption(bool& Value);
+	virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder);
+	virtual void Parse(sol::object Object);
+	bool&		 Value;
+};
+
+template<Integral T>
+class NumberOption : public OptionValue
+{
+public:
+	explicit NumberOption(T& Value) : Value(Value) {}
+	virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override { StringBuilder.Append(fmt::format("{}", Value)); }
+	virtual void Parse(sol::object Object) override { Value = Object.as<T>(); }
+	T&			 Value;
+};
+
+class LuaContainerWriter
+{
+public:
+	LuaContainerWriter(zen::StringBuilderBase& StringBuilder, std::string_view Indent);
+	~LuaContainerWriter();
+	void BeginContainer(std::string_view Name);
+	void WriteValue(std::string_view Name, std::string_view Value);
+	void EndContainer();
+
+private:
+	zen::StringBuilderBase& StringBuilder;
+	const std::size_t		InitialIndent;
+	std::string				LocalIndent;
+};
+
+class StringArrayOption : public OptionValue
+{
+public:
+	explicit StringArrayOption(std::vector<std::string>& Value);
+	virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override;
+	virtual void Parse(sol::object Object) override;
+
+private:
+	std::vector<std::string>& Value;
+};
+
+std::shared_ptr<OptionValue> MakeOption(std::string& Value);
+std::shared_ptr<OptionValue> MakeOption(std::filesystem::path& Value);
+
+template<Integral T>
+std::shared_ptr<OptionValue>
+MakeOption(T& Value)
+{
+	return std::make_shared<NumberOption<T>>(Value);
+};
+
+std::shared_ptr<OptionValue> MakeOption(bool& Value);
+std::shared_ptr<OptionValue> MakeOption(std::vector<std::string>& Value);
+
+struct Option
+{
+	std::string					 CommandLineOptionName;
+	std::shared_ptr<OptionValue> Value;
+};
+
+struct Options
+{
+public:
+	template<typename T>
+	void AddOption(std::string_view Key, T& Value, std::string_view CommandLineOptionName = "")
+	{
+		OptionMap.insert_or_assign(std::string(Key),
+								   Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)});
+	};
+
+	void Parse(const std::filesystem::path& Path, const cxxopts::ParseResult& CmdLineResult);
+	void Parse(const sol::state& LuaState, const cxxopts::ParseResult& CmdLineResult);
+	void Touch(std::string_view Key);
+	void Print(zen::StringBuilderBase& SB, const cxxopts::ParseResult& CmdLineResult);
+
+private:
+	void Traverse(sol::table Table, std::string_view PathPrefix, const cxxopts::ParseResult& CmdLineResult);
+
+	std::unordered_map<std::string, Option> OptionMap;
+	std::unordered_set<std::string>			UsedKeys;
+};
+
+}  // namespace zen::LuaConfig
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index e2d57b840..dc1675819 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -42,6 +42,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																	   /* max files */ 16,
 																	   /* rotate on open */ true);
 	auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
+	spdlog::apply_logger_env_levels(HttpLogger);
 	spdlog::register_logger(HttpLogger);
 
 	// Cache request logging
@@ -53,16 +54,19 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																		/* max files */ 16,
 																		/* rotate on open */ false);
 	auto CacheLogger = std::make_shared<spdlog::logger>("z$", CacheSink);
+	spdlog::apply_logger_env_levels(CacheLogger);
 	spdlog::register_logger(CacheLogger);
 
 	// Jupiter - only log upstream HTTP traffic to file
 
 	auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
+	spdlog::apply_logger_env_levels(JupiterLogger);
 	spdlog::register_logger(JupiterLogger);
 
 	// Zen - only log upstream HTTP traffic to file
 
 	auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
+	spdlog::apply_logger_env_levels(ZenClientLogger);
 	spdlog::register_logger(ZenClientLogger);
 
 	FinishInitializeLogging(LogOptions);
diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp
index 8c8e5cb9c..9bc408711 100644
--- a/src/zenserver/frontend/frontend.cpp
+++ b/src/zenserver/frontend/frontend.cpp
@@ -14,6 +14,9 @@ ZEN_THIRD_PARTY_INCLUDES_START
 #endif
 ZEN_THIRD_PARTY_INCLUDES_END
 
+static unsigned char gHtmlZipData[] = {
+#include <html.zip.h>
+};
 namespace zen {
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -22,8 +25,8 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di
 	std::filesystem::path SelfPath = GetRunningExecutablePath();
 
 	// Locate a .zip file appended onto the end of this binary
-	IoBuffer SelfBuffer = IoBufferBuilder::MakeFromFile(SelfPath);
-	m_ZipFs				= ZipFs(std::move(SelfBuffer));
+	IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1);
+	m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer));
 
 	if (m_Directory.empty() && !m_ZipFs)
 	{
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
new file mode 100644
index 000000000..fa2f2febf
--- /dev/null
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 69cc2bbf5..be2cdcc2d 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -39,6 +39,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #if ZEN_WITH_TESTS
 #	define ZEN_TEST_WITH_RUNNER 1
 #	include <zencore/testing.h>
+#	include <zenutil/zenutil.h>
 #endif
 
 #include <memory>
@@ -198,13 +199,15 @@ ZenEntryPoint::Run()
 		ShutdownThread.reset(new std::thread{[&] {
 			SetCurrentThreadName("shutdown_monitor");
 
-			ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName);
+			ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}",
+					 ShutdownEventName,
+					 zen::GetCurrentProcessId());
 
 			if (ShutdownEvent->Wait())
 			{
 				if (!IsApplicationExitRequested())
 				{
-					ZEN_INFO("shutdown signal received");
+					ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId());
 					Server.RequestExit(0);
 				}
 			}
@@ -244,7 +247,7 @@ ZenEntryPoint::Run()
 	}
 	catch (std::exception& e)
 	{
-		ZEN_CRITICAL("Caught exception in main: {}", e.what());
+		ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what());
 		if (!IsApplicationExitRequested())
 		{
 			RequestApplicationExit(1);
@@ -293,6 +296,7 @@ test_main(int argc, char** argv)
 	zen::zencore_forcelinktests();
 	zen::zenhttp_forcelinktests();
 	zen::zenstore_forcelinktests();
+	zen::zenutil_forcelinktests();
 	zen::z$_forcelink();
 	zen::z$service_forcelink();
 
@@ -334,9 +338,24 @@ main(int argc, char* argv[])
 		ZenServerOptions ServerOptions;
 		ParseCliOptions(argc, argv, ServerOptions);
 
+		std::string_view DeleteReason;
+
 		if (ServerOptions.IsCleanStart)
 		{
-			DeleteDirectories(ServerOptions.DataDir);
+			DeleteReason = "clean start requested"sv;
+		}
+		else if (!ServerOptions.BaseSnapshotDir.empty())
+		{
+			DeleteReason = "will initialize state from base snapshot"sv;
+		}
+
+		if (!DeleteReason.empty())
+		{
+			if (std::filesystem::exists(ServerOptions.DataDir))
+			{
+				ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason);
+				DeleteDirectories(ServerOptions.DataDir);
+			}
 		}
 
 		if (!std::filesystem::exists(ServerOptions.DataDir))
@@ -345,18 +364,24 @@ main(int argc, char* argv[])
 			std::filesystem::create_directories(ServerOptions.DataDir);
 		}
 
+		if (!ServerOptions.BaseSnapshotDir.empty())
+		{
+			ZEN_CONSOLE_INFO("copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir);
+			CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true});
+		}
+
 #if ZEN_WITH_TRACE
 		if (ServerOptions.TraceHost.size())
 		{
-			TraceStart(ServerOptions.TraceHost.c_str(), TraceType::Network);
+			TraceStart("zenserver", ServerOptions.TraceHost.c_str(), TraceType::Network);
 		}
 		else if (ServerOptions.TraceFile.size())
 		{
-			TraceStart(ServerOptions.TraceFile.c_str(), TraceType::File);
+			TraceStart("zenserver", ServerOptions.TraceFile.c_str(), TraceType::File);
 		}
 		else
 		{
-			TraceInit();
+			TraceInit("zenserver");
 		}
 		atexit(TraceShutdown);
 #endif	// ZEN_WITH_TRACE
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 3643e8011..47ef5c8b3 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -2,14 +2,18 @@
 
 #include <objectstore/objectstore.h>
 
+#include <zencore/base64.h>
+#include <zencore/compactbinaryvalue.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include "zencore/compactbinary.h"
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
 #include "zenhttp/httpserver.h"
 
+#include <filesystem>
 #include <thread>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -23,6 +27,198 @@ using namespace std::literals;
 
 ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv);
 
+class CbXmlWriter
+{
+public:
+	explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder)
+	{
+		Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		Builder << LINE_TERMINATOR_ANSI;
+	}
+
+	void WriteField(CbFieldView Field)
+	{
+		using namespace std::literals;
+
+		bool					 SkipEndTag = false;
+		const std::u8string_view Tag		= Field.GetU8Name();
+
+		AppendBeginTag(Tag);
+
+		switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+		{
+			case CbFieldType::Null:
+				Builder << "Null"sv;
+				break;
+			case CbFieldType::Object:
+			case CbFieldType::UniformObject:
+				{
+					for (CbFieldView It : Field)
+					{
+						WriteField(It);
+					}
+				}
+				break;
+			case CbFieldType::Array:
+			case CbFieldType::UniformArray:
+				{
+					bool FirstField = true;
+					for (CbFieldView It : Field)
+					{
+						if (!FirstField)
+							AppendBeginTag(Tag);
+
+						WriteField(It);
+						AppendEndTag(Tag);
+						FirstField = false;
+					}
+					SkipEndTag = true;
+				}
+				break;
+			case CbFieldType::Binary:
+				AppendBase64String(Accessor.AsBinary());
+				break;
+			case CbFieldType::String:
+				Builder << Accessor.AsU8String();
+				break;
+			case CbFieldType::IntegerPositive:
+				Builder << Accessor.AsIntegerPositive();
+				break;
+			case CbFieldType::IntegerNegative:
+				Builder << Accessor.AsIntegerNegative();
+				break;
+			case CbFieldType::Float32:
+				{
+					const float Value = Accessor.AsFloat32();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.9g}", Value));
+					}
+					else
+					{
+						Builder << "Null"sv;
+					}
+				}
+				break;
+			case CbFieldType::Float64:
+				{
+					const double Value = Accessor.AsFloat64();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.17g}", Value));
+					}
+					else
+					{
+						Builder << "null"sv;
+					}
+				}
+				break;
+			case CbFieldType::BoolFalse:
+				Builder << "False"sv;
+				break;
+			case CbFieldType::BoolTrue:
+				Builder << "True"sv;
+				break;
+			case CbFieldType::ObjectAttachment:
+			case CbFieldType::BinaryAttachment:
+				{
+					Accessor.AsAttachment().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Hash:
+				{
+					Accessor.AsHash().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Uuid:
+				{
+					Accessor.AsUuid().ToString(Builder);
+				}
+				break;
+			case CbFieldType::DateTime:
+				Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+				break;
+			case CbFieldType::TimeSpan:
+				{
+					const TimeSpan Span(Accessor.AsTimeSpanTicks());
+					if (Span.GetDays() == 0)
+					{
+						Builder << Span.ToString("%h:%m:%s.%n");
+					}
+					else
+					{
+						Builder << Span.ToString("%d.%h:%m:%s.%n");
+					}
+					break;
+				}
+			case CbFieldType::ObjectId:
+				Accessor.AsObjectId().ToString(Builder);
+				break;
+			case CbFieldType::CustomById:
+				{
+					CbCustomById Custom = Accessor.AsCustomById();
+
+					AppendBeginTag(u8"Id"sv);
+					Builder << Custom.Id;
+					AppendEndTag(u8"Id"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			case CbFieldType::CustomByName:
+				{
+					CbCustomByName Custom = Accessor.AsCustomByName();
+
+					AppendBeginTag(u8"Name"sv);
+					Builder << Custom.Name;
+					AppendEndTag(u8"Name"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			default:
+				ZEN_ASSERT(false);
+				break;
+		}
+
+		if (!SkipEndTag)
+			AppendEndTag(Tag);
+	}
+
+private:
+	void AppendBeginTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << '<' << Tag << '>';
+		}
+	}
+
+	void AppendEndTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << "</"sv << Tag << '>';
+		}
+	}
+
+	void AppendBase64String(MemoryView Value)
+	{
+		Builder << '"';
+		ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+		const uint32_t EncodedSize	= Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+		const size_t   EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+		Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+	}
+
+private:
+	StringBuilderBase& Builder;
+};
+
 HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg))
 {
 	Inititalize();
@@ -51,64 +247,218 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
 void
 HttpObjectStoreService::Inititalize()
 {
+	namespace fs = std::filesystem;
 	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
-	for (const auto& Bucket : m_Cfg.Buckets)
+
+	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
+	if (!fs::exists(BucketsPath))
 	{
-		ZEN_LOG_INFO(LogObj, " - bucket '{}' -> '{}'", Bucket.Name, Bucket.Directory);
+		CreateDirectories(BucketsPath);
 	}
 
 	m_Router.RegisterRoute(
-		"distributionpoints/{bucket}",
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		HttpVerb::kPost | HttpVerb::kPut);
+
+	m_Router.RegisterRoute(
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		HttpVerb::kDelete);
+
+	m_Router.RegisterRoute(
+		"bucket/{path}",
 		[this](zen::HttpRouterRequest& Request) {
-			const std::string BucketName = Request.GetCapture(1);
+			const std::string Path	   = Request.GetCapture(1);
+			const auto		  Sep	   = Path.find_last_of('.');
+			const bool		  IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
 
-			ExtendableStringBuilder<1024> Json;
+			if (IsObject)
 			{
-				CbObjectWriter Writer;
-				Writer.BeginArray("distributions");
-				Writer << fmt::format("http://localhost:{}/obj/{}", m_Cfg.ServerPort, BucketName);
-				Writer.EndArray();
-				Writer.Save().ToJson(Json);
+				GetObject(Request, Path);
+			}
+			else
+			{
+				ListBucket(Request, Path);
 			}
-
-			Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Json.ToString());
 		},
-		HttpVerb::kGet);
-
-	m_Router.RegisterRoute(
-		"{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { GetBlob(Request); },
-		HttpVerb::kGet);
+		HttpVerb::kHead | HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
-		"{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { PutBlob(Request); },
+		"bucket/{bucket}/{path}",
+		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 }
 
 std::filesystem::path
 HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 {
-	std::lock_guard _(BucketsMutex);
+	{
+		std::lock_guard _(BucketsMutex);
+
+		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
+										 std::end(m_Cfg.Buckets),
+										 [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; });
+			It != std::end(m_Cfg.Buckets))
+		{
+			return It->Directory.make_preferred();
+		}
+	}
+
+	return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred();
+}
+
+void
+HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
 
-	if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
-									 std::end(m_Cfg.Buckets),
-									 [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; });
-		It != std::end(m_Cfg.Buckets))
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		return It->Directory;
+		std::lock_guard _(BucketsMutex);
+		if (!fs::exists(BucketPath))
+		{
+			CreateDirectories(BucketPath);
+			ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::Created);
+		}
 	}
 
-	return std::filesystem::path();
+	ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
 }
 
 void
-HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path)
 {
 	namespace fs = std::filesystem;
 
-	const std::string& BucketName = Request.GetCapture(1);
-	const fs::path	   BucketDir  = GetBucketDirectory(BucketName);
+	const auto		  Sep		 = Path.find_first_of('/');
+	const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+	if (BucketPrefix.empty())
+	{
+		const auto QueryParms = Request.ServerRequest().GetQueryParams();
+		if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false)
+		{
+			BucketPrefix = PrefixParam;
+		}
+	}
+	BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/'));
+	BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\'));
+
+	const fs::path BucketRoot		  = GetBucketDirectory(BucketName);
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
+	const fs::path FullPath			  = BucketRoot / RelativeBucketPath;
+
+	struct Visitor : FileSystemTraversal::TreeVisitor
+	{
+		Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path)
+		{
+			Writer.BeginObject("ListBucketResult"sv);
+			Writer << "Name"sv << BucketName;
+			std::string Tmp = Prefix.string();
+			std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
+			Writer << "Prefix"sv << Tmp;
+			Writer.BeginArray("Contents"sv);
+		}
+
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			const fs::path FullPath		= Parent / fs::path(File);
+			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
+
+			std::string Key = RelativePath.string();
+			std::replace(Key.begin(), Key.end(), '\\', '/');
+
+			Writer.BeginObject();
+			Writer << "Key"sv << Key;
+			Writer << "Size"sv << FileSize;
+			Writer.EndObject();
+		}
+
+		bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; }
+
+		CbObject GetResult()
+		{
+			Writer.EndArray();
+			Writer.EndObject();
+			return Writer.Save();
+		}
+
+		CbObjectWriter Writer;
+		fs::path	   BucketPath;
+	};
+
+	Visitor				FileVisitor(BucketName, BucketRoot, RelativeBucketPath);
+	FileSystemTraversal Traversal;
+
+	{
+		std::lock_guard _(BucketsMutex);
+		Traversal.TraverseFileSystem(FullPath, FileVisitor);
+	}
+	CbObject Result = FileVisitor.GetResult();
+
+	if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON)
+	{
+		ExtendableStringBuilder<1024> Sb;
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView());
+	}
+
+	ExtendableStringBuilder<1024> Xml;
+	CbXmlWriter					  XmlWriter(Xml);
+	XmlWriter.WriteField(Result.AsFieldView());
+
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView());
+}
+
+void
+HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
+	{
+		std::lock_guard _(BucketsMutex);
+		DeleteDirectories(BucketPath);
+	}
+
+	ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path)
+{
+	namespace fs = std::filesystem;
+
+	const auto		  Sep		 = Path.find_first_of('/');
+	const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
+	const std::string BucketPrefix =
+		Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+
+	const fs::path BucketDir = GetBucketDirectory(BucketName);
 
 	if (BucketDir.empty())
 	{
@@ -116,7 +466,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	const fs::path RelativeBucketPath = Request.GetCapture(2);
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
 
 	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
 	{
@@ -124,8 +474,8 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
 	}
 
-	fs::path FilePath = BucketDir / RelativeBucketPath;
-	if (fs::exists(FilePath) == false)
+	const fs::path FilePath = BucketDir / RelativeBucketPath;
+	if (!fs::exists(FilePath))
 	{
 		ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath);
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
@@ -138,7 +488,12 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
 	}
 
-	FileContents File = ReadFile(FilePath);
+	FileContents File;
+	{
+		std::lock_guard _(BucketsMutex);
+		File = ReadFile(FilePath);
+	}
+
 	if (File.ErrorCode)
 	{
 		ZEN_LOG_WARN(LogObj,
@@ -194,7 +549,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -207,7 +562,7 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	const fs::path RelativeBucketPath = Request.GetCapture(2);
+	const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred();
 
 	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
 	{
@@ -215,17 +570,32 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
 	}
 
-	fs::path	   FilePath = BucketDir / RelativeBucketPath;
-	const IoBuffer FileBuf	= Request.ServerRequest().ReadPayload();
+	const fs::path FilePath		 = BucketDir / RelativeBucketPath;
+	const fs::path FileDirectory = FilePath.parent_path();
 
-	if (FileBuf.Size() == 0)
 	{
-		ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [FAILED], empty file", BucketName, FilePath);
-		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		std::lock_guard _(BucketsMutex);
+
+		if (!fs::exists(FileDirectory))
+		{
+			CreateDirectories(FileDirectory);
+		}
+
+		const IoBuffer FileBuf = Request.ServerRequest().ReadPayload();
+
+		if (FileBuf.Size() == 0)
+		{
+			ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		}
+
+		WriteFile(FilePath, FileBuf);
+		ZEN_LOG_DEBUG(LogObj,
+					  "PUT - '{}' [OK] ({})",
+					  (fs::path(BucketName) / RelativeBucketPath).make_preferred(),
+					  NiceBytes(FileBuf.Size()));
 	}
 
-	WriteFile(FilePath, FileBuf);
-	ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [OK] ({})", BucketName, RelativeBucketPath, NiceBytes(FileBuf.Size()));
 	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
 }
 
diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h
index 0fec59b03..c905ceab3 100644
--- a/src/zenserver/objectstore/objectstore.h
+++ b/src/zenserver/objectstore/objectstore.h
@@ -21,7 +21,6 @@ struct ObjectStoreConfig
 
 	std::filesystem::path	  RootDirectory;
 	std::vector<BucketConfig> Buckets;
-	uint16_t				  ServerPort{8558};
 };
 
 class HttpObjectStoreService final : public zen::HttpService
@@ -36,8 +35,11 @@ public:
 private:
 	void				  Inititalize();
 	std::filesystem::path GetBucketDirectory(std::string_view BucketName);
-	void				  GetBlob(zen::HttpRouterRequest& Request);
-	void				  PutBlob(zen::HttpRouterRequest& Request);
+	void				  CreateBucket(zen::HttpRouterRequest& Request);
+	void				  ListBucket(zen::HttpRouterRequest& Request, const std::string& Path);
+	void				  DeleteBucket(zen::HttpRouterRequest& Request);
+	void				  GetObject(zen::HttpRouterRequest& Request, const std::string& Path);
+	void				  PutObject(zen::HttpRouterRequest& Request);
 
 	ObjectStoreConfig	 m_Cfg;
 	std::mutex			 BucketsMutex;
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 261485834..0ba49cf8a 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -276,6 +276,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
+		"{project}/oplog/{log}/chunkinfos",
+		[this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{chunk}/info",
 		[this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); },
 		HttpVerb::kGet);
@@ -643,6 +648,41 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req)
 }
 
 void
+HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("ProjectService::ChunkInfos");
+
+	HttpServerRequest& HttpReq = Req.ServerRequest();
+
+	const auto& ProjectId = Req.GetCapture(1);
+	const auto& OplogId	  = Req.GetCapture(2);
+
+	CbObject								 ResponsePayload;
+	std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetProjectChunkInfos(ProjectId, OplogId, ResponsePayload);
+	if (Result.first == HttpResponseCode::OK)
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
+	}
+	else
+	{
+		if (Result.first == HttpResponseCode::BadRequest)
+		{
+			m_ProjectStats.BadRequestCount++;
+		}
+		ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+				  ToString(HttpReq.RequestVerb()),
+				  HttpReq.QueryString(),
+				  static_cast<int>(Result.first),
+				  Result.second);
+	}
+	if (Result.second.empty())
+	{
+		return HttpReq.WriteResponse(Result.first);
+	}
+	return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+}
+
+void
 HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ChunkInfo");
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 9998ae83e..9990ee264 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -64,6 +64,7 @@ private:
 	void HandleProjectListRequest(HttpRouterRequest& Req);
 	void HandleChunkBatchRequest(HttpRouterRequest& Req);
 	void HandleFilesRequest(HttpRouterRequest& Req);
+	void HandleChunkInfosRequest(HttpRouterRequest& Req);
 	void HandleChunkInfoRequest(HttpRouterRequest& Req);
 	void HandleChunkByIdRequest(HttpRouterRequest& Req);
 	void HandleChunkByCidRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 9fedd9165..73cb35fb8 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -2,6 +2,7 @@
 
 #include "projectstore.h"
 
+#include <zencore/assertfmt.h>
 #include <zencore/compactbinarybuilder.h>
 #include <zencore/compactbinarypackage.h>
 #include <zencore/compactbinaryutil.h>
@@ -298,38 +299,60 @@ struct ProjectStore::OplogStorage : public RefCounted
 
 		Stopwatch Timer;
 
-		uint64_t InvalidEntries = 0;
+		uint64_t InvalidEntries	  = 0;
+		uint64_t TombstoneEntries = 0;
 
 		std::vector<OplogEntry> OpLogEntries;
 		std::vector<size_t>		OplogOrder;
 		{
-			tsl::robin_map<XXH3_128, size_t, XXH3_128::Hasher> LatestKeys;
+			tsl::robin_map<Oid, size_t, Oid::Hasher> LatestKeys;
+			const uint64_t							 SkipEntryCount = 0;
+
 			m_Oplog.Replay(
 				[&](const OplogEntry& LogEntry) {
-					if (LogEntry.OpCoreSize == 0)
+					if (LogEntry.IsTombstone())
 					{
-						++InvalidEntries;
-						return;
+						if (auto It = LatestKeys.find(LogEntry.OpKeyHash); It == LatestKeys.end())
+						{
+							ZEN_SCOPED_WARN("found tombstone referencing unknown key {}", LogEntry.OpKeyHash);
+						}
+					}
+					else
+					{
+						if (LogEntry.OpCoreSize == 0)
+						{
+							++InvalidEntries;
+							return;
+						}
+
+						const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
+						m_NextOpsOffset =
+							Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign));
+						m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn);
 					}
-					const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
-					m_NextOpsOffset =
-						Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign));
-					m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn);
 
 					if (auto It = LatestKeys.find(LogEntry.OpKeyHash); It != LatestKeys.end())
 					{
-						OpLogEntries[It->second] = LogEntry;
+						OplogEntry& Entry = OpLogEntries[It->second];
+
+						if (LogEntry.IsTombstone() && Entry.IsTombstone())
+						{
+							ZEN_SCOPED_WARN("found double tombstone - '{}'", LogEntry.OpKeyHash);
+						}
+
+						Entry = LogEntry;
 					}
 					else
 					{
-						size_t OpIndex				   = OpLogEntries.size();
+						const size_t OpIndex		   = OpLogEntries.size();
 						LatestKeys[LogEntry.OpKeyHash] = OpIndex;
 						OplogOrder.push_back(OpIndex);
 						OpLogEntries.push_back(LogEntry);
 					}
 				},
-				0);
+				SkipEntryCount);
 		}
+
 		std::sort(OplogOrder.begin(), OplogOrder.end(), [&](size_t Lhs, size_t Rhs) {
 			const OplogEntry& LhsEntry = OpLogEntries[Lhs];
 			const OplogEntry& RhsEntry = OpLogEntries[Rhs];
@@ -342,47 +365,54 @@ struct ProjectStore::OplogStorage : public RefCounted
 		{
 			const OplogEntry& LogEntry = OpLogEntries[OplogOrderIndex];
 
-			const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
-			MemoryView	   OpBufferView = OpBlobsBuffer.MakeView(LogEntry.OpCoreSize, OpFileOffset);
-			if (OpBufferView.GetSize() == LogEntry.OpCoreSize)
+			if (LogEntry.IsTombstone())
+			{
+				TombstoneEntries++;
+			}
+			else
 			{
 				// Verify checksum, ignore op data if incorrect
-				const auto OpCoreHash = uint32_t(XXH3_64bits(OpBufferView.GetData(), LogEntry.OpCoreSize) & 0xffffFFFF);
 
-				if (OpCoreHash != LogEntry.OpCoreHash)
-				{
-					ZEN_WARN("skipping oplog entry with bad checksum!");
-					InvalidEntries++;
-					continue;
-				}
-				Handler(CbObjectView(OpBufferView.GetData()), LogEntry);
-				continue;
-			}
+				auto VerifyAndHandleOp = [&](MemoryView OpBufferView) {
+					const uint32_t OpCoreHash = uint32_t(XXH3_64bits(OpBufferView.GetData(), LogEntry.OpCoreSize) & 0xffffFFFF);
 
-			IoBuffer OpBuffer(LogEntry.OpCoreSize);
-			OpBlobsBuffer.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset);
+					if (OpCoreHash == LogEntry.OpCoreHash)
+					{
+						Handler(CbObjectView(OpBufferView.GetData()), LogEntry);
+					}
+					else
+					{
+						ZEN_WARN("skipping oplog entry with bad checksum!");
+						InvalidEntries++;
+					}
+				};
 
-			// Verify checksum, ignore op data if incorrect
-			const auto OpCoreHash = uint32_t(XXH3_64bits(OpBuffer.Data(), LogEntry.OpCoreSize) & 0xffffFFFF);
+				const uint64_t	 OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign;
+				const MemoryView OpBufferView = OpBlobsBuffer.MakeView(LogEntry.OpCoreSize, OpFileOffset);
+				if (OpBufferView.GetSize() == LogEntry.OpCoreSize)
+				{
+					VerifyAndHandleOp(OpBufferView);
+				}
+				else
+				{
+					IoBuffer OpBuffer(LogEntry.OpCoreSize);
+					OpBlobsBuffer.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset);
 
-			if (OpCoreHash != LogEntry.OpCoreHash)
-			{
-				ZEN_WARN("skipping oplog entry with bad checksum!");
-				InvalidEntries++;
-				continue;
+					VerifyAndHandleOp(OpBuffer);
+				}
 			}
-			Handler(CbObjectView(OpBuffer.Data()), LogEntry);
 		}
 
 		if (InvalidEntries)
 		{
-			ZEN_WARN("ignored {} zero-sized oplog entries", InvalidEntries);
+			ZEN_WARN("ignored {} invalid oplog entries", InvalidEntries);
 		}
 
-		ZEN_INFO("Oplog replay completed in {} - Max LSN# {}, Next offset: {}",
+		ZEN_INFO("oplog replay completed in {} - Max LSN# {}, Next offset: {}, {} tombstones",
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 				 m_MaxLsn.load(),
-				 m_NextOpsOffset.load());
+				 m_NextOpsOffset.load(),
+				 TombstoneEntries);
 	}
 
 	void ReplayLogEntries(const std::span<OplogEntryAddress> Entries, std::function<void(CbObjectView)>&& Handler)
@@ -418,7 +448,7 @@ struct ProjectStore::OplogStorage : public RefCounted
 		return CbObject(SharedBuffer(std::move(OpBuffer)));
 	}
 
-	OplogEntry AppendOp(SharedBuffer Buffer, uint32_t OpCoreHash, XXH3_128 KeyHash)
+	OplogEntry AppendOp(SharedBuffer Buffer, uint32_t OpCoreHash, Oid KeyHash)
 	{
 		ZEN_TRACE_CPU("Store::OplogStorage::AppendOp");
 
@@ -446,6 +476,14 @@ struct ProjectStore::OplogStorage : public RefCounted
 		return Entry;
 	}
 
+	void AppendTombstone(Oid KeyHash)
+	{
+		OplogEntry Entry = {.OpKeyHash = KeyHash};
+		Entry.MakeTombstone();
+
+		m_Oplog.Append(Entry);
+	}
+
 	void Flush()
 	{
 		m_Oplog.Flush();
@@ -507,9 +545,67 @@ ProjectStore::Oplog::Flush()
 }
 
 void
-ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx) const
+ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
 {
-	ZEN_UNUSED(Ctx);
+	std::vector<Oid> BadEntryKeys;
+
+	using namespace std::literals;
+
+	IterateOplogWithKey([&](int Lsn, const Oid& Key, CbObjectView Op) {
+		ZEN_UNUSED(Lsn);
+
+		std::vector<IoHash> Cids;
+		Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); });
+
+		{
+			XXH3_128Stream KeyHasher;
+			Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
+			XXH3_128 KeyHash128 = KeyHasher.GetHash();
+			Oid		 KeyHash;
+			memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+
+			ZEN_ASSERT_FORMAT(KeyHash == Key, "oplog data does not match information from index (op:{} != index:{})", KeyHash, Key);
+		}
+
+		for (const IoHash& Cid : Cids)
+		{
+			if (!m_CidStore.ContainsChunk(Cid))
+			{
+				// oplog entry references a CAS chunk which is not
+				// present
+				BadEntryKeys.push_back(Key);
+				return;
+			}
+			if (Ctx.IsBadCid(Cid))
+			{
+				// oplog entry references a CAS chunk which has been
+				// flagged as bad
+				BadEntryKeys.push_back(Key);
+				return;
+			}
+		}
+	});
+
+	if (!BadEntryKeys.empty())
+	{
+		if (Ctx.RunRecovery())
+		{
+			ZEN_WARN("scrubbing found {} bad ops in oplog @ '{}', these will be removed from the index", BadEntryKeys.size(), m_BasePath);
+
+			// Actually perform some clean-up
+			RwLock::ExclusiveLockScope _(m_OplogLock);
+
+			for (const auto& Key : BadEntryKeys)
+			{
+				m_LatestOpMap.erase(Key);
+				m_Storage->AppendTombstone(Key);
+			}
+		}
+		else
+		{
+			ZEN_WARN("scrubbing found {} bad ops in oplog @ '{}' but no cleanup will be performed", BadEntryKeys.size(), m_BasePath);
+		}
+	}
 }
 
 void
@@ -658,6 +754,8 @@ ProjectStore::Oplog::Update(const std::filesystem::path& MarkerPath)
 void
 ProjectStore::Oplog::ReplayLog()
 {
+	ZEN_LOG_SCOPE("ReplayLog '{}'", m_OplogId);
+
 	RwLock::ExclusiveLockScope OplogLock(m_OplogLock);
 	if (!m_Storage)
 	{
@@ -752,6 +850,21 @@ ProjectStore::Oplog::GetAllChunksInfo()
 }
 
 void
+ProjectStore::Oplog::IterateChunkMap(std::function<void(const Oid&, const IoHash&)>&& Fn)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	for (const auto& Kv : m_ChunkMap)
+	{
+		Fn(Kv.first, Kv.second);
+	}
+}
+
+void
 ProjectStore::Oplog::IterateFileMap(
 	std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn)
 {
@@ -803,41 +916,55 @@ ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbO
 		return;
 	}
 
-	std::vector<size_t>			   EntryIndexes;
-	std::vector<OplogEntryAddress> Entries;
-	std::vector<Oid>			   Keys;
-	std::vector<int>			   LSNs;
-	Entries.reserve(m_LatestOpMap.size());
-	EntryIndexes.reserve(m_LatestOpMap.size());
-	Keys.reserve(m_LatestOpMap.size());
-	LSNs.reserve(m_LatestOpMap.size());
+	std::vector<OplogEntryAddress> SortedEntries;
+	std::vector<Oid>			   SortedKeys;
+	std::vector<int>			   SortedLSNs;
 
-	for (const auto& Kv : m_LatestOpMap)
 	{
-		const auto AddressEntry = m_OpAddressMap.find(Kv.second);
-		ZEN_ASSERT(AddressEntry != m_OpAddressMap.end());
+		const auto TargetEntryCount = m_LatestOpMap.size();
 
-		Entries.push_back(AddressEntry->second);
-		Keys.push_back(Kv.first);
-		LSNs.push_back(Kv.second);
-		EntryIndexes.push_back(EntryIndexes.size());
-	}
+		std::vector<size_t>			   EntryIndexes;
+		std::vector<OplogEntryAddress> Entries;
+		std::vector<Oid>			   Keys;
+		std::vector<int>			   LSNs;
 
-	std::sort(EntryIndexes.begin(), EntryIndexes.end(), [&Entries](const size_t& Lhs, const size_t& Rhs) {
-		const OplogEntryAddress& LhsEntry = Entries[Lhs];
-		const OplogEntryAddress& RhsEntry = Entries[Rhs];
-		return LhsEntry.Offset < RhsEntry.Offset;
-	});
-	std::vector<OplogEntryAddress> SortedEntries;
-	SortedEntries.reserve(EntryIndexes.size());
-	for (size_t Index : EntryIndexes)
-	{
-		SortedEntries.push_back(Entries[Index]);
+		Entries.reserve(TargetEntryCount);
+		EntryIndexes.reserve(TargetEntryCount);
+		Keys.reserve(TargetEntryCount);
+		LSNs.reserve(TargetEntryCount);
+
+		for (const auto& Kv : m_LatestOpMap)
+		{
+			const auto AddressEntry = m_OpAddressMap.find(Kv.second);
+			ZEN_ASSERT(AddressEntry != m_OpAddressMap.end());
+
+			Entries.push_back(AddressEntry->second);
+			Keys.push_back(Kv.first);
+			LSNs.push_back(Kv.second);
+			EntryIndexes.push_back(EntryIndexes.size());
+		}
+
+		std::sort(EntryIndexes.begin(), EntryIndexes.end(), [&Entries](const size_t& Lhs, const size_t& Rhs) {
+			const OplogEntryAddress& LhsEntry = Entries[Lhs];
+			const OplogEntryAddress& RhsEntry = Entries[Rhs];
+			return LhsEntry.Offset < RhsEntry.Offset;
+		});
+
+		SortedEntries.reserve(EntryIndexes.size());
+		SortedKeys.reserve(EntryIndexes.size());
+		SortedLSNs.reserve(EntryIndexes.size());
+
+		for (size_t Index : EntryIndexes)
+		{
+			SortedEntries.push_back(Entries[Index]);
+			SortedKeys.push_back(Keys[Index]);
+			SortedLSNs.push_back(LSNs[Index]);
+		}
 	}
 
 	size_t EntryIndex = 0;
 	m_Storage->ReplayLogEntries(SortedEntries, [&](CbObjectView Op) {
-		Handler(LSNs[EntryIndex], Keys[EntryIndex], Op);
+		Handler(SortedLSNs[EntryIndex], SortedKeys[EntryIndex], Op);
 		EntryIndex++;
 	});
 }
@@ -1015,7 +1142,7 @@ ProjectStore::Oplog::GetMapping(CbObjectView Core)
 				}
 				if (ClientPath.empty())
 				{
-					ZEN_WARN("invalid file for entry '{}', missing 'both 'clientpath'", Id);
+					ZEN_WARN("invalid file for entry '{}', missing 'clientpath' field", Id);
 					continue;
 				}
 
@@ -1073,7 +1200,7 @@ ProjectStore::Oplog::RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock,
 	}
 
 	m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize});
-	m_LatestOpMap[OpEntry.OpKeyAsOId()] = OpEntry.OpLsn;
+	m_LatestOpMap[OpEntry.OpKeyHash] = OpEntry.OpLsn;
 
 	return OpEntry.OpLsn;
 }
@@ -1135,7 +1262,9 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObject Core)
 
 	XXH3_128Stream KeyHasher;
 	Core["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
-	XXH3_128 KeyHash = KeyHasher.GetHash();
+	XXH3_128 KeyHash128 = KeyHasher.GetHash();
+	Oid		 KeyHash;
+	memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
 
 	RefPtr<OplogStorage> Storage;
 	{
@@ -1435,31 +1564,40 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 	return nullptr;
 }
 
-void
-ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+std::filesystem::path
+ProjectStore::Project::RemoveOplog(std::string_view OplogId)
 {
+	RwLock::ExclusiveLockScope _(m_ProjectLock);
+
 	std::filesystem::path DeletePath;
+	if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
 	{
-		RwLock::ExclusiveLockScope _(m_ProjectLock);
+		std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
 
-		if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end())
+		if (Oplog::ExistsAt(OplogBasePath))
 		{
-			std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
-
-			if (Oplog::ExistsAt(OplogBasePath))
+			std::filesystem::path MovedDir;
+			if (PrepareDirectoryDelete(DeletePath, MovedDir))
 			{
-				DeletePath = OplogBasePath;
+				DeletePath = MovedDir;
 			}
 		}
-		else
-		{
-			std::unique_ptr<Oplog>& Oplog = OplogIt->second;
-			DeletePath					  = Oplog->PrepareForDelete(true);
-			m_DeletedOplogs.emplace_back(std::move(Oplog));
-			m_Oplogs.erase(OplogIt);
-		}
-		m_LastAccessTimes.erase(std::string(OplogId));
 	}
+	else
+	{
+		std::unique_ptr<Oplog>& Oplog = OplogIt->second;
+		DeletePath					  = Oplog->PrepareForDelete(true);
+		m_DeletedOplogs.emplace_back(std::move(Oplog));
+		m_Oplogs.erase(OplogIt);
+	}
+	m_LastAccessTimes.erase(std::string(OplogId));
+	return DeletePath;
+}
+
+void
+ProjectStore::Project::DeleteOplog(std::string_view OplogId)
+{
+	std::filesystem::path DeletePath = RemoveOplog(OplogId);
 
 	// Erase content on disk
 	if (!DeletePath.empty())
@@ -1521,7 +1659,7 @@ ProjectStore::Project::ScrubStorage(ScrubContext& Ctx)
 	{
 		OpenOplog(OpLogId);
 	}
-	IterateOplogs([&](const RwLock::SharedLockScope& ProjectLock, const Oplog& Ops) {
+	IterateOplogs([&](const RwLock::SharedLockScope& ProjectLock, Oplog& Ops) {
 		if (!IsExpired(ProjectLock, GcClock::TimePoint::min(), Ops))
 		{
 			Ops.ScrubStorage(Ctx);
@@ -1567,9 +1705,29 @@ ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 }
 
 uint64_t
+ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath)
+{
+	using namespace std::literals;
+
+	uint64_t			  Size				  = 0;
+	std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv;
+	if (std::filesystem::exists(AccessTimesFilePath))
+	{
+		Size += std::filesystem::file_size(AccessTimesFilePath);
+	}
+	std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv;
+	if (std::filesystem::exists(ProjectFilePath))
+	{
+		Size += std::filesystem::file_size(ProjectFilePath);
+	}
+
+	return Size;
+}
+
+uint64_t
 ProjectStore::Project::TotalSize() const
 {
-	uint64_t Result = 0;
+	uint64_t Result = TotalSize(m_OplogStoragePath);
 	{
 		std::vector<std::string> OpLogs = ScanForOplogs();
 		for (const std::string& OpLogId : OpLogs)
@@ -1730,6 +1888,10 @@ ProjectStore::DiscoverProjects()
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
 		std::string DirName = PathToUtf8(DirPath.filename());
+		if (DirName.starts_with("[dropped]"))
+		{
+			continue;
+		}
 		OpenProject(DirName);
 	}
 }
@@ -1954,7 +2116,7 @@ ProjectStore::StorageSize() const
 				std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv;
 				if (std::filesystem::exists(ProjectStateFilePath))
 				{
-					Result.DiskSize += std::filesystem::file_size(ProjectStateFilePath);
+					Result.DiskSize += Project::TotalSize(ProjectBasePath);
 					DirectoryContent DirContent;
 					GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent);
 					for (const std::filesystem::path& OplogBasePath : DirContent.Directories)
@@ -2068,12 +2230,8 @@ ProjectStore::UpdateProject(std::string_view ProjectId,
 }
 
 bool
-ProjectStore::DeleteProject(std::string_view ProjectId)
+ProjectStore::RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath)
 {
-	ZEN_TRACE_CPU("Store::DeleteProject");
-
-	ZEN_INFO("deleting project {}", ProjectId);
-
 	RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock);
 
 	auto ProjIt = m_Projects.find(std::string{ProjectId});
@@ -2083,20 +2241,34 @@ ProjectStore::DeleteProject(std::string_view ProjectId)
 		return true;
 	}
 
-	std::filesystem::path DeletePath;
-	bool				  Success = ProjIt->second->PrepareForDelete(DeletePath);
+	bool Success = ProjIt->second->PrepareForDelete(OutDeletePath);
 
 	if (!Success)
 	{
 		return false;
 	}
 	m_Projects.erase(ProjIt);
-	ProjectsLock.ReleaseNow();
+	return true;
+}
+
+bool
+ProjectStore::DeleteProject(std::string_view ProjectId)
+{
+	ZEN_TRACE_CPU("Store::DeleteProject");
+
+	ZEN_INFO("deleting project {}", ProjectId);
+
+	std::filesystem::path DeletePath;
+	if (!RemoveProject(ProjectId, DeletePath))
+	{
+		return false;
+	}
 
 	if (!DeletePath.empty())
 	{
 		DeleteDirectories(DeletePath);
 	}
+
 	return true;
 }
 
@@ -2172,9 +2344,9 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin
 }
 
 std::pair<HttpResponseCode, std::string>
-ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
+ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
 {
-	ZEN_TRACE_CPU("ProjectStore::GetProjectChunks");
+	ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos");
 
 	using namespace std::literals;
 
@@ -2192,21 +2364,22 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri
 	}
 	Project->TouchOplog(OplogId);
 
-	std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo();
+	std::vector<std::pair<Oid, IoHash>> ChunkInfos;
+	FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); });
 
 	CbObjectWriter Response;
+	Response.BeginArray("chunkinfos"sv);
 
-	Response.BeginArray("chunks"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+	for (const auto& ChunkInfo : ChunkInfos)
 	{
-		Response << Info.ChunkId;
-	}
-	Response.EndArray();
-
-	Response.BeginArray("sizes"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
-	{
-		Response << Info.ChunkSize;
+		if (IoBuffer Chunk = FoundLog->FindChunk(ChunkInfo.first))
+		{
+			Response.BeginObject();
+			Response << "id"sv << ChunkInfo.first;
+			Response << "rawhash"sv << ChunkInfo.second;
+			Response << "rawsize"sv << Chunk.GetSize();
+			Response.EndObject();
+		}
 	}
 	Response.EndArray();
 
@@ -3042,36 +3215,120 @@ ProjectStore::GetGcName(GcCtx&)
 	return fmt::format("projectstore:'{}'", m_ProjectBasePath.string());
 }
 
-void
-ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
+class ProjectStoreGcStoreCompactor : public GcStoreCompactor
 {
-	size_t	   ProjectCount		   = 0;
-	size_t	   ExpiredProjectCount = 0;
-	size_t	   OplogCount		   = 0;
-	size_t	   ExpiredOplogCount   = 0;
+public:
+	ProjectStoreGcStoreCompactor(const std::filesystem::path&		  BasePath,
+								 std::vector<std::filesystem::path>&& OplogPathsToRemove,
+								 std::vector<std::filesystem::path>&& ProjectPathsToRemove)
+	: m_BasePath(BasePath)
+	, m_OplogPathsToRemove(std::move(OplogPathsToRemove))
+	, m_ProjectPathsToRemove(std::move(ProjectPathsToRemove))
+	{
+	}
+
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
+	{
+		ZEN_TRACE_CPU("Store::CompactStore");
+
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: projectstore [COMPACT] '{}': RemovedDisk: {} in {}",
+					 m_BasePath,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		if (Ctx.Settings.IsDeleteMode)
+		{
+			for (const std::filesystem::path& OplogPath : m_OplogPathsToRemove)
+			{
+				uint64_t OplogSize = ProjectStore::Oplog::TotalSize(OplogPath);
+				if (DeleteDirectories(OplogPath))
+				{
+					ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed oplog folder '{}', removed {}",
+							  m_BasePath,
+							  OplogPath,
+							  NiceBytes(OplogSize));
+					Stats.RemovedDisk += OplogSize;
+				}
+				else
+				{
+					ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove oplog folder '{}'", m_BasePath, OplogPath);
+				}
+			}
+
+			for (const std::filesystem::path& ProjectPath : m_ProjectPathsToRemove)
+			{
+				uint64_t ProjectSize = ProjectStore::Project::TotalSize(ProjectPath);
+				if (DeleteDirectories(ProjectPath))
+				{
+					ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed project folder '{}', removed {}",
+							  m_BasePath,
+							  ProjectPath,
+							  NiceBytes(ProjectSize));
+					Stats.RemovedDisk += ProjectSize;
+				}
+				else
+				{
+					ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove project folder '{}'", m_BasePath, ProjectPath);
+				}
+			}
+		}
+		else
+		{
+			ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': Skipped deleting of {} oplogs and {} projects",
+					  m_BasePath,
+					  m_OplogPathsToRemove.size(),
+					  m_ProjectPathsToRemove.size());
+		}
+
+		m_ProjectPathsToRemove.clear();
+		m_OplogPathsToRemove.clear();
+	}
+
+private:
+	std::filesystem::path			   m_BasePath;
+	std::vector<std::filesystem::path> m_OplogPathsToRemove;
+	std::vector<std::filesystem::path> m_ProjectPathsToRemove;
+};
+
+GcStoreCompactor*
+ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
+{
+	ZEN_TRACE_CPU("Store::RemoveExpiredData");
+
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
 		if (!Ctx.Settings.Verbose)
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+		ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {} in {}",
 				 m_ProjectBasePath,
-				 Stats.Count,
-				 Stats.Expired,
-				 Stats.Deleted,
-				 NiceBytes(Stats.RemovedDisk),
-				 NiceBytes(Stats.RemovedMemory),
+				 Stats.CheckedCount,
+				 Stats.FoundCount,
+				 Stats.DeletedCount,
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
+	std::vector<std::filesystem::path> OplogPathsToRemove;
+	std::vector<std::filesystem::path> ProjectPathsToRemove;
+
 	std::vector<Ref<Project>> ExpiredProjects;
 	std::vector<Ref<Project>> Projects;
 
+	DiscoverProjects();
+
 	{
 		RwLock::SharedLockScope Lock(m_ProjectsLock);
 		for (auto& Kv : m_Projects)
 		{
+			Stats.CheckedCount++;
 			if (Kv.second->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime))
 			{
 				ExpiredProjects.push_back(Kv.second);
@@ -3083,12 +3340,30 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 
 	for (const Ref<Project>& Project : Projects)
 	{
+		std::vector<std::string> OpLogs = Project->ScanForOplogs();
+		for (const std::string& OpLogId : OpLogs)
+		{
+			Project->OpenOplog(OpLogId);
+			if (Ctx.IsCancelledFlag)
+			{
+				return nullptr;
+			}
+		}
+	}
+
+	size_t ExpiredOplogCount = 0;
+	for (const Ref<Project>& Project : Projects)
+	{
+		if (Ctx.IsCancelledFlag)
+		{
+			break;
+		}
+
 		std::vector<std::string> ExpiredOplogs;
 		{
-			RwLock::ExclusiveLockScope __(m_ProjectsLock);
 			Project->IterateOplogs(
-				[&Ctx, &Project, &ExpiredOplogs, &OplogCount](const RwLock::SharedLockScope& Lock, ProjectStore::Oplog& Oplog) {
-					OplogCount++;
+				[&Ctx, &Stats, &Project, &ExpiredOplogs](const RwLock::SharedLockScope& Lock, ProjectStore::Oplog& Oplog) {
+					Stats.CheckedCount++;
 					if (Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime, Oplog))
 					{
 						ExpiredOplogs.push_back(Oplog.OplogId());
@@ -3101,105 +3376,112 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats)
 		{
 			for (const std::string& OplogId : ExpiredOplogs)
 			{
-				std::filesystem::path OplogBasePath = ProjectPath / OplogId;
-				uint64_t			  OplogSize		= Oplog::TotalSize(OplogBasePath);
-				ZEN_DEBUG("gc project store '{}': garbage collected oplog '{}' in project '{}'. Removing storage on disk",
-						  m_ProjectBasePath,
-						  OplogId,
-						  Project->Identifier);
-				Project->DeleteOplog(OplogId);
-				Stats.RemovedDisk += OplogSize;
+				std::filesystem::path RemovePath = Project->RemoveOplog(OplogId);
+				if (!RemovePath.empty())
+				{
+					OplogPathsToRemove.push_back(RemovePath);
+				}
 			}
-			Stats.Deleted += ExpiredOplogs.size();
+			Stats.DeletedCount += ExpiredOplogs.size();
 			Project->Flush();
 		}
 	}
-	ProjectCount = Projects.size();
-	Stats.Count += ProjectCount + OplogCount;
-	ExpiredProjectCount = ExpiredProjects.size();
 
-	if (ExpiredProjects.empty())
+	if (ExpiredProjects.empty() && ExpiredOplogCount == 0)
 	{
-		ZEN_DEBUG("gc project store '{}': no expired projects found", m_ProjectBasePath);
-		return;
+		ZEN_DEBUG("GCV2: projectstore [REMOVE EXPIRED] '{}': no expired projects found", m_ProjectBasePath);
+		return nullptr;
 	}
 
 	if (Ctx.Settings.IsDeleteMode)
 	{
 		for (const Ref<Project>& Project : ExpiredProjects)
 		{
-			std::filesystem::path PathToRemove;
-			std::string			  ProjectId = Project->Identifier;
+			std::string ProjectId = Project->Identifier;
 			{
 				{
 					RwLock::SharedLockScope Lock(m_ProjectsLock);
 					if (!Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime))
 					{
-						ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project no longer expired.",
-								  m_ProjectBasePath,
-								  ProjectId);
+						ZEN_DEBUG(
+							"GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project no longer "
+							"expired.",
+							m_ProjectBasePath,
+							ProjectId);
 						continue;
 					}
 				}
-				RwLock::ExclusiveLockScope __(m_ProjectsLock);
-				bool					   Success = Project->PrepareForDelete(PathToRemove);
+				std::filesystem::path RemovePath;
+				bool				  Success = RemoveProject(ProjectId, RemovePath);
 				if (!Success)
 				{
-					ZEN_DEBUG("gc project store '{}': skipped garbage collect of project '{}'. Project folder is locked.",
-							  m_ProjectBasePath,
-							  ProjectId);
+					ZEN_DEBUG(
+						"GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project folder is locked.",
+						m_ProjectBasePath,
+						ProjectId);
 					continue;
 				}
-				m_Projects.erase(ProjectId);
-			}
-
-			ZEN_DEBUG("gc project store '{}': sgarbage collected project '{}'. Removing storage on disk", m_ProjectBasePath, ProjectId);
-			if (PathToRemove.empty())
-			{
-				continue;
+				if (!RemovePath.empty())
+				{
+					ProjectPathsToRemove.push_back(RemovePath);
+				}
 			}
-
-			DeleteDirectories(PathToRemove);
 		}
-		Stats.Deleted += ExpiredProjects.size();
+		Stats.DeletedCount += ExpiredProjects.size();
 	}
 
-	Stats.Expired += ExpiredOplogCount + ExpiredProjectCount;
+	size_t ExpiredProjectCount = ExpiredProjects.size();
+	Stats.FoundCount += ExpiredOplogCount + ExpiredProjectCount;
+	if (!OplogPathsToRemove.empty() || !ProjectPathsToRemove.empty())
+	{
+		return new ProjectStoreGcStoreCompactor(m_ProjectBasePath, std::move(OplogPathsToRemove), std::move(ProjectPathsToRemove));
+	}
+	return nullptr;
 }
 
 class ProjectStoreReferenceChecker : public GcReferenceChecker
 {
 public:
-	ProjectStoreReferenceChecker(GcCtx& Ctx, ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner)
+	ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner), m_PreCache(PreCache) {}
+
+	virtual ~ProjectStoreReferenceChecker() {}
+
+	virtual void PreCache(GcCtx& Ctx) override
 	{
-		if (PreCache)
+		if (m_PreCache)
 		{
+			ZEN_TRACE_CPU("Store::PreCache");
+
 			Stopwatch  Timer;
 			const auto _ = MakeGuard([&] {
 				if (!Ctx.Settings.Verbose)
 				{
 					return;
 				}
-				ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': precached {} references in {} from {}/{}",
+				ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': precached {} references in {} from {}/{}",
 						 m_Oplog.m_BasePath,
-						 m_UncachedReferences.size(),
+						 m_References.size(),
 						 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 						 m_Oplog.m_OuterProject->Identifier,
 						 m_Oplog.OplogId());
 			});
 
 			RwLock::SharedLockScope __(m_Oplog.m_OplogLock);
+			if (Ctx.IsCancelledFlag)
+			{
+				return;
+			}
 			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
+				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
 			});
 			m_PreCachedLsn = m_Oplog.GetMaxOpIndex();
 		}
 	}
 
-	virtual ~ProjectStoreReferenceChecker() {}
-
 	virtual void LockState(GcCtx& Ctx) override
 	{
+		ZEN_TRACE_CPU("Store::LockState");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
@@ -3208,7 +3490,7 @@ public:
 			}
 			ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}",
 					 m_Oplog.m_BasePath,
-					 m_UncachedReferences.size(),
+					 m_References.size(),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
 					 m_Oplog.m_OuterProject->Identifier,
 					 m_Oplog.OplogId());
@@ -3219,29 +3501,56 @@ public:
 		{
 			// TODO: Maybe we could just check the added oplog entries - we might get a few extra references from obsolete entries
 			// but I don't think that would be critical
-			m_UncachedReferences.clear();
+			m_References.resize(0);
 			m_Oplog.IterateOplog([&](CbObjectView Op) {
-				Op.IterateAttachments([&](CbFieldView Visitor) { m_UncachedReferences.insert(Visitor.AsAttachment()); });
+				Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); });
 			});
 		}
 	}
 
-	virtual void RemoveUsedReferencesFromSet(GcCtx&, HashSet& IoCids) override
+	virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override
 	{
-		for (const IoHash& ReferenceHash : m_UncachedReferences)
+		ZEN_TRACE_CPU("Store::RemoveUsedReferencesFromSet");
+
+		ZEN_ASSERT(m_OplogLock);
+
+		size_t	   InitialCount = IoCids.size();
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: projectstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}",
+					 m_Oplog.m_BasePath,
+					 InitialCount - IoCids.size(),
+					 InitialCount,
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		for (const IoHash& ReferenceHash : m_References)
 		{
-			IoCids.erase(ReferenceHash);
+			if (IoCids.erase(ReferenceHash) == 1)
+			{
+				if (IoCids.empty())
+				{
+					return;
+				}
+			}
 		}
 	}
 	ProjectStore::Oplog&					 m_Oplog;
+	bool									 m_PreCache;
 	std::unique_ptr<RwLock::SharedLockScope> m_OplogLock;
-	HashSet									 m_UncachedReferences;
+	std::vector<IoHash>						 m_References;
 	int										 m_PreCachedLsn = -1;
 };
 
 std::vector<GcReferenceChecker*>
 ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
 {
+	ZEN_TRACE_CPU("Store::CreateReferenceCheckers");
+
 	size_t ProjectCount = 0;
 	size_t OplogCount	= 0;
 
@@ -3283,7 +3592,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx)
 				ProjectStore::Oplog* Oplog		 = Project->OpenOplog(OpLogId);
 				GcClock::TimePoint	 Now		 = GcClock::Now();
 				bool				 TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5));
-				Checkers.emplace_back(new ProjectStoreReferenceChecker(Ctx, *Oplog, TryPreCache));
+				Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache));
 			}
 			OplogCount += OpLogs.size();
 		}
@@ -3478,11 +3787,17 @@ TEST_CASE("project.store.gc")
 		BasicFile ProjectFile;
 		ProjectFile.Open(Project2FilePath, BasicFile::Mode::kTruncate);
 	}
-	std::filesystem::path Project2OplogPath = TempDir.Path() / "game1" / "saves" / "cooked" / ".projectstore";
+	std::filesystem::path Project2Oplog1Path = TempDir.Path() / "game1" / "saves" / "cooked" / ".projectstore";
+	{
+		CreateDirectories(Project2Oplog1Path.parent_path());
+		BasicFile OplogFile;
+		OplogFile.Open(Project2Oplog1Path, BasicFile::Mode::kTruncate);
+	}
+	std::filesystem::path Project2Oplog2Path = TempDir.Path() / "game2" / "saves" / "cooked" / ".projectstore";
 	{
-		CreateDirectories(Project2OplogPath.parent_path());
+		CreateDirectories(Project2Oplog2Path.parent_path());
 		BasicFile OplogFile;
-		OplogFile.Open(Project2OplogPath, BasicFile::Mode::kTruncate);
+		OplogFile.Open(Project2Oplog2Path, BasicFile::Mode::kTruncate);
 	}
 
 	{
@@ -3508,94 +3823,212 @@ TEST_CASE("project.store.gc")
 																	EngineRootDir.string(),
 																	Project2RootDir.string(),
 																	Project2FilePath.string()));
-		ProjectStore::Oplog*	   Oplog = Project2->NewOplog("oplog2", Project2OplogPath);
-		CHECK(Oplog != nullptr);
-
-		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
-		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177})));
-		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96})));
-		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221})));
+		{
+			ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog2", Project2Oplog1Path);
+			CHECK(Oplog != nullptr);
+
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177})));
+			Oplog->AppendNewOplogEntry(
+				CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96})));
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221})));
+		}
+		{
+			ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog3", Project2Oplog2Path);
+			CHECK(Oplog != nullptr);
+
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137})));
+			Oplog->AppendNewOplogEntry(
+				CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98})));
+			Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271})));
+		}
 	}
 
+	SUBCASE("v1")
 	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 14);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
-	}
+		{
+			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 21);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
 
-	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 14);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
-	}
+		{
+			GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 21);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
 
-	std::filesystem::remove(Project1FilePath);
+		std::filesystem::remove(Project1FilePath);
 
-	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 14);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
-	}
+		{
+			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 21);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
 
-	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 7);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(!ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
-	}
+		{
+			GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 14);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
 
-	std::filesystem::remove(Project2OplogPath);
-	{
-		GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 7);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(!ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
-	}
+		std::filesystem::remove(Project2Oplog1Path);
+		{
+			GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 14);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
 
-	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 0);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(!ProjectStore.OpenProject("proj1"sv));
-		CHECK(ProjectStore.OpenProject("proj2"sv));
+		{
+			GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 7);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		std::filesystem::remove(Project2FilePath);
+		{
+			GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
+			ProjectStore.GatherReferences(GcCtx);
+			size_t RefCount = 0;
+			GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+			CHECK(RefCount == 0);
+			ProjectStore.CollectGarbage(GcCtx);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(!ProjectStore.OpenProject("proj2"sv));
+		}
 	}
 
-	std::filesystem::remove(Project2FilePath);
+	SUBCASE("v2")
 	{
-		GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24));
-		ProjectStore.GatherReferences(GcCtx);
-		size_t RefCount = 0;
-		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
-		CHECK(RefCount == 0);
-		ProjectStore.CollectGarbage(GcCtx);
-		CHECK(!ProjectStore.OpenProject("proj1"sv));
-		CHECK(!ProjectStore.OpenProject("proj2"sv));
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		std::filesystem::remove(Project1FilePath);
+
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
+								   .CollectSmallObjects	   = true,
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(7u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		std::filesystem::remove(Project2Oplog1Path);
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() - std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24),
+								   .CollectSmallObjects	   = true,
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(3u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
+								   .CollectSmallObjects	   = true,
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(3u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(ProjectStore.OpenProject("proj2"sv));
+		}
+
+		std::filesystem::remove(Project2FilePath);
+		{
+			GcSettings Settings = {.CacheExpireTime		   = GcClock::Now() + std::chrono::hours(24),
+								   .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24),
+								   .CollectSmallObjects	   = true,
+								   .IsDeleteMode		   = true};
+			GcResult   Result	= Gc.CollectGarbage(Settings);
+			CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount);
+			CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount);
+			CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount);
+			CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount);
+			CHECK(!ProjectStore.OpenProject("proj1"sv));
+			CHECK(!ProjectStore.OpenProject("proj2"sv));
+		}
 	}
 }
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index fe1068485..5ebcd420c 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -31,14 +31,11 @@ struct OplogEntry
 	uint32_t OpCoreOffset;	// note: Multiple of alignment!
 	uint32_t OpCoreSize;
 	uint32_t OpCoreHash;  // Used as checksum
-	XXH3_128 OpKeyHash;	  // XXH128_canonical_t
+	Oid		 OpKeyHash;
+	uint32_t Reserved;
 
-	inline Oid OpKeyAsOId() const
-	{
-		Oid Id;
-		memcpy(Id.OidBits, &OpKeyHash, sizeof Id.OidBits);
-		return Id;
-	}
+	inline bool IsTombstone() const { return OpCoreOffset == 0 && OpCoreSize == 0 && OpLsn == 0; }
+	inline void MakeTombstone() { OpLsn = OpCoreOffset = OpCoreSize = OpCoreHash = Reserved = 0; }
 };
 
 struct OplogEntryAddress
@@ -93,6 +90,7 @@ public:
 		};
 
 		std::vector<ChunkInfo> GetAllChunksInfo();
+		void				   IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn);
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObjectView)>&& Fn);
 		void IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Fn);
@@ -126,7 +124,7 @@ public:
 
 		LoggerRef		Log() { return m_OuterProject->Log(); }
 		void			Flush();
-		void			ScrubStorage(ScrubContext& Ctx) const;
+		void			ScrubStorage(ScrubContext& Ctx);
 		void			GatherReferences(GcContext& GcCtx);
 		static uint64_t TotalSize(const std::filesystem::path& BasePath);
 		uint64_t		TotalSize() const;
@@ -225,6 +223,7 @@ public:
 		Oplog*					 NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath);
 		Oplog*					 OpenOplog(std::string_view OplogId);
 		void					 DeleteOplog(std::string_view OplogId);
+		std::filesystem::path	 RemoveOplog(std::string_view OplogId);
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const;
 		void					 IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn);
 		std::vector<std::string> ScanForOplogs() const;
@@ -245,6 +244,7 @@ public:
 		void					  ScrubStorage(ScrubContext& Ctx);
 		LoggerRef				  Log();
 		void					  GatherReferences(GcContext& GcCtx);
+		static uint64_t			  TotalSize(const std::filesystem::path& BasePath);
 		uint64_t				  TotalSize() const;
 		bool					  PrepareForDelete(std::filesystem::path& OutDeletePath);
 
@@ -280,6 +280,7 @@ public:
 							   std::string_view EngineRootDir,
 							   std::string_view ProjectRootDir,
 							   std::string_view ProjectFilePath);
+	bool		 RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath);
 	bool		 DeleteProject(std::string_view ProjectId);
 	bool		 Exists(std::string_view ProjectId);
 	void		 Flush();
@@ -295,7 +296,7 @@ public:
 	virtual GcStorageSize StorageSize() const override;
 
 	virtual std::string						 GetGcName(GcCtx& Ctx) override;
-	virtual void							 RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) override;
+	virtual GcStoreCompactor*				 RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override;
 	virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override;
 
 	CbArray									 GetProjectsList();
@@ -303,9 +304,9 @@ public:
 															 const std::string_view OplogId,
 															 bool					FilterClient,
 															 CbObject&				OutPayload);
-	std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId,
-															  const std::string_view OplogId,
-															  CbObject&				 OutPayload);
+	std::pair<HttpResponseCode, std::string> GetProjectChunkInfos(const std::string_view ProjectId,
+																  const std::string_view OplogId,
+																  CbObject&				 OutPayload);
 	std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId,
 														  const std::string_view OplogId,
 														  const std::string_view ChunkId,
@@ -379,6 +380,8 @@ private:
 	const DiskWriteBlocker*				m_DiskWriteBlocker = nullptr;
 
 	std::filesystem::path BasePathForProject(std::string_view ProjectId);
+
+	friend class ProjectStoreGcStoreCompactor;
 };
 
 void prj_forcelink();
diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp
index d5d229e42..826c8ff51 100644
--- a/src/zenserver/projectstore/remoteprojectstore.cpp
+++ b/src/zenserver/projectstore/remoteprojectstore.cpp
@@ -13,6 +13,7 @@
 #include <zencore/timer.h>
 #include <zencore/workthreadpool.h>
 #include <zenstore/cidstore.h>
+#include <zenutil/workerpools.h>
 
 #include <unordered_map>
 
@@ -802,10 +803,7 @@ BuildContainer(CidStore&																	ChunkStore,
 			   const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks,
 			   tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>*							OutOptionalTempAttachments)
 {
-	// We are creating a worker thread pool here since we are uploading a lot of attachments in one go and we dont want to keep a
-	// WorkerThreadPool alive
-	size_t			 WorkerCount = Min(std::thread::hardware_concurrency(), 16u);
-	WorkerThreadPool WorkerPool(gsl::narrow<int>(WorkerCount));
+	WorkerThreadPool& WorkerPool = GetSmallWorkerPool();
 
 	AsyncRemoteResult RemoteResult;
 	CbObject		  ContainerObject = BuildContainer(ChunkStore,
@@ -1153,10 +1151,7 @@ SaveOplog(CidStore&				 ChunkStore,
 
 	Stopwatch Timer;
 
-	// We are creating a worker thread pool here since we are uploading a lot of attachments in one go
-	// Doing upload is a rare and transient occation so we don't want to keep a WorkerThreadPool alive.
-	size_t			 WorkerCount = Min(std::thread::hardware_concurrency(), 16u);
-	WorkerThreadPool WorkerPool(gsl::narrow<int>(WorkerCount), "oplog_upload"sv);
+	WorkerThreadPool& WorkerPool = GetSmallWorkerPool();
 
 	std::filesystem::path AttachmentTempPath;
 	if (UseTempBlocks)
@@ -1528,10 +1523,7 @@ LoadOplog(CidStore&			   ChunkStore,
 
 	Stopwatch Timer;
 
-	// We are creating a worker thread pool here since we are download a lot of attachments in one go and we dont want to keep a
-	// WorkerThreadPool alive
-	size_t			 WorkerCount = Min(std::thread::hardware_concurrency(), 16u);
-	WorkerThreadPool WorkerPool(gsl::narrow<int>(WorkerCount));
+	WorkerThreadPool& WorkerPool = GetSmallWorkerPool();
 
 	std::unordered_set<IoHash, IoHash::Hasher> Attachments;
 	std::vector<std::vector<IoHash>>		   ChunksInBlocks;
diff --git a/src/zenserver/sentryintegration.cpp b/src/zenserver/sentryintegration.cpp
index 755fe97db..11bf78a75 100644
--- a/src/zenserver/sentryintegration.cpp
+++ b/src/zenserver/sentryintegration.cpp
@@ -231,14 +231,25 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, std::string Sentry
 		if (m_AllowPII)
 		{
 #	if ZEN_PLATFORM_WINDOWS
-			CHAR  UserNameBuffer[511 + 1];
-			DWORD UserNameLength = sizeof(UserNameBuffer) / sizeof(CHAR);
-			BOOL  OK			 = GetUserNameA(UserNameBuffer, &UserNameLength);
-			if (OK && UserNameLength)
+			CHAR  Buffer[511 + 1];
+			DWORD BufferLength = sizeof(Buffer) / sizeof(CHAR);
+			BOOL  OK		   = GetUserNameA(Buffer, &BufferLength);
+			if (OK && BufferLength)
 			{
-				m_SentryUserName = std::string(UserNameBuffer, UserNameLength - 1);
+				m_SentryUserName = std::string(Buffer, BufferLength - 1);
+			}
+			BufferLength = sizeof(Buffer) / sizeof(CHAR);
+			OK			 = GetComputerNameA(Buffer, &BufferLength);
+			if (OK && BufferLength)
+			{
+				m_SentryHostName = std::string(Buffer, BufferLength);
+			}
+			else
+			{
+				m_SentryHostName = "unknown";
 			}
 #	endif	// ZEN_PLATFORM_WINDOWS
+
 #	if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC)
 			uid_t		   uid = geteuid();
 			struct passwd* pw  = getpwuid(uid);
@@ -246,9 +257,24 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, std::string Sentry
 			{
 				m_SentryUserName = std::string(pw->pw_name);
 			}
+			else
+			{
+				m_SentryUserName = "unknown";
+			}
+			char HostNameBuffer[1023 + 1];
+			int	 err = gethostname(HostNameBuffer, sizeof(HostNameBuffer));
+			if (err == 0)
+			{
+				m_SentryHostName = std::string(HostNameBuffer);
+			}
+			else
+			{
+				m_SentryHostName = "unknown";
+			}
 #	endif
+			m_SentryId						= fmt::format("{}@{}", m_SentryUserName, m_SentryHostName);
 			sentry_value_t SentryUserObject = sentry_value_new_object();
-			sentry_value_set_by_key(SentryUserObject, "id", sentry_value_new_string(m_SentryUserName.c_str()));
+			sentry_value_set_by_key(SentryUserObject, "id", sentry_value_new_string(m_SentryId.c_str()));
 			sentry_value_set_by_key(SentryUserObject, "username", sentry_value_new_string(m_SentryUserName.c_str()));
 			sentry_value_set_by_key(SentryUserObject, "ip_address", sentry_value_new_string("{{auto}}"));
 			sentry_set_user(SentryUserObject);
@@ -266,13 +292,29 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, std::string Sentry
 void
 SentryIntegration::LogStartupInformation()
 {
+#	if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC)
+	uid_t		   uid = geteuid();
+	struct passwd* pw  = getpwuid(uid);
+	if (pw)
+	{
+		m_SentryUserName = std::string(pw->pw_name);
+	}
+	ZEN_INFO("Username: '{}'", m_SentryUserName);
+
+	char HostNameBuffer[1023 + 1];
+	int	 err = gethostname(HostNameBuffer, sizeof(HostNameBuffer));
+	if (err == 0)
+	{
+		ZEN_INFO("Hostname: '{}'", HostNameBuffer);
+	}
+#	endif
 	if (m_IsInitialized)
 	{
 		if (m_SentryErrorCode == 0)
 		{
 			if (m_AllowPII)
 			{
-				ZEN_INFO("sentry initialized, username: '{}'", m_SentryUserName);
+				ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId);
 			}
 			else
 			{
diff --git a/src/zenserver/sentryintegration.h b/src/zenserver/sentryintegration.h
index fddba8882..dd8b87ab7 100644
--- a/src/zenserver/sentryintegration.h
+++ b/src/zenserver/sentryintegration.h
@@ -46,6 +46,8 @@ private:
 	bool									  m_AllowPII		= false;
 	std::unique_ptr<sentry::SentryAssertImpl> m_SentryAssert;
 	std::string								  m_SentryUserName;
+	std::string								  m_SentryHostName;
+	std::string								  m_SentryId;
 	std::shared_ptr<spdlog::logger>			  m_SentryLogger;
 };
 
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 127213ebd..c42f305ee 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -9,7 +9,9 @@ target("zenserver")
              "zenutil",
              "zenvfs")
     add_headerfiles("**.h")
+    add_rules("utils.bin2c", {extensions = {".zip"}})
     add_files("**.cpp")
+    add_files("frontend/*.zip")
     add_files("zenserver.cpp", {unity_ignored  = true })
     add_includedirs(".")
     set_symbols("debug")
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 7111900ec..336f715f4 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -24,6 +24,7 @@
 #include <zenstore/cidstore.h>
 #include <zenstore/scrubcontext.h>
 #include <zenutil/basicfile.h>
+#include <zenutil/workerpools.h>
 #include <zenutil/zenserverprocess.h>
 
 #if ZEN_PLATFORM_WINDOWS
@@ -117,7 +118,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_UseSentry				 = ServerOptions.NoSentry == false;
 	m_ServerEntry			 = ServerEntry;
 	m_DebugOptionForcedCrash = ServerOptions.ShouldCrash;
+	m_IsPowerCycle			 = ServerOptions.IsPowerCycle;
 	const int ParentPid		 = ServerOptions.OwnerPid;
+	m_StartupScrubOptions	 = ServerOptions.ScrubOptions;
 
 	if (ParentPid)
 	{
@@ -160,7 +163,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	// Ok so now we're configured, let's kick things off
 
 	m_Http				  = CreateHttpServer(ServerOptions.HttpServerConfig);
-	int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort);
+	int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir);
 
 	// Setup authentication manager
 	{
@@ -253,7 +256,6 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	{
 		ObjectStoreConfig ObjCfg;
 		ObjCfg.RootDirectory = m_DataRoot / "obj";
-		ObjCfg.ServerPort	 = static_cast<uint16_t>(EffectiveBasePort);
 
 		for (const auto& Bucket : ServerOptions.ObjectStoreConfig.Buckets)
 		{
@@ -289,7 +291,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 							   .DiskSizeSoftLimit		= ServerOptions.GcConfig.DiskSizeSoftLimit,
 							   .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites,
 							   .LightweightInterval				  = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds),
-							   .UseGCVersion					  = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1};
+							   .UseGCVersion					  = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1,
+							   .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent,
+							   .Verbose							  = ServerOptions.GcConfig.Verbose};
 	m_GcScheduler.Initialize(GcConfig);
 
 	// Create and register admin interface last to make sure all is properly initialized
@@ -364,9 +368,24 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions)
 
 				if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION)
 				{
-					WipeState = true;
-					WipeReason =
-						fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION);
+					std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch";
+					if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath))
+					{
+						ZEN_INFO(
+							"Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating "
+							"schema version",
+							ManifestVersion,
+							ManifestPath,
+							ZEN_CFG_SCHEMA_VERSION,
+							ManifestSkipSchemaChangePath);
+						UpdateManifest = true;
+					}
+					else
+					{
+						WipeState = true;
+						WipeReason =
+							fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION);
+					}
 				}
 			}
 		}
@@ -473,7 +492,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 						const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls);
 						if (Err)
 						{
-							ZEN_ERROR("resolve FAILED, reason '{}'", Err.message());
+							ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message());
 						}
 					}
 				}
@@ -536,10 +555,6 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 void
 ZenServer::Run()
 {
-	// This is disabled for now, awaiting better scheduling
-	//
-	// ScrubStorage();
-
 	if (m_ProcessMonitor.IsActive())
 	{
 		CheckOwnerPid();
@@ -547,12 +562,13 @@ ZenServer::Run()
 
 	if (!m_TestMode)
 	{
-		ZEN_INFO("__________                _________ __                        ");
-		ZEN_INFO("\\____    /____   ____    /   _____//  |_  ___________   ____  ");
-		ZEN_INFO("  /     // __ \\ /    \\   \\_____  \\\\   __\\/  _ \\_  __ \\_/ __ \\ ");
-		ZEN_INFO(" /     /\\  ___/|   |  \\  /        \\|  | (  <_> )  | \\/\\  ___/ ");
-		ZEN_INFO("/_______ \\___  >___|  / /_______  /|__|  \\____/|__|    \\___  >");
-		ZEN_INFO("        \\/   \\/     \\/          \\/                         \\/ ");
+		ZEN_INFO(
+			"__________                _________ __                        \n"
+			"\\____    /____   ____    /   _____//  |_  ___________   ____  \n"
+			"  /     // __ \\ /    \\   \\_____  \\\\   __\\/  _ \\_  __ \\_/ __ \\ \n"
+			" /     /\\  ___/|   |  \\  /        \\|  | (  <_> )  | \\/\\  ___/ \n"
+			"/_______ \\___  >___|  / /_______  /|__|  \\____/|__|    \\___  >\n"
+			"        \\/   \\/     \\/          \\/                         \\/ \n");
 	}
 
 	ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
@@ -575,13 +591,81 @@ ZenServer::Run()
 
 	OnReady();
 
+	if (!m_StartupScrubOptions.empty())
+	{
+		using namespace std::literals;
+
+		ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions);
+
+		bool							DoScrub = true;
+		bool							DoWait	= false;
+		GcScheduler::TriggerScrubParams ScrubParams;
+
+		ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) {
+			if (Token == "nocas"sv)
+			{
+				ScrubParams.SkipCas = true;
+			}
+			else if (Token == "nodelete"sv)
+			{
+				ScrubParams.SkipDelete = true;
+			}
+			else if (Token == "nogc"sv)
+			{
+				ScrubParams.SkipGc = true;
+			}
+			else if (Token == "no"sv)
+			{
+				DoScrub = false;
+			}
+			else if (Token == "wait"sv)
+			{
+				DoWait = true;
+			}
+			return true;
+		});
+
+		if (DoScrub)
+		{
+			m_GcScheduler.TriggerScrub(ScrubParams);
+
+			if (DoWait)
+			{
+				auto State = m_GcScheduler.Status();
+
+				while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped))
+				{
+					Sleep(500);
+
+					State = m_GcScheduler.Status();
+				}
+
+				ZEN_INFO("waiting for Scrub/GC to complete...");
+
+				while (State == GcSchedulerStatus::kRunning)
+				{
+					Sleep(500);
+
+					State = m_GcScheduler.Status();
+				}
+
+				ZEN_INFO("Scrub/GC completed");
+			}
+		}
+	}
+
+	if (m_IsPowerCycle)
+	{
+		ZEN_INFO("Power cycle mode enabled -- shutting down");
+
+		RequestExit(0);
+	}
+
 	m_Http->Run(IsInteractiveMode);
 
 	SetNewState(kShuttingDown);
 
 	ZEN_INFO(ZEN_APP_NAME " exiting");
-
-	Flush();
 }
 
 void
@@ -616,8 +700,12 @@ ZenServer::Cleanup()
 		}
 
 		m_StatsReporter.Shutdown();
-
 		m_GcScheduler.Shutdown();
+
+		Flush();
+
+		ShutdownWorkerPools();
+
 		m_AdminService.reset();
 		m_VfsService.reset();
 		m_ObjStoreService.reset();
@@ -720,7 +808,7 @@ ZenServer::CheckSigInt()
 {
 	if (utils::SignalCounter[SIGINT] > 0)
 	{
-		ZEN_INFO("SIGINT triggered (Ctrl+C), exiting");
+		ZEN_INFO("SIGINT triggered (Ctrl+C) for process {}, exiting", zen::GetCurrentProcessId());
 		RequestExit(128 + SIGINT);
 		return;
 	}
@@ -768,7 +856,7 @@ ZenServer::ScrubStorage()
 	Stopwatch Timer;
 	ZEN_INFO("Storage validation STARTING");
 
-	WorkerThreadPool ThreadPool{1};
+	WorkerThreadPool ThreadPool{1, "Scrub"};
 	ScrubContext	 Ctx{ThreadPool};
 	m_CidStore->ScrubStorage(Ctx);
 	m_ProjectStore->ScrubStorage(Ctx);
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 7da536708..1afd70b3e 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -36,7 +36,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include "vfs/vfsservice.h"
 
 #ifndef ZEN_APP_NAME
-#	define ZEN_APP_NAME "Zen store"
+#	define ZEN_APP_NAME "Unreal Zen Storage Server"
 #endif
 
 namespace zen {
@@ -88,6 +88,7 @@ private:
 	ZenServerState::ZenServerEntry* m_ServerEntry	  = nullptr;
 	bool							m_IsDedicatedMode = false;
 	bool							m_TestMode		  = false;
+	bool							m_IsPowerCycle	  = false;
 	CbObject						m_RootManifest;
 	std::filesystem::path			m_DataRoot;
 	std::filesystem::path			m_ContentRoot;
@@ -138,6 +139,8 @@ private:
 
 	bool m_DebugOptionForcedCrash = false;
 	bool m_UseSentry			  = false;
+
+	std::string m_StartupScrubOptions;
 };
 
 }  // namespace zen
diff --git a/src/zenserver/zenserver.rc b/src/zenserver/zenserver.rc
index 6d31e2c6e..e0003ea8f 100644
--- a/src/zenserver/zenserver.rc
+++ b/src/zenserver/zenserver.rc
@@ -90,11 +90,11 @@ PRODUCTVERSION ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER
         BLOCK "040904b0"
         {
             VALUE "CompanyName",        "Epic Games Inc\0"
-            VALUE "FileDescription",    "Local Storage Service for Unreal Engine\0"
+            VALUE "FileDescription",    "Unreal Zen Storage Service\0"
             VALUE "FileVersion",        ZEN_CFG_VERSION "\0"
             VALUE "LegalCopyright",     "Copyright Epic Games Inc. All Rights Reserved\0"
             VALUE "OriginalFilename",   "zenserver.exe\0"
-            VALUE "ProductName",        "Zen Storage Server\0"
+            VALUE "ProductName",        "Unreal Zen Storage Server\0"
 			VALUE "ProductVersion",		ZEN_CFG_VERSION_BUILD_STRING_FULL "\0"
         }
     }
diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp
index 00c1136b6..6ef311324 100644
--- a/src/zenstore-test/zenstore-test.cpp
+++ b/src/zenstore-test/zenstore-test.cpp
@@ -4,6 +4,7 @@
 #include <zencore/logging.h>
 #include <zencore/zencore.h>
 #include <zenstore/zenstore.h>
+#include <zenutil/zenutil.h>
 
 #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
 #	include <sys/time.h>
@@ -21,6 +22,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[])
 {
 #if ZEN_WITH_TESTS
 	zen::zenstore_forcelinktests();
+	zen::zenutil_forcelinktests();
 
 	zen::logging::InitializeLogging();
 	zen::MaximizeOpenFileCount();
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp
index 063d38707..71e306eca 100644
--- a/src/zenstore/blockstore.cpp
+++ b/src/zenstore/blockstore.cpp
@@ -15,6 +15,7 @@
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
 #include <tsl/robin_set.h>
+#include <gsl/gsl-lite.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
 #if ZEN_WITH_TESTS
@@ -226,7 +227,17 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max
 }
 
 void
-BlockStore::SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& KnownLocations)
+BlockStore::BlockIndexSet::Add(uint32_t BlockIndex)
+{
+	if (!std::binary_search(begin(BlockIndexes), end(BlockIndexes), BlockIndex))
+	{
+		auto It = std::lower_bound(begin(BlockIndexes), end(BlockIndexes), BlockIndex);
+		BlockIndexes.insert(It, BlockIndex);
+	}
+}
+
+void
+BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations)
 {
 	ZEN_TRACE_CPU("BlockStore::SyncExistingBlocksOnDisk");
 
@@ -239,14 +250,18 @@ BlockStore::SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& Know
 	{
 		DeleteBlocks.insert(It.first);
 	}
-	for (const auto& Entry : KnownLocations)
+
+	for (const uint32_t BlockIndex : KnownLocations.GetBlockIndices())
 	{
-		DeleteBlocks.erase(Entry.BlockIndex);
-		if (auto It = m_ChunkBlocks.find(Entry.BlockIndex); It != m_ChunkBlocks.end() && !It->second.IsNull())
+		DeleteBlocks.erase(BlockIndex);
+		if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end() && !It->second.IsNull())
 		{
 			continue;
 		}
-		MissingBlocks.insert(Entry.BlockIndex);
+		else
+		{
+			MissingBlocks.insert(BlockIndex);
+		}
 	}
 	for (std::uint32_t BlockIndex : MissingBlocks)
 	{
@@ -267,6 +282,66 @@ BlockStore::SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& Know
 	}
 }
 
+BlockStore::BlockEntryCountMap
+BlockStore::GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent)
+{
+	BlockEntryCountMap Result;
+	{
+		RwLock::SharedLockScope InsertLock(m_InsertLock);
+		for (const auto& It : m_ChunkBlocks)
+		{
+			uint32_t BlockIndex = It.first;
+			if ((BlockIndex == m_WriteBlockIndex.load()) && m_WriteBlock)
+			{
+				continue;
+			}
+			if (std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), BlockIndex) != m_ActiveWriteBlocks.end())
+			{
+				continue;
+			}
+
+			uint64_t UsedSize  = 0;
+			uint32_t UsedCount = 0;
+			if (auto UsageIt = BlockUsage.find(BlockIndex); UsageIt != BlockUsage.end())
+			{
+				UsedSize  = UsageIt->second.DiskUsage;
+				UsedCount = UsageIt->second.EntryCount;
+			}
+
+			uint64_t BlockSize = It.second ? It.second->FileSize() : 0u;
+			if (BlockSize == 0)
+			{
+				Result.insert_or_assign(BlockIndex, UsedCount);
+				continue;
+			}
+
+			if (BlockUsageThresholdPercent == 100)
+			{
+				if (UsedSize < BlockSize)
+				{
+					Result.insert_or_assign(BlockIndex, UsedCount);
+				}
+			}
+			else if (BlockUsageThresholdPercent == 0)
+			{
+				if (UsedSize == 0)
+				{
+					Result.insert_or_assign(BlockIndex, UsedCount);
+				}
+			}
+			else
+			{
+				const uint32_t UsedPercent = UsedSize < BlockSize ? gsl::narrow<uint32_t>((100 * UsedSize) / BlockSize) : 100u;
+				if (UsedPercent < BlockUsageThresholdPercent)
+				{
+					Result.insert_or_assign(BlockIndex, UsedCount);
+				}
+			}
+		}
+	}
+	return Result;
+}
+
 void
 BlockStore::Close()
 {
@@ -312,7 +387,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&,
 }
 
 void
-BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback)
+BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback)
 {
 	ZEN_TRACE_CPU("BlockStore::WriteChunk");
 
@@ -321,12 +396,14 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 	ZEN_ASSERT(Size <= m_MaxBlockSize);
 	ZEN_ASSERT(Alignment > 0u);
 
+	uint32_t ChunkSize = gsl::narrow<uint32_t>(Size);
+
 	RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
 
 	uint32_t WriteBlockIndex	 = m_WriteBlockIndex.load(std::memory_order_acquire);
 	bool	 IsWriting			 = !!m_WriteBlock;
-	uint64_t AlignedInsertOffset = RoundUp(m_CurrentInsertOffset, Alignment);
-	if (!IsWriting || (AlignedInsertOffset + Size) > m_MaxBlockSize)
+	uint32_t AlignedInsertOffset = RoundUp(m_CurrentInsertOffset, Alignment);
+	if (!IsWriting || (AlignedInsertOffset + ChunkSize) > m_MaxBlockSize)
 	{
 		if (m_WriteBlock)
 		{
@@ -351,16 +428,16 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, cons
 		m_CurrentInsertOffset = 0;
 		AlignedInsertOffset	  = 0;
 	}
-	uint64_t AlignedWriteSize	   = AlignedInsertOffset - m_CurrentInsertOffset + Size;
-	m_CurrentInsertOffset		   = AlignedInsertOffset + Size;
+	uint32_t AlignedWriteSize	   = AlignedInsertOffset - m_CurrentInsertOffset + ChunkSize;
+	m_CurrentInsertOffset		   = AlignedInsertOffset + ChunkSize;
 	Ref<BlockStoreFile> WriteBlock = m_WriteBlock;
 	m_ActiveWriteBlocks.push_back(WriteBlockIndex);
 	InsertLock.ReleaseNow();
 
-	WriteBlock->Write(Data, Size, AlignedInsertOffset);
+	WriteBlock->Write(Data, ChunkSize, AlignedInsertOffset);
 	m_TotalSize.fetch_add(AlignedWriteSize, std::memory_order::relaxed);
 
-	Callback({.BlockIndex = WriteBlockIndex, .Offset = AlignedInsertOffset, .Size = Size});
+	Callback({.BlockIndex = WriteBlockIndex, .Offset = AlignedInsertOffset, .Size = ChunkSize});
 
 	{
 		RwLock::ExclusiveLockScope _(m_InsertLock);
@@ -433,7 +510,7 @@ void
 BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 						 const std::vector<BlockStoreLocation>& ChunkLocations,
 						 const ChunkIndexArray&					KeepChunkIndexes,
-						 uint64_t								PayloadAlignment,
+						 uint32_t								PayloadAlignment,
 						 bool									DryRun,
 						 const ReclaimCallback&					ChangeCallback,
 						 const ClaimDiskReserveCallback&		DiskReserveCallback)
@@ -682,9 +759,9 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 				{
 					const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
 					Chunk.resize(ChunkLocation.Size);
-					OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+					OldBlockFile->Read(Chunk.data(), ChunkLocation.Size, ChunkLocation.Offset);
 
-					if (!NewBlockFile || (WriteOffset + Chunk.size() > m_MaxBlockSize))
+					if (!NewBlockFile || (WriteOffset + ChunkLocation.Size > m_MaxBlockSize))
 					{
 						uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
 
@@ -758,10 +835,12 @@ BlockStore::ReclaimSpace(const ReclaimSnapshotState&			Snapshot,
 						WriteOffset	  = 0;
 					}
 
-					NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-					MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
+					NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+					MovedChunks.push_back(
+						{ChunkIndex,
+						 {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
 					uint64_t OldOffset = WriteOffset;
-					WriteOffset		   = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
+					WriteOffset		   = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
 					m_TotalSize.fetch_add(WriteOffset - OldOffset, std::memory_order::relaxed);
 				}
 				Chunk.clear();
@@ -961,7 +1040,7 @@ BlockStore::IterateChunks(const std::vector<BlockStoreLocation>& ChunkLocations,
 
 void
 BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
-						  uint64_t						  PayloadAlignment,
+						  uint32_t						  PayloadAlignment,
 						  const CompactCallback&		  ChangeCallback,
 						  const ClaimDiskReserveCallback& DiskReserveCallback)
 {
@@ -971,7 +1050,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 
 	Stopwatch  TotalTimer;
 	const auto _ = MakeGuard([&] {
-		ZEN_DEBUG("compact blocks for '{}' DONE after {}, deleted {} and moved {} chunks ({}) ",
+		ZEN_DEBUG("Compact blocks for '{}' DONE after {}, deleted {} and moved {} chunks ({}) ",
 				  m_BlocksBasePath,
 				  NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()),
 				  NiceBytes(DeletedSize),
@@ -983,13 +1062,14 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 	uint32_t		 NewBlockIndex = 0;
 	MovedChunksArray MovedChunks;
 
+	uint64_t AddedSize	 = 0;
 	uint64_t RemovedSize = 0;
 
 	Ref<BlockStoreFile> NewBlockFile;
 	auto				NewBlockFileGuard = MakeGuard([&]() {
 		   if (NewBlockFile)
 		   {
-			   ZEN_DEBUG("dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
+			   ZEN_DEBUG("Dropping incomplete cas block store file '{}'", NewBlockFile->GetPath());
 			   {
 				   RwLock::ExclusiveLockScope _l(m_InsertLock);
 				   if (m_ChunkBlocks[NewBlockIndex] == NewBlockFile)
@@ -1001,140 +1081,174 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 		   }
 	   });
 
+	auto ReportChanges = [&]() -> bool {
+		bool Continue = true;
+		if (!MovedChunks.empty() || RemovedSize > 0)
+		{
+			Continue = ChangeCallback(MovedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0);
+			DeletedSize += RemovedSize;
+			RemovedSize = 0;
+			AddedSize	= 0;
+			MovedCount += MovedChunks.size();
+			MovedChunks.clear();
+		}
+		return Continue;
+	};
+
 	std::vector<uint32_t> RemovedBlocks;
 
-	CompactState.IterateBlocks(
-		[&](uint32_t BlockIndex, const std::vector<size_t>& KeepChunkIndexes, const std::vector<BlockStoreLocation>& ChunkLocations) {
-			Ref<BlockStoreFile> OldBlockFile;
+	CompactState.IterateBlocks([&](uint32_t								  BlockIndex,
+								   const std::vector<size_t>&			  KeepChunkIndexes,
+								   const std::vector<BlockStoreLocation>& ChunkLocations) -> bool {
+		Ref<BlockStoreFile> OldBlockFile;
+		{
+			RwLock::SharedLockScope _(m_InsertLock);
+			if ((BlockIndex == m_WriteBlockIndex.load()) && m_WriteBlock)
 			{
-				RwLock::SharedLockScope _(m_InsertLock);
-				if ((BlockIndex == m_WriteBlockIndex.load()) && m_WriteBlock)
-				{
-					// You are trying to collect the currently writing block, Report error?
-					return;
-				}
-				auto It = m_ChunkBlocks.find(BlockIndex);
-				if (It == m_ChunkBlocks.end())
-				{
-					// This block has unknown, we can't move anything. Report error?
-					return;
-				}
-				if (!It->second)
-				{
-					// This block has been removed, we can't move anything. Report error?
-					return;
-				}
-				OldBlockFile = It->second;
+				ZEN_ERROR("Compact Block was requested to rewrite the currently active write block in '{}', Block index {}",
+						  m_BlocksBasePath,
+						  BlockIndex);
+				return false;
 			}
-			ZEN_ASSERT(OldBlockFile);
-
-			uint64_t OldBlockSize = OldBlockFile->FileSize();
+			auto It = m_ChunkBlocks.find(BlockIndex);
+			if (It == m_ChunkBlocks.end())
+			{
+				ZEN_WARN("Compact Block was requested to rewrite an unknown block in '{}', Block index {}", m_BlocksBasePath, BlockIndex);
+				return true;
+			}
+			if (!It->second)
+			{
+				ZEN_WARN("Compact Block was requested to rewrite a deleted block in '{}', Block index {}", m_BlocksBasePath, BlockIndex);
+				return true;
+			}
+			OldBlockFile = It->second;
+		}
+		ZEN_ASSERT(OldBlockFile);
 
-			// TODO: Add heuristics for determining if it is worth to compact a block (if only a very small part is removed)
+		uint64_t OldBlockSize = OldBlockFile->FileSize();
 
-			std::vector<uint8_t> Chunk;
-			for (const size_t& ChunkIndex : KeepChunkIndexes)
+		std::vector<uint8_t> Chunk;
+		for (const size_t& ChunkIndex : KeepChunkIndexes)
+		{
+			const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
+			if (ChunkLocation.Offset + ChunkLocation.Size > OldBlockSize)
 			{
-				const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex];
-				Chunk.resize(ChunkLocation.Size);
-				OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
+				ZEN_WARN(
+					"Compact Block skipping chunk outside of block range in '{}', Chunk start {}, Chunk size {} in Block {}, Block "
+					"size {}",
+					m_BlocksBasePath,
+					ChunkLocation.Offset,
+					ChunkLocation.Size,
+					OldBlockFile->GetPath(),
+					OldBlockSize);
+				continue;
+			}
+
+			Chunk.resize(ChunkLocation.Size);
+			OldBlockFile->Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset);
 
-				if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
+			if ((WriteOffset + Chunk.size()) > m_MaxBlockSize)
+			{
+				if (NewBlockFile)
 				{
-					if (NewBlockFile)
-					{
-						NewBlockFile->Flush();
-						MovedSize += NewBlockFile->FileSize();
-						NewBlockFile = nullptr;
+					NewBlockFile->Flush();
+					MovedSize += NewBlockFile->FileSize();
+					NewBlockFile = nullptr;
 
-						ZEN_ASSERT(!MovedChunks.empty() || RemovedSize > 0);  // We should not have a new block if we haven't moved anything
+					ZEN_ASSERT(!MovedChunks.empty() || RemovedSize > 0);  // We should not have a new block if we haven't moved anything
 
-						ChangeCallback(MovedChunks, RemovedSize);
-						DeletedSize += RemovedSize;
-						RemovedSize = 0;
-						MovedCount += MovedChunks.size();
-						MovedChunks.clear();
+					if (!ReportChanges())
+					{
+						return false;
 					}
+				}
 
-					uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
+				uint32_t NextBlockIndex = m_WriteBlockIndex.load(std::memory_order_relaxed);
+				{
+					RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
+					std::filesystem::path	   NewBlockPath;
+					NextBlockIndex = GetFreeBlockIndex(NextBlockIndex, InsertLock, NewBlockPath);
+					if (NextBlockIndex == (uint32_t)m_MaxBlockCount)
 					{
-						RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
-						std::filesystem::path	   NewBlockPath;
-						NextBlockIndex = GetFreeBlockIndex(NextBlockIndex, InsertLock, NewBlockPath);
-						if (NextBlockIndex == (uint32_t)m_MaxBlockCount)
-						{
-							ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
-									  m_BlocksBasePath,
-									  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
-							return;
-						}
-
-						NewBlockFile				  = new BlockStoreFile(NewBlockPath);
-						m_ChunkBlocks[NextBlockIndex] = NewBlockFile;
+						ZEN_ERROR("unable to allocate a new block in '{}', count limit {} exeeded",
+								  m_BlocksBasePath,
+								  static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1);
+						return false;
 					}
-					ZEN_ASSERT(NewBlockFile);
 
-					std::error_code Error;
-					DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
-					if (Error)
+					NewBlockFile				  = new BlockStoreFile(NewBlockPath);
+					m_ChunkBlocks[NextBlockIndex] = NewBlockFile;
+				}
+				ZEN_ASSERT(NewBlockFile);
+
+				std::error_code Error;
+				DiskSpace		Space = DiskSpaceInfo(m_BlocksBasePath, Error);
+				if (Error)
+				{
+					ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
 					{
-						ZEN_ERROR("get disk space in '{}' FAILED, reason: '{}'", m_BlocksBasePath, Error.message());
-						return;
+						RwLock::ExclusiveLockScope _l(m_InsertLock);
+						ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
+						m_ChunkBlocks.erase(NextBlockIndex);
 					}
+					NewBlockFile->MarkAsDeleteOnClose();
+					NewBlockFile = nullptr;
+					return false;
+				}
 
-					if (Space.Free < m_MaxBlockSize)
+				if (Space.Free < m_MaxBlockSize)
+				{
+					uint64_t ReclaimedSpace = DiskReserveCallback();
+					if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
 					{
-						uint64_t ReclaimedSpace = DiskReserveCallback();
-						if (Space.Free + ReclaimedSpace < m_MaxBlockSize)
-						{
-							ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
-									 m_BlocksBasePath,
-									 m_MaxBlockSize,
-									 NiceBytes(Space.Free + ReclaimedSpace));
-							{
-								RwLock::ExclusiveLockScope _l(m_InsertLock);
-								ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
-								m_ChunkBlocks.erase(NextBlockIndex);
-							}
-							NewBlockFile = nullptr;
-							return;
-						}
-
-						ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
+						ZEN_WARN("garbage collect for '{}' FAILED, required disk space {}, free {}",
 								 m_BlocksBasePath,
-								 ReclaimedSpace,
+								 m_MaxBlockSize,
 								 NiceBytes(Space.Free + ReclaimedSpace));
+						{
+							RwLock::ExclusiveLockScope _l(m_InsertLock);
+							ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile);
+							m_ChunkBlocks.erase(NextBlockIndex);
+						}
+						NewBlockFile->MarkAsDeleteOnClose();
+						NewBlockFile = nullptr;
+						return false;
 					}
-					NewBlockFile->Create(m_MaxBlockSize);
-					NewBlockIndex = NextBlockIndex;
-					WriteOffset	  = 0;
-				}
 
-				NewBlockFile->Write(Chunk.data(), Chunk.size(), WriteOffset);
-				MovedChunks.push_back({ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = WriteOffset, .Size = Chunk.size()}});
-				WriteOffset = RoundUp(WriteOffset + Chunk.size(), PayloadAlignment);
+					ZEN_INFO("using gc reserve for '{}', reclaimed {}, disk free {}",
+							 m_BlocksBasePath,
+							 ReclaimedSpace,
+							 NiceBytes(Space.Free + ReclaimedSpace));
+				}
+				NewBlockFile->Create(m_MaxBlockSize);
+				NewBlockIndex = NextBlockIndex;
+				WriteOffset	  = 0;
 			}
-			Chunk.clear();
 
-			// Report what we have moved so we can purge the old block
-			if (!MovedChunks.empty() || RemovedSize > 0)
-			{
-				ChangeCallback(MovedChunks, RemovedSize);
-				DeletedSize += RemovedSize;
-				RemovedSize = 0;
-				MovedCount += MovedChunks.size();
-				MovedChunks.clear();
-			}
+			NewBlockFile->Write(Chunk.data(), ChunkLocation.Size, WriteOffset);
+			MovedChunks.push_back(
+				{ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow<uint32_t>(WriteOffset), .Size = ChunkLocation.Size}});
+			WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment);
+			AddedSize += Chunk.size();
+		}
+		Chunk.clear();
+
+		if (!ReportChanges())
+		{
+			return false;
+		}
+
+		{
+			RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
+			ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
+			OldBlockFile->MarkAsDeleteOnClose();
+			m_ChunkBlocks.erase(BlockIndex);
+			m_TotalSize.fetch_sub(OldBlockSize);
+			RemovedSize += OldBlockSize;
+		}
+		return true;
+	});
 
-			{
-				RwLock::ExclusiveLockScope InsertLock(m_InsertLock);
-				ZEN_DEBUG("marking cas block store file '{}' for delete, block #{}", OldBlockFile->GetPath(), BlockIndex);
-				OldBlockFile->MarkAsDeleteOnClose();
-				m_ChunkBlocks.erase(BlockIndex);
-				m_TotalSize.fetch_sub(OldBlockSize);
-				RemovedSize += OldBlockSize;
-			}
-		});
 	if (NewBlockFile)
 	{
 		NewBlockFile->Flush();
@@ -1142,14 +1256,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState&	  CompactState,
 		NewBlockFile = nullptr;
 	}
 
-	if (!MovedChunks.empty() || RemovedSize > 0)
-	{
-		ChangeCallback(MovedChunks, RemovedSize);
-		DeletedSize += RemovedSize;
-		RemovedSize = 0;
-		MovedCount += MovedChunks.size();
-		MovedChunks.clear();
-	}
+	ReportChanges();
 }
 
 const char*
@@ -1175,6 +1282,17 @@ BlockStore::GetBlockPath(const std::filesystem::path& BlocksBasePath, const uint
 	return Path.ToPath();
 }
 
+Ref<BlockStoreFile>
+BlockStore::GetBlockFile(uint32_t BlockIndex)
+{
+	RwLock::SharedLockScope _(m_InsertLock);
+	if (auto It = m_ChunkBlocks.find(BlockIndex); It != m_ChunkBlocks.end())
+	{
+		return It->second;
+	}
+	return {};
+}
+
 #if ZEN_WITH_TESTS
 
 TEST_CASE("blockstore.blockstoredisklocation")
@@ -1293,7 +1411,7 @@ TEST_CASE("blockstore.blockfile")
 }
 
 namespace blockstore::impl {
-	BlockStoreLocation WriteStringAsChunk(BlockStore& Store, std::string_view String, size_t PayloadAlignment)
+	BlockStoreLocation WriteStringAsChunk(BlockStore& Store, std::string_view String, uint32_t PayloadAlignment)
 	{
 		BlockStoreLocation Location;
 		Store.WriteChunk(String.data(), String.length(), PayloadAlignment, [&](const BlockStoreLocation& L) { Location = L; });
@@ -1392,7 +1510,12 @@ TEST_CASE("blockstore.clean.stray.blocks")
 	CHECK(!ThirdChunk);
 
 	// Recreate a fake block for a missing chunk location
-	Store.SyncExistingBlocksOnDisk({FirstChunkLocation, SecondChunkLocation, ThirdChunkLocation});
+	BlockStore::BlockIndexSet KnownBlocks;
+	KnownBlocks.Add(FirstChunkLocation.BlockIndex);
+	KnownBlocks.Add(SecondChunkLocation.BlockIndex);
+	KnownBlocks.Add(ThirdChunkLocation.BlockIndex);
+	Store.SyncExistingBlocksOnDisk(KnownBlocks);
+
 	// We create a fake block for the location - we should still not be able to get the chunk
 	CHECK(GetDirectoryContent(RootDirectory / "store", true, false).size() == 2);
 	ThirdChunk = Store.TryGetChunk(ThirdChunkLocation);
@@ -1760,7 +1883,10 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray&, uint64_t) { CHECK(false); },
+			[&](const BlockStore::MovedChunksArray&, uint64_t) {
+				CHECK(false);
+				return true;
+			},
 			[]() {
 				CHECK(false);
 				return 0;
@@ -1785,6 +1911,7 @@ TEST_CASE("blockstore.compact.blocks")
 			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
 				RemovedSize += Removed;
 				CHECK(Moved.empty());
+				return true;
 			},
 			[]() { return 0; });
 		CHECK_EQ(RemovedSize, PreSize);
@@ -1810,6 +1937,7 @@ TEST_CASE("blockstore.compact.blocks")
 			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
 				RemovedSize += Removed;
 				CHECK(Moved.empty());
+				return true;
 			},
 			[]() { return 0; });
 		CHECK_EQ(Store.TotalSize() + RemovedSize, PreSize);
@@ -1830,7 +1958,10 @@ TEST_CASE("blockstore.compact.blocks")
 		Store.CompactBlocks(
 			State,
 			Alignment,
-			[&](const BlockStore::MovedChunksArray&, uint64_t) { CHECK(false); },
+			[&](const BlockStore::MovedChunksArray&, uint64_t) {
+				CHECK(false);
+				return true;
+			},
 			[]() {
 				CHECK(false);
 				return 0;
@@ -1862,6 +1993,7 @@ TEST_CASE("blockstore.compact.blocks")
 			[&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) {
 				CHECK(Moved.empty());
 				RemovedSize += Removed;
+				return true;
 			},
 			[]() {
 				CHECK(false);
@@ -1905,6 +2037,7 @@ TEST_CASE("blockstore.compact.blocks")
 					(*It) = Move.second;
 				}
 				RemovedSize += Removed;
+				return true;
 			},
 			[]() {
 				CHECK(false);
@@ -1981,6 +2114,7 @@ TEST_CASE("blockstore.compact.blocks")
 					(*It) = Move.second;
 				}
 				RemovedSize += Removed;
+				return true;
 			},
 			[]() {
 				CHECK(false);
diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp
index fc549a729..d38099117 100644
--- a/src/zenstore/cas.cpp
+++ b/src/zenstore/cas.cpp
@@ -12,6 +12,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/memory.h>
+#include <zencore/scopeguard.h>
 #include <zencore/string.h>
 #include <zencore/testing.h>
 #include <zencore/testutils.h>
@@ -22,6 +23,7 @@
 #include <zenstore/cidstore.h>
 #include <zenstore/gc.h>
 #include <zenstore/scrubcontext.h>
+#include <zenutil/workerpools.h>
 
 #include <gsl/gsl-lite.hpp>
 
@@ -104,10 +106,22 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig)
 	const bool IsNewStore = OpenOrCreateManifest();
 
 	// Initialize payload storage
-
-	m_LargeStrategy.Initialize(m_Config.RootDirectory, IsNewStore);
-	m_TinyStrategy.Initialize(m_Config.RootDirectory, "tobs", 1u << 28, 16, IsNewStore);	 // 256 Mb per block
-	m_SmallStrategy.Initialize(m_Config.RootDirectory, "sobs", 1u << 30, 4096, IsNewStore);	 // 1 Gb per block
+	{
+		WorkerThreadPool&			   WorkerPool = GetSmallWorkerPool();
+		std::vector<std::future<void>> Work;
+		Work.emplace_back(
+			WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() { m_LargeStrategy.Initialize(m_Config.RootDirectory, IsNewStore); }}));
+		Work.emplace_back(WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() {
+			m_TinyStrategy.Initialize(m_Config.RootDirectory, "tobs", 1u << 28, 16, IsNewStore);  // 256 Mb per block
+		}}));
+		Work.emplace_back(WorkerPool.EnqueueTask(std::packaged_task<void()>{[&]() {
+			m_SmallStrategy.Initialize(m_Config.RootDirectory, "sobs", 1u << 30, 4096, IsNewStore);	 // 1 Gb per block
+		}}));
+		for (std::future<void>& Result : Work)
+		{
+			Result.get();
+		}
+	}
 }
 
 bool
diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp
index c04324fbc..2c26e522f 100644
--- a/src/zenstore/caslog.cpp
+++ b/src/zenstore/caslog.cpp
@@ -188,20 +188,30 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntr
 	LogBaseOffset += SkipEntryCount * m_RecordSize;
 	LogEntryCount -= SkipEntryCount;
 
-	// This should really be streaming the data rather than just
-	// reading it into memory, though we don't tend to get very
-	// large logs so it may not matter
+	const uint64_t LogDataSize	 = LogEntryCount * m_RecordSize;
+	uint64_t	   LogDataRemain = LogDataSize;
 
-	const uint64_t LogDataSize = LogEntryCount * m_RecordSize;
+	const uint64_t MaxBufferSize = 1024 * 1024;
 
 	std::vector<uint8_t> ReadBuffer;
-	ReadBuffer.resize(LogDataSize);
+	ReadBuffer.resize((Min(LogDataSize, MaxBufferSize) / m_RecordSize) * m_RecordSize);
 
-	m_File.Read(ReadBuffer.data(), LogDataSize, LogBaseOffset);
+	uint64_t ReadOffset = 0;
 
-	for (int i = 0; i < int(LogEntryCount); ++i)
+	while (LogDataRemain)
 	{
-		Handler(ReadBuffer.data() + (i * m_RecordSize));
+		const uint64_t BytesToRead	 = Min(ReadBuffer.size(), LogDataRemain);
+		const uint64_t EntriesToRead = BytesToRead / m_RecordSize;
+
+		m_File.Read(ReadBuffer.data(), BytesToRead, LogBaseOffset + ReadOffset);
+
+		for (int i = 0; i < int(EntriesToRead); ++i)
+		{
+			Handler(ReadBuffer.data() + (i * m_RecordSize));
+		}
+
+		LogDataRemain -= BytesToRead;
+		ReadOffset += BytesToRead;
 	}
 
 	m_AppendOffset = LogBaseOffset + (m_RecordSize * LogEntryCount);
@@ -219,7 +229,8 @@ CasLogFile::Append(const void* DataPointer, uint64_t DataSize)
 
 	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to write to log file '{}'", PathFromHandle(m_File.Handle())));
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to log file '{}'", PathFromHandle(m_File.Handle(), Dummy)));
 	}
 }
 
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 00a018948..b21f9f8d8 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -25,6 +25,9 @@
 #	include <zenstore/cidstore.h>
 #	include <algorithm>
 #	include <random>
+ZEN_THIRD_PARTY_INCLUDES_START
+#	include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 #endif
 
 //////////////////////////////////////////////////////////////////////////
@@ -114,8 +117,14 @@ namespace {
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get("containercas")), m_Gc(Gc)
 {
+	m_LocationMap.min_load_factor(IndexMinLoadFactor);
+	m_LocationMap.max_load_factor(IndexMaxLoadFactor);
+
 	m_Gc.AddGcStorage(this);
 	m_Gc.AddGcReferenceStore(*this);
 }
@@ -130,7 +139,7 @@ void
 CasContainerStrategy::Initialize(const std::filesystem::path& RootDirectory,
 								 const std::string_view		  ContainerBaseName,
 								 uint32_t					  MaxBlockSize,
-								 uint64_t					  Alignment,
+								 uint32_t					  Alignment,
 								 bool						  IsNewStore)
 {
 	ZEN_ASSERT(IsPow2(Alignment));
@@ -245,6 +254,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 {
 	ZEN_TRACE_CPU("CasContainer::ScrubStorage");
 
+	if (Ctx.IsSkipCas())
+	{
+		ZEN_INFO("SKIPPED scrubbing: '{}'", m_BlocksBasePath);
+		return;
+	}
+
 	ZEN_INFO("scrubbing '{}'", m_BlocksBasePath);
 
 	std::vector<IoHash>				BadKeys;
@@ -288,21 +303,12 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 			uint64_t RawSize;
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHash ComputedHash = IoHash::HashBuffer(Data, Size);
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -317,26 +323,15 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 
 			IoHash	 RawHash;
 			uint64_t RawSize;
-			// TODO: Add API to verify compressed buffer without having to memorymap the whole file
+			// TODO: Add API to verify compressed buffer without having to memory-map the whole file
 			if (CompressedBuffer::ValidateCompressedHeader(Buffer, RawHash, RawSize))
 			{
-				if (RawHash != Hash)
+				if (RawHash == Hash)
 				{
-					// Hash mismatch
-					BadKeys.push_back(Hash);
+					// TODO: this should also hash the (decompressed) contents
 					return;
 				}
-				return;
-			}
-#if ZEN_WITH_TESTS
-			IoHashStream Hasher;
-			File.StreamByteRange(Offset, Size, [&](const void* Data, size_t Size) { Hasher.Append(Data, Size); });
-			IoHash ComputedHash = Hasher.GetHash();
-			if (ComputedHash == Hash)
-			{
-				return;
 			}
-#endif
 			BadKeys.push_back(Hash);
 		};
 
@@ -389,7 +384,7 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx)
 		Ctx.ReportBadCidChunks(BadKeys);
 	}
 
-	ZEN_INFO("compact cas scrubbed: {} chunks ({})", ChunkCount, NiceBytes(ChunkBytes));
+	ZEN_INFO("scrubbed {} chunks ({}) in '{}'", ChunkCount, NiceBytes(ChunkBytes), m_RootDirectory / m_ContainerBaseName);
 }
 
 void
@@ -553,82 +548,143 @@ CasContainerStrategy::CollectGarbage(GcContext& GcCtx)
 	GcCtx.AddDeletedCids(DeletedChunks);
 }
 
-class CasContainerStoreCompactor : public GcReferenceStoreCompactor
+class CasContainerStoreCompactor : public GcStoreCompactor
 {
 public:
-	CasContainerStoreCompactor(CasContainerStrategy&	Owner,
-							   BlockStoreCompactState&& CompactState,
-							   std::vector<IoHash>&&	CompactStateKeys,
-							   std::vector<IoHash>&&	PrunedKeys)
-	: m_CasContainerStrategy(Owner)
-	, m_CompactState(std::move(CompactState))
-	, m_CompactStateKeys(std::move(CompactStateKeys))
-	, m_PrunedKeys(std::move(PrunedKeys))
-	{
-	}
+	CasContainerStoreCompactor(CasContainerStrategy& Owner) : m_CasContainerStrategy(Owner) {}
 
-	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override
 	{
+		ZEN_TRACE_CPU("CasContainer::CompactStore");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+			ZEN_INFO("GCV2: compactcas [COMPACT] '{}': RemovedDisk: {} in {}",
 					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					 Stats.Count,
-					 Stats.Pruned,
-					 Stats.Compacted,
 					 NiceBytes(Stats.RemovedDisk),
-					 NiceBytes(Stats.RemovedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
-		if (Ctx.Settings.IsDeleteMode && Ctx.Settings.CollectSmallObjects)
+		if (Ctx.Settings.CollectSmallObjects)
 		{
-			// Compact block store
-			m_CasContainerStrategy.m_BlockStore.CompactBlocks(
-				m_CompactState,
-				m_CasContainerStrategy.m_PayloadAlignment,
-				[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
-					std::vector<CasDiskIndexEntry> MovedEntries;
-					RwLock::ExclusiveLockScope	   _(m_CasContainerStrategy.m_LocationMapLock);
-					for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+			BlockStore::BlockUsageMap BlockUsage;
+			{
+				RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+				if (Ctx.IsCancelledFlag.load())
+				{
+					return;
+				}
+
+				for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
+				{
+					size_t						  Index = Entry.second;
+					const BlockStoreDiskLocation& Loc	= m_CasContainerStrategy.m_Locations[Index];
+
+					uint32_t BlockIndex = Loc.GetBlockIndex();
+					uint64_t ChunkSize	= RoundUp(Loc.GetSize(), m_CasContainerStrategy.m_PayloadAlignment);
+					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
+					{
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
+					}
+					else
 					{
-						size_t		  ChunkIndex = Moved.first;
-						const IoHash& Key		 = m_CompactStateKeys[ChunkIndex];
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
+					}
+				}
+			}
+
+			{
+				BlockStoreCompactState BlockCompactState;
+				std::vector<IoHash>	   BlockCompactStateKeys;
 
-						if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key); It != m_CasContainerStrategy.m_LocationMap.end())
+				BlockStore::BlockEntryCountMap BlocksToCompact =
+					m_CasContainerStrategy.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
+				BlockCompactState.IncludeBlocks(BlocksToCompact);
+
+				if (BlocksToCompact.size() > 0)
+				{
+					{
+						RwLock::SharedLockScope __(m_CasContainerStrategy.m_LocationMapLock);
+						for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
 						{
-							BlockStoreDiskLocation&	  Location	  = m_CasContainerStrategy.m_Locations[It->second];
-							const BlockStoreLocation& OldLocation = m_CompactState.GetLocation(ChunkIndex);
-							if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation)
+							size_t						  Index = Entry.second;
+							const BlockStoreDiskLocation& Loc	= m_CasContainerStrategy.m_Locations[Index];
+
+							if (!BlockCompactState.AddKeepLocation(Loc.Get(m_CasContainerStrategy.m_PayloadAlignment)))
 							{
-								// Someone has moved our chunk so lets just skip the new location we were provided, it will be GC:d at a
-								// later time
 								continue;
 							}
-
-							const BlockStoreLocation& NewLocation = Moved.second;
-							Location = BlockStoreDiskLocation(NewLocation, m_CasContainerStrategy.m_PayloadAlignment);
-							MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location});
+							BlockCompactStateKeys.push_back(Entry.first);
 						}
 					}
-					m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
-					Stats.RemovedDisk += FreedDiskSpace;
-				},
-				[&]() { return 0; });
 
-			Stats.Compacted +=
-				m_PrunedKeys.size();  // Slightly missleading, it might not be compacted if the block is the currently writing block
+					if (Ctx.Settings.IsDeleteMode)
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': compacting {} blocks",
+									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 BlocksToCompact.size());
+						}
+
+						m_CasContainerStrategy.m_BlockStore.CompactBlocks(
+							BlockCompactState,
+							m_CasContainerStrategy.m_PayloadAlignment,
+							[&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) {
+								std::vector<CasDiskIndexEntry> MovedEntries;
+								RwLock::ExclusiveLockScope	   _(m_CasContainerStrategy.m_LocationMapLock);
+								for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray)
+								{
+									size_t		  ChunkIndex = Moved.first;
+									const IoHash& Key		 = BlockCompactStateKeys[ChunkIndex];
+
+									if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key);
+										It != m_CasContainerStrategy.m_LocationMap.end())
+									{
+										BlockStoreDiskLocation&	  Location	  = m_CasContainerStrategy.m_Locations[It->second];
+										const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex);
+										if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation)
+										{
+											// Someone has moved our chunk so lets just skip the new location we were provided, it will be
+											// GC:d at a later time
+											continue;
+										}
+										const BlockStoreLocation& NewLocation = Moved.second;
+
+										Location = BlockStoreDiskLocation(NewLocation, m_CasContainerStrategy.m_PayloadAlignment);
+										MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location});
+									}
+								}
+								m_CasContainerStrategy.m_CasLog.Append(MovedEntries);
+								Stats.RemovedDisk += FreedDiskSpace;
+								if (Ctx.IsCancelledFlag.load())
+								{
+									return false;
+								}
+								return true;
+							},
+							ClaimDiskReserveCallback);
+					}
+					else
+					{
+						if (Ctx.Settings.Verbose)
+						{
+							ZEN_INFO("GCV2: compactcas [COMPACT] '{}': skipped compacting of {} eligible blocks",
+									 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
+									 BlocksToCompact.size());
+						}
+					}
+				}
+			}
 		}
 	}
 
-	CasContainerStrategy&  m_CasContainerStrategy;
-	BlockStoreCompactState m_CompactState;
-	std::vector<IoHash>	   m_CompactStateKeys;
-	std::vector<IoHash>	   m_PrunedKeys;
+	CasContainerStrategy& m_CasContainerStrategy;
 };
 
 class CasContainerReferencePruner : public GcReferencePruner
@@ -640,27 +696,27 @@ public:
 	{
 	}
 
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
-															  GcReferenceStoreStats&		 Stats,
-															  const GetUnusedReferencesFunc& GetUnusedReferences)
+	virtual GcStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, GcStats& Stats, const GetUnusedReferencesFunc& GetUnusedReferences)
 	{
+		ZEN_TRACE_CPU("CasContainer::RemoveUnreferencedData");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+			ZEN_INFO("GCV2: compactcas [PRUNE] '{}': Checked: {}, Deleted: {}, FreedMemory: {} in {}",
 					 m_CasContainerStrategy.m_RootDirectory / m_CasContainerStrategy.m_ContainerBaseName,
-					 Stats.Count,
-					 Stats.Pruned,
-					 Stats.Compacted,
-					 NiceBytes(Stats.RemovedDisk),
-					 NiceBytes(Stats.RemovedMemory),
+					 Stats.CheckedCount,
+					 Stats.DeletedCount,
+					 NiceBytes(Stats.FreedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
+		Stats.CheckedCount			   = m_Cids.size();
+		Stats.FoundCount			   = UnusedCids.size();
 
 		if (UnusedCids.empty())
 		{
@@ -668,18 +724,14 @@ public:
 			return nullptr;
 		}
 
-		BlockStoreCompactState			 CompactState;
-		BlockStore::ReclaimSnapshotState BlockSnapshotState;
-		std::vector<IoHash>				 CompactStateKeys;
-		std::vector<CasDiskIndexEntry>	 ExpiredEntries;
+		std::vector<CasDiskIndexEntry> ExpiredEntries;
 		ExpiredEntries.reserve(UnusedCids.size());
-		tsl::robin_set<IoHash, IoHash::Hasher> UnusedKeys;
 
 		{
 			RwLock::ExclusiveLockScope __(m_CasContainerStrategy.m_LocationMapLock);
-			if (Ctx.Settings.CollectSmallObjects)
+			if (Ctx.IsCancelledFlag.load())
 			{
-				BlockSnapshotState = m_CasContainerStrategy.m_BlockStore.GetReclaimSnapshotState();
+				return nullptr;
 			}
 
 			for (const IoHash& Cid : UnusedCids)
@@ -689,59 +741,28 @@ public:
 				{
 					continue;
 				}
-				CasDiskIndexEntry			  ExpiredEntry	= {.Key		 = Cid,
-															   .Location = m_CasContainerStrategy.m_Locations[It->second],
-															   .Flags	 = CasDiskIndexEntry::kTombstone};
-				const BlockStoreDiskLocation& Location		= m_CasContainerStrategy.m_Locations[It->second];
-				BlockStoreLocation			  BlockLocation = Location.Get(m_CasContainerStrategy.m_PayloadAlignment);
 				if (Ctx.Settings.CollectSmallObjects)
 				{
-					UnusedKeys.insert(Cid);
-					uint32_t BlockIndex			= BlockLocation.BlockIndex;
-					bool	 IsActiveWriteBlock = BlockSnapshotState.m_ActiveWriteBlocks.contains(BlockIndex);
-					if (!IsActiveWriteBlock)
-					{
-						CompactState.IncludeBlock(BlockIndex);
-					}
+					CasDiskIndexEntry ExpiredEntry = {.Key		= Cid,
+													  .Location = m_CasContainerStrategy.m_Locations[It->second],
+													  .Flags	= CasDiskIndexEntry::kTombstone};
 					ExpiredEntries.push_back(ExpiredEntry);
 				}
 			}
 
-			// Get all locations we need to keep for affected blocks
-			if (Ctx.Settings.CollectSmallObjects && !UnusedKeys.empty())
-			{
-				for (const auto& Entry : m_CasContainerStrategy.m_LocationMap)
-				{
-					const IoHash& Key = Entry.first;
-					if (UnusedKeys.contains(Key))
-					{
-						continue;
-					}
-					const BlockStoreDiskLocation& Location		= m_CasContainerStrategy.m_Locations[Entry.second];
-					BlockStoreLocation			  BlockLocation = Location.Get(m_CasContainerStrategy.m_PayloadAlignment);
-					if (CompactState.AddKeepLocation(BlockLocation))
-					{
-						CompactStateKeys.push_back(Key);
-					}
-				}
-			}
-
 			if (Ctx.Settings.IsDeleteMode)
 			{
 				for (const CasDiskIndexEntry& Entry : ExpiredEntries)
 				{
 					m_CasContainerStrategy.m_LocationMap.erase(Entry.Key);
+					Stats.DeletedCount++;
 				}
 				m_CasContainerStrategy.m_CasLog.Append(ExpiredEntries);
 				m_CasContainerStrategy.m_CasLog.Flush();
 			}
 		}
 
-		Stats.Pruned += UnusedKeys.size();
-		return new CasContainerStoreCompactor(m_CasContainerStrategy,
-											  std::move(CompactState),
-											  std::move(CompactStateKeys),
-											  std::vector<IoHash>(UnusedKeys.begin(), UnusedKeys.end()));
+		return new CasContainerStoreCompactor(m_CasContainerStrategy);
 	}
 
 private:
@@ -756,21 +777,18 @@ CasContainerStrategy::GetGcName(GcCtx&)
 }
 
 GcReferencePruner*
-CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&)
 {
+	ZEN_TRACE_CPU("CasContainer::CreateReferencePruner");
+
 	Stopwatch  Timer;
 	const auto _ = MakeGuard([&] {
 		if (!Ctx.Settings.Verbose)
 		{
 			return;
 		}
-		ZEN_INFO("GCV2: compactcas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+		ZEN_INFO("GCV2: compactcas [CREATE PRUNER] '{}' in {}",
 				 m_RootDirectory / m_ContainerBaseName,
-				 Stats.Count,
-				 Stats.Pruned,
-				 Stats.Compacted,
-				 NiceBytes(Stats.RemovedDisk),
-				 NiceBytes(Stats.RemovedMemory),
 				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
@@ -781,13 +799,17 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& S
 		{
 			return {};
 		}
+		if (Ctx.IsCancelledFlag.load())
+		{
+			return nullptr;
+		}
+
 		CidsToCheck.reserve(m_LocationMap.size());
 		for (const auto& It : m_LocationMap)
 		{
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	Stats.Count += CidsToCheck.size();
 	return new CasContainerReferencePruner(*this, std::move(CidsToCheck));
 }
 
@@ -863,9 +885,11 @@ CasContainerStrategy::MakeIndexSnapshot()
 
 		// Write the current state of the location map to a new index state
 		std::vector<CasDiskIndexEntry> Entries;
+		uint64_t					   IndexLogPosition = 0;
 
 		{
 			RwLock::SharedLockScope ___(m_LocationMapLock);
+			IndexLogPosition = m_CasLog.GetLogCount();
 			Entries.resize(m_LocationMap.size());
 
 			uint64_t EntryIndex = 0;
@@ -880,7 +904,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		BasicFile ObjectIndexFile;
 		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kTruncate);
 		CasDiskIndexHeader Header = {.EntryCount	   = Entries.size(),
-									 .LogPosition	   = LogCount,
+									 .LogPosition	   = IndexLogPosition,
 									 .PayloadAlignment = gsl::narrow<uint32_t>(m_PayloadAlignment)};
 
 		Header.Checksum = CasDiskIndexHeader::ComputeChecksum(Header);
@@ -890,7 +914,7 @@ CasContainerStrategy::MakeIndexSnapshot()
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.Close();
 		EntryCount		   = Entries.size();
-		m_LogFlushPosition = LogCount;
+		m_LogFlushPosition = IndexLogPosition;
 	}
 	catch (std::exception& Err)
 	{
@@ -924,10 +948,10 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 {
 	ZEN_TRACE_CPU("CasContainer::ReadIndexFile");
 
-	std::vector<CasDiskIndexEntry> Entries;
-	Stopwatch					   Timer;
-	const auto					   _ = MakeGuard([&] {
-		ZEN_INFO("read store '{}' index containing {} entries in {}", IndexPath, Entries.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+	uint64_t   EntryCount = 0;
+	Stopwatch  Timer;
+	const auto _ = MakeGuard([&] {
+		ZEN_INFO("read store '{}' index containing {} entries in {}", IndexPath, EntryCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
 	BasicFile ObjectIndexFile;
@@ -942,21 +966,40 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 			(Header.Checksum == CasDiskIndexHeader::ComputeChecksum(Header)) && (Header.PayloadAlignment > 0) &&
 			(Header.EntryCount <= ExpectedEntryCount))
 		{
-			Entries.resize(Header.EntryCount);
-			ObjectIndexFile.Read(Entries.data(), Header.EntryCount * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
 			m_PayloadAlignment = Header.PayloadAlignment;
 
-			std::string InvalidEntryReason;
-			for (const CasDiskIndexEntry& Entry : Entries)
+			m_Locations.reserve(ExpectedEntryCount);
+			m_LocationMap.reserve(ExpectedEntryCount);
+
+			std::vector<CasDiskIndexEntry> Entries;
+			Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry));
+
+			uint64_t RemainingEntries = Header.EntryCount;
+			uint64_t ReadOffset		  = sizeof(CasDiskIndexHeader);
+
+			do
 			{
-				if (!ValidateEntry(Entry, InvalidEntryReason))
+				const uint64_t NumToRead = Min(RemainingEntries, Entries.size());
+				Entries.resize(NumToRead);
+
+				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), ReadOffset);
+
+				std::string InvalidEntryReason;
+				for (const CasDiskIndexEntry& Entry : Entries)
 				{
-					ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
-					continue;
+					if (!ValidateEntry(Entry, InvalidEntryReason))
+					{
+						ZEN_WARN("skipping invalid entry in '{}', reason: '{}'", IndexPath, InvalidEntryReason);
+						continue;
+					}
+					m_LocationMap[Entry.Key] = m_Locations.size();
+					m_Locations.push_back(Entry.Location);
+					++EntryCount;
 				}
-				m_LocationMap[Entry.Key] = m_Locations.size();
-				m_Locations.push_back(Entry.Location);
-			}
+
+				RemainingEntries -= NumToRead;
+				ReadOffset += NumToRead * sizeof(CasDiskIndexEntry);
+			} while (RemainingEntries);
 
 			OutVersion = CasDiskIndexHeader::CurrentVersion;
 			return Header.LogPosition;
@@ -1076,16 +1119,16 @@ CasContainerStrategy::OpenContainer(bool IsNewStore)
 
 	m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite);
 
-	std::vector<BlockStoreLocation> KnownLocations;
-	KnownLocations.reserve(m_LocationMap.size());
+	BlockStore::BlockIndexSet KnownBlocks;
+
 	for (const auto& Entry : m_LocationMap)
 	{
 		const BlockStoreDiskLocation& DiskLocation	= m_Locations[Entry.second];
 		BlockStoreLocation			  BlockLocation = DiskLocation.Get(m_PayloadAlignment);
-		KnownLocations.emplace_back(std::move(BlockLocation));
+		KnownBlocks.Add(BlockLocation.BlockIndex);
 	}
 
-	m_BlockStore.SyncExistingBlocksOnDisk(KnownLocations);
+	m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks);
 
 	if (IsNewStore || (LogEntryCount > 0))
 	{
diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h
index 3ed883801..932844da7 100644
--- a/src/zenstore/compactcas.h
+++ b/src/zenstore/compactcas.h
@@ -58,7 +58,7 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore
 	void				   Initialize(const std::filesystem::path& RootDirectory,
 									  const std::string_view	   ContainerBaseName,
 									  uint32_t					   MaxBlockSize,
-									  uint64_t					   Alignment,
+									  uint32_t					   Alignment,
 									  bool						   IsNewStore);
 	void				   Flush();
 
@@ -84,7 +84,7 @@ private:
 	LoggerRef					   m_Log;
 	GcManager&					   m_Gc;
 	std::filesystem::path		   m_RootDirectory;
-	uint64_t					   m_PayloadAlignment = 1u << 4;
+	uint32_t					   m_PayloadAlignment = 1u << 4;
 	uint64_t					   m_MaxBlockSize	  = 1u << 28;
 	bool						   m_IsInitialized	  = false;
 	TCasLogFile<CasDiskIndexEntry> m_CasLog;
diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp
index a72619e4b..f18509758 100644
--- a/src/zenstore/filecas.cpp
+++ b/src/zenstore/filecas.cpp
@@ -44,6 +44,15 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
+namespace {
+	template<typename T>
+	void Reset(T& V)
+	{
+		T Tmp;
+		V.swap(Tmp);
+	}
+}  // namespace
+
 namespace filecas::impl {
 	const char* IndexExtension = ".uidx";
 	const char* LogExtension   = ".ulog";
@@ -119,8 +128,14 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 FileCasStrategy::FileCasStrategy(GcManager& Gc) : m_Log(logging::Get("filecas")), m_Gc(Gc)
 {
+	m_Index.min_load_factor(IndexMinLoadFactor);
+	m_Index.max_load_factor(IndexMaxLoadFactor);
+
 	m_Gc.AddGcStorage(this);
 	m_Gc.AddGcReferenceStore(*this);
 }
@@ -831,6 +846,13 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 {
 	ZEN_TRACE_CPU("FileCas::ScrubStorage");
 
+	if (Ctx.IsSkipCas())
+	{
+		ZEN_INFO("SKIPPED scrubbing: '{}'", m_RootDirectory);
+		return;
+	}
+
+	Stopwatch Timer;
 	ZEN_INFO("scrubbing file CAS @ '{}'", m_RootDirectory);
 
 	ZEN_ASSERT(m_IsInitialized);
@@ -838,6 +860,8 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 	std::vector<IoHash> BadHashes;
 	uint64_t			ChunkCount{0}, ChunkBytes{0};
 
+	int DiscoveredFilesNotInIndex = 0;
+
 	{
 		std::vector<FileCasStrategy::FileCasIndexEntry> ScannedEntries = FileCasStrategy::ScanFolderForCasFiles(m_RootDirectory);
 		RwLock::ExclusiveLockScope						_(m_Lock);
@@ -847,10 +871,13 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 			{
 				m_TotalSize.fetch_add(static_cast<uint64_t>(Entry.Size), std::memory_order::relaxed);
 				m_CasLog.Append({.Key = Entry.Key, .Size = Entry.Size});
+				++DiscoveredFilesNotInIndex;
 			}
 		}
 	}
 
+	ZEN_INFO("discovered {} files @ '{}' ({} not in index), scrubbing", m_Index.size(), m_RootDirectory, DiscoveredFilesNotInIndex);
+
 	IterateChunks([&](const IoHash& Hash, IoBuffer&& Payload) {
 		if (!Payload)
 		{
@@ -860,25 +887,65 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 		++ChunkCount;
 		ChunkBytes += Payload.GetSize();
 
+		IoBuffer InMemoryBuffer = IoBufferBuilder::ReadFromFileMaybe(Payload);
+
 		IoHash	 RawHash;
 		uint64_t RawSize;
-		if (CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize))
+		if (CompressedBuffer::ValidateCompressedHeader(Payload, /* out */ RawHash, /* out */ RawSize))
 		{
-			if (RawHash != Hash)
+			if (RawHash == Hash)
 			{
-				// Hash mismatch
-				BadHashes.push_back(Hash);
-				return;
+				// Header hash matches the file name, full validation requires that
+				// we check that the decompressed data hash also matches
+
+				CompressedBuffer CompBuffer = CompressedBuffer::FromCompressedNoValidate(std::move(InMemoryBuffer));
+
+				OodleCompressor		  Compressor;
+				OodleCompressionLevel CompressionLevel;
+				uint64_t			  BlockSize;
+				if (CompBuffer.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize))
+				{
+					if (BlockSize == 0)
+					{
+						BlockSize = 256 * 1024;
+					}
+					else if (BlockSize < (1024 * 1024))
+					{
+						BlockSize = BlockSize * (1024 * 1024 / BlockSize);
+					}
+
+					std::unique_ptr<uint8_t[]> DecompressionBuffer(new uint8_t[BlockSize]);
+
+					IoHashStream Hasher;
+
+					uint64_t RawOffset = 0;
+					while (RawSize)
+					{
+						const uint64_t DecompressedBlockSize = Min(BlockSize, RawSize);
+
+						bool Ok = CompBuffer.TryDecompressTo(MutableMemoryView((void*)DecompressionBuffer.get(), DecompressedBlockSize),
+															 RawOffset);
+
+						if (Ok)
+						{
+							Hasher.Append(DecompressionBuffer.get(), DecompressedBlockSize);
+						}
+
+						RawSize -= DecompressedBlockSize;
+						RawOffset += DecompressedBlockSize;
+					}
+
+					const IoHash FinalHash = Hasher.GetHash();
+
+					if (FinalHash == Hash)
+					{
+						// all good
+						return;
+					}
+				}
 			}
-			return;
 		}
-#if ZEN_WITH_TESTS
-		IoHash ComputedHash = IoHash::HashBuffer(CompositeBuffer(SharedBuffer(std::move(Payload))));
-		if (ComputedHash == Hash)
-		{
-			return;
-		}
-#endif
+
 		BadHashes.push_back(Hash);
 	});
 
@@ -886,7 +953,7 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 
 	if (!BadHashes.empty())
 	{
-		ZEN_WARN("file CAS scrubbing: {} bad chunks found", BadHashes.size());
+		ZEN_WARN("file CAS scrubbing: {} bad chunks found @ '{}'", BadHashes.size(), m_RootDirectory);
 
 		if (Ctx.RunRecovery())
 		{
@@ -899,10 +966,14 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 
 				if (Ec)
 				{
-					ZEN_WARN("failed to delete file for chunk {}", Hash);
+					ZEN_WARN("failed to delete file for chunk {}: {}", Hash, Ec.message());
 				}
 			}
 		}
+		else
+		{
+			ZEN_WARN("recovery: NOT deleting backing files for {} bad chunks", BadHashes.size());
+		}
 	}
 
 	// Let whomever it concerns know about the bad chunks. This could
@@ -910,7 +981,11 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx)
 	// than a full validation pass might be able to do
 	Ctx.ReportBadCidChunks(BadHashes);
 
-	ZEN_INFO("file CAS scrubbed: {} chunks ({})", ChunkCount, NiceBytes(ChunkBytes));
+	ZEN_INFO("file CAS @ '{}' scrubbed: {} chunks ({}), took {}",
+			 m_RootDirectory,
+			 ChunkCount,
+			 NiceBytes(ChunkBytes),
+			 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 }
 
 void
@@ -1088,8 +1163,11 @@ FileCasStrategy::MakeIndexSnapshot()
 
 		// Write the current state of the location map to a new index state
 		std::vector<FileCasIndexEntry> Entries;
+		uint64_t					   IndexLogPosition = 0;
 
 		{
+			RwLock::SharedLockScope __(m_Lock);
+			IndexLogPosition = m_CasLog.GetLogCount();
 			Entries.resize(m_Index.size());
 
 			uint64_t EntryIndex = 0;
@@ -1103,7 +1181,7 @@ FileCasStrategy::MakeIndexSnapshot()
 
 		BasicFile ObjectIndexFile;
 		ObjectIndexFile.Open(IndexPath, BasicFile::Mode::kTruncate);
-		filecas::impl::FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = LogCount};
+		filecas::impl::FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = IndexLogPosition};
 
 		Header.Checksum = filecas::impl::FileCasIndexHeader::ComputeChecksum(Header);
 
@@ -1112,7 +1190,7 @@ FileCasStrategy::MakeIndexSnapshot()
 		ObjectIndexFile.Flush();
 		ObjectIndexFile.Close();
 		EntryCount		   = Entries.size();
-		m_LogFlushPosition = LogCount;
+		m_LogFlushPosition = IndexLogPosition;
 	}
 	catch (std::exception& Err)
 	{
@@ -1331,35 +1409,34 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir)
 	return Entries;
 };
 
-class FileCasStoreCompactor : public GcReferenceStoreCompactor
+class FileCasStoreCompactor : public GcStoreCompactor
 {
 public:
 	FileCasStoreCompactor(FileCasStrategy& Owner, std::vector<IoHash>&& ReferencesToClean)
 	: m_FileCasStrategy(Owner)
 	, m_ReferencesToClean(std::move(ReferencesToClean))
 	{
+		m_ReferencesToClean.shrink_to_fit();
 	}
 
-	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&)
 	{
-		Stopwatch			Timer;
-		const auto			_ = MakeGuard([&] {
-			 if (!Ctx.Settings.Verbose)
-			 {
-				 return;
-			 }
-			 ZEN_INFO("GCV2: filecas [COMPACT] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
-					  m_FileCasStrategy.m_RootDirectory,
-					  Stats.Count,
-					  Stats.Pruned,
-					  Stats.Compacted,
-					  NiceBytes(Stats.RemovedDisk),
-					  NiceBytes(Stats.RemovedMemory),
-					  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
-		 });
-		std::vector<IoHash> ReferencedCleaned;
-		ReferencedCleaned.reserve(m_ReferencesToClean.size());
+		ZEN_TRACE_CPU("FileCas::CompactStore");
 
+		Stopwatch  Timer;
+		const auto _ = MakeGuard([&] {
+			Reset(m_ReferencesToClean);
+			if (!Ctx.Settings.Verbose)
+			{
+				return;
+			}
+			ZEN_INFO("GCV2: filecas [COMPACT] '{}': RemovedDisk: {} in {}",
+					 m_FileCasStrategy.m_RootDirectory,
+					 NiceBytes(Stats.RemovedDisk),
+					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		});
+
+		size_t Skipped = 0;
 		for (const IoHash& ChunkHash : m_ReferencesToClean)
 		{
 			FileCasStrategy::ShardingHelper Name(m_FileCasStrategy.m_RootDirectory.c_str(), ChunkHash);
@@ -1370,9 +1447,16 @@ public:
 					// Not regarded as pruned, leave it be
 					continue;
 				}
+				if (Ctx.IsCancelledFlag.load())
+				{
+					return;
+				}
+
 				if (Ctx.Settings.IsDeleteMode)
 				{
-					ZEN_DEBUG("deleting CAS payload file '{}'", Name.ShardedPath.ToUtf8());
+					ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Deleting CAS payload file '{}'",
+							  m_FileCasStrategy.m_RootDirectory,
+							  Name.ShardedPath.ToUtf8());
 					std::error_code Ec;
 					uint64_t		SizeOnDisk = std::filesystem::file_size(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
@@ -1382,7 +1466,10 @@ public:
 					bool Existed = std::filesystem::remove(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
 					{
-						ZEN_WARN("failed deleting CAS payload file '{}'. Reason '{}'", Name.ShardedPath.ToUtf8(), Ec.message());
+						ZEN_WARN("GCV2: filecas [COMPACT] '{}': Failed deleting CAS payload file '{}'. Reason '{}'",
+								 m_FileCasStrategy.m_RootDirectory,
+								 Name.ShardedPath.ToUtf8(),
+								 Ec.message());
 						continue;
 					}
 					if (!Existed)
@@ -1397,18 +1484,27 @@ public:
 					bool			Existed = std::filesystem::is_regular_file(Name.ShardedPath.c_str(), Ec);
 					if (Ec)
 					{
-						ZEN_WARN("failed checking CAS payload file '{}'. Reason '{}'", Name.ShardedPath.ToUtf8(), Ec.message());
+						ZEN_WARN("GCV2: filecas [COMPACT] '{}': Failed checking CAS payload file '{}'. Reason '{}'",
+								 m_FileCasStrategy.m_RootDirectory,
+								 Name.ShardedPath.ToUtf8(),
+								 Ec.message());
 						continue;
 					}
 					if (!Existed)
 					{
 						continue;
 					}
+					Skipped++;
 				}
-				ReferencedCleaned.push_back(ChunkHash);
 			}
 		}
-		Stats.Compacted += ReferencedCleaned.size();
+
+		if (Skipped > 0)
+		{
+			ZEN_DEBUG("GCV2: filecas [COMPACT] '{}': Skipped deleting of {} eligible files", m_FileCasStrategy.m_RootDirectory, Skipped);
+		}
+
+		Reset(m_ReferencesToClean);
 	}
 
 private:
@@ -1421,33 +1517,39 @@ class FileCasReferencePruner : public GcReferencePruner
 public:
 	FileCasReferencePruner(FileCasStrategy& Owner, std::vector<IoHash>&& Cids) : m_FileCasStrategy(Owner), m_Cids(std::move(Cids)) {}
 
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
-															  GcReferenceStoreStats&		 Stats,
-															  const GetUnusedReferencesFunc& GetUnusedReferences)
+	virtual GcStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, GcStats& Stats, const GetUnusedReferencesFunc& GetUnusedReferences)
 	{
+		ZEN_TRACE_CPU("FileCas::RemoveUnreferencedData");
+
 		Stopwatch  Timer;
 		const auto _ = MakeGuard([&] {
 			if (!Ctx.Settings.Verbose)
 			{
 				return;
 			}
-			ZEN_INFO("GCV2: filecas [PRUNE] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
+			ZEN_INFO("GCV2: filecas [PRUNE] '{}': Count: {}, Unreferenced: {}, FreedMemory: {} in {}",
 					 m_FileCasStrategy.m_RootDirectory,
-					 Stats.Count,
-					 Stats.Pruned,
-					 Stats.Compacted,
-					 NiceBytes(Stats.RemovedDisk),
-					 NiceBytes(Stats.RemovedMemory),
+					 Stats.CheckedCount,
+					 Stats.FoundCount,
+					 NiceBytes(Stats.FreedMemory),
 					 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 		});
 
 		std::vector<IoHash> UnusedCids = GetUnusedReferences(m_Cids);
+		Stats.CheckedCount			   = m_Cids.size();
 		if (UnusedCids.empty())
 		{
 			// Nothing to collect
 			return nullptr;
 		}
 
+		Stats.FoundCount += UnusedCids.size();
+
+		if (!Ctx.Settings.IsDeleteMode)
+		{
+			return nullptr;
+		}
+
 		std::vector<IoHash> PrunedReferences;
 		PrunedReferences.reserve(UnusedCids.size());
 		{
@@ -1459,19 +1561,21 @@ public:
 				{
 					continue;
 				}
-				if (Ctx.Settings.IsDeleteMode)
-				{
-					uint64_t FileSize = It->second.Size;
-					m_FileCasStrategy.m_Index.erase(It);
-					m_FileCasStrategy.m_CasLog.Append(
-						{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
-					m_FileCasStrategy.m_TotalSize.fetch_sub(It->second.Size, std::memory_order_relaxed);
-				}
+				uint64_t FileSize = It->second.Size;
+				m_FileCasStrategy.m_Index.erase(It);
+				m_FileCasStrategy.m_CasLog.Append(
+					{.Key = ChunkHash, .Flags = FileCasStrategy::FileCasIndexEntry::kTombStone, .Size = FileSize});
+				m_FileCasStrategy.m_TotalSize.fetch_sub(It->second.Size, std::memory_order_relaxed);
 				PrunedReferences.push_back(ChunkHash);
+				Stats.DeletedCount++;
 			}
 		}
 
-		Stats.Pruned += PrunedReferences.size();
+		if (PrunedReferences.empty())
+		{
+			return nullptr;
+		}
+
 		return new FileCasStoreCompactor(m_FileCasStrategy, std::move(PrunedReferences));
 	}
 
@@ -1487,22 +1591,17 @@ FileCasStrategy::GetGcName(GcCtx&)
 }
 
 GcReferencePruner*
-FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
+FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&)
 {
+	ZEN_TRACE_CPU("FileCas::CreateReferencePruner");
+
 	Stopwatch			Timer;
 	const auto			_ = MakeGuard([&] {
 		 if (!Ctx.Settings.Verbose)
 		 {
 			 return;
 		 }
-		 ZEN_INFO("GCV2: filecas [CREATE PRUNERS] '{}': Count: {}, Pruned: {}, Compacted: {}, RemovedDisk: {}, RemovedMemory: {} in {}",
-				  m_RootDirectory,
-				  Stats.Count,
-				  Stats.Pruned,
-				  Stats.Compacted,
-				  NiceBytes(Stats.RemovedDisk),
-				  NiceBytes(Stats.RemovedMemory),
-				  NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		 ZEN_INFO("GCV2: filecas [CREATE PRUNER] '{}' in {}", m_RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	 });
 	std::vector<IoHash> CidsToCheck;
 	{
@@ -1511,13 +1610,16 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats)
 		{
 			return {};
 		}
+		if (Ctx.IsCancelledFlag.load())
+		{
+			return nullptr;
+		}
 		CidsToCheck.reserve(m_Index.size());
 		for (const auto& It : m_Index)
 		{
 			CidsToCheck.push_back(It.first);
 		}
 	}
-	Stats.Count += CidsToCheck.size();
 	return new FileCasReferencePruner(*this, std::move(CidsToCheck));
 }
 
diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h
index cb1347580..70cd4ef5a 100644
--- a/src/zenstore/filecas.h
+++ b/src/zenstore/filecas.h
@@ -16,6 +16,10 @@
 #include <atomic>
 #include <functional>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
 namespace zen {
 
 class BasicFile;
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index 778a47626..de653b0e3 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -18,6 +18,7 @@
 #include <zencore/workthreadpool.h>
 #include <zenstore/cidstore.h>
 #include <zenstore/scrubcontext.h>
+#include <zenutil/workerpools.h>
 
 #include "cas.h"
 
@@ -173,166 +174,6 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 
 //////////////////////////////////////////////////////////////////////////
 
-void
-WriteReferencerStats(CbObjectWriter& Writer, const GcReferencerStats& Stats, bool HumanReadable)
-{
-	if (Stats.Count == 0)
-	{
-		return;
-	}
-	Writer << "Count" << Stats.Count;
-	Writer << "Expired" << Stats.Expired;
-	Writer << "Deleted" << Stats.Deleted;
-
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
-		Writer << "RemovedMemoryBytes" << Stats.RemovedMemory;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "RemoveExpiredData" << NiceTimeSpanMs(Stats.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckers" << NiceTimeSpanMs(Stats.CreateReferenceCheckersMS.count());
-		Writer << "LockState" << NiceTimeSpanMs(Stats.LockStateMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "RemoveExpiredDataMS" << gsl::narrow<int64_t>(Stats.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckersMS" << gsl::narrow<int64_t>(Stats.CreateReferenceCheckersMS.count());
-		Writer << "LockStateMS" << gsl::narrow<int64_t>(Stats.LockStateMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Stats.ElapsedMS.count());
-	}
-};
-
-void
-WriteReferenceStoreStats(CbObjectWriter& Writer, const GcReferenceStoreStats& Stats, bool HumanReadable)
-{
-	if (Stats.Count == 0)
-	{
-		return;
-	}
-	Writer << "Count" << Stats.Count;
-	Writer << "Pruned" << Stats.Pruned;
-	Writer << "Compacted" << Stats.Compacted;
-
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Stats.RemovedMemory);
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
-		Writer << "RemovedMemoryBytes" << Stats.RemovedMemory;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "CreateReferencePruner" << NiceTimeSpanMs(Stats.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedData" << NiceTimeSpanMs(Stats.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStore" << NiceTimeSpanMs(Stats.CompactReferenceStoreMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Stats.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "CreateReferencePrunerMS" << gsl::narrow<int64_t>(Stats.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedDataMS" << gsl::narrow<int64_t>(Stats.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStoreMS" << gsl::narrow<int64_t>(Stats.CompactReferenceStoreMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Stats.ElapsedMS.count());
-	}
-};
-
-void
-WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable, bool IncludeDetails)
-{
-	if (HumanReadable)
-	{
-		Writer << "RemovedDisk" << NiceBytes(Result.RemovedDisk);
-		Writer << "RemovedMemory" << NiceBytes(Result.RemovedMemory);
-		Writer << "WriteBlock" << NiceTimeSpanMs(Result.WriteBlockMS.count());
-		Writer << "Elapsed" << NiceTimeSpanMs(Result.ElapsedMS.count());
-	}
-	else
-	{
-		Writer << "RemovedDiskBytes" << gsl::narrow<int64_t>(Result.RemovedDisk);
-		Writer << "RemovedMemoryBytes" << gsl::narrow<int64_t>(Result.RemovedMemory);
-		Writer << "WriteBlockMS" << gsl::narrow<int64_t>(Result.WriteBlockMS.count());
-		Writer << "ElapsedMS" << gsl::narrow<int64_t>(Result.ElapsedMS.count());
-	}
-
-	if (!IncludeDetails)
-	{
-		return;
-	}
-
-	if (HumanReadable)
-	{
-		Writer << "RemoveExpiredData" << NiceTimeSpanMs(Result.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckers" << NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count());
-		Writer << "LockState" << NiceTimeSpanMs(Result.LockStateMS.count());
-
-		Writer << "CreateReferencePruner" << NiceTimeSpanMs(Result.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedData" << NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStore" << NiceTimeSpanMs(Result.CompactReferenceStoreMS.count());
-	}
-	else
-	{
-		Writer << "RemoveExpiredDataMS" << gsl::narrow<int64_t>(Result.RemoveExpiredDataMS.count());
-		Writer << "CreateReferenceCheckersMS" << gsl::narrow<int64_t>(Result.CreateReferenceCheckersMS.count());
-		Writer << "LockStateMS" << gsl::narrow<int64_t>(Result.LockStateMS.count());
-
-		Writer << "CreateReferencePrunerMS" << gsl::narrow<int64_t>(Result.CreateReferencePrunerMS.count());
-		Writer << "RemoveUnreferencedDataMS" << gsl::narrow<int64_t>(Result.RemoveUnreferencedDataMS.count());
-		Writer << "CompactReferenceStoreMS" << gsl::narrow<int64_t>(Result.CompactReferenceStoreMS.count());
-	}
-
-	Writer.BeginObject("ReferencerStats");
-	{
-		WriteReferencerStats(Writer, Result.ReferencerStat, HumanReadable);
-	}
-	Writer.EndObject();
-
-	Writer.BeginObject("ReferenceStoreStats");
-	{
-		WriteReferenceStoreStats(Writer, Result.ReferenceStoreStat, HumanReadable);
-	}
-	Writer.EndObject();
-
-	if (!Result.ReferencerStats.empty())
-	{
-		Writer.BeginArray("Referencers");
-		{
-			for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
-			{
-				Writer.BeginObject();
-				Writer << "Name" << It.first;
-				WriteReferencerStats(Writer, It.second, HumanReadable);
-				Writer.EndObject();
-			}
-		}
-		Writer.EndArray();
-	}
-	if (!Result.ReferenceStoreStats.empty())
-	{
-		Writer.BeginArray("ReferenceStores");
-		for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
-		{
-			Writer.BeginObject();
-			Writer << "Name" << It.first;
-			WriteReferenceStoreStats(Writer, It.second, HumanReadable);
-			Writer.EndObject();
-		}
-		Writer.EndArray();
-	}
-};
-
 struct GcContext::GcState
 {
 	using CacheKeyContexts = std::unordered_map<std::string, std::vector<IoHash>>;
@@ -490,44 +331,243 @@ GcManager::~GcManager()
 //////// Begin GC V2
 
 void
-GcResult::Sum()
+WriteGcStats(CbObjectWriter& Writer, const GcStats& Stats, bool HumanReadable)
 {
-	for (std::pair<std::string, GcReferencerStats>& Referencer : ReferencerStats)
+	Writer << "Checked" << Stats.CheckedCount;
+	Writer << "Found" << Stats.FoundCount;
+	Writer << "Deleted" << Stats.DeletedCount;
+	if (HumanReadable)
 	{
-		GcReferencerStats& SubStat = Referencer.second;
-		ReferencerStat.Count += SubStat.Count;
-		ReferencerStat.Expired += SubStat.Expired;
-		ReferencerStat.Deleted += SubStat.Deleted;
-		ReferencerStat.RemovedDisk += SubStat.RemovedDisk;
-		ReferencerStat.RemovedMemory += SubStat.RemovedMemory;
-		SubStat.ElapsedMS = SubStat.RemoveExpiredDataMS + SubStat.CreateReferenceCheckersMS + SubStat.LockStateMS;
+		Writer << "FreedMemory" << NiceBytes(Stats.FreedMemory);
+	}
+	else
+	{
+		Writer << "FreedMemoryBytes" << Stats.FreedMemory;
+	}
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+}
 
-		ReferencerStat.RemoveExpiredDataMS += SubStat.RemoveExpiredDataMS;
-		ReferencerStat.CreateReferenceCheckersMS += SubStat.CreateReferenceCheckersMS;
-		ReferencerStat.LockStateMS += SubStat.LockStateMS;
-		ReferencerStat.ElapsedMS += SubStat.ElapsedMS;
+void
+WriteCompactStoreStats(CbObjectWriter& Writer, const GcCompactStoreStats& Stats, bool HumanReadable)
+{
+	if (HumanReadable)
+	{
+		Writer << "RemovedDisk" << NiceBytes(Stats.RemovedDisk);
+	}
+	else
+	{
+		Writer << "RemovedDiskBytes" << Stats.RemovedDisk;
+	}
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+}
 
-		RemovedDisk += SubStat.RemovedDisk;
-		RemovedMemory += SubStat.RemovedMemory;
+void
+WriteReferencerStats(CbObjectWriter& Writer, const GcReferencerStats& Stats, bool HumanReadable)
+{
+	if (Stats.RemoveExpiredDataStats.CheckedCount == 0)
+	{
+		return;
 	}
-	for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : ReferenceStoreStats)
+	Writer.BeginObject("RemoveExpired");
 	{
-		GcReferenceStoreStats& SubStat = ReferenceStore.second;
-		ReferenceStoreStat.Count += SubStat.Count;
-		ReferenceStoreStat.Pruned += SubStat.Pruned;
-		ReferenceStoreStat.Compacted += SubStat.Compacted;
-		ReferenceStoreStat.RemovedDisk += SubStat.RemovedDisk;
-		ReferenceStoreStat.RemovedMemory += SubStat.RemovedMemory;
-		SubStat.ElapsedMS = SubStat.CreateReferencePrunerMS + SubStat.RemoveUnreferencedDataMS + SubStat.CompactReferenceStoreMS;
+		WriteGcStats(Writer, Stats.RemoveExpiredDataStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Stats.CompactStoreStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "CreateReferenceCheckers" << ToTimeSpan(Stats.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Stats.PreCacheStateMS);
+	Writer << "LockState" << ToTimeSpan(Stats.LockStateMS);
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+};
+
+void
+WriteReferenceStoreStats(CbObjectWriter& Writer, const GcReferenceStoreStats& Stats, bool HumanReadable)
+{
+	if (Stats.RemoveUnreferencedDataStats.CheckedCount == 0)
+	{
+		return;
+	}
+	Writer.BeginObject("RemoveUnreferenced");
+	{
+		WriteGcStats(Writer, Stats.RemoveUnreferencedDataStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Stats.CompactStoreStats, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "CreateReferencePruners" << ToTimeSpan(Stats.CreateReferencePrunersMS);
+	Writer << "Elapsed" << ToTimeSpan(Stats.ElapsedMS);
+};
+
+void
+WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanReadable, bool IncludeDetails)
+{
+	if (!IncludeDetails)
+	{
+		if (HumanReadable)
+		{
+			Writer << "RemovedDisk" << NiceBytes(Result.CompactStoresStatSum.RemovedDisk);
+			Writer << "FreedMemory" << NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
+		}
+		else
+		{
+			Writer << "RemovedDiskBytes" << gsl::narrow<int64_t>(Result.CompactStoresStatSum.RemovedDisk);
+			Writer << "RemovedMemoryBytes" << gsl::narrow<int64_t>(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory);
+		}
+		Writer << "WriteBlock" << ToTimeSpan(Result.WriteBlockMS);
+		Writer << "Elapsed" << ToTimeSpan(Result.ElapsedMS);
+		Writer << "Cancelled" << Result.WasCancelled;
+		return;
+	}
+
+	Writer.BeginObject("Referencer");
+	{
+		WriteReferencerStats(Writer, Result.ReferencerStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("ReferenceStore");
+	{
+		WriteReferenceStoreStats(Writer, Result.ReferenceStoreStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer.BeginObject("Compact");
+	{
+		WriteCompactStoreStats(Writer, Result.CompactStoresStatSum, HumanReadable);
+	}
+	Writer.EndObject();
+
+	Writer << "RemoveExpiredData" << ToTimeSpan(Result.RemoveExpiredDataMS);
+	Writer << "CreateReferenceCheckers" << ToTimeSpan(Result.CreateReferenceCheckersMS);
+	Writer << "PreCacheState" << ToTimeSpan(Result.PreCacheStateMS);
+	Writer << "LockState" << ToTimeSpan(Result.LockStateMS);
+
+	Writer << "CreateReferencePruners" << ToTimeSpan(Result.CreateReferencePrunersMS);
+	Writer << "RemoveUnreferencedData" << ToTimeSpan(Result.RemoveUnreferencedDataMS);
+	Writer << "CompactStores" << ToTimeSpan(Result.CompactStoresMS);
+	Writer << "WriteBlock" << ToTimeSpan(Result.WriteBlockMS);
+	Writer << "Elapsed" << ToTimeSpan(Result.ElapsedMS);
+
+	if (!Result.ReferencerStats.empty())
+	{
+		Writer.BeginArray("Referencers");
+		{
+			for (const std::pair<std::string, GcReferencerStats>& It : Result.ReferencerStats)
+			{
+				Writer.BeginObject();
+				Writer << "Name" << It.first;
+				WriteReferencerStats(Writer, It.second, HumanReadable);
+				Writer.EndObject();
+			}
+		}
+		Writer.EndArray();
+	}
+	if (!Result.ReferenceStoreStats.empty())
+	{
+		Writer.BeginArray("ReferenceStores");
+		for (const std::pair<std::string, GcReferenceStoreStats>& It : Result.ReferenceStoreStats)
+		{
+			Writer.BeginObject();
+			Writer << "Name" << It.first;
+			WriteReferenceStoreStats(Writer, It.second, HumanReadable);
+			Writer.EndObject();
+		}
+		Writer.EndArray();
+	}
+};
+
+void
+Add(GcCompactStoreStats& Sum, const GcCompactStoreStats& Sub)
+{
+	Sum.RemovedDisk += Sub.RemovedDisk;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
 
-		ReferenceStoreStat.CreateReferencePrunerMS += SubStat.CreateReferencePrunerMS;
-		ReferenceStoreStat.RemoveUnreferencedDataMS += SubStat.RemoveUnreferencedDataMS;
-		ReferenceStoreStat.CompactReferenceStoreMS += SubStat.CompactReferenceStoreMS;
-		ReferenceStoreStat.ElapsedMS += SubStat.ElapsedMS;
+void
+Add(GcStats& Sum, const GcStats& Sub)
+{
+	Sum.CheckedCount += Sub.CheckedCount;
+	Sum.FoundCount += Sub.FoundCount;
+	Sum.DeletedCount += Sub.DeletedCount;
+	Sum.FreedMemory += Sub.FreedMemory;
 
-		RemovedDisk += SubStat.RemovedDisk;
-		RemovedMemory += SubStat.RemovedMemory;
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+void
+Sum(GcReferencerStats& Stat)
+{
+	Stat.ElapsedMS = Stat.RemoveExpiredDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferenceCheckersMS +
+					 Stat.PreCacheStateMS + Stat.LockStateMS;
+}
+
+void
+Add(GcReferencerStats& Sum, const GcReferencerStats& Sub)
+{
+	Add(Sum.RemoveExpiredDataStats, Sub.RemoveExpiredDataStats);
+	Add(Sum.CompactStoreStats, Sub.CompactStoreStats);
+
+	Sum.CreateReferenceCheckersMS += Sub.CreateReferenceCheckersMS;
+	Sum.PreCacheStateMS += Sub.PreCacheStateMS;
+	Sum.LockStateMS += Sub.LockStateMS;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+void
+Sum(GcReferenceStoreStats& Stat)
+{
+	Stat.ElapsedMS = Stat.RemoveUnreferencedDataStats.ElapsedMS + Stat.CompactStoreStats.ElapsedMS + Stat.CreateReferencePrunersMS;
+}
+
+void
+Add(GcReferenceStoreStats& Sum, const GcReferenceStoreStats& Sub)
+{
+	Add(Sum.RemoveUnreferencedDataStats, Sub.RemoveUnreferencedDataStats);
+	Add(Sum.CompactStoreStats, Sub.CompactStoreStats);
+
+	Sum.CreateReferencePrunersMS += Sub.CreateReferencePrunersMS;
+
+	Sum.ElapsedMS += Sub.ElapsedMS;
+}
+
+GcResult&
+Sum(GcResult& Stat, bool Cancelled = false)
+{
+	for (std::pair<std::string, GcReferencerStats>& Referencer : Stat.ReferencerStats)
+	{
+		GcReferencerStats& SubStat = Referencer.second;
+		Sum(SubStat);
+		Add(Stat.ReferencerStatSum, SubStat);
 	}
+	for (std::pair<std::string, GcReferenceStoreStats>& ReferenceStore : Stat.ReferenceStoreStats)
+	{
+		GcReferenceStoreStats& SubStat = ReferenceStore.second;
+		Sum(SubStat);
+		Add(Stat.ReferenceStoreStatSum, SubStat);
+	}
+
+	Sum(Stat.ReferencerStatSum);
+	Sum(Stat.ReferenceStoreStatSum);
+
+	Add(Stat.CompactStoresStatSum, Stat.ReferencerStatSum.CompactStoreStats);
+	Add(Stat.CompactStoresStatSum, Stat.ReferenceStoreStatSum.CompactStoreStats);
+
+	Stat.WasCancelled = Cancelled;
+
+	return Stat;
 }
 
 void
@@ -563,7 +603,9 @@ GcManager::RemoveGcReferenceStore(GcReferenceStore& ReferenceStore)
 GcResult
 GcManager::CollectGarbage(const GcSettings& Settings)
 {
-	GcCtx	 Ctx{.Settings = Settings};
+	ZEN_TRACE_CPU("GcV2::CollectGarbage");
+
+	GcCtx	 Ctx{.Settings = Settings, .IsCancelledFlag = m_CancelGC};
 	GcResult Result;
 
 	{
@@ -572,256 +614,417 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 
 		RwLock::SharedLockScope GcLock(m_Lock);
 
-		int WorkerThreadPoolCount = 0;
-		if (!Settings.SingleThread)
-		{
-			const size_t MaxHwTreadUse = Max((std::thread::hardware_concurrency() / 4u), 1u);
-			WorkerThreadPoolCount	   = gsl::narrow<int>(Min(MaxHwTreadUse, m_GcReferencers.size()));
-		}
-
 		Result.ReferencerStats.resize(m_GcReferencers.size());
 
-		WorkerThreadPool ThreadPool(WorkerThreadPoolCount);
+		std::unordered_map<std::unique_ptr<GcStoreCompactor>, GcCompactStoreStats*> StoreCompactors;
+		RwLock																		StoreCompactorsLock;
+		WorkerThreadPool& ThreadPool = Settings.SingleThread ? GetSyncWorkerPool() : GetSmallWorkerPool();
 
 		ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size());
 		if (!m_GcReferencers.empty())
 		{
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+			ZEN_TRACE_CPU("GcV2::RemoveExpiredData");
+
 			Latch WorkLeft(1);
-			// First remove any cache keys that may own references
-			SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 			{
-				GcReferencer*							   Owner = m_GcReferencers[Index];
-				std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, Owner, &Stats, &WorkLeft]() {
-					auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					Stats.first = Owner->GetGcName(Ctx);
-					SCOPED_TIMER(Stats.second.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-					Owner->RemoveExpiredData(Ctx, Stats.second);
+				// First remove any cache keys that may own references
+				SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
+					ZEN_INFO("GCV2: Removed epxired data for {} referenceners in {}",
+							 m_GcReferencers.size(),
+							 NiceTimeSpanMs(Result.RemoveExpiredDataMS.count()));
 				});
+				for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+				{
+					if (CheckGCCancel())
+					{
+						WorkLeft.CountDown();
+						WorkLeft.Wait();
+						return Sum(Result, true);
+					}
+					GcReferencer*							   Owner = m_GcReferencers[Index];
+					std::pair<std::string, GcReferencerStats>& Stats = Result.ReferencerStats[Index];
+					WorkLeft.AddCount(1);
+					ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Owner, &Stats, &StoreCompactorsLock, &StoreCompactors]() {
+						auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+						Stats.first = Owner->GetGcName(Ctx);
+						SCOPED_TIMER(Stats.second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						std::unique_ptr<GcStoreCompactor> StoreCompactor(
+							Owner->RemoveExpiredData(Ctx, Stats.second.RemoveExpiredDataStats));
+						if (StoreCompactor)
+						{
+							RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+							StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.second.CompactStoreStats);
+						}
+					});
+				}
+				WorkLeft.CountDown();
+				WorkLeft.Wait();
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
 		}
 
-		if (Ctx.Settings.SkipCidDelete)
+		if (!Ctx.Settings.SkipCidDelete)
 		{
-			Result.Sum();
-			return Result;
-		}
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
 
-		Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
+			Result.ReferenceStoreStats.resize(m_GcReferenceStores.size());
 
-		ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
-		std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
-		if (!m_GcReferenceStores.empty())
-		{
-			ReferencePruners.reserve(m_GcReferenceStores.size());
-			Latch  WorkLeft(1);
-			RwLock ReferencePrunersLock;
-			// CreateReferencePruner is usually not very heavy but big data sets change that
-			SCOPED_TIMER(Result.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
+			ZEN_INFO("GCV2: Creating reference pruners from {} reference stores", m_GcReferenceStores.size());
+			std::unordered_map<size_t, std::unique_ptr<GcReferencePruner>> ReferencePruners;
+			if (!m_GcReferenceStores.empty())
 			{
-				GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
-				std::pair<std::string, GcReferenceStoreStats>& Stats		  = Result.ReferenceStoreStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
-					auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					Stats.first = ReferenceStore->GetGcName(Ctx);
-					std::unique_ptr<GcReferencePruner> ReferencePruner;
-					{
-						SCOPED_TIMER(Stats.second.CreateReferencePrunerMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						// The ReferenceStore will pick a list of CId entries to check, returning a collector
-						ReferencePruner = std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
-					}
-					if (ReferencePruner)
+				ZEN_TRACE_CPU("GcV2::CreateReferencePruners");
+
+				ReferencePruners.reserve(m_GcReferenceStores.size());
+				Latch  WorkLeft(1);
+				RwLock ReferencePrunersLock;
+				{
+					// CreateReferencePruner is usually not very heavy but big data sets change that
+					SCOPED_TIMER(Result.CreateReferencePrunersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+								 if (Ctx.Settings.Verbose) {
+									 ZEN_INFO("GCV2: Created {} reference pruners using {} referencer stores in {}",
+											  ReferencePruners.size(),
+											  m_GcReferenceStores.size(),
+											  NiceTimeSpanMs(Result.CreateReferencePrunersMS.count()));
+								 });
+					for (size_t Index = 0; Index < m_GcReferenceStores.size(); Index++)
 					{
-						RwLock::ExclusiveLockScope __(ReferencePrunersLock);
-						ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+						if (CheckGCCancel())
+						{
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+							return Sum(Result, true);
+						}
+
+						GcReferenceStore*							   ReferenceStore = m_GcReferenceStores[Index];
+						std::pair<std::string, GcReferenceStoreStats>& Stats		  = Result.ReferenceStoreStats[Index];
+						WorkLeft.AddCount(1);
+						ThreadPool.ScheduleWork(
+							[&Ctx, ReferenceStore, &Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() {
+								auto _		= MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+								Stats.first = ReferenceStore->GetGcName(Ctx);
+								std::unique_ptr<GcReferencePruner> ReferencePruner;
+								{
+									SCOPED_TIMER(Stats.second.CreateReferencePrunersMS =
+													 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									// The ReferenceStore will pick a list of CId entries to check, returning a collector
+									ReferencePruner =
+										std::unique_ptr<GcReferencePruner>(ReferenceStore->CreateReferencePruner(Ctx, Stats.second));
+								}
+								if (ReferencePruner)
+								{
+									RwLock::ExclusiveLockScope __(ReferencePrunersLock);
+									ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner));
+								}
+							});
 					}
-				});
+					WorkLeft.CountDown();
+					WorkLeft.Wait();
+				}
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
-		}
 
-		ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
-		std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
-		if (!m_GcReferencers.empty())
-		{
-			ReferenceCheckers.reserve(m_GcReferencers.size());
-			Latch  WorkLeft(1);
-			RwLock ReferenceCheckersLock;
-			SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			// Lock all reference owners from changing the reference data and get access to check for referenced data
-			for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
+			if (!ReferencePruners.empty())
 			{
-				GcReferencer*							   Referencer = m_GcReferencers[Index];
-				std::pair<std::string, GcReferencerStats>& Stats	  = Result.ReferencerStats[Index];
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
-					auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					// The Referencer will create a reference checker that guarrantees that the references do not change as long as it lives
-					std::vector<GcReferenceChecker*> Checkers;
-					{
-						SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						Checkers = Referencer->CreateReferenceCheckers(Ctx);
-					}
-					try
+				if (CheckGCCancel())
+				{
+					return Sum(Result, true);
+				}
+
+				ZEN_INFO("GCV2: Creating reference checkers from {} referencers", m_GcReferencers.size());
+				std::unordered_map<std::unique_ptr<GcReferenceChecker>, size_t> ReferenceCheckers;
+				if (!m_GcReferencers.empty())
+				{
+					ZEN_TRACE_CPU("GcV2::CreateReferenceCheckers");
+
+					ReferenceCheckers.reserve(m_GcReferencers.size());
+					Latch  WorkLeft(1);
+					RwLock ReferenceCheckersLock;
 					{
-						if (!Checkers.empty())
+						SCOPED_TIMER(Result.CreateReferenceCheckersMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+									 if (Ctx.Settings.Verbose) {
+										 ZEN_INFO("GCV2: Created {} reference checkers using {} referencers in {}",
+												  ReferenceCheckers.size(),
+												  m_GcReferencers.size(),
+												  NiceTimeSpanMs(Result.CreateReferenceCheckersMS.count()));
+									 });
+						// Lock all reference owners from changing the reference data and get access to check for referenced data
+						for (size_t Index = 0; Index < m_GcReferencers.size(); Index++)
 						{
-							RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
-							for (auto& Checker : Checkers)
+							if (CheckGCCancel())
 							{
-								ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
-								Checker = nullptr;
+								WorkLeft.CountDown();
+								WorkLeft.Wait();
+								return Sum(Result, true);
 							}
+
+							GcReferencer*							   Referencer = m_GcReferencers[Index];
+							std::pair<std::string, GcReferencerStats>& Stats	  = Result.ReferencerStats[Index];
+							WorkLeft.AddCount(1);
+							ThreadPool.ScheduleWork(
+								[&Ctx, &WorkLeft, Referencer, Index, &Stats, &ReferenceCheckersLock, &ReferenceCheckers]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									// The Referencer will create a reference checker that guarrantees that the references do not change as
+									// long as it lives
+									std::vector<GcReferenceChecker*> Checkers;
+									{
+										SCOPED_TIMER(Stats.second.CreateReferenceCheckersMS =
+														 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+										Checkers = Referencer->CreateReferenceCheckers(Ctx);
+									}
+									try
+									{
+										if (!Checkers.empty())
+										{
+											RwLock::ExclusiveLockScope __(ReferenceCheckersLock);
+											for (auto& Checker : Checkers)
+											{
+												ReferenceCheckers.insert_or_assign(std::unique_ptr<GcReferenceChecker>(Checker), Index);
+												Checker = nullptr;
+											}
+										}
+									}
+									catch (std::exception&)
+									{
+										while (!Checkers.empty())
+										{
+											delete Checkers.back();
+											Checkers.pop_back();
+										}
+										throw;
+									}
+								});
 						}
+						WorkLeft.CountDown();
+						WorkLeft.Wait();
 					}
-					catch (std::exception&)
+				}
+
+				{
+					ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size());
+					if (!ReferenceCheckers.empty())
 					{
-						while (!Checkers.empty())
+						if (CheckGCCancel())
 						{
-							delete Checkers.back();
-							Checkers.pop_back();
+							return Sum(Result, true);
 						}
-						throw;
-					}
-				});
-			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
-		}
-
-		std::unordered_map<std::unique_ptr<GcReferenceStoreCompactor>, size_t> ReferenceStoreCompactors;
-		ReferenceStoreCompactors.reserve(ReferencePruners.size());
+						ZEN_TRACE_CPU("GcV2::PreCache");
 
-		ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
-		{
-			SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
-						 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
-			if (!ReferenceCheckers.empty())
-			{
-				// Locking all references checkers so we have a steady state of which references are used
-				// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
-				// we delete the ReferenceCheckers
-				Latch WorkLeft(1);
+						Latch WorkLeft(1);
 
-				SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-				for (auto& It : ReferenceCheckers)
-				{
-					GcReferenceChecker*						   Checker = It.first.get();
-					size_t									   Index   = It.second;
-					std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
-					WorkLeft.AddCount(1);
-					ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-						Checker->LockState(Ctx);
-					});
+						{
+							SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Precached state using {} reference checkers in {}",
+													  ReferenceCheckers.size(),
+													  NiceTimeSpanMs(Result.PreCacheStateMS.count()));
+										 });
+							for (auto& It : ReferenceCheckers)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferenceChecker*						   Checker = It.first.get();
+								size_t									   Index   = It.second;
+								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									SCOPED_TIMER(Stats.second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									Checker->PreCache(Ctx);
+								});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
+						}
+					}
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
-			}
 
-			ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
-			if (!ReferencePruners.empty())
-			{
-				const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
-					HashSet UnusedCids(References.begin(), References.end());
-					for (const auto& It : ReferenceCheckers)
+				SCOPED_TIMER(uint64_t ElapsedMS = Timer.GetElapsedTimeMs(); Result.WriteBlockMS = std::chrono::milliseconds(ElapsedMS);
+							 ZEN_INFO("GCV2: Writes blocked for {}", NiceTimeSpanMs(ElapsedMS)));
+				{
+					ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size());
+					if (!ReferenceCheckers.empty())
 					{
-						GcReferenceChecker* ReferenceChecker = It.first.get();
-						ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
-						if (UnusedCids.empty())
+						if (CheckGCCancel())
 						{
-							return {};
+							return Sum(Result, true);
 						}
-					}
-					return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
-				};
-
-				// checking all Cids agains references in cache
-				// Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight operation
-				// that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+						ZEN_TRACE_CPU("GcV2::LockState");
 
-				Latch  WorkLeft(1);
-				RwLock ReferenceStoreCompactorsLock;
+						// Locking all references checkers so we have a steady state of which references are used
+						// From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until
+						// we delete the ReferenceCheckers
+						Latch WorkLeft(1);
 
-				SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-				for (auto& It : ReferencePruners)
-				{
-					GcReferencePruner*	   Pruner = It.second.get();
-					size_t				   Index  = It.first;
-					GcReferenceStoreStats& Stats  = Result.ReferenceStoreStats[Index].second;
-					WorkLeft.AddCount(1);
-					ThreadPool.ScheduleWork([&Ctx,
-											 Pruner,
-											 &Stats,
-											 &WorkLeft,
-											 Index,
-											 &GetUnusedReferences,
-											 &ReferenceStoreCompactorsLock,
-											 &ReferenceStoreCompactors]() {
-						auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
-						std::unique_ptr<GcReferenceStoreCompactor> ReferenceCompactor;
 						{
-							SCOPED_TIMER(Stats.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-							ReferenceCompactor =
-								std::unique_ptr<GcReferenceStoreCompactor>(Pruner->RemoveUnreferencedData(Ctx, Stats, GetUnusedReferences));
+							SCOPED_TIMER(Result.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Locked state using {} reference checkers in {}",
+													  ReferenceCheckers.size(),
+													  NiceTimeSpanMs(Result.LockStateMS.count()));
+										 });
+							for (auto& It : ReferenceCheckers)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferenceChecker*						   Checker = It.first.get();
+								size_t									   Index   = It.second;
+								std::pair<std::string, GcReferencerStats>& Stats   = Result.ReferencerStats[Index];
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork([&Ctx, Checker, Index, &Stats, &WorkLeft]() {
+									auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+									SCOPED_TIMER(Stats.second.LockStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+									Checker->LockState(Ctx);
+								});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
 						}
-						if (ReferenceCompactor)
+					}
+				}
+				{
+					ZEN_INFO("GCV2: Removing unreferenced data for {} reference pruners", ReferencePruners.size());
+					{
+						const auto GetUnusedReferences = [&ReferenceCheckers, &Ctx](std::span<IoHash> References) -> std::vector<IoHash> {
+							HashSet UnusedCids(References.begin(), References.end());
+							for (const auto& It : ReferenceCheckers)
+							{
+								GcReferenceChecker* ReferenceChecker = It.first.get();
+								ReferenceChecker->RemoveUsedReferencesFromSet(Ctx, UnusedCids);
+								if (UnusedCids.empty())
+								{
+									return {};
+								}
+							}
+							return std::vector<IoHash>(UnusedCids.begin(), UnusedCids.end());
+						};
+
+						// checking all Cids agains references in cache
+						// Ask stores to remove data that the ReferenceCheckers says are not referenced - this should be a lightweight
+						// operation that only updates in-memory index, actual disk changes should be done by the ReferenceStoreCompactors
+
+						ZEN_TRACE_CPU("GcV2::RemoveUnreferencedData");
+
+						Latch WorkLeft(1);
+
 						{
-							RwLock::ExclusiveLockScope __(ReferenceStoreCompactorsLock);
-							ReferenceStoreCompactors.insert_or_assign(std::move(ReferenceCompactor), Index);
+							SCOPED_TIMER(Result.RemoveUnreferencedDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());
+										 if (Ctx.Settings.Verbose) {
+											 ZEN_INFO("GCV2: Removed unused data using {} pruners in {}",
+													  ReferencePruners.size(),
+													  NiceTimeSpanMs(Result.RemoveUnreferencedDataMS.count()));
+										 });
+							for (auto& It : ReferencePruners)
+							{
+								if (CheckGCCancel())
+								{
+									WorkLeft.CountDown();
+									WorkLeft.Wait();
+									return Sum(Result, true);
+								}
+
+								GcReferencePruner*	   Pruner = It.second.get();
+								size_t				   Index  = It.first;
+								GcReferenceStoreStats& Stats  = Result.ReferenceStoreStats[Index].second;
+								WorkLeft.AddCount(1);
+								ThreadPool.ScheduleWork(
+									[&Ctx, Pruner, &Stats, &WorkLeft, &GetUnusedReferences, &StoreCompactorsLock, &StoreCompactors]() {
+										auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
+										// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are
+										// referenced or not.
+										std::unique_ptr<GcStoreCompactor> StoreCompactor;
+										{
+											SCOPED_TIMER(Stats.RemoveUnreferencedDataStats.ElapsedMS =
+															 std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+											StoreCompactor = std::unique_ptr<GcStoreCompactor>(
+												Pruner->RemoveUnreferencedData(Ctx,
+																			   Stats.RemoveUnreferencedDataStats,
+																			   GetUnusedReferences));
+										}
+										if (StoreCompactor)
+										{
+											RwLock::ExclusiveLockScope __(StoreCompactorsLock);
+											StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats.CompactStoreStats);
+										}
+									});
+							}
+							WorkLeft.CountDown();
+							WorkLeft.Wait();
 						}
-					});
+					}
+					// Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
+					ReferenceCheckers.clear();
+					ReferencePruners.clear();
 				}
-				WorkLeft.CountDown();
-				WorkLeft.Wait();
 			}
-			// Let the GcReferencers add new data, we will only change on-disk data at this point, adding new data is allowed
-			ReferenceCheckers.clear();
 		}
 
-		// Let go of the pruners
-		ReferencePruners.clear();
-
-		ZEN_INFO("GCV2: Compacting reference stores for {} reference store compactors", ReferenceStoreCompactors.size());
-		if (!ReferenceStoreCompactors.empty())
+		ZEN_INFO("GCV2: Compacting using {} store compactors", StoreCompactors.size());
+		if (!StoreCompactors.empty())
 		{
-			Latch WorkLeft(1);
+			if (CheckGCCancel())
+			{
+				return Sum(Result, true);
+			}
+
+			ZEN_TRACE_CPU("GcV2::CompactStores");
 
+			auto ClaimDiskReserve = [&]() -> uint64_t {
+				if (!std::filesystem::is_regular_file(Settings.DiskReservePath))
+				{
+					return 0;
+				}
+				uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath);
+				if (std::filesystem::remove(Settings.DiskReservePath))
+				{
+					return ReclaimedSize;
+				}
+				return 0;
+			};
 			// Remove the stuff we deemed unreferenced from disk - may be heavy operation
-			SCOPED_TIMER(Result.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-			for (auto& It : ReferenceStoreCompactors)
+			// Don't do in parallel, we don't want to steal CPU/Disk from regular operation
 			{
-				GcReferenceStoreCompactor* Compactor = It.first.get();
-				size_t					   Index	 = It.second;
-				GcReferenceStoreStats&	   Stats	 = Result.ReferenceStoreStats[Index].second;
-				WorkLeft.AddCount(1);
-				ThreadPool.ScheduleWork([&Ctx, Compactor, &Stats, &WorkLeft]() {
-					auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); });
-					// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
-					SCOPED_TIMER(Stats.CompactReferenceStoreMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
-					Compactor->CompactReferenceStore(Ctx, Stats);
+				SCOPED_TIMER(Result.CompactStoresMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) {
+					ZEN_INFO("GCV2: Compacted {} stores in {}", StoreCompactors.size(), NiceTimeSpanMs(Result.CompactStoresMS.count()));
 				});
+				for (auto& It : StoreCompactors)
+				{
+					if (CheckGCCancel())
+					{
+						return Sum(Result, true);
+					}
+
+					GcStoreCompactor*	 Compactor = It.first.get();
+					GcCompactStoreStats& Stats	   = *It.second;
+					{
+						// Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or not.
+						SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()););
+						Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve);
+					}
+				}
 			}
-			WorkLeft.CountDown();
-			WorkLeft.Wait();
+			StoreCompactors.clear();
 		}
 
-		ReferenceStoreCompactors.clear();
-
 		ZEN_INFO("GCV2: Completed in {}", NiceTimeSpanMs(TotalTimer.GetElapsedTimeMs()));
 	}
 
-	Result.Sum();
-	return Result;
+	return Sum(Result);
 }
 
 #undef SCOPED_TIMER
@@ -829,6 +1032,12 @@ GcManager::CollectGarbage(const GcSettings& Settings)
 //////// End GC V2
 
 void
+GcManager::SetCancelGC(bool CancelFlag)
+{
+	m_CancelGC.store(CancelFlag);
+}
+
+void
 GcManager::AddGcContributor(GcContributor* Contributor)
 {
 	RwLock::ExclusiveLockScope _(m_Lock);
@@ -884,6 +1093,10 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		const auto Guard = MakeGuard([&] { ZEN_INFO("gathered references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
 		for (GcContributor* Contributor : m_GcContribs)
 		{
+			if (CheckGCCancel())
+			{
+				return GCTotalSizeDiff;
+			}
 			Contributor->GatherReferences(GcCtx);
 		}
 	}
@@ -901,6 +1114,11 @@ GcManager::CollectGarbage(GcContext& GcCtx)
 		});
 		for (GcStorage* Storage : m_GcStorage)
 		{
+			if (CheckGCCancel())
+			{
+				break;
+			}
+
 			const auto PreSize = Storage->StorageSize();
 			Storage->CollectGarbage(GcCtx);
 			const auto PostSize = Storage->StorageSize();
@@ -1127,7 +1345,12 @@ GcScheduler::Shutdown()
 	if (static_cast<uint32_t>(GcSchedulerStatus::kStopped) != m_Status)
 	{
 		bool GcIsRunning = m_Status == static_cast<uint32_t>(GcSchedulerStatus::kRunning);
-		m_Status		 = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
+		if (GcIsRunning)
+		{
+			ZEN_INFO("Requesting cancel running garbage collection");
+		}
+		m_GcManager.SetCancelGC(true);
+		m_Status = static_cast<uint32_t>(GcSchedulerStatus::kStopped);
 		m_GcSignal.notify_one();
 
 		if (m_GcThread.joinable())
@@ -1183,6 +1406,20 @@ GcScheduler::TriggerScrub(const TriggerScrubParams& Params)
 	return false;
 }
 
+bool
+GcScheduler::CancelGC()
+{
+	std::unique_lock Lock(m_GcMutex);
+
+	if (static_cast<uint32_t>(GcSchedulerStatus::kRunning) == m_Status)
+	{
+		ZEN_INFO("Cancel requested for running garbage collection");
+		m_GcManager.SetCancelGC(true);
+		return true;
+	}
+	return false;
+}
+
 DiskSpace
 GcScheduler::CheckDiskSpace()
 {
@@ -1227,22 +1464,17 @@ GcScheduler::AppendGCLog(GcClock::TimePoint StartTime, const GcSettings& Setting
 			std::string	   Id = fmt::format("{}", gsl::narrow<int64_t>(StartTime.time_since_epoch().count()));
 			Writer.BeginObject(Id);
 			{
-				Writer << "StartTimeSec"sv
-					   << gsl::narrow<int64_t>(std::chrono::duration_cast<std::chrono::seconds>(StartTime.time_since_epoch()).count());
+				Writer << "StartTime"sv << ToDateTime(StartTime);
 				Writer.BeginObject("Settings"sv);
 				{
-					Writer << "CacheExpireTimeSec"sv
-						   << gsl::narrow<int64_t>(
-								  std::chrono::duration_cast<std::chrono::seconds>(Settings.CacheExpireTime.time_since_epoch()).count());
-					Writer << "ProjectStoreExpireTimeSec"sv
-						   << gsl::narrow<int64_t>(
-								  std::chrono::duration_cast<std::chrono::seconds>(Settings.ProjectStoreExpireTime.time_since_epoch())
-									  .count());
+					Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime);
+					Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime);
 					Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects;
 					Writer << "IsDeleteMode"sv << Settings.IsDeleteMode;
 					Writer << "SkipCidDelete"sv << Settings.SkipCidDelete;
 					Writer << "Verbose"sv << Settings.Verbose;
 					Writer << "SingleThread"sv << Settings.SingleThread;
+					Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent;
 				}
 				Writer.EndObject();
 
@@ -1417,18 +1649,20 @@ GcScheduler::SchedulerThread()
 
 		try
 		{
-			bool				 DoGc					 = m_Config.Enabled;
-			bool				 DoScrubbing			 = false;
-			std::chrono::seconds ScrubTimeslice			 = std::chrono::seconds::max();
-			bool				 DoDelete				 = true;
-			bool				 CollectSmallObjects	 = m_Config.CollectSmallObjects;
-			std::chrono::seconds GcInterval				 = m_Config.Interval;
-			std::chrono::seconds LightweightGcInterval	 = m_Config.LightweightInterval;
-			std::chrono::seconds MaxCacheDuration		 = m_Config.MaxCacheDuration;
-			std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration;
-			uint64_t			 DiskSizeSoftLimit		 = m_Config.DiskSizeSoftLimit;
-			bool				 SkipCid				 = false;
-			GcVersion			 UseGCVersion			 = m_Config.UseGCVersion;
+			bool				 DoGc							   = m_Config.Enabled;
+			bool				 DoScrubbing					   = false;
+			std::chrono::seconds ScrubTimeslice					   = std::chrono::seconds::max();
+			bool				 DoDelete						   = true;
+			bool				 CollectSmallObjects			   = m_Config.CollectSmallObjects;
+			std::chrono::seconds GcInterval						   = m_Config.Interval;
+			std::chrono::seconds LightweightGcInterval			   = m_Config.LightweightInterval;
+			std::chrono::seconds MaxCacheDuration				   = m_Config.MaxCacheDuration;
+			std::chrono::seconds MaxProjectStoreDuration		   = m_Config.MaxProjectStoreDuration;
+			uint64_t			 DiskSizeSoftLimit				   = m_Config.DiskSizeSoftLimit;
+			bool				 SkipCid						   = false;
+			GcVersion			 UseGCVersion					   = m_Config.UseGCVersion;
+			uint32_t			 CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent;
+			bool				 Verbose						   = m_Config.Verbose;
 
 			bool DiskSpaceGCTriggered = false;
 			bool TimeBasedGCTriggered = false;
@@ -1463,7 +1697,10 @@ GcScheduler::SchedulerThread()
 					DoDelete = false;
 				}
 				UseGCVersion = TriggerParams.ForceGCVersion.value_or(UseGCVersion);
-				DoGc		 = true;
+				CompactBlockUsageThresholdPercent =
+					TriggerParams.CompactBlockUsageThresholdPercent.value_or(CompactBlockUsageThresholdPercent);
+				Verbose = TriggerParams.Verbose.value_or(Verbose);
+				DoGc	= true;
 			}
 
 			if (m_TriggerScrubParams)
@@ -1475,12 +1712,18 @@ GcScheduler::SchedulerThread()
 					DoGc = false;
 				}
 
+				if (m_TriggerScrubParams->SkipCas)
+				{
+					SkipCid = true;
+				}
+
+				DoDelete	   = !m_TriggerScrubParams->SkipDelete;
 				ScrubTimeslice = m_TriggerScrubParams->MaxTimeslice;
 			}
 
 			if (DoScrubbing)
 			{
-				ScrubStorage(DoDelete, ScrubTimeslice);
+				ScrubStorage(DoDelete, SkipCid, ScrubTimeslice);
 				m_TriggerScrubParams.reset();
 			}
 
@@ -1668,7 +1911,16 @@ GcScheduler::SchedulerThread()
 				}
 			}
 
-			CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, UseGCVersion);
+			CollectGarbage(CacheExpireTime,
+						   ProjectStoreExpireTime,
+						   DoDelete,
+						   CollectSmallObjects,
+						   SkipCid,
+						   UseGCVersion,
+						   CompactBlockUsageThresholdPercent,
+						   Verbose);
+
+			m_GcManager.SetCancelGC(false);
 
 			uint32_t RunningState = static_cast<uint32_t>(GcSchedulerStatus::kRunning);
 			if (!m_Status.compare_exchange_strong(RunningState, static_cast<uint32_t>(GcSchedulerStatus::kIdle)))
@@ -1715,7 +1967,7 @@ GcScheduler::SchedulerThread()
 }
 
 void
-GcScheduler::ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice)
+GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds TimeSlice)
 {
 	const std::chrono::steady_clock::time_point TimeNow	 = std::chrono::steady_clock::now();
 	std::chrono::steady_clock::time_point		Deadline = TimeNow + TimeSlice;
@@ -1726,13 +1978,14 @@ GcScheduler::ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice)
 	}
 
 	Stopwatch Timer;
-	ZEN_INFO("scrubbing STARTING (delete mode => {})", DoDelete);
+	ZEN_INFO("scrubbing STARTING (delete mode => {}, skip CID => {})", DoDelete, SkipCid);
 
-	WorkerThreadPool ThreadPool{4, "scrubber"};
-	ScrubContext	 Ctx{ThreadPool, Deadline};
+	WorkerThreadPool& ThreadPool = GetSmallWorkerPool();
+	ScrubContext	  Ctx{ThreadPool, Deadline};
 
 	try
 	{
+		Ctx.SetSkipCas(SkipCid);
 		Ctx.SetShouldDelete(DoDelete);
 		m_GcManager.ScrubStorage(Ctx);
 	}
@@ -1750,7 +2003,9 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 							bool					  Delete,
 							bool					  CollectSmallObjects,
 							bool					  SkipCid,
-							GcVersion				  UseGCVersion)
+							GcVersion				  UseGCVersion,
+							uint32_t				  CompactBlockUsageThresholdPercent,
+							bool					  Verbose)
 {
 	ZEN_TRACE_CPU("GcScheduler::CollectGarbage");
 
@@ -1813,30 +2068,33 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 				break;
 			case GcVersion::kV2:
 				{
-					const GcSettings   Settings	   = {.CacheExpireTime		  = CacheExpireTime,
-													  .ProjectStoreExpireTime = ProjectStoreExpireTime,
-													  .CollectSmallObjects	  = CollectSmallObjects,
-													  .IsDeleteMode			  = Delete,
-													  .SkipCidDelete		  = SkipCid};
+					const GcSettings   Settings	   = {.CacheExpireTime					 = CacheExpireTime,
+													  .ProjectStoreExpireTime			 = ProjectStoreExpireTime,
+													  .CollectSmallObjects				 = CollectSmallObjects,
+													  .IsDeleteMode						 = Delete,
+													  .SkipCidDelete					 = SkipCid,
+													  .Verbose							 = Verbose,
+													  .CompactBlockUsageThresholdPercent = CompactBlockUsageThresholdPercent,
+													  .DiskReservePath					 = m_Config.RootDirectory / "reserve.gc"};
 					GcClock::TimePoint GcStartTime = GcClock::Now();
 					GcResult		   Result	   = m_GcManager.CollectGarbage(Settings);
 
 					ZEN_INFO(
-						"GCV2: Removed {} items out of {}, deleted {} out of {}. Pruned {} Cid entries out of {}, compacted {} Cid entries "
-						"out of {}, "
-						"freed "
-						"{} on disk and {} of memory in {}. CacheExpireTime: {}, ProjectStoreExpireTime: {}, CollectSmallObjects: {}, "
+						"GCV2: Found {} expired items out of {}, deleted {}. "
+						"Found {} unreferenced Cid entries out of {}, deleted {}. "
+						"Freed {} on disk and {} of memory in {}. "
+						"CacheExpireTime: {}, ProjectStoreExpireTime: {}, CollectSmallObjects: {}, "
 						"IsDeleteMode: {}, SkipCidDelete: {}",
-						Result.ReferencerStat.Expired,
-						Result.ReferencerStat.Count,
-						Result.ReferencerStat.Deleted,
-						Result.ReferencerStat.Expired,
-						Result.ReferenceStoreStat.Pruned,
-						Result.ReferenceStoreStat.Count,
-						Result.ReferenceStoreStat.Compacted,
-						Result.ReferenceStoreStat.Pruned,
-						NiceBytes(Result.RemovedDisk),
-						NiceBytes(Result.RemovedMemory),
+						Result.ReferencerStatSum.RemoveExpiredDataStats.FoundCount,
+						Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount,
+						Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount,
+
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.FoundCount,
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount,
+						Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount,
+
+						NiceBytes(Result.CompactStoresStatSum.RemovedDisk),
+						NiceBytes(Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory),
 						NiceTimeSpanMs(Result.ElapsedMS.count()),
 						Settings.CacheExpireTime,
 						Settings.ProjectStoreExpireTime,
@@ -1854,8 +2112,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
 					{
 						m_LastFullGCV2Result = Result;
 					}
-					Diff.DiskSize	= Result.RemovedDisk;
-					Diff.MemorySize = Result.RemovedMemory;
+					Diff.DiskSize	= Result.CompactStoresStatSum.RemovedDisk;
+					Diff.MemorySize = Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory;
 				}
 				break;
 		}
diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h
index 75accd9b8..786780b5e 100644
--- a/src/zenstore/include/zenstore/blockstore.h
+++ b/src/zenstore/include/zenstore/blockstore.h
@@ -16,8 +16,8 @@ namespace zen {
 struct BlockStoreLocation
 {
 	uint32_t BlockIndex;
-	uint64_t Offset;
-	uint64_t Size;
+	uint32_t Offset;
+	uint32_t Size;
 
 	inline auto operator<=>(const BlockStoreLocation& Rhs) const = default;
 };
@@ -32,19 +32,19 @@ struct BlockStoreDiskLocation
 	constexpr static uint32_t MaxBlockIndex		= (1ul << BlockStoreDiskLocation::MaxBlockIndexBits) - 1ul;
 	constexpr static uint32_t MaxOffset			= (1ul << BlockStoreDiskLocation::MaxOffsetBits) - 1ul;
 
-	BlockStoreDiskLocation(const BlockStoreLocation& Location, uint64_t OffsetAlignment)
+	BlockStoreDiskLocation(const BlockStoreLocation& Location, uint32_t OffsetAlignment)
 	{
 		Init(Location.BlockIndex, Location.Offset / OffsetAlignment, Location.Size);
 	}
 
 	BlockStoreDiskLocation() = default;
 
-	inline BlockStoreLocation Get(uint64_t OffsetAlignment) const
+	inline BlockStoreLocation Get(uint32_t OffsetAlignment) const
 	{
 		uint64_t PackedOffset = 0;
 		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
-		return {.BlockIndex = static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits),
-				.Offset		= (PackedOffset & MaxOffset) * OffsetAlignment,
+		return {.BlockIndex = static_cast<uint32_t>(PackedOffset >> MaxOffsetBits),
+				.Offset		= static_cast<uint32_t>((PackedOffset & MaxOffset) * OffsetAlignment),
 				.Size		= GetSize()};
 	}
 
@@ -55,14 +55,14 @@ struct BlockStoreDiskLocation
 		return static_cast<std::uint32_t>(PackedOffset >> MaxOffsetBits);
 	}
 
-	inline uint64_t GetOffset(uint64_t OffsetAlignment) const
+	inline uint32_t GetOffset(uint32_t OffsetAlignment) const
 	{
 		uint64_t PackedOffset = 0;
 		memcpy(&PackedOffset, &m_Offset, sizeof m_Offset);
-		return (PackedOffset & MaxOffset) * OffsetAlignment;
+		return static_cast<uint32_t>((PackedOffset & MaxOffset) * OffsetAlignment);
 	}
 
-	inline uint64_t GetSize() const { return m_Size; }
+	inline uint32_t GetSize() const { return m_Size; }
 
 	inline auto operator<=>(const BlockStoreDiskLocation& Rhs) const = default;
 
@@ -126,21 +126,39 @@ public:
 	typedef std::vector<size_t>								   ChunkIndexArray;
 
 	typedef std::function<void(const MovedChunksArray& MovedChunks, const ChunkIndexArray& RemovedChunks)> ReclaimCallback;
-	typedef std::function<void(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)>			   CompactCallback;
+	typedef std::function<bool(const MovedChunksArray& MovedChunks, uint64_t FreedDiskSpace)>			   CompactCallback;
 	typedef std::function<uint64_t()>																	   ClaimDiskReserveCallback;
 	typedef std::function<void(size_t ChunkIndex, const void* Data, uint64_t Size)>						   IterateChunksSmallSizeCallback;
 	typedef std::function<void(size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size)>   IterateChunksLargeSizeCallback;
 	typedef std::function<void(const BlockStoreLocation& Location)>										   WriteChunkCallback;
 
+	struct BlockUsageInfo
+	{
+		uint64_t DiskUsage;
+		uint32_t EntryCount;
+	};
+	typedef std::unordered_map<uint32_t, BlockUsageInfo> BlockUsageMap;
+	typedef std::unordered_map<uint32_t, uint32_t>		 BlockEntryCountMap;
+
 	void Initialize(const std::filesystem::path& BlocksBasePath, uint64_t MaxBlockSize, uint64_t MaxBlockCount);
 
+	struct BlockIndexSet
+	{
+		void					  Add(uint32_t BlockIndex);
+		std::span<const uint32_t> GetBlockIndices() const { return BlockIndexes; }
+
+	private:
+		std::vector<uint32_t> BlockIndexes;
+	};
+
 	// Ask the store to create empty blocks for all locations that does not have a block
 	// Remove any block that is not referenced
-	void SyncExistingBlocksOnDisk(const std::vector<BlockStoreLocation>& KnownLocations);
+	void			   SyncExistingBlocksOnDisk(const BlockIndexSet& KnownLocations);
+	BlockEntryCountMap GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent);
 
 	void Close();
 
-	void WriteChunk(const void* Data, uint64_t Size, uint64_t Alignment, const WriteChunkCallback& Callback);
+	void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback);
 
 	IoBuffer TryGetChunk(const BlockStoreLocation& Location) const;
 	void	 Flush(bool ForceNewBlock);
@@ -150,7 +168,7 @@ public:
 						const ReclaimSnapshotState&			   Snapshot,
 						const std::vector<BlockStoreLocation>& ChunkLocations,
 						const ChunkIndexArray&				   KeepChunkIndexes,
-						uint64_t							   PayloadAlignment,
+						uint32_t							   PayloadAlignment,
 						bool								   DryRun,
 						const ReclaimCallback&				   ChangeCallback	   = [](const MovedChunksArray&, const ChunkIndexArray&) {},
 						const ClaimDiskReserveCallback&		   DiskReserveCallback = []() { return 0; });
@@ -161,8 +179,8 @@ public:
 
 	void CompactBlocks(
 		const BlockStoreCompactState&	CompactState,
-		uint64_t						PayloadAlignment,
-		const CompactCallback&			ChangeCallback		= [](const MovedChunksArray&, uint64_t) {},
+		uint32_t						PayloadAlignment,
+		const CompactCallback&			ChangeCallback		= [](const MovedChunksArray&, uint64_t) { return true; },
 		const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; });
 
 	static const char*			 GetBlockFileExtension();
@@ -170,6 +188,8 @@ public:
 
 	inline uint64_t TotalSize() const { return m_TotalSize.load(std::memory_order::relaxed); }
 
+	Ref<BlockStoreFile> GetBlockFile(uint32_t BlockIndex);
+
 private:
 	uint32_t GetFreeBlockIndex(uint32_t StartProbeIndex, RwLock::ExclusiveLockScope&, std::filesystem::path& OutBlockPath) const;
 
@@ -177,7 +197,7 @@ private:
 
 	mutable RwLock		  m_InsertLock;	 // used to serialize inserts
 	Ref<BlockStoreFile>	  m_WriteBlock;
-	std::uint64_t		  m_CurrentInsertOffset = 0;
+	std::uint32_t		  m_CurrentInsertOffset = 0;
 	std::atomic_uint32_t  m_WriteBlockIndex{};
 	std::vector<uint32_t> m_ActiveWriteBlocks;
 
@@ -193,10 +213,25 @@ class BlockStoreCompactState
 public:
 	BlockStoreCompactState() = default;
 
+	void IncludeBlocks(const BlockStore::BlockEntryCountMap& BlockEntryCountMap)
+	{
+		size_t EntryCountTotal = 0;
+		for (auto& BlockUsageIt : BlockEntryCountMap)
+		{
+			uint32_t BlockIndex = BlockUsageIt.first;
+			ZEN_ASSERT(m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end());
+
+			m_KeepChunks.emplace_back(std::vector<size_t>());
+			m_KeepChunks.back().reserve(BlockUsageIt.second);
+			m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1);
+			EntryCountTotal += BlockUsageIt.second;
+		}
+		m_ChunkLocations.reserve(EntryCountTotal);
+	}
+
 	void IncludeBlock(uint32_t BlockIndex)
 	{
-		auto It = m_BlockIndexToChunkMapIndex.find(BlockIndex);
-		if (It == m_BlockIndexToChunkMapIndex.end())
+		if (m_BlockIndexToChunkMapIndex.find(BlockIndex) == m_BlockIndexToChunkMapIndex.end())
 		{
 			m_KeepChunks.emplace_back(std::vector<size_t>());
 			m_BlockIndexToChunkMapIndex.insert_or_assign(BlockIndex, m_KeepChunks.size() - 1);
@@ -220,14 +255,18 @@ public:
 
 	const BlockStoreLocation& GetLocation(size_t Index) const { return m_ChunkLocations[Index]; }
 
-	void IterateBlocks(std::function<void(uint32_t								 BlockIndex,
+	void IterateBlocks(std::function<bool(uint32_t								 BlockIndex,
 										  const std::vector<size_t>&			 KeepChunkIndexes,
 										  const std::vector<BlockStoreLocation>& ChunkLocations)> Callback) const
 	{
 		for (auto It : m_BlockIndexToChunkMapIndex)
 		{
 			size_t ChunkMapIndex = It.second;
-			Callback(It.first, m_KeepChunks[ChunkMapIndex], m_ChunkLocations);
+			bool   Continue		 = Callback(It.first, m_KeepChunks[ChunkMapIndex], m_ChunkLocations);
+			if (!Continue)
+			{
+				break;
+			}
 		}
 	}
 
diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h
index 319683dcb..4c9f30608 100644
--- a/src/zenstore/include/zenstore/cidstore.h
+++ b/src/zenstore/include/zenstore/cidstore.h
@@ -9,10 +9,6 @@
 #include <zenstore/hashkeyset.h>
 #include <zenutil/statsreporter.h>
 
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <tsl/robin_map.h>
-ZEN_THIRD_PARTY_INCLUDES_END
-
 #include <filesystem>
 
 namespace zen {
diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h
index d4c7bba25..30dd97ce8 100644
--- a/src/zenstore/include/zenstore/gc.h
+++ b/src/zenstore/include/zenstore/gc.h
@@ -60,61 +60,71 @@ struct GcSettings
 	bool			   SkipCidDelete		  = false;
 	bool			   Verbose				  = false;
 	bool			   SingleThread			  = false;
+	uint32_t		   CompactBlockUsageThresholdPercent =
+		90;	 // 0 = compact only empty eligible blocks, 100 = compact all non-full eligible blocks, 1-99 = compact eligible blocks with less
+			 // usage than CompactBlockUsageThresholdPercent
+	std::filesystem::path DiskReservePath;
+};
+
+struct GcCompactStoreStats
+{
+	std::uint64_t			  RemovedDisk = 0;
+	std::chrono::milliseconds ElapsedMS	  = {};
+};
+
+struct GcStats
+{
+	std::uint64_t			  CheckedCount = 0;
+	std::uint64_t			  FoundCount   = 0;
+	std::uint64_t			  DeletedCount = 0;
+	std::uint64_t			  FreedMemory  = 0;
+	std::chrono::milliseconds ElapsedMS	   = {};
 };
 
 struct GcReferencerStats
 {
-	std::uint64_t Count			= 0;
-	std::uint64_t Expired		= 0;
-	std::uint64_t Deleted		= 0;
-	std::uint64_t RemovedDisk	= 0;
-	std::uint64_t RemovedMemory = 0;
+	GcStats				RemoveExpiredDataStats;
+	GcCompactStoreStats CompactStoreStats;
 
-	std::chrono::milliseconds RemoveExpiredDataMS		= {};
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds PreCacheStateMS			= {};
 	std::chrono::milliseconds LockStateMS				= {};
 	std::chrono::milliseconds ElapsedMS					= {};
 };
 
 struct GcReferenceStoreStats
 {
-	std::uint64_t Count			= 0;
-	std::uint64_t Pruned		= 0;
-	std::uint64_t Compacted		= 0;
-	std::uint64_t RemovedDisk	= 0;
-	std::uint64_t RemovedMemory = 0;
+	GcStats				RemoveUnreferencedDataStats;
+	GcCompactStoreStats CompactStoreStats;
 
-	std::chrono::milliseconds CreateReferencePrunerMS  = {};
-	std::chrono::milliseconds RemoveUnreferencedDataMS = {};
-	std::chrono::milliseconds CompactReferenceStoreMS  = {};
+	std::chrono::milliseconds CreateReferencePrunersMS = {};
 	std::chrono::milliseconds ElapsedMS				   = {};
 };
 
 struct GcResult
 {
-	GcReferencerStats	  ReferencerStat;
-	GcReferenceStoreStats ReferenceStoreStat;
-
-	std::uint64_t RemovedDisk	= 0;
-	std::uint64_t RemovedMemory = 0;
-
 	std::vector<std::pair<std::string, GcReferencerStats>>	   ReferencerStats;
 	std::vector<std::pair<std::string, GcReferenceStoreStats>> ReferenceStoreStats;
 
+	GcReferencerStats	  ReferencerStatSum;
+	GcReferenceStoreStats ReferenceStoreStatSum;
+	GcCompactStoreStats	  CompactStoresStatSum;
+
 	// Wall times, not sum of each
 	std::chrono::milliseconds RemoveExpiredDataMS		= {};
 	std::chrono::milliseconds CreateReferenceCheckersMS = {};
+	std::chrono::milliseconds PreCacheStateMS			= {};
 	std::chrono::milliseconds LockStateMS				= {};
 
-	std::chrono::milliseconds CreateReferencePrunerMS  = {};
+	std::chrono::milliseconds CreateReferencePrunersMS = {};
 	std::chrono::milliseconds RemoveUnreferencedDataMS = {};
-	std::chrono::milliseconds CompactReferenceStoreMS  = {};
+	std::chrono::milliseconds CompactStoresMS		   = {};
 
 	std::chrono::milliseconds WriteBlockMS = {};
 
 	std::chrono::milliseconds ElapsedMS = {};
 
-	void Sum();
+	bool WasCancelled = false;
 };
 
 class CbObjectWriter;
@@ -123,28 +133,30 @@ void WriteGCResult(CbObjectWriter& Writer, const GcResult& Result, bool HumanRea
 
 struct GcCtx
 {
-	const GcSettings Settings;
+	const GcSettings  Settings;
+	std::atomic_bool& IsCancelledFlag;
 };
 
 typedef tsl::robin_set<IoHash> HashSet;
 
 /**
- * @brief An interface to remove the stored data on disk after a GcReferencePruner::RemoveUnreferencedData
+ * @brief An interface to remove the stored data on disk after a GcReferencer::RemoveExpiredData and
+ * GcReferencePruner::RemoveUnreferencedData
  *
- * CompactReferenceStore is called after pruning (GcReferencePruner::RemoveUnreferencedData) and state locking is
- * complete so implementor must take care to only remove data that has not been altered since the prune operation.
+ * CompactStore is called after state locking is complete so implementor must take care to only remove
+ * data that has not been altered since the prune operation.
  *
- * Instance will be deleted after CompactReferenceStore has completed execution.
+ * Instance will be deleted after CompactStore has completed execution.
  *
  * The subclass constructor should be provided with information on what is intended to be removed.
  */
-class GcReferenceStoreCompactor
+class GcStoreCompactor
 {
 public:
-	virtual ~GcReferenceStoreCompactor() = default;
+	virtual ~GcStoreCompactor() = default;
 
 	// Remove data on disk based on results from GcReferencePruner::RemoveUnreferencedData
-	virtual void CompactReferenceStore(GcCtx& Ctx, GcReferenceStoreStats& Stats) = 0;
+	virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) = 0;
 };
 
 /**
@@ -161,6 +173,8 @@ public:
 	// Destructor should unlock what was locked in LockState
 	virtual ~GcReferenceChecker() = default;
 
+	virtual void PreCache(GcCtx& Ctx) = 0;
+
 	// Lock the state and make sure no references changes, usually a read-lock is taken until the destruction
 	// of the instance. Called once before any calls to RemoveUsedReferencesFromSet
 	// The implementation should be as fast as possible as LockState is part of a stop the world (from changes)
@@ -175,10 +189,6 @@ public:
 /**
  * @brief Interface to handle GC of data that references Cid data
  *
- * TODO: Maybe we should split up being a referencer and something that holds cache values?
- *
- * GcCacheStore and GcReferencer?
- *
  * This interface is registered/unregistered to GcManager vua AddGcReferencer() and RemoveGcReferencer()
  */
 class GcReferencer
@@ -190,10 +200,7 @@ public:
 	virtual std::string GetGcName(GcCtx& Ctx) = 0;
 
 	// Remove expired data based on either GcCtx::Settings CacheExpireTime/ProjectExpireTime
-	// TODO: For disk layer we need to first update it with access times from the memory layer
-	// The implementer of GcReferencer (in our case a disk bucket) does not know about any
-	// potential memory cache layer :(
-	virtual void RemoveExpiredData(GcCtx& Ctx, GcReferencerStats& Stats) = 0;
+	virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) = 0;
 
 	// Create 0-n GcReferenceChecker for this GcReferencer. Caller will manage lifetime of
 	// returned instances
@@ -213,14 +220,12 @@ public:
 	// Check a set of references to see if they are in use.
 	// Use the GetUnusedReferences input function to check if references are used and update any pointers
 	// so any query for references determined to be unreferences will not be found.
-	// If any references a found to be unused, return a GcReferenceStoreCompactor instance which will
+	// If any references a found to be unused, return a GcStoreCompactor instance which will
 	// clean up any stored bulk data mapping to the pruned references.
 	// Caller will manage lifetime of returned instance
 	// This function should execute as fast as possible, so try to prepare a list of references to check ahead of
 	// call to this function and make sure the removal of unreferences items is as lightweight as possible.
-	virtual GcReferenceStoreCompactor* RemoveUnreferencedData(GcCtx&						 Ctx,
-															  GcReferenceStoreStats&		 Stats,
-															  const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
+	virtual GcStoreCompactor* RemoveUnreferencedData(GcCtx& Ctx, GcStats& Stats, const GetUnusedReferencesFunc& GetUnusedReferences) = 0;
 };
 
 /**
@@ -343,6 +348,7 @@ public:
 	void RemoveGcReferenceStore(GcReferenceStore& ReferenceStore);
 
 	GcResult CollectGarbage(const GcSettings& Settings);
+	void	 SetCancelGC(bool CancelFlag);
 
 	//////// End GC V2
 
@@ -361,6 +367,7 @@ public:
 	void					SetDiskWriteBlocker(const DiskWriteBlocker* Monitor) { m_DiskWriteBlocker = Monitor; }
 
 private:
+	bool						CheckGCCancel() { return m_CancelGC.load(); }
 	LoggerRef					Log() { return m_Log; }
 	LoggerRef					m_Log;
 	mutable RwLock				m_Lock;
@@ -371,6 +378,8 @@ private:
 
 	std::vector<GcReferencer*>	   m_GcReferencers;
 	std::vector<GcReferenceStore*> m_GcReferenceStores;
+
+	std::atomic_bool m_CancelGC{false};
 };
 
 enum class GcSchedulerStatus : uint32_t
@@ -399,7 +408,9 @@ struct GcSchedulerConfig
 	uint64_t			  DiskSizeSoftLimit					= 0;
 	uint64_t			  MinimumFreeDiskSpaceToAllowWrites = 1ul << 28;
 	std::chrono::seconds  LightweightInterval{};
-	GcVersion			  UseGCVersion = GcVersion::kV1;
+	GcVersion			  UseGCVersion						= GcVersion::kV1;
+	uint32_t			  CompactBlockUsageThresholdPercent = 90;
+	bool				  Verbose							= false;
 };
 
 struct GcSchedulerState
@@ -471,6 +482,8 @@ public:
 		bool					 SkipCid				 = false;
 		bool					 SkipDelete				 = false;
 		std::optional<GcVersion> ForceGCVersion;
+		std::optional<uint32_t>	 CompactBlockUsageThresholdPercent;
+		std::optional<bool>		 Verbose;
 	};
 
 	bool TriggerGc(const TriggerGcParams& Params);
@@ -479,10 +492,14 @@ public:
 	{
 		bool				 SkipGc		  = false;
 		std::chrono::seconds MaxTimeslice = std::chrono::seconds::max();
+		bool				 SkipDelete	  = false;
+		bool				 SkipCas	  = false;
 	};
 
 	bool TriggerScrub(const TriggerScrubParams& Params);
 
+	bool CancelGC();
+
 private:
 	void		 SchedulerThread();
 	void		 CollectGarbage(const GcClock::TimePoint& CacheExpireTime,
@@ -490,8 +507,10 @@ private:
 								bool					  Delete,
 								bool					  CollectSmallObjects,
 								bool					  SkipCid,
-								GcVersion				  UseGCVersion);
-	void		 ScrubStorage(bool DoDelete, std::chrono::seconds TimeSlice);
+								GcVersion				  UseGCVersion,
+								uint32_t				  CompactBlockUsageThresholdPercent,
+								bool					  Verbose);
+	void		 ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds TimeSlice);
 	LoggerRef	 Log() { return m_Log; }
 	virtual bool AreDiskWritesAllowed() const override { return !m_AreDiskWritesBlocked.load(); }
 	DiskSpace	 CheckDiskSpace();
diff --git a/src/zenstore/include/zenstore/scrubcontext.h b/src/zenstore/include/zenstore/scrubcontext.h
index cefaf0888..2f28cfec7 100644
--- a/src/zenstore/include/zenstore/scrubcontext.h
+++ b/src/zenstore/include/zenstore/scrubcontext.h
@@ -38,15 +38,20 @@ public:
 	inline uint64_t ScrubbedBytes() const { return m_ByteCount; }
 
 	HashKeySet BadCids() const;
+	bool	   IsBadCid(const IoHash& Cid) const;
 
 	inline bool RunRecovery() const { return m_Recover; }
 	inline void SetShouldDelete(bool DoDelete) { m_Recover = DoDelete; }
 
+	inline bool IsSkipCas() const { return m_SkipCas; }
+	inline void SetSkipCas(bool DoSkipCas) { m_SkipCas = DoSkipCas; }
+
 	inline WorkerThreadPool& ThreadPool() { return m_WorkerThreadPool; }
 
 private:
 	uint64_t							  m_ScrubTime = GetHifreqTimerValue();
 	bool								  m_Recover	  = true;
+	bool								  m_SkipCas	  = false;
 	std::atomic<uint64_t>				  m_ChunkCount{0};
 	std::atomic<uint64_t>				  m_ByteCount{0};
 	mutable RwLock						  m_Lock;
diff --git a/src/zenstore/scrubcontext.cpp b/src/zenstore/scrubcontext.cpp
index f5a3784c3..fbcd7d33c 100644
--- a/src/zenstore/scrubcontext.cpp
+++ b/src/zenstore/scrubcontext.cpp
@@ -33,6 +33,13 @@ ScrubContext::BadCids() const
 	return m_BadCid;
 }
 
+bool
+ScrubContext::IsBadCid(const IoHash& Cid) const
+{
+	RwLock::SharedLockScope _(m_Lock);
+	return m_BadCid.ContainsHash(Cid);
+}
+
 void
 ScrubContext::ReportBadCidChunks(std::span<IoHash> BadCasChunks)
 {
diff --git a/src/zenstore/zenstore.cpp b/src/zenstore/zenstore.cpp
index d87652fde..60dabe31f 100644
--- a/src/zenstore/zenstore.cpp
+++ b/src/zenstore/zenstore.cpp
@@ -7,7 +7,6 @@
 #	include <zenstore/blockstore.h>
 #	include <zenstore/gc.h>
 #	include <zenstore/hashkeyset.h>
-#	include <zenutil/basicfile.h>
 
 #	include "cas.h"
 #	include "compactcas.h"
@@ -18,7 +17,6 @@ namespace zen {
 void
 zenstore_forcelinktests()
 {
-	basicfile_forcelink();
 	CAS_forcelink();
 	filecas_forcelink();
 	blockstore_forcelink();
diff --git a/src/zenutil/basicfile.cpp b/src/zenutil/basicfile.cpp
index 1dce71e60..819d0805d 100644
--- a/src/zenutil/basicfile.cpp
+++ b/src/zenutil/basicfile.cpp
@@ -76,16 +76,15 @@ BasicFile::Open(const std::filesystem::path& FileName, Mode InMode, std::error_c
 
 	const DWORD dwShareMode = FILE_SHARE_READ | (EnumHasAllFlags(InMode, Mode::kPreventWrite) ? 0 : FILE_SHARE_WRITE) |
 							  (EnumHasAllFlags(InMode, Mode::kPreventDelete) ? 0 : FILE_SHARE_DELETE);
-	const DWORD dwFlagsAndAttributes =
-		FILE_ATTRIBUTE_NORMAL | (EnumHasAllFlags(InMode, Mode::kDeleteOnClose) ? FILE_FLAG_DELETE_ON_CLOSE : 0);
-	const HANDLE hTemplateFile = nullptr;
-	const HANDLE FileHandle	   = CreateFile(FileName.c_str(),
-											dwDesiredAccess,
-											dwShareMode,
-											/* lpSecurityAttributes */ nullptr,
-											dwCreationDisposition,
-											dwFlagsAndAttributes,
-											hTemplateFile);
+	const DWORD	 dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL;
+	const HANDLE hTemplateFile		  = nullptr;
+	const HANDLE FileHandle			  = CreateFile(FileName.c_str(),
+										   dwDesiredAccess,
+										   dwShareMode,
+										   /* lpSecurityAttributes */ nullptr,
+										   dwCreationDisposition,
+										   dwFlagsAndAttributes,
+										   hTemplateFile);
 
 	if (FileHandle == INVALID_HANDLE_VALUE)
 	{
@@ -192,7 +191,8 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 
 		if (!Success)
 		{
-			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowLastError(fmt::format("Failed to read from file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
 		}
 
 		BytesToRead -= NumberOfBytesToRead;
@@ -325,7 +325,8 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t Offset)
 
 	if (Ec)
 	{
-		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle)));
+		std::error_code Dummy;
+		throw std::system_error(Ec, fmt::format("Failed to write to file '{}'", zen::PathFromHandle(m_FileHandle, Dummy)));
 	}
 }
 
@@ -357,7 +358,8 @@ BasicFile::FileSize()
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	return uint64_t(liFileSize.QuadPart);
@@ -367,7 +369,8 @@ BasicFile::FileSize()
 	struct stat Stat;
 	if (fstat(Fd, &Stat) == -1)
 	{
-		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle)));
+		std::error_code Dummy;
+		ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy)));
 	}
 	return uint64_t(Stat.st_size);
 #endif
@@ -414,7 +417,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set file pointer to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	OK = ::SetEndOfFile(m_FileHandle);
@@ -423,7 +428,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set end of file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #elif ZEN_PLATFORM_MAC
@@ -433,7 +440,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #else
@@ -443,7 +452,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = zen::GetLastError();
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to set truncate file to {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 	if (FileSize > 0)
@@ -451,7 +462,9 @@ BasicFile::SetFileSize(uint64_t FileSize)
 		int Error = posix_fallocate64(Fd, 0, (off64_t)FileSize);
 		if (Error)
 		{
-			ThrowSystemError(Error, fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle)));
+			std::error_code Dummy;
+			ThrowSystemError(Error,
+							 fmt::format("Failed to allocate space of {} for file {}", FileSize, PathFromHandle(m_FileHandle, Dummy)));
 		}
 	}
 #endif
@@ -588,6 +601,8 @@ LockFile::Update(CbObject Payload, std::error_code& Ec)
 	BasicFile::Write(Payload.GetBuffer(), 0, Ec);
 }
 
+//////////////////////////////////////////////////////////////////////////
+
 BasicFileBuffer::BasicFileBuffer(BasicFile& Base, uint64_t BufferSize)
 : m_Base(Base)
 , m_Buffer(nullptr)
@@ -662,6 +677,79 @@ BasicFileBuffer::MakeView(uint64_t Size, uint64_t FileOffset)
 	return MemoryView(m_Buffer + (FileOffset - m_BufferStart), Size);
 }
 
+//////////////////////////////////////////////////////////////////////////
+
+BasicFileWriter::BasicFileWriter(BasicFile& Base, uint64_t BufferSize)
+: m_Base(Base)
+, m_Buffer(nullptr)
+, m_BufferSize(BufferSize)
+, m_BufferStart(0)
+, m_BufferEnd(0)
+{
+	m_Buffer = (uint8_t*)Memory::Alloc(m_BufferSize);
+}
+
+BasicFileWriter::~BasicFileWriter()
+{
+	Flush();
+	Memory::Free(m_Buffer);
+}
+
+void
+BasicFileWriter::Write(void* Data, uint64_t Size, uint64_t FileOffset)
+{
+	if (m_Buffer == nullptr || (Size >= m_BufferSize))
+	{
+		m_Base.Write(Data, Size, FileOffset);
+		return;
+	}
+
+	// Note that this only supports buffering of sequential writes!
+
+	if (FileOffset != m_BufferEnd)
+	{
+		Flush();
+		m_BufferStart = m_BufferEnd = FileOffset;
+	}
+
+	while (Size)
+	{
+		const uint64_t RemainingBufferCapacity = m_BufferStart + m_BufferSize - m_BufferEnd;
+		const uint64_t BlockWriteBytes		   = Min(RemainingBufferCapacity, Size);
+		const uint64_t BufferWriteOffset	   = FileOffset - m_BufferStart;
+
+		ZEN_ASSERT_SLOW(BufferWriteOffset < m_BufferSize);
+		ZEN_ASSERT_SLOW((BufferWriteOffset + BlockWriteBytes) <= m_BufferSize);
+
+		memcpy(m_Buffer + BufferWriteOffset, Data, BlockWriteBytes);
+
+		Size -= BlockWriteBytes;
+		m_BufferEnd += BlockWriteBytes;
+		FileOffset += BlockWriteBytes;
+
+		if ((m_BufferEnd - m_BufferStart) == m_BufferSize)
+		{
+			Flush();
+		}
+	}
+}
+
+void
+BasicFileWriter::Flush()
+{
+	const uint64_t BufferedBytes = m_BufferEnd - m_BufferStart;
+
+	if (BufferedBytes == 0)
+		return;
+
+	const uint64_t WriteOffset = m_BufferStart;
+	m_BufferStart			   = m_BufferEnd;
+
+	m_Base.Write(m_Buffer, BufferedBytes, WriteOffset);
+}
+
+//////////////////////////////////////////////////////////////////////////
+
 /*
 		___________              __
 		\__    ___/___   _______/  |_  ______
diff --git a/src/zenutil/include/zenutil/basicfile.h b/src/zenutil/include/zenutil/basicfile.h
index 7797258e8..f25d9f23c 100644
--- a/src/zenutil/include/zenutil/basicfile.h
+++ b/src/zenutil/include/zenutil/basicfile.h
@@ -44,7 +44,6 @@ public:
 		kModeMask	   = 0x0007,
 		kPreventDelete = 0x1000'0000,  // Do not open with delete sharing mode (prevent other processes from deleting file while open)
 		kPreventWrite  = 0x2000'0000,  // Do not open with write sharing mode (prevent other processes from writing to file while open)
-		kDeleteOnClose = 0x4000'0000,  // File should be deleted when the last handle is closed
 	};
 
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
@@ -138,6 +137,13 @@ public:
 	void	   Read(void* Data, uint64_t Size, uint64_t FileOffset);
 	MemoryView MakeView(uint64_t Size, uint64_t FileOffset);
 
+	template<typename T>
+	const T* MakeView(uint64_t FileOffset)
+	{
+		MemoryView View = MakeView(sizeof(T), FileOffset);
+		return reinterpret_cast<const T*>(View.GetData());
+	}
+
 private:
 	BasicFile&	   m_Base;
 	uint8_t*	   m_Buffer;
@@ -147,6 +153,29 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
+/** Adds a layer of buffered writing to a BasicFile
+
+This class is not intended for concurrent access, it is not thread safe.
+
+*/
+
+class BasicFileWriter
+{
+public:
+	BasicFileWriter(BasicFile& Base, uint64_t BufferSize);
+	~BasicFileWriter();
+
+	void Write(void* Data, uint64_t Size, uint64_t FileOffset);
+	void Flush();
+
+private:
+	BasicFile&	   m_Base;
+	uint8_t*	   m_Buffer;
+	const uint64_t m_BufferSize;
+	uint64_t	   m_BufferStart;
+	uint64_t	   m_BufferEnd;
+};
+
 ZENCORE_API void basicfile_forcelink();
 
 }  // namespace zen
diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h
index 498ecd143..146fea7a0 100644
--- a/src/zenutil/include/zenutil/logging/fullformatter.h
+++ b/src/zenutil/include/zenutil/logging/fullformatter.h
@@ -16,136 +16,175 @@ namespace zen::logging {
 class full_formatter final : public spdlog::formatter
 {
 public:
-	full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch) : m_Epoch(Epoch), m_LogId(LogId) {}
+	full_formatter(std::string_view LogId, std::chrono::time_point<std::chrono::system_clock> Epoch)
+	: m_Epoch(Epoch)
+	, m_LogId(LogId)
+	, m_LinePrefix(128, ' ')
+	, m_UseFullDate(false)
+	{
+	}
 
-	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
+	full_formatter(std::string_view LogId) : m_LogId(LogId), m_LinePrefix(128, ' '), m_UseFullDate(true) {}
 
-	static constexpr bool UseDate = false;
+	virtual std::unique_ptr<formatter> clone() const override { return std::make_unique<full_formatter>(m_LogId, m_Epoch); }
 
-	virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& dest) override
+	virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& OutBuffer) override
 	{
-		using namespace std::literals;
-
-		if constexpr (UseDate)
-		{
-			auto secs = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
-			if (secs != m_LastLogSecs)
-			{
-				m_CachedTm	  = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time));
-				m_LastLogSecs = secs;
-			}
-		}
+		// Note that the sink is responsible for ensuring there is only ever a
+		// single caller in here
 
-		const auto& tm_time = m_CachedTm;
+		using namespace std::literals;
 
-		// cache the date/time part for the next second.
-		auto duration = msg.time - m_Epoch;
-		auto secs	  = std::chrono::duration_cast<std::chrono::seconds>(duration);
+		std::chrono::seconds TimestampSeconds;
 
-		if (m_CacheTimestamp != secs || m_CachedDatetime.size() == 0)
+		if (m_UseFullDate)
 		{
-			m_CachedDatetime.clear();
-			m_CachedDatetime.push_back('[');
-
-			if constexpr (UseDate)
+			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(msg.time.time_since_epoch());
+			if (TimestampSeconds != m_LastLogSecs)
 			{
-				spdlog::details::fmt_helper::append_int(tm_time.tm_year + 1900, m_CachedDatetime);
-				m_CachedDatetime.push_back('-');
+				m_LastLogSecs = TimestampSeconds;
 
-				spdlog::details::fmt_helper::pad2(tm_time.tm_mon + 1, m_CachedDatetime);
+				m_CachedLocalTm = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time));
+				m_CachedDatetime.clear();
+				m_CachedDatetime.push_back('[');
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_year % 100, m_CachedDatetime);
 				m_CachedDatetime.push_back('-');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_mday, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mon + 1, m_CachedDatetime);
+				m_CachedDatetime.push_back('-');
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mday, m_CachedDatetime);
 				m_CachedDatetime.push_back(' ');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_hour, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_hour, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_min, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_min, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
-
-				spdlog::details::fmt_helper::pad2(tm_time.tm_sec, m_CachedDatetime);
+				spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime);
+				m_CachedDatetime.push_back('.');
 			}
-			else
+		}
+		else
+		{
+			auto ElapsedTime = msg.time - m_Epoch;
+			TimestampSeconds = std::chrono::duration_cast<std::chrono::seconds>(ElapsedTime);
+
+			// cache the date/time part for the next second.
+
+			if (m_CacheTimestamp != TimestampSeconds || m_CachedDatetime.size() == 0)
 			{
-				int Count = int(secs.count());
+				m_CacheTimestamp = TimestampSeconds;
 
+				int		  Count	  = int(TimestampSeconds.count());
 				const int LogSecs = Count % 60;
 				Count /= 60;
-
 				const int LogMins = Count % 60;
 				Count /= 60;
-
 				const int LogHours = Count;
 
+				m_CachedDatetime.clear();
+				m_CachedDatetime.push_back('[');
 				spdlog::details::fmt_helper::pad2(LogHours, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
 				spdlog::details::fmt_helper::pad2(LogMins, m_CachedDatetime);
 				m_CachedDatetime.push_back(':');
 				spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime);
+				m_CachedDatetime.push_back('.');
 			}
-
-			m_CachedDatetime.push_back('.');
-
-			m_CacheTimestamp = secs;
 		}
 
-		dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
+		OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end());
 
 		auto millis = spdlog::details::fmt_helper::time_fraction<std::chrono::milliseconds>(msg.time);
-		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), dest);
-		dest.push_back(']');
-		dest.push_back(' ');
+		spdlog::details::fmt_helper::pad3(static_cast<uint32_t>(millis.count()), OutBuffer);
+		OutBuffer.push_back(']');
+		OutBuffer.push_back(' ');
 
 		if (!m_LogId.empty())
 		{
-			dest.push_back('[');
-			spdlog::details::fmt_helper::append_string_view(m_LogId, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			OutBuffer.push_back('[');
+			spdlog::details::fmt_helper::append_string_view(m_LogId, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
 		// append logger name if exists
 		if (msg.logger_name.size() > 0)
 		{
-			dest.push_back('[');
-			spdlog::details::fmt_helper::append_string_view(msg.logger_name, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			OutBuffer.push_back('[');
+			spdlog::details::fmt_helper::append_string_view(msg.logger_name, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
-		dest.push_back('[');
+		OutBuffer.push_back('[');
 		// wrap the level name with color
-		msg.color_range_start = dest.size();
-		spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), dest);
-		msg.color_range_end = dest.size();
-		dest.push_back(']');
-		dest.push_back(' ');
+		msg.color_range_start = OutBuffer.size();
+		spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), OutBuffer);
+		msg.color_range_end = OutBuffer.size();
+		OutBuffer.push_back(']');
+		OutBuffer.push_back(' ');
 
 		// add source location if present
 		if (!msg.source.empty())
 		{
-			dest.push_back('[');
+			OutBuffer.push_back('[');
 			const char* filename =
 				spdlog::details::short_filename_formatter<spdlog::details::null_scoped_padder>::basename(msg.source.filename);
-			spdlog::details::fmt_helper::append_string_view(filename, dest);
-			dest.push_back(':');
-			spdlog::details::fmt_helper::append_int(msg.source.line, dest);
-			dest.push_back(']');
-			dest.push_back(' ');
+			spdlog::details::fmt_helper::append_string_view(filename, OutBuffer);
+			OutBuffer.push_back(':');
+			spdlog::details::fmt_helper::append_int(msg.source.line, OutBuffer);
+			OutBuffer.push_back(']');
+			OutBuffer.push_back(' ');
 		}
 
-		spdlog::details::fmt_helper::append_string_view(msg.payload, dest);
-		spdlog::details::fmt_helper::append_string_view("\n"sv, dest);
+		// Handle newlines in single log call by prefixing each additional line to make
+		// subsequent lines align with the first line in the message
+
+		const size_t LinePrefixCount = Min<size_t>(OutBuffer.size(), m_LinePrefix.size());
+
+		auto ItLineBegin  = msg.payload.begin();
+		auto ItMessageEnd = msg.payload.end();
+		bool IsFirstline  = true;
+
+		{
+			auto ItLineEnd = ItLineBegin;
+
+			auto EmitLine = [&] {
+				if (IsFirstline)
+				{
+					IsFirstline = false;
+				}
+				else
+				{
+					spdlog::details::fmt_helper::append_string_view(std::string_view(m_LinePrefix.data(), LinePrefixCount), OutBuffer);
+				}
+				spdlog::details::fmt_helper::append_string_view(spdlog::string_view_t(&*ItLineBegin, ItLineEnd - ItLineBegin), OutBuffer);
+			};
+
+			while (ItLineEnd != ItMessageEnd)
+			{
+				if (*ItLineEnd++ == '\n')
+				{
+					EmitLine();
+					ItLineBegin = ItLineEnd;
+				}
+			}
+
+			if (ItLineBegin != ItMessageEnd)
+			{
+				EmitLine();
+				spdlog::details::fmt_helper::append_string_view("\n"sv, OutBuffer);
+			}
+		}
 	}
 
 private:
 	std::chrono::time_point<std::chrono::system_clock> m_Epoch;
-	std::tm											   m_CachedTm;
+	std::tm											   m_CachedLocalTm;
 	std::chrono::seconds							   m_LastLogSecs;
 	std::chrono::seconds							   m_CacheTimestamp{0};
 	spdlog::memory_buf_t							   m_CachedDatetime;
 	std::string										   m_LogId;
+	std::string										   m_LinePrefix;
+	bool											   m_UseFullDate = true;
 };
 
 }  // namespace zen::logging
diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h
new file mode 100644
index 000000000..339120ece
--- /dev/null
+++ b/src/zenutil/include/zenutil/workerpools.h
@@ -0,0 +1,21 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/workthreadpool.h>
+
+namespace zen {
+
+// Worker pool with std::thread::hardware_concurrency() worker threads
+WorkerThreadPool& GetLargeWorkerPool();
+
+// Worker pool with std::thread::hardware_concurrency() / 4 worker threads
+WorkerThreadPool& GetSmallWorkerPool();
+
+// Special worker pool that does not use worker thread but issues all scheduled work on the calling thread
+// This is useful for debugging when multiple async thread can make stepping in debugger complicated
+WorkerThreadPool& GetSyncWorkerPool();
+
+void ShutdownWorkerPools();
+
+}  // namespace zen
diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h
index 60adfba54..15138341c 100644
--- a/src/zenutil/include/zenutil/zenserverprocess.h
+++ b/src/zenutil/include/zenutil/zenserverprocess.h
@@ -4,6 +4,7 @@
 
 #include <zencore/enumflags.h>
 #include <zencore/logging.h>
+#include <zencore/process.h>
 #include <zencore/thread.h>
 #include <zencore/uid.h>
 
@@ -38,6 +39,7 @@ public:
 	inline bool				IsInitialized() const { return m_IsInitialized; }
 	inline bool				IsTestEnvironment() const { return m_IsTestInstance; }
 	inline std::string_view GetServerClass() const { return m_ServerClass; }
+	inline uint16_t			GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); }
 
 private:
 	std::filesystem::path m_ProgramBaseDir;
@@ -45,6 +47,7 @@ private:
 	bool				  m_IsInitialized  = false;
 	bool				  m_IsTestInstance = false;
 	std::string			  m_ServerClass;
+	std::atomic_uint16_t  m_NextPortNumber{20000};
 };
 
 /** Zen Server Instance management
@@ -63,16 +66,29 @@ struct ZenServerInstance
 
 	void			   Shutdown();
 	void			   SignalShutdown();
-	void			   WaitUntilReady();
+	uint16_t		   WaitUntilReady();
 	[[nodiscard]] bool WaitUntilReady(int Timeout);
 	void			   EnableTermination() { m_Terminate = true; }
 	void			   DisableShutdownOnDestroy() { m_ShutdownOnDestroy = false; }
 	void			   Detach();
 	inline int		   GetPid() { return m_Process.Pid(); }
 	inline void		   SetOwnerPid(int Pid) { m_OwnerPid = Pid; }
+	bool			   IsRunning();
 
-	void		SetTestDir(std::filesystem::path TestDir);
-	inline void SpawnServer(int BasePort = 0, std::string_view AdditionalServerArgs = std::string_view())
+	void SetTestDir(std::filesystem::path TestDir);
+
+	inline void SpawnServer(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 0);
+	}
+
+	inline uint16_t SpawnServerAndWaitUntilReady(std::string_view AdditionalServerArgs = std::string_view())
+	{
+		SpawnServer(m_Env.GetNewPortNumber(), AdditionalServerArgs, /* WaitTimeoutMs */ 100'000);
+		return GetBasePort();
+	}
+
+	inline void SpawnServer(int BasePort, std::string_view AdditionalServerArgs = std::string_view())
 	{
 		SpawnServer(BasePort, AdditionalServerArgs, /* WaitTimeoutMs */ 0);
 	}
@@ -84,6 +100,7 @@ struct ZenServerInstance
 
 	void		AttachToRunningServer(int BasePort = 0);
 	std::string GetBaseUri() const;
+	uint16_t	GetBasePort() const { return m_BasePort; }
 
 private:
 	ZenServerEnvironment& m_Env;
@@ -93,11 +110,13 @@ private:
 	bool				  m_Terminate		  = false;
 	bool				  m_ShutdownOnDestroy = true;
 	std::filesystem::path m_TestDir;
-	int					  m_BasePort = 0;
+	uint16_t			  m_BasePort = 0;
 	std::optional<int>	  m_OwnerPid;
+	std::string			  m_Name;
 
 	void CreateShutdownEvent(int BasePort);
 	void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs);
+	void OnServerReady();
 };
 
 /** Shared system state
diff --git a/src/zenutil/include/zenutil/zenutil.h b/src/zenutil/include/zenutil/zenutil.h
index 14d21ea0d..662743de8 100644
--- a/src/zenutil/include/zenutil/zenutil.h
+++ b/src/zenutil/include/zenutil/zenutil.h
@@ -1,3 +1,9 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #pragma once
+
+namespace zen {
+
+void zenutil_forcelinktests();
+
+}
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index 512c7901c..fedfdc7e8 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenutil/logging/fullformatter.h>
 #include <zenutil/logging/jsonformatter.h>
@@ -152,24 +153,25 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 void
 FinishInitializeLogging(const LoggingOptions& LogOptions)
 {
-	spdlog::level::level_enum LogLevel = spdlog::level::info;
+	logging::level::LogLevel LogLevel = logging::level::Info;
 
 	if (LogOptions.IsDebug)
 	{
-		LogLevel = spdlog::level::debug;
+		LogLevel = logging::level::Debug;
 	}
 
 	if (LogOptions.IsTest)
 	{
-		LogLevel = spdlog::level::trace;
+		LogLevel = logging::level::Trace;
 	}
 
 	// Configure all registered loggers according to settings
 
-	spdlog::set_level(LogLevel);
+	logging::RefreshLogLevels(LogLevel);
 	spdlog::flush_on(spdlog::level::err);
 	spdlog::flush_every(std::chrono::seconds{2});
-	spdlog::set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));
+	spdlog::set_formatter(
+		std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));	 // default to duration prefix
 
 	if (g_FileSink)
 	{
@@ -179,7 +181,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions)
 		}
 		else
 		{
-			g_FileSink->set_pattern("[%C-%m-%d.%e %T] [%n] [%l] %v");
+			g_FileSink->set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId));	 // this will have a date prefix
 		}
 	}
 
diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp
new file mode 100644
index 000000000..3ae302064
--- /dev/null
+++ b/src/zenutil/workerpools.cpp
@@ -0,0 +1,95 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenutil/workerpools.h"
+
+#include <zencore/intmath.h>
+#include <zencore/thread.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <gsl/gsl-lite.hpp>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+namespace {
+	const int LargeWorkerThreadPoolTreadCount = gsl::narrow<int>(std::thread::hardware_concurrency());
+	const int SmallWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 4u), 1u));
+
+	static bool IsShutDown = false;
+
+	RwLock PoolLock;
+
+	std::unique_ptr<WorkerThreadPool> LargeWorkerPool;
+	std::unique_ptr<WorkerThreadPool> SmallWorkerPool;
+	std::unique_ptr<WorkerThreadPool> SyncWorkerPool;
+}  // namespace
+
+WorkerThreadPool&
+GetLargeWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (LargeWorkerPool)
+		{
+			return *LargeWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (LargeWorkerPool)
+	{
+		return *LargeWorkerPool;
+	}
+	LargeWorkerPool.reset(new WorkerThreadPool(LargeWorkerThreadPoolTreadCount, "LargeThreadPool"));
+	return *LargeWorkerPool;
+}
+
+WorkerThreadPool&
+GetSmallWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (SmallWorkerPool)
+		{
+			return *SmallWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (SmallWorkerPool)
+	{
+		return *SmallWorkerPool;
+	}
+	SmallWorkerPool.reset(new WorkerThreadPool(SmallWorkerThreadPoolTreadCount, "SmallThreadPool"));
+	return *SmallWorkerPool;
+}
+
+WorkerThreadPool&
+GetSyncWorkerPool()
+{
+	{
+		RwLock::SharedLockScope _(PoolLock);
+		if (SyncWorkerPool)
+		{
+			return *SyncWorkerPool;
+		}
+	}
+	RwLock::ExclusiveLockScope _(PoolLock);
+	ZEN_ASSERT(!IsShutDown);
+	if (SyncWorkerPool)
+	{
+		return *SyncWorkerPool;
+	}
+	SyncWorkerPool.reset(new WorkerThreadPool(0, "SyncThreadPool"));
+	return *SyncWorkerPool;
+}
+
+void
+ShutdownWorkerPools()
+{
+	RwLock::ExclusiveLockScope _(PoolLock);
+	IsShutDown = true;
+	LargeWorkerPool.reset();
+	SmallWorkerPool.reset();
+	SyncWorkerPool.reset();
+}
+}  // namespace zen
diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp
index 83c6668ba..909692fbc 100644
--- a/src/zenutil/zenserverprocess.cpp
+++ b/src/zenutil/zenserverprocess.cpp
@@ -12,6 +12,8 @@
 
 #include <atomic>
 
+#include <gsl/gsl-lite.hpp>
+
 #if ZEN_PLATFORM_WINDOWS
 #	include <zencore/windows.h>
 #else
@@ -468,7 +470,14 @@ ZenServerInstance::ZenServerInstance(ZenServerEnvironment& TestEnvironment) : m_
 
 ZenServerInstance::~ZenServerInstance()
 {
-	Shutdown();
+	try
+	{
+		Shutdown();
+	}
+	catch (const std::exception& Err)
+	{
+		ZEN_ERROR("Shutting down zenserver instance failed, reason: '{}'", Err.what());
+	}
 }
 
 void
@@ -480,19 +489,33 @@ ZenServerInstance::SignalShutdown()
 void
 ZenServerInstance::Shutdown()
 {
-	if (m_Process.IsValid() && m_ShutdownOnDestroy)
+	if (m_Process.IsValid())
 	{
-		if (m_Terminate)
+		if (m_ShutdownOnDestroy)
 		{
-			ZEN_INFO("Terminating zenserver process");
-			m_Process.Terminate(111);
-			m_Process.Reset();
+			if (m_Terminate)
+			{
+				ZEN_INFO("Terminating zenserver process {}", m_Name);
+				m_Process.Terminate(111);
+				m_Process.Reset();
+				ZEN_DEBUG("zenserver process {} ({}) terminated", m_Name, m_Process.Pid());
+			}
+			else
+			{
+				ZEN_DEBUG("Requesting zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				SignalShutdown();
+				ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid());
+				while (!m_Process.Wait(5000))
+				{
+					ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid());
+				}
+				m_Process.Reset();
+			}
+			ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid());
 		}
 		else
 		{
-			SignalShutdown();
-			m_Process.Wait();
-			m_Process.Reset();
+			ZEN_DEBUG("Detached from zenserver process {} ({})", m_Name, m_Process.Pid());
 		}
 	}
 }
@@ -509,10 +532,9 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	ChildEventName << "Zen_Child_" << ChildId;
 	NamedEvent ChildEvent{ChildEventName};
 
-	CreateShutdownEvent(BasePort);
-
 	ExtendableStringBuilder<32> LogId;
 	LogId << "Zen" << ChildId;
+	m_Name = LogId.ToString();
 
 	ExtendableStringBuilder<512> CommandLine;
 	CommandLine << "zenserver" ZEN_EXE_SUFFIX_LITERAL;	// see CreateProc() call for actual binary path
@@ -526,7 +548,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 			m_OwnerPid = MyPid;
 		}
 
-		CommandLine << " --test --log-id " << LogId;
+		CommandLine << " --test --log-id " << m_Name;
+		CommandLine << " --no-sentry";
 	}
 
 	if (m_OwnerPid.has_value())
@@ -544,7 +567,7 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	if (BasePort)
 	{
 		CommandLine << " --port " << BasePort;
-		m_BasePort = BasePort;
+		m_BasePort = gsl::narrow_cast<uint16_t>(BasePort);
 	}
 
 	if (!m_TestDir.empty())
@@ -604,41 +627,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr
 	{
 		if (!WaitUntilReady(WaitTimeoutMs))
 		{
-			throw std::runtime_error(fmt::format("server start timeout after {}", NiceTimeSpanMs(WaitTimeoutMs)));
+			throw std::runtime_error(fmt::format("server start of {} timeout after {}", m_Name, NiceTimeSpanMs(WaitTimeoutMs)));
 		}
-
-		// Determine effective base port
-
-		ZenServerState State;
-		if (!State.InitializeReadOnly())
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no zen state found");
-		}
-
-		const ZenServerState::ZenServerEntry* Entry = nullptr;
-
-		if (BasePort)
-		{
-			Entry = State.Lookup(BasePort);
-		}
-		else
-		{
-			State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
-				if (InEntry.Pid == static_cast<uint32_t>(GetProcessId(ChildPid)))
-				{
-					Entry = &InEntry;
-				}
-			});
-		}
-
-		if (!Entry)
-		{
-			// TODO: return success/error code instead?
-			throw std::runtime_error("no server entry found");
-		}
-
-		m_BasePort = Entry->EffectiveListenPort;
 	}
 }
 
@@ -694,23 +684,73 @@ ZenServerInstance::Detach()
 	}
 }
 
-void
+uint16_t
 ZenServerInstance::WaitUntilReady()
 {
 	while (m_ReadyEvent.Wait(100) == false)
 	{
 		if (!m_Process.IsRunning() || !m_Process.IsValid())
 		{
-			ZEN_INFO("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
-			return;
+			ZEN_WARN("Wait abandoned by invalid process (running={})", m_Process.IsRunning());
+
+			return 0;
 		}
 	}
+
+	OnServerReady();
+
+	return m_BasePort;
 }
 
 bool
 ZenServerInstance::WaitUntilReady(int Timeout)
 {
-	return m_ReadyEvent.Wait(Timeout);
+	if (m_ReadyEvent.Wait(Timeout))
+	{
+		OnServerReady();
+
+		return true;
+	}
+
+	return false;
+}
+
+void
+ZenServerInstance::OnServerReady()
+{
+	// Determine effective base port
+
+	ZenServerState State;
+	if (!State.InitializeReadOnly())
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no zen state found");
+	}
+
+	const ZenServerState::ZenServerEntry* Entry = nullptr;
+
+	if (m_BasePort)
+	{
+		Entry = State.Lookup(m_BasePort);
+	}
+	else
+	{
+		State.Snapshot([&](const ZenServerState::ZenServerEntry& InEntry) {
+			if (InEntry.Pid == (uint32_t)m_Process.Pid())
+			{
+				Entry = &InEntry;
+			}
+		});
+	}
+
+	if (!Entry)
+	{
+		// TODO: return success/error code instead?
+		throw std::runtime_error("no server entry found");
+	}
+
+	m_BasePort = Entry->EffectiveListenPort;
+	CreateShutdownEvent(m_BasePort);
 }
 
 std::string
@@ -728,4 +768,14 @@ ZenServerInstance::SetTestDir(std::filesystem::path TestDir)
 	m_TestDir = TestDir;
 }
 
+bool
+ZenServerInstance::IsRunning()
+{
+	if (!m_Process.IsValid())
+	{
+		return false;
+	}
+	return m_Process.IsRunning();
+}
+
 }  // namespace zen
diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp
new file mode 100644
index 000000000..df075ea3f
--- /dev/null
+++ b/src/zenutil/zenutil.cpp
@@ -0,0 +1,19 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "zenutil/zenutil.h"
+
+#if ZEN_WITH_TESTS
+
+#	include <zenutil/basicfile.h>
+
+namespace zen {
+
+void
+zenutil_forcelinktests()
+{
+	basicfile_forcelink();
+}
+
+}  // namespace zen
+
+#endif