Merge remote-tracking branch 'origin/main' into 273-integrated-memory-tracking

13 files changed, 400 insertions, 206 deletions

diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h
index d14d1ab8d..8b76d754c 100644
--- a/src/zencore/include/zencore/logging.h
+++ b/src/zencore/include/zencore/logging.h
@@ -35,6 +35,10 @@ LoggerRef ErrorLog();
 void	  SetErrorLog(std::string_view LoggerId);
 LoggerRef Get(std::string_view Name);
 
+void ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers);
+void RefreshLogLevels();
+void RefreshLogLevels(level::LogLevel DefaultLevel);
+
 struct LogCategory
 {
 	inline LogCategory(std::string_view InCategory) : CategoryName(InCategory) {}
diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp
index 025ed4262..0bf07affd 100644
--- a/src/zencore/logging.cpp
+++ b/src/zencore/logging.cpp
@@ -4,10 +4,14 @@
 
 #include <zencore/string.h>
 #include <zencore/testing.h>
+#include <zencore/thread.h>
 
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <spdlog/details/registry.h>
 #include <spdlog/sinks/null_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 #include <spdlog/spdlog.h>
+ZEN_THIRD_PARTY_INCLUDES_END
 
 #if ZEN_PLATFORM_WINDOWS
 #	pragma section(".zlog$a", read)
@@ -46,6 +50,8 @@ LoggingContext::~LoggingContext()
 {
 }
 
+//////////////////////////////////////////////////////////////////////////
+
 static inline bool
 IsErrorLevel(int LogLevel)
 {
@@ -176,8 +182,77 @@ ToStringView(level::LogLevel Level)
 
 }  // namespace zen::logging::level
 
+//////////////////////////////////////////////////////////////////////////
+
 namespace zen::logging {
 
+RwLock		LogLevelsLock;
+std::string LogLevels[level::LogLevelCount];
+
+void
+ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers)
+{
+	RwLock::ExclusiveLockScope _(LogLevelsLock);
+	LogLevels[Level] = Loggers;
+}
+
+void
+RefreshLogLevels(level::LogLevel* DefaultLevel)
+{
+	spdlog::details::registry::log_levels Levels;
+
+	{
+		RwLock::SharedLockScope _(LogLevelsLock);
+
+		for (int i = 0; i < level::LogLevelCount; ++i)
+		{
+			level::LogLevel CurrentLevel{i};
+
+			std::string_view Spec = LogLevels[i];
+
+			while (!Spec.empty())
+			{
+				std::string LoggerName;
+
+				if (auto CommaPos = Spec.find_first_of(','); CommaPos != std::string_view::npos)
+				{
+					LoggerName = Spec.substr(CommaPos + 1);
+					Spec.remove_prefix(CommaPos + 1);
+				}
+				else
+				{
+					LoggerName = Spec;
+					Spec	   = {};
+				}
+
+				Levels[LoggerName] = to_spdlog_level(CurrentLevel);
+			}
+		}
+	}
+
+	if (DefaultLevel)
+	{
+		spdlog::level::level_enum SpdDefaultLevel = to_spdlog_level(*DefaultLevel);
+		spdlog::details::registry::instance().set_levels(Levels, &SpdDefaultLevel);
+	}
+	else
+	{
+		spdlog::details::registry::instance().set_levels(Levels, nullptr);
+	}
+}
+
+void
+RefreshLogLevels(level::LogLevel DefaultLevel)
+{
+	RefreshLogLevels(&DefaultLevel);
+}
+
+void
+RefreshLogLevels()
+{
+	RefreshLogLevels(nullptr);
+}
+
 void
 SetLogLevel(level::LogLevel NewLogLevel)
 {
@@ -240,6 +315,7 @@ Get(std::string_view Name)
 	if (!Logger)
 	{
 		Logger = Default().SpdLogger->clone(std::string(Name));
+		spdlog::apply_logger_env_levels(Logger);
 		spdlog::register_logger(Logger);
 	}
 
@@ -262,6 +338,7 @@ ConsoleLog()
 		if (!ConLogger)
 		{
 			ConLogger = spdlog::stdout_color_mt("console");
+			spdlog::apply_logger_env_levels(ConLogger);
 
 			ConLogger->set_pattern("%v");
 		}
diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index a4a37b0af..13f3c9e58 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -246,7 +246,8 @@ public:
 		return OutVersion == CurrentDiskBucketVersion;
 	}
 
-	void ParseManifest(ZenCacheDiskLayer::CacheBucket&							   Bucket,
+	void ParseManifest(RwLock::ExclusiveLockScope&								   BucketLock,
+					   ZenCacheDiskLayer::CacheBucket&							   Bucket,
 					   std::filesystem::path									   ManifestPath,
 					   ZenCacheDiskLayer::CacheBucket::IndexMap&				   Index,
 					   std::vector<AccessTime>&									   AccessTimes,
@@ -256,13 +257,15 @@ public:
 
 	IoBuffer MakeSidecarManifest(const Oid& BucketId, uint64_t EntryCount);
 	uint64_t GetSidecarSize() const { return m_ManifestEntryCount * sizeof(ManifestData); }
-	void	 WriteSidecarFile(const std::filesystem::path&										 SidecarPath,
+	void	 WriteSidecarFile(RwLock::SharedLockScope&											 BucketLock,
+							  const std::filesystem::path&										 SidecarPath,
 							  uint64_t															 SnapshotLogPosition,
 							  const ZenCacheDiskLayer::CacheBucket::IndexMap&					 Index,
 							  const std::vector<AccessTime>&									 AccessTimes,
 							  const std::vector<ZenCacheDiskLayer::CacheBucket::BucketPayload>&	 Payloads,
 							  const std::vector<ZenCacheDiskLayer::CacheBucket::BucketMetaData>& MetaDatas);
-	bool	 ReadSidecarFile(ZenCacheDiskLayer::CacheBucket&							 Bucket,
+	bool	 ReadSidecarFile(RwLock::ExclusiveLockScope&								 BucketLock,
+							 ZenCacheDiskLayer::CacheBucket&							 Bucket,
 							 std::filesystem::path										 SidecarPath,
 							 ZenCacheDiskLayer::CacheBucket::IndexMap&					 Index,
 							 std::vector<AccessTime>&									 AccessTimes,
@@ -309,7 +312,8 @@ private:
 };
 
 void
-BucketManifestSerializer::ParseManifest(ZenCacheDiskLayer::CacheBucket&								Bucket,
+BucketManifestSerializer::ParseManifest(RwLock::ExclusiveLockScope&									BucketLock,
+										ZenCacheDiskLayer::CacheBucket&								Bucket,
 										std::filesystem::path										ManifestPath,
 										ZenCacheDiskLayer::CacheBucket::IndexMap&					Index,
 										std::vector<AccessTime>&									AccessTimes,
@@ -317,7 +321,7 @@ BucketManifestSerializer::ParseManifest(ZenCacheDiskLayer::CacheBucket&								B
 {
 	if (Manifest["UsingMetaFile"sv].AsBool())
 	{
-		ReadSidecarFile(Bucket, GetMetaPath(Bucket.m_BucketDir, Bucket.m_BucketName), Index, AccessTimes, Payloads);
+		ReadSidecarFile(BucketLock, Bucket, GetMetaPath(Bucket.m_BucketDir, Bucket.m_BucketName), Index, AccessTimes, Payloads);
 
 		return;
 	}
@@ -373,7 +377,7 @@ BucketManifestSerializer::ParseManifest(ZenCacheDiskLayer::CacheBucket&								B
 				if (RawSize != 0 || RawHash != IoHash::Zero)
 				{
 					BucketPayload& Payload = Payloads[KeyIndex];
-					Bucket.SetMetaData(Payload, BucketMetaData{.RawSize = RawSize, .RawHash = RawHash});
+					Bucket.SetMetaData(BucketLock, Payload, BucketMetaData{.RawSize = RawSize, .RawHash = RawHash});
 				}
 			}
 
@@ -496,7 +500,8 @@ BucketManifestSerializer::MakeSidecarManifest(const Oid& BucketId, uint64_t Entr
 }
 
 bool
-BucketManifestSerializer::ReadSidecarFile(ZenCacheDiskLayer::CacheBucket&							  Bucket,
+BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope&								  BucketLock,
+										  ZenCacheDiskLayer::CacheBucket&							  Bucket,
 										  std::filesystem::path										  SidecarPath,
 										  ZenCacheDiskLayer::CacheBucket::IndexMap&					  Index,
 										  std::vector<AccessTime>&									  AccessTimes,
@@ -567,7 +572,7 @@ BucketManifestSerializer::ReadSidecarFile(ZenCacheDiskLayer::CacheBucket&
 
 			if (Entry->RawSize && Entry->RawHash != IoHash::Zero)
 			{
-				Bucket.SetMetaData(PayloadEntry, BucketMetaData{.RawSize = Entry->RawSize, .RawHash = Entry->RawHash});
+				Bucket.SetMetaData(BucketLock, PayloadEntry, BucketMetaData{.RawSize = Entry->RawSize, .RawHash = Entry->RawHash});
 			}
 		}
 
@@ -580,7 +585,8 @@ BucketManifestSerializer::ReadSidecarFile(ZenCacheDiskLayer::CacheBucket&
 }
 
 void
-BucketManifestSerializer::WriteSidecarFile(const std::filesystem::path&										  SidecarPath,
+BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&,
+										   const std::filesystem::path&										  SidecarPath,
 										   uint64_t															  SnapshotLogPosition,
 										   const ZenCacheDiskLayer::CacheBucket::IndexMap&					  Index,
 										   const std::vector<AccessTime>&									  AccessTimes,
@@ -696,6 +702,10 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 	using namespace std::literals;
 
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::OpenOrCreate");
+	ZEN_ASSERT(m_IsFlushing.load());
+
+	// We want to take the lock here since we register as a GC referencer a construction
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 
 	ZEN_LOG_SCOPE("opening cache bucket '{}'", BucketDir);
 
@@ -738,20 +748,25 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo
 		return false;
 	}
 
-	InitializeIndexFromDisk(IsNew);
+	InitializeIndexFromDisk(IndexLock, IsNew);
+
+	auto _ = MakeGuard([&]() {
+		// We are now initialized, allow flushing when we exit
+		m_IsFlushing.store(false);
+	});
 
 	if (IsNew)
 	{
 		return true;
 	}
 
-	ManifestReader.ParseManifest(*this, ManifestPath, m_Index, m_AccessTimes, m_Payloads);
+	ManifestReader.ParseManifest(IndexLock, *this, ManifestPath, m_Index, m_AccessTimes, m_Payloads);
 
 	return true;
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc)
+ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(const std::function<uint64_t()>& ClaimDiskReserveFunc)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::WriteIndexSnapshot");
 
@@ -861,7 +876,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshot(const std::function<uint64_t(
 }
 
 uint64_t
-ZenCacheDiskLayer::CacheBucket::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion)
+ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const std::filesystem::path& IndexPath, uint32_t& OutVersion)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::ReadIndexFile");
 
@@ -950,7 +965,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(const std::filesystem::path& Index
 }
 
 uint64_t
-ZenCacheDiskLayer::CacheBucket::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
+ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::ReadLog");
 
@@ -1020,7 +1035,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(const std::filesystem::path& LogPath, ui
 };
 
 void
-ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(const bool IsNew)
+ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockScope& IndexLock, const bool IsNew)
 {
 	ZEN_TRACE_CPU("Z$::Disk::Bucket::OpenLog");
 
@@ -1055,7 +1070,7 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(const bool IsNew)
 	if (std::filesystem::is_regular_file(IndexPath))
 	{
 		uint32_t IndexVersion = 0;
-		m_LogFlushPosition	  = ReadIndexFile(IndexPath, IndexVersion);
+		m_LogFlushPosition	  = ReadIndexFile(IndexLock, IndexPath, IndexVersion);
 		if (IndexVersion == 0)
 		{
 			ZEN_WARN("removing invalid index file at '{}'", IndexPath);
@@ -1068,7 +1083,7 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(const bool IsNew)
 	{
 		if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath))
 		{
-			LogEntryCount = ReadLog(LogPath, m_LogFlushPosition);
+			LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition);
 		}
 		else if (fs::is_regular_file(LogPath))
 		{
@@ -1100,7 +1115,7 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(const bool IsNew)
 
 	if (IsNew || LogEntryCount > 0)
 	{
-		WriteIndexSnapshot();
+		WriteIndexSnapshot(IndexLock);
 	}
 }
 
@@ -1218,14 +1233,14 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 				{
 					ZEN_TRACE_CPU("Z$::Disk::Bucket::Get::MemCache");
 					OutValue.Value = IoBufferBuilder::ReadFromFileMaybe(OutValue.Value);
-					RwLock::ExclusiveLockScope _(m_IndexLock);
+					RwLock::ExclusiveLockScope UpdateIndexLock(m_IndexLock);
 					if (auto UpdateIt = m_Index.find(HashKey); UpdateIt != m_Index.end())
 					{
-						BucketPayload& WritePayload = m_Payloads[EntryIndex];
+						BucketPayload& WritePayload = m_Payloads[UpdateIt->second];
 						// Only update if it has not already been updated by other thread
 						if (!WritePayload.MemCached)
 						{
-							SetMemCachedData(WritePayload, OutValue.Value);
+							SetMemCachedData(UpdateIndexLock, WritePayload, OutValue.Value);
 						}
 					}
 				}
@@ -1250,7 +1265,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			OutValue.RawHash = IoHash::HashBuffer(OutValue.Value);
 			OutValue.RawSize = OutValue.Value.GetSize();
 		}
-		RwLock::ExclusiveLockScope __(m_IndexLock);
+		RwLock::ExclusiveLockScope UpdateIndexLock(m_IndexLock);
 		if (auto WriteIt = m_Index.find(HashKey); WriteIt != m_Index.end())
 		{
 			BucketPayload& WritePayload = m_Payloads[WriteIt.value()];
@@ -1258,7 +1273,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 			// Only set if no other path has already updated the meta data
 			if (!WritePayload.MetaData)
 			{
-				SetMetaData(WritePayload, {.RawSize = OutValue.RawSize, .RawHash = OutValue.RawHash});
+				SetMetaData(UpdateIndexLock, WritePayload, {.RawSize = OutValue.RawSize, .RawHash = OutValue.RawHash});
 			}
 		}
 	}
@@ -1297,7 +1312,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
 {
 	GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
-	RwLock::ExclusiveLockScope _(m_IndexLock);
+	RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 	if (m_MemCachedPayloads.empty())
 	{
 		return;
@@ -1312,7 +1327,7 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
 		}
 		if (m_AccessTimes[Index] < ExpireTicks)
 		{
-			RemoveMemCachedData(Payload);
+			RemoveMemCachedData(IndexLock, Payload);
 		}
 	}
 	m_MemCachedPayloads.shrink_to_fit();
@@ -1434,7 +1449,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 
 			{
 				RwLock::SharedLockScope IndexLock(m_IndexLock);
-				WriteIndexSnapshot();
+				WriteIndexSnapshot(IndexLock);
 				// Note: this copy could be eliminated on shutdown to
 				// reduce memory usage and execution time
 				Index		= m_Index;
@@ -1474,7 +1489,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 		else
 		{
 			RwLock::SharedLockScope IndexLock(m_IndexLock);
-			WriteIndexSnapshot();
+			WriteIndexSnapshot(IndexLock);
 			const uint64_t EntryCount = m_Index.size();
 			Buffer					  = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount);
 			uint64_t SidecarSize	  = ManifestWriter.GetSidecarSize();
@@ -1502,7 +1517,8 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function<uint64_t()>& Cl
 				return;
 			}
 
-			ManifestWriter.WriteSidecarFile(GetMetaPath(m_BucketDir, m_BucketName),
+			ManifestWriter.WriteSidecarFile(IndexLock,
+											GetMetaPath(m_BucketDir, m_BucketName),
 											m_LogFlushPosition,
 											m_Index,
 											m_AccessTimes,
@@ -1776,8 +1792,8 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 						m_StandaloneSize.fetch_sub(Location.Size(), std::memory_order::relaxed);
 					}
 
-					RemoveMemCachedData(Payload);
-					RemoveMetaData(Payload);
+					RemoveMemCachedData(IndexLock, Payload);
+					RemoveMetaData(IndexLock, Payload);
 
 					Location.Flags |= DiskLocation::kTombStone;
 					LogEntries.push_back(DiskIndexEntry{.Key = BadKey, .Location = Location});
@@ -1813,7 +1829,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx)
 
 				{
 					RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-					CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+					CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
 				}
 			}
 		}
@@ -1875,6 +1891,10 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 			WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 		});
 #endif	// CALCULATE_BLOCKING_TIME
+		if (m_Index.empty())
+		{
+			return;
+		}
 		Index				= m_Index;
 		AccessTimes			= m_AccessTimes;
 		Payloads			= m_Payloads;
@@ -1954,10 +1974,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	for (const auto& Entry : StructuredItemsWithUnknownAttachments)
 	{
-		const IoHash&		Key			 = Entry.first;
-		size_t				PayloadIndex = Entry.second;
-		BucketPayload&		Payload		 = Payloads[PayloadIndex];
-		const DiskLocation& Loc			 = Payload.Location;
+		const IoHash&		Key		= Entry.first;
+		BucketPayload&		Payload = Payloads[Entry.second];
+		const DiskLocation& Loc		= Payload.Location;
 		{
 			IoBuffer Buffer;
 			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
@@ -1980,7 +1999,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 #endif	// CALCULATE_BLOCKING_TIME
 				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
-					const BucketPayload& CachedPayload = Payloads[PayloadIndex];
+					const BucketPayload& CachedPayload = Payloads[It->second];
 					if (CachedPayload.MemCached)
 					{
 						Buffer = m_MemCachedPayloads[CachedPayload.MemCached];
@@ -2098,8 +2117,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		}
 	});
 
-	m_SlogFile.Flush();
-
 	auto __ = MakeGuard([&]() {
 		if (!DeletedChunks.empty())
 		{
@@ -2118,13 +2135,20 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					WriteBlockTimeUs += ElapsedUs;
 					WriteBlockLongestTimeUs = std::max(ElapsedUs, WriteBlockLongestTimeUs);
 				});
-				CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+				CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
 			}
 			GcCtx.AddDeletedCids(std::vector<IoHash>(DeletedChunks.begin(), DeletedChunks.end()));
 		}
 	});
 
-	std::span<const IoHash>					   ExpiredCacheKeySpan = GcCtx.ExpiredCacheKeys(m_BucketDir.string());
+	std::span<const IoHash> ExpiredCacheKeySpan = GcCtx.ExpiredCacheKeys(m_BucketDir.string());
+	if (ExpiredCacheKeySpan.empty())
+	{
+		return;
+	}
+
+	m_SlogFile.Flush();
+
 	std::unordered_set<IoHash, IoHash::Hasher> ExpiredCacheKeys(ExpiredCacheKeySpan.begin(), ExpiredCacheKeySpan.end());
 
 	std::vector<DiskIndexEntry>		 ExpiredStandaloneEntries;
@@ -2292,7 +2316,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			std::vector<DiskIndexEntry> LogEntries;
 			LogEntries.reserve(MovedChunks.size() + RemovedChunks.size());
 			{
-				RwLock::ExclusiveLockScope __(m_IndexLock);
+				RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
 				Stopwatch				   Timer;
 				const auto				   ____ = MakeGuard([&] {
 					uint64_t ElapsedUs = Timer.GetElapsedTimeUs();
@@ -2330,8 +2354,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 																   m_Configuration.PayloadAlignment,
 																   OldDiskLocation.GetFlags() | DiskLocation::kTombStone)});
 
-					RemoveMemCachedData(Payload);
-					RemoveMetaData(Payload);
+					RemoveMemCachedData(IndexLock, Payload);
+					RemoveMetaData(IndexLock, Payload);
 
 					m_Index.erase(ChunkHash);
 					DeletedChunks.insert(ChunkHash);
@@ -2368,7 +2392,7 @@ ZenCacheDiskLayer::CacheBucket::EntryCount() const
 }
 
 CacheValueDetails::ValueDetails
-ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex Index) const
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const IoHash& Key, PayloadIndex Index) const
 {
 	std::vector<IoHash>	 Attachments;
 	const BucketPayload& Payload = m_Payloads[Index];
@@ -2380,7 +2404,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex
 		CbObjectView Obj(Value.GetData());
 		Obj.IterateAttachments([&Attachments](CbFieldView Field) { Attachments.emplace_back(Field.AsAttachment()); });
 	}
-	BucketMetaData MetaData = GetMetaData(Payload);
+	BucketMetaData MetaData = GetMetaData(IndexLock, Payload);
 	return CacheValueDetails::ValueDetails{.Size		= Payload.Location.Size(),
 										   .RawSize		= MetaData.RawSize,
 										   .RawHash		= MetaData.RawHash,
@@ -2390,7 +2414,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const IoHash& Key, PayloadIndex
 }
 
 CacheValueDetails::BucketDetails
-ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilter) const
+ZenCacheDiskLayer::CacheBucket::GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const
 {
 	CacheValueDetails::BucketDetails Details;
 	RwLock::SharedLockScope			 _(m_IndexLock);
@@ -2399,7 +2423,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
 		Details.Values.reserve(m_Index.size());
 		for (const auto& It : m_Index)
 		{
-			Details.Values.insert_or_assign(It.first, GetValueDetails(It.first, It.second));
+			Details.Values.insert_or_assign(It.first, GetValueDetails(IndexLock, It.first, It.second));
 		}
 	}
 	else
@@ -2407,7 +2431,7 @@ ZenCacheDiskLayer::CacheBucket::GetValueDetails(const std::string_view ValueFilt
 		IoHash Key = IoHash::FromHexString(ValueFilter);
 		if (auto It = m_Index.find(Key); It != m_Index.end())
 		{
-			Details.Values.insert_or_assign(It->first, GetValueDetails(It->first, It->second));
+			Details.Values.insert_or_assign(It->first, GetValueDetails(IndexLock, It->first, It->second));
 		}
 	}
 	return Details;
@@ -2417,10 +2441,10 @@ void
 ZenCacheDiskLayer::CacheBucket::EnumerateBucketContents(
 	std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const
 {
-	RwLock::SharedLockScope _(m_IndexLock);
+	RwLock::SharedLockScope IndexLock(m_IndexLock);
 	for (const auto& It : m_Index)
 	{
-		CacheValueDetails::ValueDetails Vd = GetValueDetails(It.first, It.second);
+		CacheValueDetails::ValueDetails Vd = GetValueDetails(IndexLock, It.first, It.second);
 
 		Fn(It.first, Vd);
 	}
@@ -2595,16 +2619,16 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 			SetReferences(IndexLock, m_FirstReferenceIndex[EntryIndex], References);
 		}
 		m_AccessTimes[EntryIndex] = GcClock::TickCount();
-		RemoveMemCachedData(Payload);
+		RemoveMemCachedData(IndexLock, Payload);
 		m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed);
 	}
 	if (Value.RawSize != 0 || Value.RawHash != IoHash::Zero)
 	{
-		SetMetaData(m_Payloads[EntryIndex], {.RawSize = Value.RawSize, .RawHash = Value.RawHash});
+		SetMetaData(IndexLock, m_Payloads[EntryIndex], {.RawSize = Value.RawSize, .RawHash = Value.RawHash});
 	}
 	else
 	{
-		RemoveMetaData(m_Payloads[EntryIndex]);
+		RemoveMetaData(IndexLock, m_Payloads[EntryIndex]);
 	}
 
 	m_SlogFile.Append({.Key = HashKey, .Location = Loc});
@@ -2612,7 +2636,9 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::SetMetaData(BucketPayload& Payload, const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData)
+ZenCacheDiskLayer::CacheBucket::SetMetaData(RwLock::ExclusiveLockScope&,
+											BucketPayload&										  Payload,
+											const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData)
 {
 	if (Payload.MetaData)
 	{
@@ -2635,7 +2661,7 @@ ZenCacheDiskLayer::CacheBucket::SetMetaData(BucketPayload& Payload, const ZenCac
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::RemoveMetaData(BucketPayload& Payload)
+ZenCacheDiskLayer::CacheBucket::RemoveMetaData(RwLock::ExclusiveLockScope&, BucketPayload& Payload)
 {
 	if (Payload.MetaData)
 	{
@@ -2645,7 +2671,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveMetaData(BucketPayload& Payload)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffer& MemCachedData)
+ZenCacheDiskLayer::CacheBucket::SetMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload, IoBuffer& MemCachedData)
 {
 	uint64_t PayloadSize = MemCachedData.GetSize();
 	ZEN_ASSERT(PayloadSize != 0);
@@ -2670,7 +2696,7 @@ ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffe
 }
 
 size_t
-ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(BucketPayload& Payload)
+ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload)
 {
 	if (Payload.MemCached)
 	{
@@ -2685,7 +2711,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(BucketPayload& Payload)
 }
 
 ZenCacheDiskLayer::CacheBucket::BucketMetaData
-ZenCacheDiskLayer::CacheBucket::GetMetaData(const BucketPayload& Payload) const
+ZenCacheDiskLayer::CacheBucket::GetMetaData(RwLock::SharedLockScope&, const BucketPayload& Payload) const
 {
 	if (Payload.MetaData)
 	{
@@ -2728,8 +2754,8 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 									ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size()));
 									BucketPayload& Payload = m_Payloads[EntryIndex];
 
-									RemoveMemCachedData(Payload);
-									RemoveMetaData(Payload);
+									RemoveMemCachedData(IndexLock, Payload);
+									RemoveMetaData(IndexLock, Payload);
 
 									Payload					  = (BucketPayload{.Location = Location});
 									m_AccessTimes[EntryIndex] = GcClock::TickCount();
@@ -3027,6 +3053,10 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		{
 			return nullptr;
 		}
+		if (m_Index.empty())
+		{
+			return nullptr;
+		}
 
 		TotalEntries = m_Index.size();
 
@@ -3072,8 +3102,8 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 				auto It = m_Index.find(Entry.Key);
 				ZEN_ASSERT(It != m_Index.end());
 				BucketPayload& Payload = m_Payloads[It->second];
-				RemoveMetaData(Payload);
-				Stats.FreedMemory += RemoveMemCachedData(Payload);
+				RemoveMetaData(IndexLock, Payload);
+				Stats.FreedMemory += RemoveMemCachedData(IndexLock, Payload);
 				m_Index.erase(It);
 				Stats.DeletedCount++;
 			}
@@ -3092,7 +3122,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats)
 		IndexMap					Index;
 		{
 			RwLock::ExclusiveLockScope IndexLock(m_IndexLock);
-			CompactState(Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
+			CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, FirstReferenceIndex, Index, IndexLock);
 		}
 	}
 
@@ -3486,6 +3516,14 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx)
 		ZEN_INFO("GCV2: cachebucket [CREATE CHECKERS] '{}': completed in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	});
 
+	{
+		RwLock::SharedLockScope __(m_IndexLock);
+		if (m_Index.empty())
+		{
+			return {};
+		}
+	}
+
 	return {new DiskBucketReferenceChecker(*this)};
 }
 
@@ -3666,7 +3704,8 @@ ZenCacheDiskLayer::CacheBucket::ClearReferenceCache()
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloads,
+ZenCacheDiskLayer::CacheBucket::CompactState(RwLock::ExclusiveLockScope&,
+											 std::vector<BucketPayload>&  Payloads,
 											 std::vector<AccessTime>&	  AccessTimes,
 											 std::vector<BucketMetaData>& MetaDatas,
 											 std::vector<IoBuffer>&		  MemCachedPayloads,
@@ -3749,124 +3788,99 @@ ZenCacheDiskLayer::ZenCacheDiskLayer(GcManager& Gc, JobQueue& JobQueue, const st
 
 ZenCacheDiskLayer::~ZenCacheDiskLayer()
 {
-}
-
-bool
-ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
-{
-	ZEN_TRACE_CPU("Z$::Disk::Get");
-
-	const auto	 BucketName = std::string(InBucket);
-	CacheBucket* Bucket		= nullptr;
-
+	try
 	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto It = m_Buckets.find(BucketName);
-
-		if (It != m_Buckets.end())
 		{
-			Bucket = It->second.get();
+			RwLock::ExclusiveLockScope _(m_Lock);
+			for (auto& It : m_Buckets)
+			{
+				m_DroppedBuckets.emplace_back(std::move(It.second));
+			}
+			m_Buckets.clear();
 		}
+		// We destroy the buckets without holding a lock since destructor calls GcManager::RemoveGcReferencer which takes an exclusive lock.
+		// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
+		m_DroppedBuckets.clear();
 	}
-
-	if (Bucket == nullptr)
+	catch (std::exception& Ex)
 	{
-		// Bucket needs to be opened/created
+		ZEN_ERROR("~ZenCacheDiskLayer() failed. Reason: '{}'", Ex.what());
+	}
+}
 
-		RwLock::ExclusiveLockScope _(m_Lock);
+ZenCacheDiskLayer::CacheBucket*
+ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket)
+{
+	ZEN_TRACE_CPU("Z$::Disk::GetOrCreateBucket");
+	const auto BucketName = std::string(InBucket);
 
+	{
+		RwLock::SharedLockScope SharedLock(m_Lock);
 		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 		{
-			Bucket = It->second.get();
-		}
-		else
-		{
-			auto InsertResult =
-				m_Buckets.emplace(BucketName,
-								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
-			Bucket = InsertResult.first->second.get();
-
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName;
-
-			if (!Bucket->OpenOrCreate(BucketPath))
-			{
-				m_Buckets.erase(InsertResult.first);
-				return false;
-			}
+			return It->second.get();
 		}
 	}
 
-	ZEN_ASSERT(Bucket != nullptr);
-	if (Bucket->Get(HashKey, OutValue))
+	// We create the bucket without holding a lock since contructor calls GcManager::AddGcReferencer which takes an exclusive lock.
+	// This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock
+	std::unique_ptr<CacheBucket> Bucket(
+		std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
+
+	RwLock::ExclusiveLockScope Lock(m_Lock);
+	if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
 	{
-		TryMemCacheTrim();
-		return true;
+		return It->second.get();
 	}
-	return false;
-}
-
-void
-ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
-{
-	ZEN_TRACE_CPU("Z$::Disk::Put");
-
-	const auto	 BucketName = std::string(InBucket);
-	CacheBucket* Bucket		= nullptr;
 
+	std::filesystem::path BucketPath = m_RootDir;
+	BucketPath /= BucketName;
+	try
 	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		auto It = m_Buckets.find(BucketName);
-
-		if (It != m_Buckets.end())
+		if (!Bucket->OpenOrCreate(BucketPath))
 		{
-			Bucket = It->second.get();
+			ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
+			return nullptr;
 		}
 	}
-
-	if (Bucket == nullptr)
+	catch (const std::exception& Err)
 	{
-		// New bucket needs to be created
+		ZEN_WARN("Creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
+		throw;
+	}
 
-		RwLock::ExclusiveLockScope _(m_Lock);
+	CacheBucket* Result = Bucket.get();
+	m_Buckets.emplace(BucketName, std::move(Bucket));
 
-		if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end())
-		{
-			Bucket = It->second.get();
-		}
-		else
-		{
-			auto InsertResult =
-				m_Buckets.emplace(BucketName,
-								  std::make_unique<CacheBucket>(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig));
-			Bucket = InsertResult.first->second.get();
+	return Result;
+}
 
-			std::filesystem::path BucketPath = m_RootDir;
-			BucketPath /= BucketName;
+bool
+ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCacheValue& OutValue)
+{
+	ZEN_TRACE_CPU("Z$::Disk::Get");
 
-			try
-			{
-				if (!Bucket->OpenOrCreate(BucketPath))
-				{
-					ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir);
-					m_Buckets.erase(InsertResult.first);
-					return;
-				}
-			}
-			catch (const std::exception& Err)
-			{
-				ZEN_WARN("creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what());
-				throw;
-			}
+	if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr)
+	{
+		if (Bucket->Get(HashKey, OutValue))
+		{
+			TryMemCacheTrim();
+			return true;
 		}
 	}
+	return false;
+}
 
-	ZEN_ASSERT(Bucket != nullptr);
+void
+ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References)
+{
+	ZEN_TRACE_CPU("Z$::Disk::Put");
 
-	Bucket->Put(HashKey, Value, References);
-	TryMemCacheTrim();
+	if (CacheBucket* Bucket = GetOrCreateBucket(InBucket); Bucket != nullptr)
+	{
+		Bucket->Put(HashKey, Value, References);
+		TryMemCacheTrim();
+	}
 }
 
 void
@@ -4027,10 +4041,6 @@ ZenCacheDiskLayer::Flush()
 
 	{
 		RwLock::SharedLockScope __(m_Lock);
-		if (m_Buckets.empty())
-		{
-			return;
-		}
 		Buckets.reserve(m_Buckets.size());
 		for (auto& Kv : m_Buckets)
 		{
@@ -4061,7 +4071,6 @@ void
 ZenCacheDiskLayer::ScrubStorage(ScrubContext& Ctx)
 {
 	RwLock::SharedLockScope _(m_Lock);
-
 	{
 		std::vector<std::future<void>> Results;
 		Results.reserve(m_Buckets.size());
@@ -4182,20 +4191,22 @@ ZenCacheDiskLayer::EnumerateBucketContents(std::string_view
 CacheValueDetails::NamespaceDetails
 ZenCacheDiskLayer::GetValueDetails(const std::string_view BucketFilter, const std::string_view ValueFilter) const
 {
-	RwLock::SharedLockScope				_(m_Lock);
 	CacheValueDetails::NamespaceDetails Details;
-	if (BucketFilter.empty())
 	{
-		Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
-		for (auto& Kv : m_Buckets)
+		RwLock::SharedLockScope IndexLock(m_Lock);
+		if (BucketFilter.empty())
+		{
+			Details.Buckets.reserve(BucketFilter.empty() ? m_Buckets.size() : 1);
+			for (auto& Kv : m_Buckets)
+			{
+				Details.Buckets[Kv.first] = Kv.second->GetValueDetails(IndexLock, ValueFilter);
+			}
+		}
+		else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
 		{
-			Details.Buckets[Kv.first] = Kv.second->GetValueDetails(ValueFilter);
+			Details.Buckets[It->first] = It->second->GetValueDetails(IndexLock, ValueFilter);
 		}
 	}
-	else if (auto It = m_Buckets.find(std::string(BucketFilter)); It != m_Buckets.end())
-	{
-		Details.Buckets[It->first] = It->second->GetValueDetails(ValueFilter);
-	}
 	return Details;
 }
 
diff --git a/src/zenserver/cache/cachedisklayer.h b/src/zenserver/cache/cachedisklayer.h
index 11f474d5a..277371f2c 100644
--- a/src/zenserver/cache/cachedisklayer.h
+++ b/src/zenserver/cache/cachedisklayer.h
@@ -215,7 +215,7 @@ public:
 		uint64_t	EntryCount() const;
 		BucketStats Stats();
 
-		CacheValueDetails::BucketDetails GetValueDetails(const std::string_view ValueFilter) const;
+		CacheValueDetails::BucketDetails GetValueDetails(RwLock::SharedLockScope& IndexLock, const std::string_view ValueFilter) const;
 		void EnumerateBucketContents(std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>& Fn) const;
 
 		void GetUsageByAccess(GcClock::TimePoint TickStart, GcClock::Duration SectionLength, std::vector<uint64_t>& InOutUsageSlots);
@@ -301,7 +301,7 @@ public:
 		BucketConfiguration	  m_Configuration;
 		BlockStore			  m_BlockStore;
 		Oid					  m_BucketId;
-		std::atomic_bool	  m_IsFlushing{};
+		std::atomic_bool	  m_IsFlushing{true};  // Don't allow flush until we are properly initialized
 
 		// These files are used to manage storage of small objects for this bucket
 
@@ -342,7 +342,7 @@ public:
 		IoBuffer GetStandaloneCacheValue(ZenContentType ContentType, const IoHash& HashKey) const;
 		void	 PutInlineCacheValue(const IoHash& HashKey, const ZenCacheValue& Value, std::span<IoHash> References);
 		IoBuffer GetInlineCacheValue(const DiskLocation& Loc) const;
-		CacheValueDetails::ValueDetails GetValueDetails(const IoHash& Key, PayloadIndex Index) const;
+		CacheValueDetails::ValueDetails GetValueDetails(RwLock::SharedLockScope&, const IoHash& Key, PayloadIndex Index) const;
 
 		void		CompactReferences(RwLock::ExclusiveLockScope&);
 		void		SetReferences(RwLock::ExclusiveLockScope&, ReferenceIndex& FirstReferenceIndex, std::span<IoHash> References);
@@ -359,20 +359,35 @@ public:
 		bool		   LockedGetReferences(ReferenceIndex FirstReferenceIndex, std::vector<IoHash>& OutReferences) const;
 		void		   ClearReferenceCache();
 
-		void		   SetMetaData(BucketPayload& Payload, const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
-		void		   RemoveMetaData(BucketPayload& Payload);
-		BucketMetaData GetMetaData(const BucketPayload& Payload) const;
-		void		   SetMemCachedData(BucketPayload& Payload, IoBuffer& MemCachedData);
-		size_t		   RemoveMemCachedData(BucketPayload& Payload);
+		void		   SetMetaData(RwLock::ExclusiveLockScope&,
+								   BucketPayload&										 Payload,
+								   const ZenCacheDiskLayer::CacheBucket::BucketMetaData& MetaData);
+		void		   RemoveMetaData(RwLock::ExclusiveLockScope&, BucketPayload& Payload);
+		BucketMetaData GetMetaData(RwLock::SharedLockScope&, const BucketPayload& Payload) const;
+		void		   SetMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload, IoBuffer& MemCachedData);
+		size_t		   RemoveMemCachedData(RwLock::ExclusiveLockScope&, BucketPayload& Payload);
 
-		void	 InitializeIndexFromDisk(bool IsNew);
-		uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion);
-		uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition);
+		void	 InitializeIndexFromDisk(RwLock::ExclusiveLockScope&, bool IsNew);
+		uint64_t ReadIndexFile(RwLock::ExclusiveLockScope&, const std::filesystem::path& IndexPath, uint32_t& OutVersion);
+		uint64_t ReadLog(RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t LogPosition);
 
 		void SaveSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
-		void WriteIndexSnapshot(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
+		void WriteIndexSnapshot(
+			RwLock::ExclusiveLockScope&,
+			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
+		{
+			WriteIndexSnapshotLocked(ClaimDiskReserveFunc);
+		}
+		void WriteIndexSnapshot(
+			RwLock::SharedLockScope&,
+			const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; })
+		{
+			WriteIndexSnapshotLocked(ClaimDiskReserveFunc);
+		}
+		void WriteIndexSnapshotLocked(const std::function<uint64_t()>& ClaimDiskReserveFunc = []() { return 0; });
 
-		void CompactState(std::vector<BucketPayload>&  Payloads,
+		void CompactState(RwLock::ExclusiveLockScope&,
+						  std::vector<BucketPayload>&  Payloads,
 						  std::vector<AccessTime>&	   AccessTimes,
 						  std::vector<BucketMetaData>& MetaDatas,
 						  std::vector<IoBuffer>&	   MemCachedPayloads,
@@ -406,7 +421,8 @@ public:
 	};
 
 private:
-	inline void TryMemCacheTrim()
+	CacheBucket* GetOrCreateBucket(std::string_view InBucket);
+	inline void	 TryMemCacheTrim()
 	{
 		if (m_Configuration.MemCacheTargetFootprintBytes == 0)
 		{
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index fe92613f4..3fe0b0c63 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -9,6 +9,7 @@
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
 #include <zenutil/basicfile.h>
@@ -519,7 +520,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 						  cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false"));
 	options.add_options()("help", "Show command line help");
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
-	options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
 	options.add_options()("snapshot-dir",
 						  "Specify a snapshot of server state to mirror into the persistence root at startup",
@@ -528,7 +528,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("powercycle",
 						  "Exit immediately after initialization is complete",
 						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
-	options.add_options()("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
@@ -537,7 +536,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("sentry-allow-personal-info",
 						  "Allow personally identifiable information in sentry crash reports",
 						  cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false"));
-	options.add_options()("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"));
+
+	// clang-format off
+	options.add_options("logging")
+		("abslog",			"Path to log file",								cxxopts::value<std::string>(AbsLogFile))
+		("log-id",			"Specify id for adding context to log output",	cxxopts::value<std::string>(ServerOptions.LogId))
+		("quiet", 			"Disable console logging", 						cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+		("log-trace", 		"Change selected loggers to level TRACE", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+		("log-debug", 		"Change selected loggers to level DEBUG", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+		("log-info", 		"Change selected loggers to level INFO", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+		("log-warn", 		"Change selected loggers to level WARN", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+		("log-error", 		"Change selected loggers to level ERROR", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+		("log-critical", 	"Change selected loggers to level CRITICAL", 	cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+		("log-off", 		"Change selected loggers to level OFF", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+	;
+	// clang-format on
 
 	options.add_option("security",
 					   "",
@@ -952,6 +965,12 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			exit(0);
 		}
 
+		for (int i = 0; i < logging::level::LogLevelCount; ++i)
+		{
+			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
+		}
+		logging::RefreshLogLevels();
+
 		ServerOptions.DataDir						  = MakeSafePath(DataDir);
 		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
 		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 8135bf8f0..11311f9d8 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/logbase.h>
 #include <zencore/zencore.h>
 #include <zenhttp/httpserver.h>
 #include <filesystem>
@@ -151,6 +152,7 @@ struct ZenServerOptions
 	bool					 SentryAllowPII		= false;  // Allow personally identifiable information in sentry crash reports
 	bool					 ObjectStoreEnabled = false;
 	bool					 NoConsoleOutput	= false;  // Control default use of stdout for diagnostics
+	std::string				 Loggers[zen::logging::level::LogLevelCount];
 #if ZEN_WITH_TRACE
 	std::string TraceHost;	// Host name or IP address to send trace data to
 	std::string TraceFile;	// Path of a file to write a trace
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index e2d57b840..dc1675819 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -42,6 +42,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																	   /* max files */ 16,
 																	   /* rotate on open */ true);
 	auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
+	spdlog::apply_logger_env_levels(HttpLogger);
 	spdlog::register_logger(HttpLogger);
 
 	// Cache request logging
@@ -53,16 +54,19 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																		/* max files */ 16,
 																		/* rotate on open */ false);
 	auto CacheLogger = std::make_shared<spdlog::logger>("z$", CacheSink);
+	spdlog::apply_logger_env_levels(CacheLogger);
 	spdlog::register_logger(CacheLogger);
 
 	// Jupiter - only log upstream HTTP traffic to file
 
 	auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
+	spdlog::apply_logger_env_levels(JupiterLogger);
 	spdlog::register_logger(JupiterLogger);
 
 	// Zen - only log upstream HTTP traffic to file
 
 	auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
+	spdlog::apply_logger_env_levels(ZenClientLogger);
 	spdlog::register_logger(ZenClientLogger);
 
 	FinishInitializeLogging(LogOptions);
diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp
index 261485834..0ba49cf8a 100644
--- a/src/zenserver/projectstore/httpprojectstore.cpp
+++ b/src/zenserver/projectstore/httpprojectstore.cpp
@@ -276,6 +276,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects,
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
+		"{project}/oplog/{log}/chunkinfos",
+		[this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
 		"{project}/oplog/{log}/{chunk}/info",
 		[this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); },
 		HttpVerb::kGet);
@@ -643,6 +648,41 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req)
 }
 
 void
+HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req)
+{
+	ZEN_TRACE_CPU("ProjectService::ChunkInfos");
+
+	HttpServerRequest& HttpReq = Req.ServerRequest();
+
+	const auto& ProjectId = Req.GetCapture(1);
+	const auto& OplogId	  = Req.GetCapture(2);
+
+	CbObject								 ResponsePayload;
+	std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetProjectChunkInfos(ProjectId, OplogId, ResponsePayload);
+	if (Result.first == HttpResponseCode::OK)
+	{
+		return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload);
+	}
+	else
+	{
+		if (Result.first == HttpResponseCode::BadRequest)
+		{
+			m_ProjectStats.BadRequestCount++;
+		}
+		ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`",
+				  ToString(HttpReq.RequestVerb()),
+				  HttpReq.QueryString(),
+				  static_cast<int>(Result.first),
+				  Result.second);
+	}
+	if (Result.second.empty())
+	{
+		return HttpReq.WriteResponse(Result.first);
+	}
+	return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second);
+}
+
+void
 HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req)
 {
 	ZEN_TRACE_CPU("ProjectService::ChunkInfo");
diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h
index 9998ae83e..9990ee264 100644
--- a/src/zenserver/projectstore/httpprojectstore.h
+++ b/src/zenserver/projectstore/httpprojectstore.h
@@ -64,6 +64,7 @@ private:
 	void HandleProjectListRequest(HttpRouterRequest& Req);
 	void HandleChunkBatchRequest(HttpRouterRequest& Req);
 	void HandleFilesRequest(HttpRouterRequest& Req);
+	void HandleChunkInfosRequest(HttpRouterRequest& Req);
 	void HandleChunkInfoRequest(HttpRouterRequest& Req);
 	void HandleChunkByIdRequest(HttpRouterRequest& Req);
 	void HandleChunkByCidRequest(HttpRouterRequest& Req);
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index c6097dea2..9ba8e3a19 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -752,6 +752,21 @@ ProjectStore::Oplog::GetAllChunksInfo()
 }
 
 void
+ProjectStore::Oplog::IterateChunkMap(std::function<void(const Oid&, const IoHash&)>&& Fn)
+{
+	RwLock::SharedLockScope _(m_OplogLock);
+	if (!m_Storage)
+	{
+		return;
+	}
+
+	for (const auto& Kv : m_ChunkMap)
+	{
+		Fn(Kv.first, Kv.second);
+	}
+}
+
+void
 ProjectStore::Oplog::IterateFileMap(
 	std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn)
 {
@@ -2215,9 +2230,9 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin
 }
 
 std::pair<HttpResponseCode, std::string>
-ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
+ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload)
 {
-	ZEN_TRACE_CPU("ProjectStore::GetProjectChunks");
+	ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos");
 
 	using namespace std::literals;
 
@@ -2235,21 +2250,22 @@ ProjectStore::GetProjectChunks(const std::string_view ProjectId, const std::stri
 	}
 	Project->TouchOplog(OplogId);
 
-	std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfo = FoundLog->GetAllChunksInfo();
+	std::vector<std::pair<Oid, IoHash>> ChunkInfos;
+	FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); });
 
 	CbObjectWriter Response;
+	Response.BeginArray("chunkinfos"sv);
 
-	Response.BeginArray("chunks"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
+	for (const auto& ChunkInfo : ChunkInfos)
 	{
-		Response << Info.ChunkId;
-	}
-	Response.EndArray();
-
-	Response.BeginArray("sizes"sv);
-	for (ProjectStore::Oplog::ChunkInfo& Info : ChunkInfo)
-	{
-		Response << Info.ChunkSize;
+		if (IoBuffer Chunk = FoundLog->FindChunk(ChunkInfo.first))
+		{
+			Response.BeginObject();
+			Response << "id"sv << ChunkInfo.first;
+			Response << "rawhash"sv << ChunkInfo.second;
+			Response << "rawsize"sv << Chunk.GetSize();
+			Response.EndObject();
+		}
 	}
 	Response.EndArray();
 
diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h
index 555f8bdf2..57cda8ae7 100644
--- a/src/zenserver/projectstore/projectstore.h
+++ b/src/zenserver/projectstore/projectstore.h
@@ -93,6 +93,7 @@ public:
 		};
 
 		std::vector<ChunkInfo> GetAllChunksInfo();
+		void				   IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn);
 		void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn);
 		void IterateOplog(std::function<void(CbObjectView)>&& Fn);
 		void IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Fn);
@@ -306,9 +307,9 @@ public:
 															 const std::string_view OplogId,
 															 bool					FilterClient,
 															 CbObject&				OutPayload);
-	std::pair<HttpResponseCode, std::string> GetProjectChunks(const std::string_view ProjectId,
-															  const std::string_view OplogId,
-															  CbObject&				 OutPayload);
+	std::pair<HttpResponseCode, std::string> GetProjectChunkInfos(const std::string_view ProjectId,
+																  const std::string_view OplogId,
+																  CbObject&				 OutPayload);
 	std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId,
 														  const std::string_view OplogId,
 														  const std::string_view ChunkId,
diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp
index 42302c4a9..96ab65a5f 100644
--- a/src/zenstore/compactcas.cpp
+++ b/src/zenstore/compactcas.cpp
@@ -989,13 +989,14 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 			Entries.resize(128 * 1024 / sizeof(CasDiskIndexEntry));
 
 			uint64_t RemainingEntries = Header.EntryCount;
+			uint64_t ReadOffset		  = sizeof(CasDiskIndexHeader);
 
 			do
 			{
 				const uint64_t NumToRead = Min(RemainingEntries, Entries.size());
 				Entries.resize(NumToRead);
 
-				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), sizeof(CasDiskIndexHeader));
+				ObjectIndexFile.Read(Entries.data(), Entries.size() * sizeof(CasDiskIndexEntry), ReadOffset);
 
 				std::string InvalidEntryReason;
 				for (const CasDiskIndexEntry& Entry : Entries)
@@ -1011,6 +1012,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint
 				}
 
 				RemainingEntries -= NumToRead;
+				ReadOffset += NumToRead * sizeof(CasDiskIndexEntry);
 			} while (RemainingEntries);
 
 			OutVersion = CasDiskIndexHeader::CurrentVersion;
diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp
index d0a6ac0b4..d82789e42 100644
--- a/src/zenutil/logging.cpp
+++ b/src/zenutil/logging.cpp
@@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <zencore/compactbinary.h>
 #include <zencore/filesystem.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenutil/logging/fullformatter.h>
 #include <zenutil/logging/jsonformatter.h>
@@ -152,21 +153,21 @@ BeginInitializeLogging(const LoggingOptions& LogOptions)
 void
 FinishInitializeLogging(const LoggingOptions& LogOptions)
 {
-	spdlog::level::level_enum LogLevel = spdlog::level::info;
+	logging::level::LogLevel LogLevel = logging::level::Info;
 
 	if (LogOptions.IsDebug)
 	{
-		LogLevel = spdlog::level::debug;
+		LogLevel = logging::level::Debug;
 	}
 
 	if (LogOptions.IsTest)
 	{
-		LogLevel = spdlog::level::trace;
+		LogLevel = logging::level::Trace;
 	}
 
 	// Configure all registered loggers according to settings
 
-	spdlog::set_level(LogLevel);
+	logging::RefreshLogLevels(LogLevel);
 	spdlog::flush_on(spdlog::level::err);
 	spdlog::flush_every(std::chrono::seconds{2});
 	spdlog::set_formatter(std::make_unique<logging::full_formatter>(LogOptions.LogId, std::chrono::system_clock::now()));