Merged main

4 files changed, 419 insertions, 221 deletions

diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp
index dc96aecae..4a2a3748a 100644
--- a/zenserver/cache/structuredcache.cpp
+++ b/zenserver/cache/structuredcache.cpp
@@ -12,6 +12,7 @@
 #include <zenhttp/httpserver.h>
 #include <zenstore/CAS.h>
 
+#include "monitoring/httpstats.h"
 #include "structuredcache.h"
 #include "structuredcachestore.h"
 #include "upstream/jupiter.h"
@@ -149,13 +150,19 @@ ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams)
 HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore&				  InCacheStore,
 													   CasStore&					  InStore,
 													   CidStore&					  InCidStore,
+													   HttpStatsService&			  StatsService,
+													   HttpStatusService&			  StatusService,
 													   std::unique_ptr<UpstreamCache> UpstreamCache)
 : m_Log(logging::Get("cache"))
 , m_CacheStore(InCacheStore)
+, m_StatsService(StatsService)
+, m_StatusService(StatusService)
 , m_CasStore(InStore)
 , m_CidStore(InCidStore)
 , m_UpstreamCache(std::move(UpstreamCache))
 {
+	StatsService.RegisterHandler("z$", *this);
+	StatusService.RegisterHandler("z$", *this);
 }
 
 HttpStructuredCacheService::~HttpStructuredCacheService()
@@ -200,11 +207,6 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request)
 	{
 		std::string_view Key = Request.RelativeUri();
 
-		if (Key.empty())
-		{
-			return HandleStatusRequest(Request);
-		}
-
 		if (std::all_of(begin(Key), end(Key), [](const char c) { return std::isalnum(c); }))
 		{
 			// Bucket reference
@@ -270,10 +272,6 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request,
 		case kHead:
 		case kGet:
 			{
-				if (Verb == kHead)
-				{
-					Request.SetSuppressResponseBody();
-				}
 				HandleGetCacheRecord(Request, Ref, Policy);
 			}
 			break;
@@ -288,28 +286,104 @@ HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request,
 void
 HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy)
 {
-	const ZenContentType AcceptType = Request.AcceptContentType();
+	const ZenContentType AcceptType		 = Request.AcceptContentType();
+	const bool			 SkipData		 = (Policy & CachePolicy::SkipData) == CachePolicy::SkipData;
+	const bool			 SkipAttachments = (Policy & CachePolicy::SkipAttachments) == CachePolicy::SkipAttachments;
+	const bool			 QueryUpstream	 = m_UpstreamCache && ((Policy & CachePolicy::QueryRemote) == CachePolicy::QueryRemote);
 
-	ZenCacheValue Value;
-	bool		  Success		  = m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, /* out */ Value);
-	bool		  InUpstreamCache = false;
+	bool		  Success = false;
+	ZenCacheValue LocalCacheValue;
 
-	const bool QueryUpstream = !Success && m_UpstreamCache && (CachePolicy::QueryRemote == (Policy & CachePolicy::QueryRemote));
+	if (m_CacheStore.Get(Ref.BucketSegment, Ref.HashKey, LocalCacheValue))
+	{
+		Success = true;
 
-	if (QueryUpstream)
+		if (!SkipData && AcceptType == ZenContentType::kCbPackage)
+		{
+			CbPackage	 Package;
+			CbObjectView CacheRecord(LocalCacheValue.Value.Data());
+			uint32_t	 AttachmentCount = 0;
+			uint32_t	 ValidCount		 = 0;
+
+			if (!SkipAttachments)
+			{
+				CacheRecord.IterateAttachments([this, &Ref, &Package, &AttachmentCount, &ValidCount](CbFieldView AttachmentHash) {
+					if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
+					{
+						Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
+						ValidCount++;
+					}
+					AttachmentCount++;
+				});
+
+				if (ValidCount != AttachmentCount)
+				{
+					Success = false;
+					ZEN_WARN("GET - '{}/{}' '{}' FAILED, found '{}' of '{}' attachments",
+							 Ref.BucketSegment,
+							 Ref.HashKey,
+							 ToString(AcceptType),
+							 ValidCount,
+							 AttachmentCount);
+				}
+			}
+
+			Package.SetObject(LoadCompactBinaryObject(LocalCacheValue.Value));
+
+			BinaryWriter MemStream;
+			Package.Save(MemStream);
+
+			LocalCacheValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size());
+			LocalCacheValue.Value.SetContentType(HttpContentType::kCbPackage);
+		}
+	}
+
+	if (Success)
 	{
-		const ZenContentType CacheRecordType = AcceptType;
+		ZEN_DEBUG("HIT - '{}/{}' {} '{}' (LOCAL)",
+				  Ref.BucketSegment,
+				  Ref.HashKey,
+				  NiceBytes(LocalCacheValue.Value.Size()),
+				  ToString(LocalCacheValue.Value.GetContentType()));
+
+		m_CacheStats.HitCount++;
+
+		if (SkipData)
+		{
+			return Request.WriteResponse(HttpResponseCode::OK);
+		}
+		else
+		{
+			return Request.WriteResponse(HttpResponseCode::OK, LocalCacheValue.Value.GetContentType(), LocalCacheValue.Value);
+		}
+	}
+	else if (!QueryUpstream)
+	{
+		ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+		m_CacheStats.MissCount++;
+		return Request.WriteResponse(HttpResponseCode::NotFound);
+	}
+
+	// Issue upstream query asynchronously in order to keep requests flowing without
+	// hogging I/O servicing threads with blocking work
+
+	Request.WriteResponseAsync([this, AcceptType, SkipData, SkipAttachments, Ref](HttpServerRequest& AsyncRequest) {
+		bool		  Success = false;
+		ZenCacheValue UpstreamCacheValue;
 
-		if (auto UpstreamResult = m_UpstreamCache->GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, CacheRecordType);
+		metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming);
+
+		if (GetUpstreamCacheResult UpstreamResult = m_UpstreamCache->GetCacheRecord({Ref.BucketSegment, Ref.HashKey}, AcceptType);
 			UpstreamResult.Success)
 		{
-			Value.Value		= UpstreamResult.Value;
-			Success			= true;
-			InUpstreamCache = true;
+			Success					 = true;
+			UpstreamCacheValue.Value = UpstreamResult.Value;
+
+			UpstreamCacheValue.Value.SetContentType(AcceptType);
 
-			if (CacheRecordType == ZenContentType::kBinary || CacheRecordType == ZenContentType::kCbObject)
+			if (AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kCbObject)
 			{
-				if (CacheRecordType == ZenContentType::kCbObject)
+				if (AcceptType == ZenContentType::kCbObject)
 				{
 					const CbValidateError ValidationResult = ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All);
 
@@ -322,7 +396,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 						CacheRecord.IterateAttachments([&](CbFieldView Attachment) { IndexData.AddHash(Attachment.AsHash()); });
 						IndexData.EndArray();
 
-						Value.IndexData = IndexData.Save();
+						UpstreamCacheValue.IndexData = IndexData.Save();
 					}
 					else
 					{
@@ -336,19 +410,17 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 
 				if (Success)
 				{
-					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, Value);
+					m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, UpstreamCacheValue);
 				}
 			}
-			else
+			else if (AcceptType == ZenContentType::kCbPackage)
 			{
-				ZEN_ASSERT(CacheRecordType == ZenContentType::kCbPackage);
-
 				CbPackage Package;
-				if (Package.TryLoad(UpstreamResult.Value))
+				if (Package.TryLoad(UpstreamCacheValue.Value))
 				{
+					CbObject CacheRecord	 = Package.GetObject();
 					uint32_t AttachmentCount = 0;
 					uint32_t ValidCount		 = 0;
-					CbObject CacheRecord	 = Package.GetObject();
 
 					CacheRecord.IterateAttachments([this, &Package, &Ref, &AttachmentCount, &ValidCount](CbFieldView AttachmentHash) {
 						if (const CbAttachment* Attachment = Package.FindAttachment(AttachmentHash.AsHash()))
@@ -373,7 +445,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 					{
 						m_CacheStore.Put(Ref.BucketSegment, Ref.HashKey, {.Value = CacheRecord.GetBuffer().AsIoBuffer()});
 
-						if (zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments))
+						if (SkipAttachments)
 						{
 							CbPackage PackageWithoutAttachments;
 							PackageWithoutAttachments.SetObject(CacheRecord);
@@ -381,7 +453,7 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 							BinaryWriter MemStream;
 							PackageWithoutAttachments.Save(MemStream);
 
-							Value.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size());
+							UpstreamCacheValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size());
 						}
 					}
 					else
@@ -400,86 +472,34 @@ HttpStructuredCacheService::HandleGetCacheRecord(zen::HttpServerRequest& Request
 				}
 			}
 		}
-	}
-
-	if (!Success)
-	{
-		ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
-		return Request.WriteResponse(HttpResponseCode::NotFound);
-	}
-
-	if (AcceptType == ZenContentType::kCbPackage && !InUpstreamCache)
-	{
-		CbObjectView CacheRecord(Value.Value.Data());
-
-		const CbValidateError ValidationResult = ValidateCompactBinary(Value.Value, CbValidateMode::All);
 
-		if (ValidationResult != CbValidateError::None)
+		if (Success)
 		{
-			ZEN_WARN("GET - '{}/{}' '{}' FAILED, invalid compact binary object", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
-			return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Invalid cache record"sv);
-		}
-
-		const bool SkipAttachments = zen::CachePolicy::SkipAttachments == (Policy & zen::CachePolicy::SkipAttachments);
-		uint32_t   AttachmentCount = 0;
-		uint32_t   ValidCount	   = 0;
-		uint64_t   AttachmentBytes = 0ull;
-
-		CbPackage Package;
+			ZEN_DEBUG("HIT - '{}/{}' {} '{}' (UPSTREAM)",
+					  Ref.BucketSegment,
+					  Ref.HashKey,
+					  NiceBytes(UpstreamCacheValue.Value.Size()),
+					  ToString(UpstreamCacheValue.Value.GetContentType()));
 
-		if (!SkipAttachments)
-		{
-			CacheRecord.IterateAttachments(
-				[this, &Ref, &Package, &AttachmentCount, &ValidCount, &AttachmentBytes](CbFieldView AttachmentHash) {
-					if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash()))
-					{
-						Package.AddAttachment(CbAttachment(CompressedBuffer::FromCompressed(SharedBuffer(Chunk))));
-						AttachmentBytes += Chunk.Size();
-						ValidCount++;
-					}
-					AttachmentCount++;
-				});
+			m_CacheStats.HitCount++;
+			m_CacheStats.UpstreamHitCount++;
 
-			if (ValidCount != AttachmentCount)
+			if (SkipData)
 			{
-				ZEN_WARN("GET - '{}/{}' '{}' FAILED, found '{}' of '{}' attachments",
-						 Ref.BucketSegment,
-						 Ref.HashKey,
-						 ToString(AcceptType),
-						 ValidCount,
-						 AttachmentCount);
-
-				return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Missing attachments"sv);
+				AsyncRequest.WriteResponse(HttpResponseCode::OK);
+			}
+			else
+			{
+				AsyncRequest.WriteResponse(HttpResponseCode::OK, UpstreamCacheValue.Value.GetContentType(), UpstreamCacheValue.Value);
 			}
 		}
-
-		Package.SetObject(LoadCompactBinaryObject(Value.Value));
-
-		ZEN_DEBUG("HIT - '{}/{}' {} '{}', {} attachments (LOCAL)",
-				  Ref.BucketSegment,
-				  Ref.HashKey,
-				  NiceBytes(AttachmentBytes + Value.Value.Size()),
-				  ToString(HttpContentType::kCbPackage),
-				  AttachmentCount);
-
-		BinaryWriter MemStream;
-		Package.Save(MemStream);
-
-		IoBuffer Response(IoBuffer::Clone, MemStream.Data(), MemStream.Size());
-
-		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, Response);
-	}
-	else
-	{
-		ZEN_DEBUG("HIT - '{}/{}' {} '{}' ({})",
-				  Ref.BucketSegment,
-				  Ref.HashKey,
-				  NiceBytes(Value.Value.Size()),
-				  ToString(Value.Value.GetContentType()),
-				  InUpstreamCache ? "UPSTREAM" : "LOCAL");
-
-		Request.WriteResponse(HttpResponseCode::OK, Value.Value.GetContentType(), Value.Value);
-	}
+		else
+		{
+			ZEN_DEBUG("MISS - '{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, ToString(AcceptType));
+			m_CacheStats.MissCount++;
+			AsyncRequest.WriteResponse(HttpResponseCode::NotFound);
+		}
+	});
 }
 
 void
@@ -668,10 +688,6 @@ HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request
 		case kGet:
 			{
 				HandleGetCachePayload(Request, Ref, Policy);
-				if (Verb == kHead)
-				{
-					Request.SetSuppressResponseBody();
-				}
 			}
 			break;
 		case kPut:
@@ -712,7 +728,8 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 
 	if (!Payload)
 	{
-		ZEN_DEBUG("MISS - '{}/{}/{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId);
+		ZEN_DEBUG("MISS - '{}/{}/{}' '{}'", Ref.BucketSegment, Ref.HashKey, Ref.PayloadId, ToString(Request.AcceptContentType()));
+		m_CacheStats.MissCount++;
 		return Request.WriteResponse(HttpResponseCode::NotFound);
 	}
 
@@ -724,7 +741,20 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques
 			  ToString(Payload.GetContentType()),
 			  InUpstreamCache ? "UPSTREAM" : "LOCAL");
 
-	Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload);
+	m_CacheStats.HitCount++;
+	if (InUpstreamCache)
+	{
+		m_CacheStats.UpstreamHitCount++;
+	}
+
+	if ((Policy & CachePolicy::SkipData) == CachePolicy::SkipData)
+	{
+		Request.WriteResponse(HttpResponseCode::OK);
+	}
+	else
+	{
+		Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Payload);
+	}
 }
 
 void
@@ -839,12 +869,25 @@ HttpStructuredCacheService::ValidateKeyUri(HttpServerRequest& Request, CacheRef&
 }
 
 void
-HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
+HttpStructuredCacheService::HandleStatsRequest(zen::HttpServerRequest& Request)
 {
 	CbObjectWriter Cbo;
-	Cbo << "ok" << true;
 
 	EmitSnapshot("requests", m_HttpRequests, Cbo);
+	EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo);
+
+	const uint64_t HitCount			= m_CacheStats.HitCount;
+	const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount;
+	const uint64_t MissCount		= m_CacheStats.MissCount;
+	const uint64_t TotalCount		= HitCount + MissCount;
+
+	Cbo.BeginObject("cache");
+	Cbo << "hits" << HitCount << "misses" << MissCount;
+	Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount) * 100.0) : 0.0);
+	Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount;
+	Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) * 100.0 : 0.0);
+	Cbo.EndObject();
+
 	if (m_UpstreamCache)
 	{
 		Cbo.BeginObject("upstream");
@@ -855,4 +898,12 @@ HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
 	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
 }
 
+void
+HttpStructuredCacheService::HandleStatusRequest(zen::HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
 }  // namespace zen
diff --git a/zenserver/cache/structuredcache.h b/zenserver/cache/structuredcache.h
index 47fc173e9..ad7253f79 100644
--- a/zenserver/cache/structuredcache.h
+++ b/zenserver/cache/structuredcache.h
@@ -5,6 +5,9 @@
 #include <zencore/stats.h>
 #include <zenhttp/httpserver.h>
 
+#include "monitoring/httpstats.h"
+#include "monitoring/httpstatus.h"
+
 #include <memory>
 
 namespace spdlog {
@@ -27,12 +30,12 @@ enum class CachePolicy : uint8_t;
  *
  *   {BucketId}/{KeyHash}
  *
- * Where BucketId is an alphanumeric string, and KeyHash is a 40-character hexadecimal
- * sequence. The hash value may be derived in any number of ways, it's up to the
- * application to pick an approach.
+ * Where BucketId is a lower-case alphanumeric string, and KeyHash is a 40-character
+ * hexadecimal sequence. The hash value may be derived in any number of ways, it's
+ * up to the application to pick an approach.
  *
  * Values may be structured or unstructured. Structured values are encoded using Unreal
- * Engine's compact binary encoding
+ * Engine's compact binary encoding (see CbObject)
  *
  * Additionally, attachments may be addressed as:
  *
@@ -47,18 +50,19 @@ enum class CachePolicy : uint8_t;
  *
  */
 
-class HttpStructuredCacheService : public zen::HttpService
+class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider
 {
 public:
 	HttpStructuredCacheService(ZenCacheStore&				  InCacheStore,
-							   zen::CasStore&				  InCasStore,
-							   zen::CidStore&				  InCidStore,
+							   CasStore&					  InCasStore,
+							   CidStore&					  InCidStore,
+							   HttpStatsService&			  StatsService,
+							   HttpStatusService&			  StatusService,
 							   std::unique_ptr<UpstreamCache> UpstreamCache);
 	~HttpStructuredCacheService();
 
 	virtual const char* BaseUri() const override;
-
-	virtual void HandleRequest(zen::HttpServerRequest& Request) override;
+	virtual void		HandleRequest(zen::HttpServerRequest& Request) override;
 
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
@@ -71,6 +75,13 @@ private:
 		IoHash		PayloadId;
 	};
 
+	struct CacheStats
+	{
+		std::atomic_uint64_t HitCount{};
+		std::atomic_uint64_t UpstreamHitCount{};
+		std::atomic_uint64_t MissCount{};
+	};
+
 	[[nodiscard]] bool ValidateKeyUri(zen::HttpServerRequest& Request, CacheRef& OutRef);
 	void			   HandleCacheRecordRequest(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandleGetCacheRecord(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
@@ -79,16 +90,21 @@ private:
 	void			   HandleGetCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandlePutCachePayload(zen::HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy);
 	void			   HandleCacheBucketRequest(zen::HttpServerRequest& Request, std::string_view Bucket);
-	void			   HandleStatusRequest(zen::HttpServerRequest& Request);
+	virtual void	   HandleStatsRequest(zen::HttpServerRequest& Request) override;
+	virtual void	   HandleStatusRequest(zen::HttpServerRequest& Request) override;
 
 	spdlog::logger&				   Log() { return m_Log; }
 	spdlog::logger&				   m_Log;
-	zen::ZenCacheStore&			   m_CacheStore;
-	zen::CasStore&				   m_CasStore;
-	zen::CidStore&				   m_CidStore;
+	ZenCacheStore&				   m_CacheStore;
+	HttpStatsService&			   m_StatsService;
+	HttpStatusService&			   m_StatusService;
+	CasStore&					   m_CasStore;
+	CidStore&					   m_CidStore;
 	std::unique_ptr<UpstreamCache> m_UpstreamCache;
 	uint64_t					   m_LastScrubTime = 0;
 	metrics::OperationTiming	   m_HttpRequests;
+	metrics::OperationTiming	   m_UpstreamGetRequestTiming;
+	CacheStats					   m_CacheStats;
 };
 
 }  // namespace zen
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 5e93ebaa9..580446473 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -32,6 +32,8 @@ ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir
 {
 	ZEN_INFO("initializing structured cache at '{}'", RootDir);
 	CreateDirectories(RootDir);
+
+	m_DiskLayer.DiscoverBuckets();
 }
 
 ZenCacheStore::~ZenCacheStore()
@@ -116,6 +118,13 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 	m_DiskLayer.Scrub(Ctx);
 	m_MemLayer.Scrub(Ctx);
 }
+
+void
+ZenCacheStore::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
 //////////////////////////////////////////////////////////////////////////
 
 ZenCacheMemoryLayer::ZenCacheMemoryLayer()
@@ -142,6 +151,10 @@ ZenCacheMemoryLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCa
 
 	_.ReleaseNow();
 
+	// There's a race here. Since the lock is released early to allow
+	// inserts, the bucket delete path could end up deleting the
+	// underlying data structure
+
 	return Bucket->Get(HashKey, OutValue);
 }
 
@@ -195,13 +208,21 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
 }
 
 void
+ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
+void
 ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 {
+	RwLock::SharedLockScope _(m_bucketLock);
+
 	std::vector<IoHash> BadHashes;
 
 	for (auto& Kv : m_cacheMap)
 	{
-		if (Kv.first != IoHash::HashBuffer(Kv.second))
+		if (Kv.first != IoHash::HashBuffer(Kv.second.Payload))
 		{
 			BadHashes.push_back(Kv.first);
 		}
@@ -209,10 +230,16 @@ ZenCacheMemoryLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 
 	if (!BadHashes.empty())
 	{
-		Ctx.ReportBadChunks(BadHashes);
+		Ctx.ReportBadCasChunks(BadHashes);
 	}
 }
 
+void
+ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
+{
+	ZEN_UNUSED(GcCtx);
+}
+
 bool
 ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
@@ -224,18 +251,26 @@ ZenCacheMemoryLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutV
 	}
 	else
 	{
-		OutValue.Value = bucketIt->second;
+		BucketValue& Value = bucketIt.value();
+		OutValue.Value	   = Value.Payload;
+		Value.LastAccess   = GetCurrentTimeStamp();
 
 		return true;
 	}
 }
 
+uint64_t
+ZenCacheMemoryLayer::CacheBucket::GetCurrentTimeStamp()
+{
+	return GetLofreqTimerValue();
+}
+
 void
 ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	RwLock::ExclusiveLockScope _(m_bucketLock);
 
-	m_cacheMap[HashKey] = Value.Value;
+	m_cacheMap.insert_or_assign(HashKey, BucketValue{.LastAccess = GetCurrentTimeStamp(), .Payload = Value.Value});
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -245,11 +280,17 @@ ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue
 
 struct DiskLocation
 {
-	uint64_t OffsetAndFlags;
-	uint32_t Size;
-	uint32_t IndexDataSize;
+	inline DiskLocation() = default;
+
+	inline DiskLocation(uint64_t Offset, uint64_t ValueSize, uint32_t IndexSize, uint64_t Flags)
+	: OffsetAndFlags(CombineOffsetAndFlags(Offset, Flags))
+	, LowerSize(ValueSize & 0xFFFFffff)
+	, IndexDataSize(IndexSize)
+	{
+	}
 
-	static const uint64_t kOffsetMask	  = 0x00FF'ffFF'ffFF'ffFFull;
+	static const uint64_t kOffsetMask	  = 0x0000'ffFF'ffFF'ffFFull;
+	static const uint64_t kSizeMask		  = 0x00FF'0000'0000'0000ull;
 	static const uint64_t kFlagsMask	  = 0xff00'0000'0000'0000ull;
 	static const uint64_t kStandaloneFile = 0x8000'0000'0000'0000ull;
 	static const uint64_t kStructured	  = 0x4000'0000'0000'0000ull;
@@ -257,6 +298,7 @@ struct DiskLocation
 	static uint64_t CombineOffsetAndFlags(uint64_t Offset, uint64_t Flags) { return Offset | Flags; }
 
 	inline uint64_t		  Offset() const { return OffsetAndFlags & kOffsetMask; }
+	inline uint64_t		  Size() const { return LowerSize; }
 	inline uint64_t		  IsFlagSet(uint64_t Flag) const { return OffsetAndFlags & Flag; }
 	inline ZenContentType GetContentType() const
 	{
@@ -269,6 +311,11 @@ struct DiskLocation
 
 		return ContentType;
 	}
+
+private:
+	uint64_t OffsetAndFlags = 0;
+	uint32_t LowerSize		= 0;
+	uint32_t IndexDataSize	= 0;
 };
 
 struct DiskIndexEntry
@@ -286,7 +333,7 @@ struct ZenCacheDiskLayer::CacheBucket
 	CacheBucket(CasStore& Cas);
 	~CacheBucket();
 
-	void		OpenOrCreate(std::filesystem::path BucketDir);
+	void		OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
 	static bool Delete(std::filesystem::path BucketDir);
 
 	bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -294,14 +341,15 @@ struct ZenCacheDiskLayer::CacheBucket
 	void Drop();
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
-	inline bool IsOk() const { return m_Ok; }
+	inline bool IsOk() const { return m_IsOk; }
 
 private:
 	CasStore&			  m_CasStore;
 	std::filesystem::path m_BucketDir;
 	Oid					  m_BucketId;
-	bool				  m_Ok					 = false;
+	bool				  m_IsOk				 = false;
 	uint64_t			  m_LargeObjectThreshold = 64 * 1024;
 
 	// These files are used to manage storage of small objects for this bucket
@@ -314,9 +362,19 @@ private:
 	uint64_t											 m_WriteCursor = 0;
 
 	void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey);
-	void PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value);
-	bool GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc);
+	void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value);
+	bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue);
 	bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue);
+
+	// These locks are here to avoid contention on file creation, therefore it's sufficient
+	// that we take the same lock for the same hash
+	//
+	// These locks are small and should really be spaced out so they don't share cache lines,
+	// but we don't currently access them at particularly high frequency so it should not be
+	// an issue in practice
+
+	RwLock		   m_ShardedLocks[256];
+	inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
 };
 
 ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
@@ -341,7 +399,7 @@ ZenCacheDiskLayer::CacheBucket::Delete(std::filesystem::path BucketDir)
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
+ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate)
 {
 	CreateDirectories(BucketDir);
 
@@ -368,17 +426,23 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
 		{
 			ManifestFile.Read(&m_BucketId, sizeof(Oid), 0);
 
-			m_Ok = true;
+			m_IsOk = true;
 		}
 
-		if (!m_Ok)
+		if (!m_IsOk)
 		{
 			ManifestFile.Close();
 		}
 	}
 
-	if (!m_Ok)
+	if (!m_IsOk)
 	{
+		if (AllowCreate == false)
+		{
+			// Invalid bucket
+			return;
+		}
+
 		// No manifest file found, this is a new bucket
 
 		ManifestFile.Open(ManifestPath, /* IsCreate */ true, Ec);
@@ -410,13 +474,13 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir)
 		m_SlogFile.Replay([&](const DiskIndexEntry& Record) {
 			m_Index[Record.Key] = Record.Location;
 
-			MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset() + Record.Location.Size);
+			MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Record.Location.Offset() + Record.Location.Size());
 		});
 
 		m_WriteCursor = (MaxFileOffset + 15) & ~15;
 	}
 
-	m_Ok = true;
+	m_IsOk = true;
 }
 
 void
@@ -437,23 +501,25 @@ ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoH
 bool
 ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue)
 {
-	if (!Loc.IsFlagSet(DiskLocation::kStandaloneFile))
+	if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 	{
-		OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), Loc.Offset(), Loc.Size);
-		OutValue.Value.SetContentType(Loc.GetContentType());
-
-		return true;
+		return false;
 	}
 
-	return false;
+	OutValue.Value = IoBufferBuilder::MakeFromFileHandle(m_SobsFile.Handle(), Loc.Offset(), Loc.Size());
+	OutValue.Value.SetContentType(Loc.GetContentType());
+
+	return true;
 }
 
 bool
-ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const IoHash& HashKey, ZenCacheValue& OutValue, const DiskLocation& Loc)
+ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue)
 {
 	WideStringBuilder<128> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
 
+	RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
+
 	if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str()))
 	{
 		OutValue.Value = Data;
@@ -468,7 +534,7 @@ ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const IoHash& HashKey, Z
 bool
 ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutValue)
 {
-	if (!m_Ok)
+	if (!m_IsOk)
 	{
 		return false;
 	}
@@ -486,7 +552,7 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 
 		_.ReleaseNow();
 
-		return GetStandaloneCacheValue(HashKey, OutValue, Loc);
+		return GetStandaloneCacheValue(Loc, HashKey, OutValue);
 	}
 
 	return false;
@@ -495,14 +561,14 @@ ZenCacheDiskLayer::CacheBucket::Get(const IoHash& HashKey, ZenCacheValue& OutVal
 void
 ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
 {
-	if (!m_Ok)
+	if (!m_IsOk)
 	{
 		return;
 	}
 
 	if (Value.Value.Size() >= m_LargeObjectThreshold)
 	{
-		return PutLargeObject(HashKey, Value);
+		return PutStandaloneCacheValue(HashKey, Value);
 	}
 	else
 	{
@@ -517,10 +583,9 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue&
 
 		RwLock::ExclusiveLockScope _(m_IndexLock);
 
-		DiskLocation Loc{.OffsetAndFlags = DiskLocation::CombineOffsetAndFlags(m_WriteCursor, EntryFlags),
-						 .Size			 = gsl::narrow<uint32_t>(Value.Value.Size())};
+		DiskLocation Loc(m_WriteCursor, Value.Value.Size(), 0, EntryFlags);
 
-		m_WriteCursor = RoundUp(m_WriteCursor + Loc.Size, 16);
+		m_WriteCursor = RoundUp(m_WriteCursor + Loc.Size(), 16);
 
 		if (auto it = m_Index.find(HashKey); it == m_Index.end())
 		{
@@ -530,11 +595,13 @@ ZenCacheDiskLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue&
 		else
 		{
 			// TODO: should check if write is idempotent and bail out if it is?
+			// this would requiring comparing contents on disk unless we add a
+			// content hash to the index entry
 			it.value() = Loc;
 		}
 
 		m_SlogFile.Append({.Key = HashKey, .Location = Loc});
-		m_SobsFile.Write(Value.Value.Data(), Loc.Size, Loc.Offset());
+		m_SobsFile.Write(Value.Value.Data(), Loc.Size(), Loc.Offset());
 	}
 }
 
@@ -558,61 +625,69 @@ ZenCacheDiskLayer::CacheBucket::Flush()
 void
 ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 {
-	std::vector<DiskIndexEntry> StandaloneFiles;
+	std::atomic<uint64_t> ScrubbedChunks{0}, ScrubbedBytes{0};
 
-	std::vector<IoHash>	  BadChunks;
-	std::vector<IoBuffer> BadStandaloneChunks;
+	std::vector<IoHash> BadChunks;
 
 	{
 		RwLock::SharedLockScope _(m_IndexLock);
 
 		for (auto& Kv : m_Index)
 		{
-			const IoHash&		Hash = Kv.first;
-			const DiskLocation& Loc	 = Kv.second;
+			const IoHash&		HashKey = Kv.first;
+			const DiskLocation& Loc		= Kv.second;
 
 			ZenCacheValue Value;
 
-			if (!GetInlineCacheValue(Loc, Value))
+			if (GetInlineCacheValue(Loc, Value))
 			{
-				ZEN_ASSERT(Loc.IsFlagSet(DiskLocation::kStandaloneFile));
-				StandaloneFiles.push_back({.Key = Hash, .Location = Loc});
+				// Validate contents
 			}
 			else
 			{
-				if (GetStandaloneCacheValue(Hash, Value, Loc))
+				if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 				{
-					// Hash contents
-
-					const IoHash ComputedHash = HashBuffer(Value.Value);
-
-					if (ComputedHash != Hash)
+					if (GetStandaloneCacheValue(Loc, HashKey, Value))
 					{
-						BadChunks.push_back(Hash);
+						// Note: we cannot currently validate contents since we don't
+						// have a content hash!
+					}
+					else
+					{
+						// Value not found
+						BadChunks.push_back(HashKey);
 					}
-				}
-				else
-				{
-					// Non-existent
 				}
 			}
 		}
 	}
 
-	if (Ctx.RunRecovery())
+	Ctx.ReportScrubbed(ScrubbedChunks, ScrubbedBytes);
+
+	if (BadChunks.empty())
 	{
-		// Clean out bad chunks
+		return;
 	}
 
-	if (!BadChunks.empty())
+	Ctx.ReportBadCasChunks(BadChunks);
+
+	if (Ctx.RunRecovery())
 	{
-		Ctx.ReportBadChunks(BadChunks);
+		// Clean out bad data
 	}
 }
 
 void
-ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenCacheValue& Value)
+ZenCacheDiskLayer::CacheBucket::GarbageCollect(GcContext& GcCtx)
 {
+	ZEN_UNUSED(GcCtx);
+}
+
+void
+ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value)
+{
+	RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey));
+
 	WideStringBuilder<128> DataFilePath;
 	BuildPath(DataFilePath, HashKey);
 
@@ -661,7 +736,7 @@ ZenCacheDiskLayer::CacheBucket::PutLargeObject(const IoHash& HashKey, const ZenC
 
 	RwLock::ExclusiveLockScope _(m_IndexLock);
 
-	DiskLocation Loc{.OffsetAndFlags = DiskLocation::CombineOffsetAndFlags(0, EntryFlags), .Size = 0};
+	DiskLocation Loc(/* Offset */ 0, Value.Value.Size(), 0, EntryFlags);
 
 	if (auto it = m_Index.find(HashKey); it == m_Index.end())
 	{
@@ -719,7 +794,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= std::string(InBucket);
 
-			Bucket->OpenOrCreate(BucketPath.c_str());
+			Bucket->OpenOrCreate(BucketPath);
 		}
 	}
 
@@ -762,7 +837,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 			std::filesystem::path bucketPath = m_RootDir;
 			bucketPath /= std::string(InBucket);
 
-			Bucket->OpenOrCreate(bucketPath.c_str());
+			Bucket->OpenOrCreate(bucketPath);
 		}
 	}
 
@@ -774,6 +849,63 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 	}
 }
 
+void
+ZenCacheDiskLayer::DiscoverBuckets()
+{
+	FileSystemTraversal Traversal;
+	struct Visitor : public FileSystemTraversal::TreeVisitor
+	{
+		virtual void VisitFile([[maybe_unused]] const std::filesystem::path& Parent,
+							   [[maybe_unused]] const path_view&			 File,
+							   [[maybe_unused]] uint64_t					 FileSize) override
+		{
+		}
+
+		virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override
+		{
+			Dirs.push_back(std::wstring(DirectoryName));
+			return false;
+		}
+
+		std::vector<std::wstring> Dirs;
+	} Visit;
+
+	Traversal.TraverseFileSystem(m_RootDir, Visit);
+
+	// Initialize buckets
+
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	for (const std::wstring& BucketName : Visit.Dirs)
+	{
+		// New bucket needs to be created
+
+		std::string BucketName8 = WideToUtf8(BucketName);
+
+		if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end())
+		{
+		}
+		else
+		{
+			auto InsertResult = m_Buckets.try_emplace(BucketName8, m_CasStore);
+
+			std::filesystem::path BucketPath = m_RootDir;
+			BucketPath /= BucketName8;
+
+			CacheBucket& Bucket = InsertResult.first->second;
+
+			Bucket.OpenOrCreate(BucketPath, /* AllowCreate */ false);
+
+			if (!Bucket.IsOk())
+			{
+				ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName8, m_RootDir);
+
+				m_Buckets.erase(InsertResult.first);
+			}
+		}
+	}
+}
+
 bool
 ZenCacheDiskLayer::DropBucket(std::string_view InBucket)
 {
@@ -830,27 +962,10 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
 	}
 }
 
-//////////////////////////////////////////////////////////////////////////
-
-ZenCacheTracker::ZenCacheTracker(ZenCacheStore& CacheStore)
-{
-	ZEN_UNUSED(CacheStore);
-}
-
-ZenCacheTracker::~ZenCacheTracker()
-{
-}
-
-void
-ZenCacheTracker::TrackAccess(std::string_view Bucket, const IoHash& HashKey)
-{
-	ZEN_UNUSED(Bucket);
-	ZEN_UNUSED(HashKey);
-}
-
 void
-ZenCacheTracker::Flush()
+ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
 {
+	ZEN_UNUSED(GcCtx);
 }
 
 }  // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index f96757409..4753af627 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -46,6 +46,11 @@ struct ZenCacheValue
 	CbObject IndexData;
 };
 
+/** In-memory cache storage
+
+	Intended for small values which are frequently accessed
+
+ */
 class ZenCacheMemoryLayer
 {
 public:
@@ -56,20 +61,41 @@ public:
 	void Put(std::string_view Bucket, const IoHash& HashKey, const ZenCacheValue& Value);
 	bool DropBucket(std::string_view Bucket);
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
+
+	struct Configuration
+	{
+		uint64_t TargetFootprintBytes = 16 * 1024 * 1024;
+		uint64_t ScavengeThreshold	  = 4 * 1024 * 1024;
+	};
+
+	const Configuration& GetConfiguration() const { return m_Configuration; }
+	void				 SetConfiguration(const Configuration& NewConfig) { m_Configuration = NewConfig; }
 
 private:
 	struct CacheBucket
 	{
-		RwLock							 m_bucketLock;
-		tsl::robin_map<IoHash, IoBuffer> m_cacheMap;
+		struct BucketValue
+		{
+			uint64_t LastAccess = 0;
+			IoBuffer Payload;
+		};
+
+		RwLock								m_bucketLock;
+		tsl::robin_map<IoHash, BucketValue> m_cacheMap;
 
 		bool Get(const IoHash& HashKey, ZenCacheValue& OutValue);
 		void Put(const IoHash& HashKey, const ZenCacheValue& Value);
 		void Scrub(ScrubContext& Ctx);
+		void GarbageCollect(GcContext& GcCtx);
+
+	private:
+		uint64_t GetCurrentTimeStamp();
 	};
 
 	RwLock										 m_Lock;
 	std::unordered_map<std::string, CacheBucket> m_Buckets;
+	Configuration								 m_Configuration;
 };
 
 class ZenCacheDiskLayer
@@ -83,6 +109,9 @@ public:
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
+
+	void DiscoverBuckets();
 
 private:
 	/** A cache bucket manages a single directory containing
@@ -107,6 +136,7 @@ public:
 	bool DropBucket(std::string_view Bucket);
 	void Flush();
 	void Scrub(ScrubContext& Ctx);
+	void GarbageCollect(GcContext& GcCtx);
 
 private:
 	std::filesystem::path m_RootDir;
@@ -116,18 +146,4 @@ private:
 	uint64_t			  m_LastScrubTime		   = 0;
 };
 
-/** Tracks cache entry access, stats and orchestrates cleanup activities
- */
-class ZenCacheTracker
-{
-public:
-	ZenCacheTracker(ZenCacheStore& CacheStore);
-	~ZenCacheTracker();
-
-	void TrackAccess(std::string_view Bucket, const IoHash& HashKey);
-	void Flush();
-
-private:
-};
-
 }  // namespace zen