per namespace/project cas prep refactor (#470)

- Refactor so we can have more than one cas store for project store and cache.
- Refactor `UpstreamCacheClient` so it is not tied to a specific CidStore
- Refactor scrub to keep the GC interface ScrubStorage function separate from scrub accessor functions (renamed to Scrub).
- Refactor storage size to keep GC interface StorageSize function separate from size accessor functions (renamed to TotalSize)
- Refactor cache storage so  `ZenCacheDiskLayer::CacheBucket` implements GcStorage interface rather than `ZenCacheNamespace`

1 files changed, 31 insertions, 57 deletions

diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index ba1413819..5cab54acc 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -239,18 +239,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	CidStoreConfiguration Config;
 	Config.RootDirectory = m_DataRoot / "cas";
 
-	m_CidStore = std::make_unique<CidStore>(m_GcManager);
-	m_CidStore->Initialize(Config);
+	m_CidStores.insert_or_assign({}, std::make_unique<CidStore>(m_GcManager));
+	m_CidStores.at({})->Initialize(Config);
 
 	ZEN_INFO("instantiating project service");
 
-	m_ProjectStore = new ProjectStore(*m_CidStore,
+	m_ProjectStore = new ProjectStore([this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
 									  m_DataRoot / "projects",
 									  m_GcManager,
 									  *m_JobQueue,
 									  *m_OpenProcessCache,
 									  ProjectStore::Configuration{});
-	m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr});
+	m_HttpProjectService.reset(new HttpProjectService(m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr));
 
 	if (ServerOptions.WorksSpacesConfig.Enabled)
 	{
@@ -362,17 +362,15 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_GcScheduler.Initialize(GcConfig);
 
 	// Create and register admin interface last to make sure all is properly initialized
-	m_AdminService =
-		std::make_unique<HttpAdminService>(m_GcScheduler,
-										   *m_JobQueue,
-										   m_CacheStore.Get(),
-										   m_CidStore.get(),
-										   m_ProjectStore,
-										   m_BuildStore.get(),
-										   HttpAdminService::LogPaths{.AbsLogPath	= ServerOptions.AbsLogFile,
-																	  .HttpLogPath	= ServerOptions.DataDir / "logs" / "http.log",
-																	  .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
-										   ServerOptions);
+	m_AdminService = std::make_unique<HttpAdminService>(
+		m_GcScheduler,
+		*m_JobQueue,
+		m_CacheStore.Get(),
+		[this]() { Flush(); },
+		HttpAdminService::LogPaths{.AbsLogPath	 = ServerOptions.AbsLogFile,
+								   .HttpLogPath	 = ServerOptions.DataDir / "logs" / "http.log",
+								   .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"},
+		ServerOptions);
 	m_Http->RegisterService(*m_AdminService);
 
 	return EffectiveBasePort;
@@ -595,7 +593,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 		UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount);
 	}
 
-	m_UpstreamCache	  = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore);
+	m_UpstreamCache	  = CreateUpstreamCache(UpstreamOptions, *m_CacheStore);
 	m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr);
 	m_UpstreamCache->Initialize();
 
@@ -659,19 +657,23 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 		}
 	}
 
-	m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore,
-																			*m_CidStore,
-																			m_StatsService,
-																			m_StatusService,
-																			*m_UpstreamCache,
-																			m_GcManager.GetDiskWriteBlocker(),
-																			*m_OpenProcessCache);
+	m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(
+		*m_CacheStore,
+		[this](std::string_view) -> CidStore& { return *m_CidStores.at({}).get(); },
+		m_StatsService,
+		m_StatusService,
+		*m_UpstreamCache,
+		m_GcManager.GetDiskWriteBlocker(),
+		*m_OpenProcessCache);
 
 	m_Http->RegisterService(*m_StructuredCacheService);
 	m_Http->RegisterService(*m_UpstreamService);
 
 	m_StatsReporter.AddProvider(m_CacheStore.Get());
-	m_StatsReporter.AddProvider(m_CidStore.get());
+	for (const auto& It : m_CidStores)
+	{
+		m_StatsReporter.AddProvider(It.second.get());
+	}
 	m_StatsReporter.AddProvider(m_BuildCidStore.get());
 }
 
@@ -857,7 +859,7 @@ ZenServer::Cleanup()
 		m_Workspaces.reset();
 		m_HttpProjectService.reset();
 		m_ProjectStore = {};
-		m_CidStore.reset();
+		m_CidStores.clear();
 		m_AuthService.reset();
 		m_AuthMgr.reset();
 		m_Http = {};
@@ -1038,40 +1040,12 @@ ZenServer::CheckOwnerPid()
 }
 
 void
-ZenServer::ScrubStorage()
-{
-	Stopwatch Timer;
-	ZEN_INFO("Storage validation STARTING");
-
-	WorkerThreadPool ThreadPool{1, "Scrub"};
-	ScrubContext	 Ctx{ThreadPool};
-
-	if (m_CidStore)
-		m_CidStore->ScrubStorage(Ctx);
-
-	if (m_ProjectStore)
-		m_ProjectStore->ScrubStorage(Ctx);
-
-	if (m_StructuredCacheService)
-		m_StructuredCacheService->ScrubStorage(Ctx);
-
-	if (m_BuildCidStore)
-		m_BuildCidStore->ScrubStorage(Ctx);
-
-	const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
-
-	ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})",
-			 NiceTimeSpanMs(ElapsedTimeMs),
-			 NiceBytes(Ctx.ScrubbedBytes()),
-			 Ctx.ScrubbedChunks(),
-			 NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs));
-}
-
-void
 ZenServer::Flush()
 {
-	if (m_CidStore)
-		m_CidStore->Flush();
+	for (auto& It : m_CidStores)
+	{
+		It.second->Flush();
+	}
 
 	if (m_StructuredCacheService)
 		m_StructuredCacheService->Flush();