From c7e22a4ef1cce7103b9afbeec487461cb32f8dbe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 17 Jun 2022 07:06:21 -0700 Subject: Make cas storage an hidden implementation detail of CidStore (#130) - Bumped ZEN_SCHEMA_VERSION - CasStore no longer a public API, it is hidden behind CidStore - Moved cas.h from public header folder - CidStore no longer maps from Cid -> Cas, we store entries in Cas under RawHash - CasStore now decompresses data to validate content (matching against RawHash) - CasChunkSet renames to HashKeySet and put in separate header/cpp file - Disabled "Chunk" command for now as it relied on CAS being exposed as a service - Changed CAS http service to Cid http server - Moved "Run" command completely inside ZEN_WITH_EXEC_SERVICES define - Removed "cas.basic" test - Uncommented ".exec.basic" test and added return-skip at start of test - Moved ScrubContext to separate header file - Renamed CasGC to GcManager - Cleaned up configuration passing in cas store classes - Removed CAS stuff from GcContext and clarified naming in class - Remove migration code --- zenserver/zenserver.cpp | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) (limited to 'zenserver/zenserver.cpp') diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 801a98523..31ca8851a 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -17,8 +17,8 @@ #include #include #include -#include #include +#include #include #if ZEN_PLATFORM_WINDOWS @@ -56,7 +56,6 @@ ZEN_THIRD_PARTY_INCLUDES_END ////////////////////////////////////////////////////////////////////////// -#include "casstore.h" #include "config.h" #include "diag/logging.h" @@ -103,6 +102,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "auth/authservice.h" #include "cache/structuredcache.h" #include "cache/structuredcachestore.h" +#include "cidstore.h" #include "compute/function.h" #include "diag/diagsvcs.h" #include "experimental/usnjournal.h" @@ -253,17 +253,16 @@ public: ZEN_INFO("initializing storage"); - zen::CasStoreConfiguration Config; + zen::CidStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; - m_CasStore->Initialize(Config); - - m_CidStore = std::make_unique(*m_CasStore, m_DataRoot / "cid"); - m_CasGc.SetCidStore(m_CidStore.get()); + m_CidStore = std::make_unique(m_GcManager); + m_CidStore->Initialize(Config); + m_CidService.reset(new zen::HttpCidService{*m_CidStore}); ZEN_INFO("instantiating project service"); - m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_CasGc); + m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager); m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore}); #if ZEN_WITH_EXEC_SERVICES @@ -274,7 +273,7 @@ public: std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; zen::CreateDirectories(SandboxDir); - m_HttpLaunchService = std::make_unique(*m_CasStore, SandboxDir); + m_HttpLaunchService = std::make_unique(*m_CidStore, SandboxDir); } else { @@ -328,7 +327,7 @@ public: m_Http->RegisterService(*m_HttpProjectService); } - m_Http->RegisterService(m_CasService); + m_Http->RegisterService(*m_CidService); #if ZEN_WITH_EXEC_SERVICES if (ServerOptions.ExecServiceEnabled) @@ -522,7 +521,6 @@ public: ZEN_INFO("Storage validation STARTING"); ScrubContext Ctx; - m_CasStore->Scrub(Ctx); m_CidStore->Scrub(Ctx); m_ProjectStore->Scrub(Ctx); m_StructuredCacheService->Scrub(Ctx); @@ -538,9 +536,6 @@ public: void Flush() { - if (m_CasStore) - m_CasStore->Flush(); - if (m_CidStore) m_CidStore->Flush(); @@ -602,14 +597,13 @@ private: std::unique_ptr m_AuthService; zen::HttpStatusService m_StatusService; zen::HttpStatsService m_StatsService; - zen::CasGc m_CasGc; - zen::GcScheduler m_GcScheduler{m_CasGc}; - std::unique_ptr m_CasStore{zen::CreateCasStore(m_CasGc)}; + zen::GcManager m_GcManager; + zen::GcScheduler m_GcScheduler{m_GcManager}; std::unique_ptr m_CidStore; std::unique_ptr m_CacheStore; zen::HttpTestService m_TestService; zen::HttpTestingService m_TestingService; - zen::HttpCasService m_CasService{*m_CasStore}; + std::unique_ptr m_CidService; zen::RefPtr m_ProjectStore; std::unique_ptr m_HttpProjectService; std::unique_ptr m_UpstreamCache; @@ -745,7 +739,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) ZEN_INFO("instantiating structured cache service"); m_CacheStore = std::make_unique( - m_CasGc, + m_GcManager, ZenCacheStore::Configuration{.BasePath = m_DataRoot / "cache", .AllowAutomaticCreationOfNamespaces = true}); const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; @@ -873,8 +867,7 @@ ZenServer::InitializeCompute(const ZenServerOptions& ServerOptions) .OpenIdProvider = UpstreamConfig.HordeConfig.StorageOpenIdProvider, .AccessToken = UpstreamConfig.HordeConfig.StorageAccessToken}; - m_HttpFunctionService = std::make_unique(*m_CasStore, - *m_CidStore, + m_HttpFunctionService = std::make_unique(*m_CidStore, ComputeOptions, StorageOptions, ComputeAuthConfig, -- cgit v1.2.3