Implemented simple persistence for CidStore, which stores CID->CAS mappings

3 files changed, 74 insertions, 16 deletions

diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 7142024ea..e57a6142d 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -77,18 +77,20 @@ public:
 		// Ok so now we're configured, let's kick things off
 
 		zen::CasStoreConfiguration Config;
-		Config.RootDirectory = m_DataRoot / "CAS";
+		Config.RootDirectory = m_DataRoot / "cas";
 
 		m_CasStore->Initialize(Config);
 
 		spdlog::info("instantiating project service");
 
-		m_ProjectStore = new zen::ProjectStore(*m_CasStore, m_DataRoot / "Builds");
+		m_ProjectStore = new zen::ProjectStore(*m_CasStore, m_DataRoot / "projects");
 		m_HttpProjectService.reset(new zen::HttpProjectService{*m_CasStore, m_ProjectStore});
 		m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore);
 
 		m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CasStore);
 
+		m_CidStore = std::make_unique<zen::CidStore>(*m_CasStore, m_DataRoot / "cid");
+
 		if (ServiceConfig.LegacyCacheEnabled)
 		{
 			spdlog::info("instantiating legacy cache service");
@@ -102,7 +104,7 @@ public:
 		if (ServiceConfig.StructuredCacheEnabled)
 		{
 			spdlog::info("instantiating structured cache service");
-			m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(m_DataRoot / "cache", *m_CasStore, m_CidStore));
+			m_StructuredCacheService.reset(new zen::HttpStructuredCacheService(m_DataRoot / "cache", *m_CasStore, *m_CidStore));
 		}
 		else
 		{
@@ -208,7 +210,7 @@ private:
 
 	zen::HttpServer									 m_Http;
 	std::unique_ptr<zen::CasStore>					 m_CasStore{zen::CreateCasStore()};
-	zen::CidStore									 m_CidStore{*m_CasStore};
+	std::unique_ptr<zen::CidStore>					 m_CidStore;
 	zen::CasGc										 m_Gc{*m_CasStore};
 	zen::CasScrubber								 m_Scrubber{*m_CasStore};
 	HttpTestService									 m_TestService;
diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 6e558e639..e041574a4 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -1,12 +1,69 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "zenstore/cidstore.h"
-#include "zenstore/CAS.h"
+
+#include <zencore/filesystem.h>
+#include <zenstore/CAS.h>
+#include <zenstore/caslog.h>
+
+#include <spdlog/spdlog.h>
+#include <filesystem>
 
 namespace zen {
 
-CidStore::CidStore(CasStore& InCasStore) : m_CasStore(InCasStore)
+struct CidStore::CidState
+{
+	CidState(CasStore& InCasStore) : m_CasStore(InCasStore) {}
+
+	struct IndexEntry
+	{
+		IoHash Uncompressed;
+		IoHash Compressed;
+	};
+
+	CasStore&				m_CasStore;
+	TCasLogFile<IndexEntry> m_LogFile;
+
+	RwLock						   m_Lock;
+	tsl::robin_map<IoHash, IoHash> m_CidMap;
+
+	void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		m_CidMap.insert_or_assign(DecompressedId, Compressed);
+		m_LogFile.Append({.Uncompressed = DecompressedId, .Compressed = Compressed});
+	}
+
+	IoBuffer FindChunkByCid(const IoHash& DecompressedId)
+	{
+		if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
+		{
+			return m_CasStore.FindChunk(It->second);
+		}
+
+		return IoBuffer();
+	}
+
+	void InitializeIndex(const std::filesystem::path& RootDir)
+	{
+		zen::CreateDirectories(RootDir);
+		std::filesystem::path SlogPath{RootDir / "cid.slog"};
+
+		bool IsNew = !std::filesystem::exists(SlogPath);
+
+		m_LogFile.Open(SlogPath, IsNew);
+
+		m_LogFile.Replay([&](const IndexEntry& Ie) { m_CidMap.insert_or_assign(Ie.Uncompressed, Ie.Compressed); });
+
+		spdlog::debug("CID index initialized: {} entries found", m_CidMap.size());
+	}
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+CidStore::CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir) : m_Impl(std::make_unique<CidState>(InCasStore))
 {
+	m_Impl->InitializeIndex(RootDir);
 }
 
 CidStore::~CidStore()
@@ -16,18 +73,13 @@ CidStore::~CidStore()
 void
 CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
 {
-	m_CidMap.insert_or_assign(DecompressedId, Compressed);
+	m_Impl->AddCompressedCid(DecompressedId, Compressed);
 }
 
 IoBuffer
 CidStore::FindChunkByCid(const IoHash& DecompressedId)
 {
-	if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
-	{
-		return m_CasStore.FindChunk(It->second);
-	}
-
-	return IoBuffer();
+	return m_Impl->FindChunkByCid(DecompressedId);
 }
 
 }  // namespace zen
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 0c8f51740..e365b198e 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -5,6 +5,10 @@
 #include <tsl/robin_map.h>
 #include <zencore/iohash.h>
 
+namespace std::filesystem {
+class path;
+}
+
 namespace zen {
 
 class CasStore;
@@ -20,15 +24,15 @@ class IoBuffer;
 class CidStore
 {
 public:
-	CidStore(CasStore& InCasStore);
+	CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir);
 	~CidStore();
 
 	void	 AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
 	IoBuffer FindChunkByCid(const IoHash& DecompressedId);
 
 private:
-	CasStore&									   m_CasStore;
-	tsl::robin_map<IoHash, IoHash, IoHash::Hasher> m_CidMap;
+	struct CidState;
+	std::unique_ptr<CidState> m_Impl;
 };
 
 }  // namespace zen