Implemented simple persistence for CidStore, which stores CID->CAS mappings

2 files changed, 68 insertions, 12 deletions

diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp
index 6e558e639..e041574a4 100644
--- a/zenstore/cidstore.cpp
+++ b/zenstore/cidstore.cpp
@@ -1,12 +1,69 @@
 // Copyright Epic Games, Inc. All Rights Reserved.
 
 #include "zenstore/cidstore.h"
-#include "zenstore/CAS.h"
+
+#include <zencore/filesystem.h>
+#include <zenstore/CAS.h>
+#include <zenstore/caslog.h>
+
+#include <spdlog/spdlog.h>
+#include <filesystem>
 
 namespace zen {
 
-CidStore::CidStore(CasStore& InCasStore) : m_CasStore(InCasStore)
+struct CidStore::CidState
+{
+	CidState(CasStore& InCasStore) : m_CasStore(InCasStore) {}
+
+	struct IndexEntry
+	{
+		IoHash Uncompressed;
+		IoHash Compressed;
+	};
+
+	CasStore&				m_CasStore;
+	TCasLogFile<IndexEntry> m_LogFile;
+
+	RwLock						   m_Lock;
+	tsl::robin_map<IoHash, IoHash> m_CidMap;
+
+	void AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
+	{
+		RwLock::ExclusiveLockScope _(m_Lock);
+		m_CidMap.insert_or_assign(DecompressedId, Compressed);
+		m_LogFile.Append({.Uncompressed = DecompressedId, .Compressed = Compressed});
+	}
+
+	IoBuffer FindChunkByCid(const IoHash& DecompressedId)
+	{
+		if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
+		{
+			return m_CasStore.FindChunk(It->second);
+		}
+
+		return IoBuffer();
+	}
+
+	void InitializeIndex(const std::filesystem::path& RootDir)
+	{
+		zen::CreateDirectories(RootDir);
+		std::filesystem::path SlogPath{RootDir / "cid.slog"};
+
+		bool IsNew = !std::filesystem::exists(SlogPath);
+
+		m_LogFile.Open(SlogPath, IsNew);
+
+		m_LogFile.Replay([&](const IndexEntry& Ie) { m_CidMap.insert_or_assign(Ie.Uncompressed, Ie.Compressed); });
+
+		spdlog::debug("CID index initialized: {} entries found", m_CidMap.size());
+	}
+};
+
+//////////////////////////////////////////////////////////////////////////
+
+CidStore::CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir) : m_Impl(std::make_unique<CidState>(InCasStore))
 {
+	m_Impl->InitializeIndex(RootDir);
 }
 
 CidStore::~CidStore()
@@ -16,18 +73,13 @@ CidStore::~CidStore()
 void
 CidStore::AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed)
 {
-	m_CidMap.insert_or_assign(DecompressedId, Compressed);
+	m_Impl->AddCompressedCid(DecompressedId, Compressed);
 }
 
 IoBuffer
 CidStore::FindChunkByCid(const IoHash& DecompressedId)
 {
-	if (auto It = m_CidMap.find(DecompressedId); It != m_CidMap.end())
-	{
-		return m_CasStore.FindChunk(It->second);
-	}
-
-	return IoBuffer();
+	return m_Impl->FindChunkByCid(DecompressedId);
 }
 
 }  // namespace zen
diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h
index 0c8f51740..e365b198e 100644
--- a/zenstore/include/zenstore/cidstore.h
+++ b/zenstore/include/zenstore/cidstore.h
@@ -5,6 +5,10 @@
 #include <tsl/robin_map.h>
 #include <zencore/iohash.h>
 
+namespace std::filesystem {
+class path;
+}
+
 namespace zen {
 
 class CasStore;
@@ -20,15 +24,15 @@ class IoBuffer;
 class CidStore
 {
 public:
-	CidStore(CasStore& InCasStore);
+	CidStore(CasStore& InCasStore, const std::filesystem::path& RootDir);
 	~CidStore();
 
 	void	 AddCompressedCid(const IoHash& DecompressedId, const IoHash& Compressed);
 	IoBuffer FindChunkByCid(const IoHash& DecompressedId);
 
 private:
-	CasStore&									   m_CasStore;
-	tsl::robin_map<IoHash, IoHash, IoHash::Hasher> m_CidMap;
+	struct CidState;
+	std::unique_ptr<CidState> m_Impl;
 };
 
 }  // namespace zen