fix concurrency issues in projectstore and enable GC (#181)

* Fix concurreny issues when deleting projects/oplogs
* remove rocksdb test code
* project store unit tests
* safer deletion of oplogs/projects
* reference count ProjectStore::Project to handle lifetime during GC
* Don't open all project oplogs unless we need them
* Don't scrub expired projects
* Don't gather references from expired projects
* added logging details for GC
* release lock as soon as folder is moved
* more tests for project store
* changelog

1 files changed, 594 insertions, 237 deletions

diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp
index ab407f868..d953651af 100644
--- a/zenserver/projectstore.cpp
+++ b/zenserver/projectstore.cpp
@@ -21,31 +21,50 @@
 
 #include "config.h"
 
-#if ZEN_PLATFORM_WINDOWS
-#	include <zencore/windows.h>
-#endif
-
-#define USE_ROCKSDB 0
-
 ZEN_THIRD_PARTY_INCLUDES_START
-
-#if USE_ROCKSDB
-#	pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
-#	include <rocksdb/db.h>
-#endif
-
 #include <xxh3.h>
-#include <asio.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
-#include <latch>
-#include <string>
+#if ZEN_WITH_TESTS
+#endif	// ZEN_WITH_TESTS
 
 namespace zen {
 
-#if USE_ROCKSDB
-namespace rocksdb = ROCKSDB_NAMESPACE;
-#endif
+namespace {
+	bool PrepareDirectoryDelete(const std::filesystem::path& Dir, std::filesystem::path& OutDeleteDir)
+	{
+		int DropIndex = 0;
+		do
+		{
+			if (!std::filesystem::exists(Dir))
+			{
+				return true;
+			}
+
+			std::string			  DroppedName		= fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex);
+			std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName;
+			if (std::filesystem::exists(DroppedBucketPath))
+			{
+				DropIndex++;
+				continue;
+			}
+
+			std::error_code Ec;
+			std::filesystem::rename(Dir, DroppedBucketPath, Ec);
+			if (!Ec)
+			{
+				OutDeleteDir = DroppedBucketPath;
+				return true;
+			}
+			if (Ec && !std::filesystem::exists(DroppedBucketPath))
+			{
+				// We can't move our folder, probably because it is busy, bail..
+				return false;
+			}
+			zen::Sleep(100);
+		} while (true);
+	}
+}  // namespace
 
 //////////////////////////////////////////////////////////////////////////
 
@@ -79,24 +98,6 @@ struct ProjectStore::OplogStorage : public RefCounted
 	{
 		ZEN_INFO("closing oplog storage at {}", m_OplogStoragePath);
 		Flush();
-
-#if USE_ROCKSDB
-		if (m_RocksDb)
-		{
-			// Column families must be torn down before database is closed
-			for (const auto& Handle : m_RocksDbColumnHandles)
-			{
-				m_RocksDb->DestroyColumnFamilyHandle(Handle);
-			}
-
-			rocksdb::Status Status = m_RocksDb->Close();
-
-			if (!Status.ok())
-			{
-				ZEN_WARN("db close error reported for '{}' : '{}'", m_OplogStoragePath, Status.getState());
-			}
-		}
-#endif
 	}
 
 	[[nodiscard]] bool		  Exists() { return Exists(m_OplogStoragePath); }
@@ -126,51 +127,6 @@ struct ProjectStore::OplogStorage : public RefCounted
 
 		ZEN_ASSERT(IsPow2(m_OpsAlign));
 		ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1)));
-
-#if USE_ROCKSDB
-		{
-			std::string RocksdbPath = PathToUtf8(m_OplogStoragePath / "ops.rdb");
-
-			ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath);
-
-			rocksdb::DB*	   Db;
-			rocksdb::DBOptions Options;
-			Options.create_if_missing = true;
-
-			std::vector<std::string> ExistingColumnFamilies;
-			rocksdb::Status			 Status = rocksdb::DB::ListColumnFamilies(Options, RocksdbPath, &ExistingColumnFamilies);
-
-			std::vector<rocksdb::ColumnFamilyDescriptor> ColumnDescriptors;
-
-			if (Status.IsPathNotFound())
-			{
-				ColumnDescriptors.emplace_back(rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName, {}});
-			}
-			else if (Status.ok())
-			{
-				for (const std::string& Column : ExistingColumnFamilies)
-				{
-					rocksdb::ColumnFamilyDescriptor ColumnFamily;
-					ColumnFamily.name = Column;
-					ColumnDescriptors.push_back(ColumnFamily);
-				}
-			}
-			else
-			{
-				throw std::runtime_error(
-					fmt::format("column family iteration failed for '{}': '{}'", RocksdbPath, Status.getState()).c_str());
-			}
-
-			Status = rocksdb::DB::Open(Options, RocksdbPath, ColumnDescriptors, &m_RocksDbColumnHandles, &Db);
-
-			if (!Status.ok())
-			{
-				throw std::runtime_error(fmt::format("database open failed for '{}': '{}'", RocksdbPath, Status.getState()).c_str());
-			}
-
-			m_RocksDb.reset(Db);
-		}
-#endif
 	}
 
 	void ReplayLog(std::function<void(CbObject, const OplogEntry&)>&& Handler)
@@ -301,11 +257,6 @@ private:
 	std::atomic<uint64_t>	m_NextOpsOffset{0};
 	uint64_t				m_OpsAlign = 32;
 	std::atomic<uint32_t>	m_MaxLsn{0};
-
-#if USE_ROCKSDB
-	std::unique_ptr<rocksdb::DB>			  m_RocksDb;
-	std::vector<rocksdb::ColumnFamilyHandle*> m_RocksDbColumnHandles;
-#endif
 };
 
 //////////////////////////////////////////////////////////////////////////
@@ -329,12 +280,16 @@ ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Stor
 
 ProjectStore::Oplog::~Oplog()
 {
-	Flush();
+	if (m_Storage)
+	{
+		Flush();
+	}
 }
 
 void
 ProjectStore::Oplog::Flush()
 {
+	ZEN_ASSERT(m_Storage);
 	m_Storage->Flush();
 }
 
@@ -350,6 +305,7 @@ ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
 	RwLock::SharedLockScope _(m_OplogLock);
 
 	std::vector<IoHash> Hashes;
+	Hashes.reserve(Max(m_ChunkMap.size(), m_MetaMap.size()));
 
 	for (const auto& Kv : m_ChunkMap)
 	{
@@ -368,6 +324,28 @@ ProjectStore::Oplog::GatherReferences(GcContext& GcCtx)
 	GcCtx.AddRetainedCids(Hashes);
 }
 
+std::filesystem::path
+ProjectStore::Oplog::PrepareForDelete(bool MoveFolder)
+{
+	RwLock::ExclusiveLockScope _(m_OplogLock);
+	m_ChunkMap.clear();
+	m_MetaMap.clear();
+	m_FileMap.clear();
+	m_OpAddressMap.clear();
+	m_LatestOpMap.clear();
+	m_Storage = {};
+	if (!MoveFolder)
+	{
+		return {};
+	}
+	std::filesystem::path MovedDir;
+	if (PrepareDirectoryDelete(m_BasePath, MovedDir))
+	{
+		return MovedDir;
+	}
+	return {};
+}
+
 bool
 ProjectStore::Oplog::ExistsAt(std::filesystem::path BasePath)
 {
@@ -383,12 +361,12 @@ ProjectStore::Oplog::ReplayLog()
 IoBuffer
 ProjectStore::Oplog::FindChunk(Oid ChunkId)
 {
-	RwLock::SharedLockScope _(m_OplogLock);
+	RwLock::SharedLockScope OplogLock(m_OplogLock);
 
 	if (auto ChunkIt = m_ChunkMap.find(ChunkId); ChunkIt != m_ChunkMap.end())
 	{
 		IoHash ChunkHash = ChunkIt->second;
-		_.ReleaseNow();
+		OplogLock.ReleaseNow();
 
 		IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkHash);
 		Chunk.SetContentType(ZenContentType::kCompressedBinary);
@@ -400,7 +378,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
 	{
 		std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath;
 
-		_.ReleaseNow();
+		OplogLock.ReleaseNow();
 
 		IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath);
 		FileChunk.SetContentType(ZenContentType::kBinary);
@@ -411,7 +389,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId)
 	if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end())
 	{
 		IoHash ChunkHash = MetaIt->second;
-		_.ReleaseNow();
+		OplogLock.ReleaseNow();
 
 		IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkHash);
 		Chunk.SetContentType(ZenContentType::kCompressedBinary);
@@ -487,10 +465,12 @@ ProjectStore::Oplog::GetOpByIndex(int Index)
 }
 
 bool
-ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view ServerPath, std::string_view ClientPath)
+ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&,
+									Oid				 FileId,
+									IoHash			 Hash,
+									std::string_view ServerPath,
+									std::string_view ClientPath)
 {
-	// NOTE: Caller must hold an exclusive lock on m_OplogLock
-
 	if (ServerPath.empty() || ClientPath.empty())
 	{
 		return false;
@@ -511,18 +491,14 @@ ProjectStore::Oplog::AddFileMapping(Oid FileId, IoHash Hash, std::string_view Se
 }
 
 void
-ProjectStore::Oplog::AddChunkMapping(Oid ChunkId, IoHash Hash)
+ProjectStore::Oplog::AddChunkMapping(const RwLock::ExclusiveLockScope&, Oid ChunkId, IoHash Hash)
 {
-	// NOTE: Caller must hold an exclusive lock on m_OplogLock
-
 	m_ChunkMap.insert_or_assign(ChunkId, Hash);
 }
 
 void
-ProjectStore::Oplog::AddMetaMapping(Oid ChunkId, IoHash Hash)
+ProjectStore::Oplog::AddMetaMapping(const RwLock::ExclusiveLockScope&, Oid ChunkId, IoHash Hash)
 {
-	// NOTE: Caller must hold an exclusive lock on m_OplogLock
-
 	m_MetaMap.insert_or_assign(ChunkId, Hash);
 }
 
@@ -536,7 +512,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 	// For now we're assuming the update is all in-memory so we can hold an exclusive lock without causing
 	// too many problems. Longer term we'll probably want to ensure we can do concurrent updates however
 
-	RwLock::ExclusiveLockScope _(m_OplogLock);
+	RwLock::ExclusiveLockScope OplogLock(m_OplogLock);
 
 	using namespace std::literals;
 
@@ -548,7 +524,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		Oid			 PackageId	 = PkgObj["id"sv].AsObjectId();
 		IoHash		 PackageHash = PkgObj["data"sv].AsBinaryAttachment();
 
-		AddChunkMapping(PackageId, PackageHash);
+		AddChunkMapping(OplogLock, PackageId, PackageHash);
 
 		ZEN_DEBUG("package data {} -> {}", PackageId, PackageHash);
 	}
@@ -560,7 +536,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		Oid	   BulkDataId	= BulkObj["id"sv].AsObjectId();
 		IoHash BulkDataHash = BulkObj["data"sv].AsBinaryAttachment();
 
-		AddChunkMapping(BulkDataId, BulkDataHash);
+		AddChunkMapping(OplogLock, BulkDataId, BulkDataHash);
 
 		ZEN_DEBUG("bulkdata {} -> {}", BulkDataId, BulkDataHash);
 	}
@@ -568,7 +544,8 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 	if (Core["files"sv])
 	{
 		Stopwatch Timer;
-		int32_t	  FileCount = 0;
+		int32_t	  FileCount	 = 0;
+		int32_t	  ChunkCount = 0;
 
 		for (CbFieldView& Entry : Core["files"sv])
 		{
@@ -578,7 +555,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 			std::string_view ServerPath	  = FileObj["serverpath"sv].AsString();
 			std::string_view ClientPath	  = FileObj["clientpath"sv].AsString();
 
-			if (AddFileMapping(FileId, FileDataHash, ServerPath, ClientPath))
+			if (AddFileMapping(OplogLock, FileId, FileDataHash, ServerPath, ClientPath))
 			{
 				++FileCount;
 			}
@@ -588,7 +565,11 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 			}
 		}
 
-		ZEN_DEBUG("added {} file(s) in {}", FileCount, NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+		ZEN_INFO("added {} file(s), {} as files and {} as chunks in {}",
+				 FileCount + ChunkCount,
+				 FileCount,
+				 ChunkCount,
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
 	}
 
 	for (CbFieldView& Entry : Core["meta"sv])
@@ -598,7 +579,7 @@ ProjectStore::Oplog::RegisterOplogEntry(CbObject Core, const OplogEntry& OpEntry
 		auto		 NameString	  = MetaObj["name"sv].AsString();
 		IoHash		 MetaDataHash = MetaObj["data"sv].AsBinaryAttachment();
 
-		AddMetaMapping(MetaId, MetaDataHash);
+		AddMetaMapping(OplogLock, MetaId, MetaDataHash);
 
 		ZEN_DEBUG("meta data ({}) {} -> {}", NameString, MetaId, MetaDataHash);
 	}
@@ -614,6 +595,8 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage)
 {
 	ZEN_TRACE_CPU("ProjectStore::Oplog::AppendNewOplogEntry");
 
+	ZEN_ASSERT(m_Storage);
+
 	using namespace std::literals;
 
 	const CbObject&	 Core	 = OpPackage.GetObject();
@@ -750,9 +733,11 @@ ProjectStore::Project::NewOplog(std::string_view OplogId)
 
 	try
 	{
-		Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, OplogBasePath).first->second;
+		Oplog* Log =
+			m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath))
+				.first->second.get();
 
-		return &Log;
+		return Log;
 	}
 	catch (std::exception&)
 	{
@@ -776,7 +761,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 
 		if (OplogIt != m_Oplogs.end())
 		{
-			return &OplogIt->second;
+			return OplogIt->second.get();
 		}
 	}
 
@@ -790,11 +775,13 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 
 		try
 		{
-			Oplog& Log = m_Oplogs.try_emplace(std::string{OplogId}, OplogId, this, m_CidStore, OplogBasePath).first->second;
+			Oplog* Log =
+				m_Oplogs.try_emplace(std::string{OplogId}, std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath))
+					.first->second.get();
 
-			Log.ReplayLog();
+			Log->ReplayLog();
 
-			return &Log;
+			return Log;
 		}
 		catch (std::exception& ex)
 		{
@@ -810,6 +797,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId)
 void
 ProjectStore::Project::DeleteOplog(std::string_view OplogId)
 {
+	std::filesystem::path DeletePath;
 	{
 		RwLock::ExclusiveLockScope _(m_ProjectLock);
 
@@ -817,83 +805,114 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId)
 
 		if (OplogIt != m_Oplogs.end())
 		{
+			std::unique_ptr<Oplog>& Oplog = OplogIt->second;
+			DeletePath					  = Oplog->PrepareForDelete(true);
+			m_DeletedOplogs.emplace_back(std::move(Oplog));
 			m_Oplogs.erase(OplogIt);
 		}
 	}
 
-	// Actually erase
-
-	std::filesystem::path OplogBasePath = BasePathForOplog(OplogId);
-
-	OplogStorage::Delete(OplogBasePath);
+	// Erase content on disk
+	if (!DeletePath.empty())
+	{
+		OplogStorage::Delete(DeletePath);
+	}
 }
 
-void
-ProjectStore::Project::DiscoverOplogs()
+std::vector<std::string>
+ProjectStore::Project::ScanForOplogs() const
 {
 	DirectoryContent DirContent;
 	GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent);
-
+	std::vector<std::string> Oplogs;
+	Oplogs.reserve(DirContent.Directories.size());
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
-		OpenOplog(PathToUtf8(DirPath.filename()));
+		Oplogs.push_back(DirPath.filename().string());
 	}
+	return Oplogs;
 }
 
 void
 ProjectStore::Project::IterateOplogs(std::function<void(const Oplog&)>&& Fn) const
 {
-	// TODO: should also iterate over oplogs which are present on disk but not yet loaded
-
 	RwLock::SharedLockScope _(m_ProjectLock);
 
 	for (auto& Kv : m_Oplogs)
 	{
-		Fn(Kv.second);
+		Fn(*Kv.second);
 	}
 }
 
 void
 ProjectStore::Project::IterateOplogs(std::function<void(Oplog&)>&& Fn)
 {
-	// TODO: should also iterate over oplogs which are present on disk but not yet loaded
-
 	RwLock::SharedLockScope _(m_ProjectLock);
 
 	for (auto& Kv : m_Oplogs)
 	{
-		Fn(Kv.second);
+		Fn(*Kv.second);
 	}
 }
 
 void
 ProjectStore::Project::Flush()
 {
-	// TODO
+	// We only need to flush oplogs that we have already loaded
+	IterateOplogs([&](Oplog& Ops) { Ops.Flush(); });
 }
 
 void
 ProjectStore::Project::Scrub(ScrubContext& Ctx)
 {
+	// Scrubbing needs to check all existing oplogs
+	std::vector<std::string> OpLogs = ScanForOplogs();
+	for (const std::string& OpLogId : OpLogs)
+	{
+		OpenOplog(OpLogId);
+	}
 	IterateOplogs([&](const Oplog& Ops) { Ops.Scrub(Ctx); });
 }
 
 void
 ProjectStore::Project::GatherReferences(GcContext& GcCtx)
 {
-	if (IsExpired())
+	ZEN_TRACE_CPU("ProjectStore::Project::GatherReferences");
+
+	Stopwatch  Timer;
+	const auto Guard = MakeGuard(
+		[&] { ZEN_INFO("gathered references from project store project {} in {}", Identifier, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
+	// GatherReferences needs to check all existing oplogs
+	std::vector<std::string> OpLogs = ScanForOplogs();
+	for (const std::string& OpLogId : OpLogs)
 	{
-		ZEN_DEBUG("Skipping reference gathering for '{}', project file '{}' no longer exist", Identifier, ProjectFilePath);
-		return;
+		OpenOplog(OpLogId);
 	}
 	IterateOplogs([&](Oplog& Ops) { Ops.GatherReferences(GcCtx); });
 }
 
-void
-ProjectStore::Project::Delete()
+bool
+ProjectStore::Project::PrepareForDelete(std::filesystem::path& OutDeletePath)
 {
 	RwLock::ExclusiveLockScope _(m_ProjectLock);
-	DeleteDirectories(m_OplogStoragePath);
+
+	for (auto& It : m_Oplogs)
+	{
+		// We don't care about the moved folder
+		It.second->PrepareForDelete(false);
+		m_DeletedOplogs.emplace_back(std::move(It.second));
+	}
+
+	m_Oplogs.clear();
+
+	bool Success = PrepareDirectoryDelete(m_OplogStoragePath, OutDeletePath);
+	if (!Success)
+	{
+		return false;
+	}
+	m_OplogStoragePath.clear();
+	return true;
 }
 
 bool
@@ -944,12 +963,7 @@ ProjectStore::DiscoverProjects()
 	for (const std::filesystem::path& DirPath : DirContent.Directories)
 	{
 		std::string DirName = PathToUtf8(DirPath.filename());
-		Project*	Project = OpenProject(DirName);
-
-		if (Project)
-		{
-			Project->DiscoverOplogs();
-		}
+		OpenProject(DirName);
 	}
 }
 
@@ -960,74 +974,166 @@ ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn)
 
 	for (auto& Kv : m_Projects)
 	{
-		Fn(Kv.second);
+		Fn(*Kv.second.Get());
 	}
 }
 
 void
 ProjectStore::Flush()
 {
-	RwLock::SharedLockScope _(m_ProjectsLock);
+	std::vector<Ref<Project>> Projects;
+	{
+		RwLock::SharedLockScope _(m_ProjectsLock);
+		Projects.reserve(m_Projects.size());
 
-	for (auto& Kv : m_Projects)
+		for (auto& Kv : m_Projects)
+		{
+			Projects.push_back(Kv.second);
+		}
+	}
+	for (const Ref<Project>& Project : Projects)
 	{
-		Kv.second.Flush();
+		Project->Flush();
 	}
 }
 
 void
 ProjectStore::Scrub(ScrubContext& Ctx)
 {
-	RwLock::SharedLockScope _(m_ProjectsLock);
+	DiscoverProjects();
 
-	for (auto& Kv : m_Projects)
+	std::vector<Ref<Project>> Projects;
 	{
-		Kv.second.Scrub(Ctx);
+		RwLock::SharedLockScope _(m_ProjectsLock);
+		Projects.reserve(m_Projects.size());
+
+		for (auto& Kv : m_Projects)
+		{
+			if (Kv.second->IsExpired())
+			{
+				continue;
+			}
+			Projects.push_back(Kv.second);
+		}
+	}
+	for (const Ref<Project>& Project : Projects)
+	{
+		Project->Scrub(Ctx);
 	}
 }
 
 void
 ProjectStore::GatherReferences(GcContext& GcCtx)
 {
+	ZEN_TRACE_CPU("ProjectStore::GatherReferences");
+
+	size_t	   ProjectCount		   = 0;
+	size_t	   ExpiredProjectCount = 0;
 	Stopwatch  Timer;
-	const auto Guard =
-		MakeGuard([&] { ZEN_INFO("project store gathered all references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+	const auto Guard = MakeGuard([&] {
+		ZEN_INFO("gathered references from '{}' in {}, found {} active projects and {} expired projects",
+				 m_ProjectBasePath.string(),
+				 NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+				 ProjectCount,
+				 ExpiredProjectCount);
+	});
 
 	DiscoverProjects();
 
-	RwLock::SharedLockScope _(m_ProjectsLock);
+	std::vector<Ref<Project>> Projects;
+	{
+		RwLock::SharedLockScope _(m_ProjectsLock);
+		Projects.reserve(m_Projects.size());
 
-	for (auto& Kv : m_Projects)
+		for (auto& Kv : m_Projects)
+		{
+			if (Kv.second->IsExpired())
+			{
+				ExpiredProjectCount++;
+				continue;
+			}
+			Projects.push_back(Kv.second);
+		}
+	}
+	ProjectCount = Projects.size();
+	for (const Ref<Project>& Project : Projects)
 	{
-		Kv.second.GatherReferences(GcCtx);
+		Project->GatherReferences(GcCtx);
 	}
 }
 
 void
 ProjectStore::CollectGarbage(GcContext& GcCtx)
 {
-	std::vector<std::string> ExpiredProjects;
+	ZEN_TRACE_CPU("ProjectStore::CollectGarbage");
+
+	size_t ProjectCount		   = 0;
+	size_t ExpiredProjectCount = 0;
+
+	Stopwatch				  Timer;
+	const auto				  Guard = MakeGuard([&] {
+		   ZEN_INFO("garbage collect from '{}' DONE after {}, found {} active projects and {} expired projects",
+					m_ProjectBasePath.string(),
+					NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+					ProjectCount,
+					ExpiredProjectCount);
+	   });
+	std::vector<Ref<Project>> ExpiredProjects;
 
 	{
 		RwLock::SharedLockScope _(m_ProjectsLock);
 		for (auto& Kv : m_Projects)
 		{
-			if (Kv.second.IsExpired())
+			if (Kv.second->IsExpired())
 			{
-				ExpiredProjects.push_back(Kv.first);
+				ExpiredProjects.push_back(Kv.second);
+				ExpiredProjectCount++;
+				continue;
 			}
+			ProjectCount++;
 		}
 	}
 
+	if (ExpiredProjects.empty())
+	{
+		ZEN_INFO("garbage collect SKIPPED, for '{}', no expired projects found", m_ProjectBasePath.string());
+		return;
+	}
+
 	if (!GcCtx.IsDeletionMode())
 	{
+		ZEN_INFO("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string());
 		return;
 	}
 
-	for (const std::string& ProjectId : ExpiredProjects)
+	for (const Ref<Project>& Project : ExpiredProjects)
 	{
-		ZEN_INFO("ProjectStore::CollectGarbage would delete '{}'. Disabled for now due to concurrency issues in ProjectStore", ProjectId);
-		//		DeleteProject(ProjectId, true);
+		std::filesystem::path PathToRemove;
+		std::string			  ProjectId;
+		{
+			RwLock::ExclusiveLockScope _(m_ProjectsLock);
+			if (!Project->IsExpired())
+			{
+				ZEN_INFO("ProjectStore::CollectGarbage skipped garbage collect of project '{}'. Project no longer expired.", ProjectId);
+				continue;
+			}
+			bool Success = Project->PrepareForDelete(PathToRemove);
+			if (!Success)
+			{
+				ZEN_INFO("ProjectStore::CollectGarbage skipped garbage collect of project '{}'. Project folder is locked.", ProjectId);
+				continue;
+			}
+			m_Projects.erase(Project->Identifier);
+			ProjectId = Project->Identifier;
+		}
+
+		ZEN_INFO("ProjectStore::CollectGarbage garbage collected project '{}'. Removing storage on disk", ProjectId);
+		if (PathToRemove.empty())
+		{
+			continue;
+		}
+
+		DeleteDirectories(PathToRemove);
 	}
 }
 
@@ -1037,7 +1143,7 @@ ProjectStore::StorageSize() const
 	return {0, 0};
 }
 
-ProjectStore::Project*
+Ref<ProjectStore::Project>
 ProjectStore::OpenProject(std::string_view ProjectId)
 {
 	{
@@ -1047,28 +1153,31 @@ ProjectStore::OpenProject(std::string_view ProjectId)
 
 		if (ProjIt != m_Projects.end())
 		{
-			return &(ProjIt->second);
+			return ProjIt->second;
 		}
 	}
 
 	RwLock::ExclusiveLockScope _(m_ProjectsLock);
 
-	std::filesystem::path ProjectBasePath = BasePathForProject(ProjectId);
+	std::filesystem::path BasePath = BasePathForProject(ProjectId);
 
-	if (Project::Exists(ProjectBasePath))
+	if (Project::Exists(BasePath))
 	{
 		try
 		{
-			ZEN_INFO("opening project {} @ {}", ProjectId, ProjectBasePath);
-
-			ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, ProjectBasePath).first->second;
-			Prj.Identifier			   = ProjectId;
-			Prj.Read();
-			return &Prj;
+			ZEN_INFO("opening project {} @ {}", ProjectId, BasePath);
+
+			Ref<Project>& Prj =
+				m_Projects
+					.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+					.first->second;
+			Prj->Identifier = ProjectId;
+			Prj->Read();
+			return Prj;
 		}
 		catch (std::exception& e)
 		{
-			ZEN_WARN("failed to open {} @ {} ({})", ProjectId, ProjectBasePath, e.what());
+			ZEN_WARN("failed to open {} @ {} ({})", ProjectId, BasePath, e.what());
 			m_Projects.erase(std::string{ProjectId});
 		}
 	}
@@ -1076,7 +1185,7 @@ ProjectStore::OpenProject(std::string_view ProjectId)
 	return nullptr;
 }
 
-ProjectStore::Project*
+Ref<ProjectStore::Project>
 ProjectStore::NewProject(std::filesystem::path BasePath,
 						 std::string_view	   ProjectId,
 						 std::string_view	   RootDir,
@@ -1086,39 +1195,48 @@ ProjectStore::NewProject(std::filesystem::path BasePath,
 {
 	RwLock::ExclusiveLockScope _(m_ProjectsLock);
 
-	ProjectStore::Project& Prj = m_Projects.try_emplace(std::string{ProjectId}, this, m_CidStore, BasePath).first->second;
-	Prj.Identifier			   = ProjectId;
-	Prj.RootDir				   = RootDir;
-	Prj.EngineRootDir		   = EngineRootDir;
-	Prj.ProjectRootDir		   = ProjectRootDir;
-	Prj.ProjectFilePath		   = ProjectFilePath;
-	Prj.Write();
-
-	return &Prj;
+	Ref<Project>& Prj =
+		m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath)))
+			.first->second;
+	Prj->Identifier		 = ProjectId;
+	Prj->RootDir		 = RootDir;
+	Prj->EngineRootDir	 = EngineRootDir;
+	Prj->ProjectRootDir	 = ProjectRootDir;
+	Prj->ProjectFilePath = ProjectFilePath;
+	Prj->Write();
+
+	return Prj;
 }
 
-void
-ProjectStore::DeleteProject(std::string_view ProjectId, bool OnlyDeleteIfExpired)
+bool
+ProjectStore::DeleteProject(std::string_view ProjectId)
 {
 	ZEN_INFO("deleting project {}", ProjectId);
 
-	RwLock::SharedLockScope _(m_ProjectsLock);
+	RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock);
 
 	auto ProjIt = m_Projects.find(std::string{ProjectId});
 
 	if (ProjIt == m_Projects.end())
 	{
-		return;
+		return true;
 	}
-	if (OnlyDeleteIfExpired && !ProjIt->second.IsExpired())
+
+	std::filesystem::path DeletePath;
+	bool				  Success = ProjIt->second->PrepareForDelete(DeletePath);
+
+	if (!Success)
 	{
-		return;
+		return false;
 	}
-	// _.ReleaseNow(); TODO: We can't release here since any changes to m_Projects map will invalidate our ProjIt pointer
-	ProjIt->second.Delete();
+	m_Projects.erase(ProjIt);
+	ProjectsLock.ReleaseNow();
 
-	m_Projects.erase(
-		ProjIt);  // TODO: We need to keep the project pointer alive, somebody else may use it after fetching it from m_Projects
+	if (!DeletePath.empty())
+	{
+		DeleteDirectories(DeletePath);
+	}
+	return true;
 }
 
 bool
@@ -1127,17 +1245,6 @@ ProjectStore::Exists(std::string_view ProjectId)
 	return Project::Exists(BasePathForProject(ProjectId));
 }
 
-ProjectStore::Oplog*
-ProjectStore::OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId)
-{
-	if (Project* ProjectIt = OpenProject(ProjectId))
-	{
-		return ProjectIt->OpenOplog(OplogId);
-	}
-
-	return nullptr;
-}
-
 //////////////////////////////////////////////////////////////////////////
 
 HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
@@ -1186,9 +1293,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
 
-			if (FoundLog == nullptr)
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1314,9 +1427,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
 
-			if (FoundLog == nullptr)
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1356,9 +1475,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& OplogId	  = Req.GetCapture(2);
 			const auto& ChunkId	  = Req.GetCapture(3);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
 
-			if (FoundLog == nullptr)
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1435,9 +1560,16 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 				AcceptType = HttpContentType::kBinary;
 			}
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				m_Log.warn("chunk - '{}/{}/{}' FAILED, missing project", ProjectId, OplogId, ChunkId);
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
 
-			if (FoundLog == nullptr)
+			if (!FoundLog)
 			{
 				m_Log.warn("chunk - '{}/{}/{}' FAILED, missing oplog", ProjectId, OplogId, ChunkId);
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
@@ -1548,9 +1680,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
 
-			if (FoundLog == nullptr)
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1611,9 +1749,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 				IsUsingSalt			= true;
 			}
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
 
-			if (FoundLog == nullptr)
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1713,9 +1857,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const std::string& OplogId	  = Req.GetCapture(2);
 			const std::string& OpIdString = Req.GetCapture(3);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
+
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
 
-			if (FoundLog == nullptr)
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1797,22 +1947,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
-			ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 
-			if (!ProjectIt)
+			if (!Project)
 			{
 				return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
 														 HttpContentType::kText,
 														 fmt::format("project {} not found", ProjectId));
 			}
 
-			ProjectStore::Project& Prj = *ProjectIt;
-
 			switch (Req.ServerRequest().RequestVerb())
 			{
 				case HttpVerb::kGet:
 					{
-						ProjectStore::Oplog* OplogIt = Prj.OpenOplog(OplogId);
+						ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
 
 						if (!OplogIt)
 						{
@@ -1824,7 +1972,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 						ProjectStore::Oplog& Log = *OplogIt;
 
 						CbObjectWriter Cb;
-						Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str();
+						Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str();
 
 						Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save());
 					}
@@ -1832,11 +1980,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 				case HttpVerb::kPost:
 					{
-						ProjectStore::Oplog* OplogIt = Prj.OpenOplog(OplogId);
+						ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId);
 
 						if (!OplogIt)
 						{
-							if (!Prj.NewOplog(OplogId))
+							if (!Project->NewOplog(OplogId))
 							{
 								// TODO: indicate why the operation failed!
 								return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError);
@@ -1858,7 +2006,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 					{
 						ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId);
 
-						ProjectIt->DeleteOplog(OplogId);
+						Project->DeleteOplog(OplogId);
 
 						return Req.ServerRequest().WriteResponse(HttpResponseCode::OK);
 					}
@@ -1878,9 +2026,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 			const auto& ProjectId = Req.GetCapture(1);
 			const auto& OplogId	  = Req.GetCapture(2);
 
-			ProjectStore::Oplog* FoundLog = m_ProjectStore->OpenProjectOplog(ProjectId, OplogId);
+			Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
+			if (!Project)
+			{
+				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
+			}
 
-			if (FoundLog == nullptr)
+			ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId);
+
+			if (!FoundLog)
 			{
 				return HttpReq.WriteResponse(HttpResponseCode::NotFound);
 			}
@@ -1954,26 +2108,27 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 				case HttpVerb::kGet:
 					{
-						ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId);
+						Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 
-						if (!ProjectIt)
+						if (!Project)
 						{
 							return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
 																	 HttpContentType::kText,
 																	 fmt::format("project {} not found", ProjectId));
 						}
 
-						const ProjectStore::Project& Prj = *ProjectIt;
+						std::vector<std::string> OpLogs = Project->ScanForOplogs();
 
 						CbObjectWriter Response;
-						Response << "id"sv << Prj.Identifier << "root"sv << PathToUtf8(Prj.RootDir);
+						Response << "id"sv << Project->Identifier << "root"sv << PathToUtf8(Project->RootDir);
 
 						Response.BeginArray("oplogs"sv);
-						Prj.IterateOplogs([&](const ProjectStore::Oplog& I) {
+						for (const std::string& OplogId : OpLogs)
+						{
 							Response.BeginObject();
-							Response << "id"sv << I.OplogId();
+							Response << "id"sv << OplogId;
 							Response.EndObject();
-						});
+						}
 						Response.EndArray();  // oplogs
 
 						Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
@@ -1982,16 +2137,21 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects)
 
 				case HttpVerb::kDelete:
 					{
-						ProjectStore::Project* ProjectIt = m_ProjectStore->OpenProject(ProjectId);
+						Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId);
 
-						if (!ProjectIt)
+						if (!Project)
 						{
 							return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound,
 																	 HttpContentType::kText,
 																	 fmt::format("project {} not found", ProjectId));
 						}
 
-						m_ProjectStore->DeleteProject(ProjectId, false);
+						if (!m_ProjectStore->DeleteProject(ProjectId))
+						{
+							return Req.ServerRequest().WriteResponse(HttpResponseCode::Locked,
+																	 HttpContentType::kText,
+																	 fmt::format("project {} is in use", ProjectId));
+						}
 
 						return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent);
 					}
@@ -2027,11 +2187,208 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request)
 
 #if ZEN_WITH_TESTS
 
-TEST_CASE("prj.store")
+namespace testutils {
+	using namespace std::literals;
+
+	CbPackage CreateOplogPackage(const Oid& Id, const std::span<const CompressedBuffer>& Attachments)
+	{
+		CbPackage	   Package;
+		CbObjectWriter Object;
+		Object << "key"sv << Id;
+		if (!Attachments.empty())
+		{
+			Object.BeginArray("bulkdata");
+			for (const CompressedBuffer& Attachment : Attachments)
+			{
+				Object.BeginObject();
+				Object << "id" << Oid::NewOid();
+				Object << "type"
+					   << "Standard";
+				Object << "data" << CbAttachment(Attachment);
+				Object.EndObject();
+
+				Package.AddAttachment(CbAttachment(Attachment));
+			}
+			Object.EndArray();
+		}
+		Package.SetObject(Object.Save());
+		return Package;
+	};
+
+	std::vector<CompressedBuffer> CreateAttachments(const std::span<const size_t>& Sizes)
+	{
+		std::vector<CompressedBuffer> Result;
+		Result.reserve(Sizes.size());
+		for (size_t Size : Sizes)
+		{
+			std::vector<uint8_t> Data;
+			Data.resize(Size);
+			for (size_t Idx = 0; Idx < Size; ++Idx)
+			{
+				Data[Idx] = Idx % 255;
+			}
+
+			Result.emplace_back(zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Data.data(), Data.size())));
+		}
+		return Result;
+	}
+
+}  // namespace testutils
+
+TEST_CASE("project.store.create")
+{
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+
+	GcManager			  Gc;
+	CidStore			  CidStore(Gc);
+	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
+	CidStore.Initialize(CidConfig);
+
+	std::string_view	  ProjectName("proj1"sv);
+	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	std::filesystem::path RootDir		  = TempDir.Path() / "root";
+	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
+	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
+	std::filesystem::path ProjectFilePath = TempDir.Path() / "game" / "game.uproject";
+
+	Ref<ProjectStore::Project> Project(ProjectStore.NewProject(BasePath / ProjectName,
+															   ProjectName,
+															   RootDir.string(),
+															   EngineRootDir.string(),
+															   ProjectRootDir.string(),
+															   ProjectFilePath.string()));
+	CHECK(ProjectStore.DeleteProject(ProjectName));
+	CHECK(!Project->Exists(BasePath));
+}
+
+TEST_CASE("project.store.lifetimes")
 {
 	using namespace std::literals;
 
 	ScopedTemporaryDirectory TempDir;
+
+	GcManager			  Gc;
+	CidStore			  CidStore(Gc);
+	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
+	CidStore.Initialize(CidConfig);
+
+	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	std::filesystem::path RootDir		  = TempDir.Path() / "root";
+	std::filesystem::path EngineRootDir	  = TempDir.Path() / "engine";
+	std::filesystem::path ProjectRootDir  = TempDir.Path() / "game";
+	std::filesystem::path ProjectFilePath = TempDir.Path() / "game" / "game.uproject";
+
+	Ref<ProjectStore::Project> Project(ProjectStore.NewProject(BasePath / "proj1"sv,
+															   "proj1"sv,
+															   RootDir.string(),
+															   EngineRootDir.string(),
+															   ProjectRootDir.string(),
+															   ProjectFilePath.string()));
+	ProjectStore::Oplog*	   Oplog = Project->NewOplog("oplog1");
+	CHECK(Oplog != nullptr);
+
+	std::filesystem::path DeletePath;
+	CHECK(Project->PrepareForDelete(DeletePath));
+	CHECK(!DeletePath.empty());
+	CHECK(Project->OpenOplog("oplog1") == nullptr);
+	// Oplog is now invalid, but pointer can still be accessed since we store old oplog pointers
+	CHECK(Oplog->OplogCount() == 0);
+	// Project is still valid since we have a Ref to it
+	CHECK(Project->Identifier == "proj1"sv);
+}
+
+TEST_CASE("project.store.gc")
+{
+	using namespace std::literals;
+	using namespace testutils;
+
+	ScopedTemporaryDirectory TempDir;
+
+	GcManager			  Gc;
+	CidStore			  CidStore(Gc);
+	CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096};
+	CidStore.Initialize(CidConfig);
+
+	std::filesystem::path BasePath = TempDir.Path() / "projectstore";
+	ProjectStore		  ProjectStore(CidStore, BasePath, Gc);
+	std::filesystem::path RootDir		= TempDir.Path() / "root";
+	std::filesystem::path EngineRootDir = TempDir.Path() / "engine";
+
+	std::filesystem::path Project1RootDir  = TempDir.Path() / "game1";
+	std::filesystem::path Project1FilePath = TempDir.Path() / "game1" / "game.uproject";
+	{
+		CreateDirectories(Project1FilePath.parent_path());
+		BasicFile ProjectFile;
+		ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate);
+	}
+
+	std::filesystem::path Project2RootDir  = TempDir.Path() / "game2";
+	std::filesystem::path Project2FilePath = TempDir.Path() / "game2" / "game.uproject";
+	{
+		CreateDirectories(Project2FilePath.parent_path());
+		BasicFile ProjectFile;
+		ProjectFile.Open(Project2FilePath, BasicFile::Mode::kTruncate);
+	}
+
+	{
+		Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv,
+																	"proj1"sv,
+																	RootDir.string(),
+																	EngineRootDir.string(),
+																	Project1RootDir.string(),
+																	Project1FilePath.string()));
+		ProjectStore::Oplog*	   Oplog = Project1->NewOplog("oplog1");
+		CHECK(Oplog != nullptr);
+
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77})));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99})));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122})));
+	}
+
+	{
+		Ref<ProjectStore::Project> Project2(ProjectStore.NewProject(BasePath / "proj2"sv,
+																	"proj2"sv,
+																	RootDir.string(),
+																	EngineRootDir.string(),
+																	Project2RootDir.string(),
+																	Project2FilePath.string()));
+		ProjectStore::Oplog*	   Oplog = Project2->NewOplog("oplog1");
+		CHECK(Oplog != nullptr);
+
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {}));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177})));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96})));
+		Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221})));
+	}
+
+	{
+		GcContext GcCtx;
+		ProjectStore.GatherReferences(GcCtx);
+		size_t RefCount = 0;
+		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+		CHECK(RefCount == 14);
+		ProjectStore.CollectGarbage(GcCtx);
+		CHECK(ProjectStore.OpenProject("proj1"sv));
+		CHECK(ProjectStore.OpenProject("proj2"sv));
+	}
+
+	std::filesystem::remove(Project1FilePath);
+
+	{
+		GcContext GcCtx;
+		ProjectStore.GatherReferences(GcCtx);
+		size_t RefCount = 0;
+		GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; });
+		CHECK(RefCount == 7);
+		ProjectStore.CollectGarbage(GcCtx);
+		CHECK(!ProjectStore.OpenProject("proj1"sv));
+		CHECK(ProjectStore.OpenProject("proj2"sv));
+	}
 }
 
 #endif