Merge pull request #58 from EpicGames/de/cas-store-with-block-store

de/cas store with block store

1 files changed, 37 insertions, 23 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 769167433..738e4c1fd 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -59,7 +59,11 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object)
 	WriteFile(Path, Object.GetBuffer().AsIoBuffer());
 }
 
-ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcStorage(Gc), GcContributor(Gc), m_DiskLayer(RootDir)
+ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir)
+: GcStorage(Gc)
+, GcContributor(Gc)
+, m_RootDir(RootDir)
+, m_DiskLayer(RootDir)
 {
 	ZEN_INFO("initializing structured cache at '{}'", RootDir);
 	CreateDirectories(RootDir);
@@ -188,6 +192,10 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 void
 ZenCacheStore::GatherReferences(GcContext& GcCtx)
 {
+	Stopwatch  Timer;
+	const auto Guard = MakeGuard(
+		[this, &Timer] { ZEN_INFO("cache gathered all references from '{}' in {}", m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
 	access_tracking::AccessTimes AccessTimes;
 	m_MemLayer.GatherAccessTimes(AccessTimes);
 
@@ -476,25 +484,27 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
 	std::filesystem::path SobsPath{BucketDir / "zen.sobs"};
 	std::filesystem::path SlogPath{BucketDir / "zen.slog"};
 
-	m_SobsFile.Open(SobsPath, IsNew);
-	m_SlogFile.Open(SlogPath, IsNew);
+	m_SobsFile.Open(SobsPath, IsNew ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite);
+	m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite);
 
-	m_SlogFile.Replay([&](const DiskIndexEntry& Entry) {
-		if (Entry.Key == IoHash::Zero)
-		{
-			++InvalidEntryCount;
-		}
-		else if (Entry.Location.IsFlagSet(DiskLocation::kTombStone))
-		{
-			m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed);
-		}
-		else
-		{
-			m_Index.insert_or_assign(Entry.Key, IndexEntry(Entry.Location, GcClock::TickCount()));
-			m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed);
-		}
-		MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Entry.Location.Offset() + Entry.Location.Size());
-	});
+	m_SlogFile.Replay(
+		[&](const DiskIndexEntry& Entry) {
+			if (Entry.Key == IoHash::Zero)
+			{
+				++InvalidEntryCount;
+			}
+			else if (Entry.Location.IsFlagSet(DiskLocation::kTombStone))
+			{
+				m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed);
+			}
+			else
+			{
+				m_Index.insert_or_assign(Entry.Key, IndexEntry(Entry.Location, GcClock::TickCount()));
+				m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed);
+			}
+			MaxFileOffset = std::max<uint64_t>(MaxFileOffset, Entry.Location.Offset() + Entry.Location.Size());
+		},
+		0);
 
 	if (InvalidEntryCount)
 	{
@@ -757,6 +767,10 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 {
 	ZEN_TRACE_CPU("Z$::DiskLayer::CacheBucket::GatherReferences");
 
+	Stopwatch  Timer;
+	const auto Guard = MakeGuard(
+		[this, &Timer] { ZEN_INFO("gathered references from '{}' in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); });
+
 	const GcClock::TimePoint ExpireTime =
 		GcCtx.MaxCacheDuration() == GcClock::Duration::max() ? GcClock::TimePoint::min() : GcCtx.Time() - GcCtx.MaxCacheDuration();
 
@@ -905,8 +919,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			m_SlogFile.Close();
 
 			const bool IsNew = true;
-			m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew);
-			m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew);
+			m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite);
+			m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite);
 
 			m_SobsCursor = 0;
 			m_TotalSize	 = 0;
@@ -967,8 +981,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			uint64_t					TmpCursor{};
 			std::vector<uint8_t>		Chunk;
 
-			TmpSobs.Open(TmpSobsPath, true);
-			TmpLog.Open(TmpSlogPath, true);
+			TmpSobs.Open(TmpSobsPath, BasicFile::Mode::kTruncate);
+			TmpLog.Open(TmpSlogPath, CasLogFile::Mode::kTruncate);
 
 			for (const auto& Entry : ValidEntries)
 			{