From 52bf08afc4b9da9ccdd73089c8ebfc7bda859bd3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Mar 2022 22:41:46 +0100 Subject: Migration now works in larger disk IO chunks BasicFile and CasLogFile now has new explicit modes instead of create true/false --- zenserver/cache/structuredcachestore.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 769167433..10a19f949 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -476,8 +476,8 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is std::filesystem::path SobsPath{BucketDir / "zen.sobs"}; std::filesystem::path SlogPath{BucketDir / "zen.slog"}; - m_SobsFile.Open(SobsPath, IsNew); - m_SlogFile.Open(SlogPath, IsNew); + m_SobsFile.Open(SobsPath, IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); + m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); m_SlogFile.Replay([&](const DiskIndexEntry& Entry) { if (Entry.Key == IoHash::Zero) @@ -905,8 +905,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) m_SlogFile.Close(); const bool IsNew = true; - m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew); - m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew); + m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); + m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); m_SobsCursor = 0; m_TotalSize = 0; @@ -967,8 +967,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) uint64_t TmpCursor{}; std::vector Chunk; - TmpSobs.Open(TmpSobsPath, true); - TmpLog.Open(TmpSlogPath, true); + TmpSobs.Open(TmpSobsPath, BasicFile::EMode::kTruncate); + TmpLog.Open(TmpSlogPath, CasLogFile::EMode::kTruncate); for (const auto& Entry : ValidEntries) { -- cgit v1.2.3 From 77924e6add0e4dc73e6855abae4b9cf8ffa9bb51 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 30 Mar 2022 22:37:52 +0200 Subject: improved logging for gc/migration --- zenserver/cache/structuredcachestore.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 10a19f949..95e9e2dba 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -59,7 +59,11 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) WriteFile(Path, Object.GetBuffer().AsIoBuffer()); } -ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcStorage(Gc), GcContributor(Gc), m_DiskLayer(RootDir) +ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) +: m_RootDir(RootDir) +, GcStorage(Gc) +, GcContributor(Gc) +, m_DiskLayer(RootDir) { ZEN_INFO("initializing structured cache at '{}'", RootDir); CreateDirectories(RootDir); @@ -188,6 +192,9 @@ ZenCacheStore::Scrub(ScrubContext& Ctx) void ZenCacheStore::GatherReferences(GcContext& GcCtx) { + Stopwatch Timer; + const auto Guard = MakeGuard([this, &Timer] { ZEN_INFO("cache gathered all references from '{}' in {}", m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + access_tracking::AccessTimes AccessTimes; m_MemLayer.GatherAccessTimes(AccessTimes); @@ -757,6 +764,10 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx) { ZEN_TRACE_CPU("Z$::DiskLayer::CacheBucket::GatherReferences"); + Stopwatch Timer; + const auto Guard = MakeGuard( + [this, &Timer] { ZEN_INFO("gathered references from '{}' in {}", m_BucketDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + const GcClock::TimePoint ExpireTime = GcCtx.MaxCacheDuration() == GcClock::Duration::max() ? GcClock::TimePoint::min() : GcCtx.Time() - GcCtx.MaxCacheDuration(); -- cgit v1.2.3 From 27224ec4da12a7d80db13d8b2f3a67ec9335ee14 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 31 Mar 2022 10:20:06 +0200 Subject: Don hard fail on removing files we no longer care about --- zenserver/cache/structuredcachestore.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 95e9e2dba..eec548f1b 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -193,7 +193,8 @@ void ZenCacheStore::GatherReferences(GcContext& GcCtx) { Stopwatch Timer; - const auto Guard = MakeGuard([this, &Timer] { ZEN_INFO("cache gathered all references from '{}' in {}", m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + const auto Guard = MakeGuard( + [this, &Timer] { ZEN_INFO("cache gathered all references from '{}' in {}", m_RootDir, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); access_tracking::AccessTimes AccessTimes; m_MemLayer.GatherAccessTimes(AccessTimes); -- cgit v1.2.3 From 345cf462e89ba37867abe4c0722ce2549dddc4ca Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 16:08:53 +0200 Subject: linux fixes --- zenserver/cache/structuredcachestore.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index eec548f1b..79bb8486e 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -60,8 +60,8 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) } ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) -: m_RootDir(RootDir) -, GcStorage(Gc) +: GcStorage(Gc) +, m_RootDir(RootDir) , GcContributor(Gc) , m_DiskLayer(RootDir) { -- cgit v1.2.3 From e51efe1674fa3d9d4af715d85a4045e1ee7450f3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 22:47:46 +0200 Subject: linux compilation fixes --- zenserver/cache/structuredcachestore.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 79bb8486e..6b448b8d2 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -61,8 +61,8 @@ SaveCompactBinaryObject(const fs::path& Path, const CbObject& Object) ZenCacheStore::ZenCacheStore(CasGc& Gc, const std::filesystem::path& RootDir) : GcStorage(Gc) -, m_RootDir(RootDir) , GcContributor(Gc) +, m_RootDir(RootDir) , m_DiskLayer(RootDir) { ZEN_INFO("initializing structured cache at '{}'", RootDir); -- cgit v1.2.3 From 34235f5628f8ec50a01d5d16b8af56792292147e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 23:49:09 +0200 Subject: rename EMode to Mode --- zenserver/cache/structuredcachestore.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 6b448b8d2..e22b06572 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -484,8 +484,8 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is std::filesystem::path SobsPath{BucketDir / "zen.sobs"}; std::filesystem::path SlogPath{BucketDir / "zen.slog"}; - m_SobsFile.Open(SobsPath, IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); - m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); + m_SobsFile.Open(SobsPath, IsNew ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite); + m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite); m_SlogFile.Replay([&](const DiskIndexEntry& Entry) { if (Entry.Key == IoHash::Zero) @@ -917,8 +917,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) m_SlogFile.Close(); const bool IsNew = true; - m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); - m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); + m_SobsFile.Open(m_BucketDir / "zen.sobs", IsNew ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite); + m_SlogFile.Open(m_BucketDir / "zen.slog", IsNew ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite); m_SobsCursor = 0; m_TotalSize = 0; @@ -979,8 +979,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) uint64_t TmpCursor{}; std::vector Chunk; - TmpSobs.Open(TmpSobsPath, BasicFile::EMode::kTruncate); - TmpLog.Open(TmpSlogPath, CasLogFile::EMode::kTruncate); + TmpSobs.Open(TmpSobsPath, BasicFile::Mode::kTruncate); + TmpLog.Open(TmpSlogPath, CasLogFile::Mode::kTruncate); for (const auto& Entry : ValidEntries) { -- cgit v1.2.3 From 605b4f330eed43b14135f37ffb58c14fa1cd79c2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 4 Apr 2022 12:05:25 +0200 Subject: always keep full log but read from index snapshot location if available --- zenserver/cache/structuredcachestore.cpp | 34 +++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'zenserver/cache/structuredcachestore.cpp') diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index e22b06572..738e4c1fd 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -487,22 +487,24 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is m_SobsFile.Open(SobsPath, IsNew ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite); m_SlogFile.Open(SlogPath, IsNew ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite); - m_SlogFile.Replay([&](const DiskIndexEntry& Entry) { - if (Entry.Key == IoHash::Zero) - { - ++InvalidEntryCount; - } - else if (Entry.Location.IsFlagSet(DiskLocation::kTombStone)) - { - m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed); - } - else - { - m_Index.insert_or_assign(Entry.Key, IndexEntry(Entry.Location, GcClock::TickCount())); - m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed); - } - MaxFileOffset = std::max(MaxFileOffset, Entry.Location.Offset() + Entry.Location.Size()); - }); + m_SlogFile.Replay( + [&](const DiskIndexEntry& Entry) { + if (Entry.Key == IoHash::Zero) + { + ++InvalidEntryCount; + } + else if (Entry.Location.IsFlagSet(DiskLocation::kTombStone)) + { + m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed); + } + else + { + m_Index.insert_or_assign(Entry.Key, IndexEntry(Entry.Location, GcClock::TickCount())); + m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed); + } + MaxFileOffset = std::max(MaxFileOffset, Entry.Location.Offset() + Entry.Location.Size()); + }, + 0); if (InvalidEntryCount) { -- cgit v1.2.3