From 52bf08afc4b9da9ccdd73089c8ebfc7bda859bd3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Mar 2022 22:41:46 +0100 Subject: Migration now works in larger disk IO chunks BasicFile and CasLogFile now has new explicit modes instead of create true/false --- zenserver/projectstore.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 58b806989..f916b2250 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -114,10 +114,10 @@ struct ProjectStore::OplogStorage : public RefCounted CreateDirectories(m_OplogStoragePath); } - m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate); + m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); m_Oplog.Initialize(); - m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate); + m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); ZEN_ASSERT(IsPow2(m_OpsAlign)); ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); @@ -653,7 +653,7 @@ ProjectStore::Project::Read() ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); BasicFile Blob; - Blob.Open(ProjectStateFilePath, false); + Blob.Open(ProjectStateFilePath, BasicFile::EMode::kRead); IoBuffer Obj = Blob.ReadAll(); CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); @@ -693,7 +693,7 @@ ProjectStore::Project::Write() ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); BasicFile Blob; - Blob.Open(ProjectStateFilePath, true); + Blob.Open(ProjectStateFilePath, BasicFile::EMode::kTruncate); Blob.Write(Mem.Data(), Mem.Size(), 0); Blob.Flush(); } -- cgit v1.2.3 From 77924e6add0e4dc73e6855abae4b9cf8ffa9bb51 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 30 Mar 2022 22:37:52 +0200 Subject: improved logging for gc/migration --- zenserver/projectstore.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index f916b2250..6b18bce3f 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -970,6 +971,11 @@ ProjectStore::Scrub(ScrubContext& Ctx) void ProjectStore::GatherReferences(GcContext& GcCtx) { + Stopwatch Timer; + const auto Guard = MakeGuard([this, &Timer] { + ZEN_INFO("project store gathered all references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + DiscoverProjects(); RwLock::SharedLockScope _(m_ProjectsLock); -- cgit v1.2.3 From 27224ec4da12a7d80db13d8b2f3a67ec9335ee14 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 31 Mar 2022 10:20:06 +0200 Subject: Don hard fail on removing files we no longer care about --- zenserver/projectstore.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 6b18bce3f..bee143bd8 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -972,9 +972,8 @@ void ProjectStore::GatherReferences(GcContext& GcCtx) { Stopwatch Timer; - const auto Guard = MakeGuard([this, &Timer] { - ZEN_INFO("project store gathered all references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); + const auto Guard = + MakeGuard([this, &Timer] { ZEN_INFO("project store gathered all references in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); DiscoverProjects(); -- cgit v1.2.3 From 34235f5628f8ec50a01d5d16b8af56792292147e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 1 Apr 2022 23:49:09 +0200 Subject: rename EMode to Mode --- zenserver/projectstore.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index bee143bd8..bb8adb19f 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -115,10 +115,10 @@ struct ProjectStore::OplogStorage : public RefCounted CreateDirectories(m_OplogStoragePath); } - m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate ? CasLogFile::EMode::kTruncate : CasLogFile::EMode::kWrite); + m_Oplog.Open(m_OplogStoragePath / "ops.zlog", IsCreate ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite); m_Oplog.Initialize(); - m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate ? BasicFile::EMode::kTruncate : BasicFile::EMode::kWrite); + m_OpBlobs.Open(m_OplogStoragePath / "ops.zops", IsCreate ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite); ZEN_ASSERT(IsPow2(m_OpsAlign)); ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); @@ -654,7 +654,7 @@ ProjectStore::Project::Read() ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); BasicFile Blob; - Blob.Open(ProjectStateFilePath, BasicFile::EMode::kRead); + Blob.Open(ProjectStateFilePath, BasicFile::Mode::kRead); IoBuffer Obj = Blob.ReadAll(); CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); @@ -694,7 +694,7 @@ ProjectStore::Project::Write() ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); BasicFile Blob; - Blob.Open(ProjectStateFilePath, BasicFile::EMode::kTruncate); + Blob.Open(ProjectStateFilePath, BasicFile::Mode::kTruncate); Blob.Write(Mem.Data(), Mem.Size(), 0); Blob.Flush(); } -- cgit v1.2.3 From 605b4f330eed43b14135f37ffb58c14fa1cd79c2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 4 Apr 2022 12:05:25 +0200 Subject: always keep full log but read from index snapshot location if available --- zenserver/projectstore.cpp | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) (limited to 'zenserver/projectstore.cpp') diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index bb8adb19f..617f50660 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -181,36 +181,39 @@ struct ProjectStore::OplogStorage : public RefCounted uint64_t InvalidEntries = 0; - m_Oplog.Replay([&](const zen::OplogEntry& LogEntry) { - if (LogEntry.OpCoreSize == 0) - { - ++InvalidEntries; + m_Oplog.Replay( + [&](const zen::OplogEntry& LogEntry) { + if (LogEntry.OpCoreSize == 0) + { + ++InvalidEntries; - return; - } + return; + } - IoBuffer OpBuffer(LogEntry.OpCoreSize); + IoBuffer OpBuffer(LogEntry.OpCoreSize); - const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign; + const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign; - m_OpBlobs.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset); + m_OpBlobs.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset); - // Verify checksum, ignore op data if incorrect - const auto OpCoreHash = uint32_t(XXH3_64bits(OpBuffer.Data(), OpBuffer.Size()) & 0xffffFFFF); + // Verify checksum, ignore op data if incorrect + const auto OpCoreHash = uint32_t(XXH3_64bits(OpBuffer.Data(), OpBuffer.Size()) & 0xffffFFFF); - if (OpCoreHash != LogEntry.OpCoreHash) - { - ZEN_WARN("skipping oplog entry with bad checksum!"); - return; - } + if (OpCoreHash != LogEntry.OpCoreHash) + { + ZEN_WARN("skipping oplog entry with bad checksum!"); + return; + } - CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); + CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); - m_NextOpsOffset = Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); + m_NextOpsOffset = + Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); - Handler(Op, LogEntry); - }); + Handler(Op, LogEntry); + }, + 0); if (InvalidEntries) { -- cgit v1.2.3