From d02b0720813a62a4f1fe875e6e784843f5c2da46 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 15 Nov 2023 11:30:56 +0100 Subject: fix race contdition when signaling shutdown of process and waiting for completion (#539) --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 7111900ec..9d3502a23 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -720,7 +720,7 @@ ZenServer::CheckSigInt() { if (utils::SignalCounter[SIGINT] > 0) { - ZEN_INFO("SIGINT triggered (Ctrl+C), exiting"); + ZEN_INFO("SIGINT triggered (Ctrl+C) for process {}, exiting", zen::GetCurrentProcessId()); RequestExit(128 + SIGINT); return; } -- cgit v1.2.3 From 8ff65cd87265b4b5065cbff3290169c417227de7 Mon Sep 17 00:00:00 2001 From: Per Larsson Date: Wed, 15 Nov 2023 12:19:21 +0100 Subject: Make object store endpoint S3 compatible. (#535) * Make object store endpoint S3 compatible. * Removed XML pretty printing and set object store endpoint disabled by default. --- src/zenserver/zenserver.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 9d3502a23..b87679b7d 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -253,7 +253,6 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen { ObjectStoreConfig ObjCfg; ObjCfg.RootDirectory = m_DataRoot / "obj"; - ObjCfg.ServerPort = static_cast(EffectiveBasePort); for (const auto& Bucket : ServerOptions.ObjectStoreConfig.Buckets) { -- cgit v1.2.3 From 58cf8407214177bd9b2accd98f0b9284f044e14b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 16 Nov 2023 18:47:04 +0100 Subject: add wipe prevention via file in data root dir (#548) * if a file named root_manifest.ignore_schema_mismatch exists in the root data dir, ignore schema mismatch --- src/zenserver/zenserver.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index b87679b7d..8359f5088 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -363,9 +363,24 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { - WipeState = true; - WipeReason = - fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION); + std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; + if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + { + ZEN_INFO( + "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " + "schema version", + ManifestVersion, + ManifestPath, + ZEN_CFG_SCHEMA_VERSION, + ManifestSkipSchemaChangePath); + UpdateManifest = true; + } + else + { + WipeState = true; + WipeReason = + fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION); + } } } } -- cgit v1.2.3 From 573907447db3e19d49c0bcaf3f659cf2d599c738 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 16 Nov 2023 18:50:27 +0100 Subject: blocking queue fix (#550) * make BlockingQueue::m_CompleteAdding non-atomic * ZenCacheDiskLayer::Flush logging * name worker threads in ZenCacheDiskLayer::DiscoverBuckets * name worker threads in gcv2 * improved logging in ZenServerInstance * scrub threadpool naming * remove waitpid handling, we should just call wait to kill zombie processes --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 8359f5088..e6724e40a 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -782,7 +782,7 @@ ZenServer::ScrubStorage() Stopwatch Timer; ZEN_INFO("Storage validation STARTING"); - WorkerThreadPool ThreadPool{1}; + WorkerThreadPool ThreadPool{1, "Scrub"}; ScrubContext Ctx{ThreadPool}; m_CidStore->ScrubStorage(Ctx); m_ProjectStore->ScrubStorage(Ctx); -- cgit v1.2.3 From 669a8869b5414c0e8708dd90b1f4aa297d091887 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 21 Nov 2023 16:58:43 +0100 Subject: add command line options for compact block threshold and gc verbose (#557) - Feature: Added new options to zenserver for GC V2 - `--gc-compactblock-threshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90% - `--gc-verbose` GCV2 - enable more verbose output when running a GC pass - Feature: Added new options to `zen gc` command for GC V2 - `--compactblockthreshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90% - `--verbose` GCV2 - enable more verbose output when running a GC pass - Feature: Added new parameters for endpoint `admin/gc` (PUT) - `compactblockthreshold` GCV2 - how much of a compact block should be used to skip compacting the block, default is 90% - `verbose` GCV2 - enable more verbose output when running a GC pass --- src/zenserver/zenserver.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index e6724e40a..a50ff1b53 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -288,7 +288,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .DiskSizeSoftLimit = ServerOptions.GcConfig.DiskSizeSoftLimit, .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, .LightweightInterval = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds), - .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1}; + .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1, + .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + .Verbose = ServerOptions.GcConfig.Verbose}; m_GcScheduler.Initialize(GcConfig); // Create and register admin interface last to make sure all is properly initialized -- cgit v1.2.3 From 48f4a8bce0450263cda89d3b33fc15e32620e6f4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 23 Nov 2023 13:43:37 +0100 Subject: added --powercycle option (#565) * added --powercycle option. when this is passed in the zenserver process will shut down immediately after initialization is complete. This is primarily useful when benchmarking init/cleanup but could also be used to verify/clean up disk state * moved EmptyStandbyList code to make it accessible to more commands --- src/zenserver/zenserver.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index a50ff1b53..ba9ff4f88 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -117,6 +117,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_UseSentry = ServerOptions.NoSentry == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; + m_IsPowerCycle = ServerOptions.IsPowerCycle; const int ParentPid = ServerOptions.OwnerPid; if (ParentPid) @@ -591,6 +592,13 @@ ZenServer::Run() OnReady(); + if (m_IsPowerCycle) + { + ZEN_INFO("Power cycle mode enabled -- shutting down"); + + RequestExit(0); + } + m_Http->Run(IsInteractiveMode); SetNewState(kShuttingDown); -- cgit v1.2.3 From 6171e981b445b1850ed7c5ba7a2a5901f2227fba Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 27 Nov 2023 05:20:24 -0500 Subject: gc stop command (#569) - Feature: New endpoint `/admin/gc-stop` to cancel a running garbage collect operation - Feature: Added `zen gc-stop` command to cancel a running garbage collect operation - Bugfix: GCv2 - make sure to discover all projects and oplogs before checking for expired data --- src/zenserver/zenserver.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index ba9ff4f88..841f19295 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -604,8 +604,6 @@ ZenServer::Run() SetNewState(kShuttingDown); ZEN_INFO(ZEN_APP_NAME " exiting"); - - Flush(); } void @@ -640,8 +638,10 @@ ZenServer::Cleanup() } m_StatsReporter.Shutdown(); - m_GcScheduler.Shutdown(); + + Flush(); + m_AdminService.reset(); m_VfsService.reset(); m_ObjStoreService.reset(); -- cgit v1.2.3 From 68b3382ef7e0f7795b9a601aae73adc2f8ef9873 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 29 Nov 2023 09:14:57 -0500 Subject: global thread worker pools (#577) - Improvement: Use two global worker thread pools instead of ad-hoc creation of worker pools --- src/zenserver/zenserver.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 841f19295..2430267c1 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #if ZEN_PLATFORM_WINDOWS @@ -659,6 +660,7 @@ ZenServer::Cleanup() m_AuthMgr.reset(); m_Http = {}; m_JobQueue.reset(); + ShutdownWorkerPools(); } catch (std::exception& Ex) { -- cgit v1.2.3 From aeecf717fee464ac2b9700e571d19c0c51794d7b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 4 Dec 2023 05:16:22 -0500 Subject: safe threadpool shutdown (#584) * shut down thread pools earlier to worker threads has a chance to terminate before main thread atexit --- src/zenserver/zenserver.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 2430267c1..f663417fb 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -643,6 +643,8 @@ ZenServer::Cleanup() Flush(); + ShutdownWorkerPools(); + m_AdminService.reset(); m_VfsService.reset(); m_ObjStoreService.reset(); @@ -660,7 +662,6 @@ ZenServer::Cleanup() m_AuthMgr.reset(); m_Http = {}; m_JobQueue.reset(); - ShutdownWorkerPools(); } catch (std::exception& Ex) { -- cgit v1.2.3 From e371da958c3ffbaded81f8c7b072041a64eaac9a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 4 Dec 2023 12:14:54 +0100 Subject: Add context to Zen upstream resolve error message (#585) --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f663417fb..fafbe8304 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -491,7 +491,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls); if (Err) { - ZEN_ERROR("resolve FAILED, reason '{}'", Err.message()); + ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message()); } } } -- cgit v1.2.3 From 0b9d89f5beb889c1785a8f22b586c59bcca0b3ea Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 5 Dec 2023 08:51:53 +0100 Subject: HTTP plugin request debug logging (#587) * added log level control/query to LoggerRef * added debug logging to http plugin implementation * added GetDebugName() to transport plugin interfaces * added debug name to log output --- src/zenserver/zenserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index fafbe8304..2aeb6a4d5 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -162,7 +162,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen // Ok so now we're configured, let's kick things off m_Http = CreateHttpServer(ServerOptions.HttpServerConfig); - int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort); + int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir); // Setup authentication manager { -- cgit v1.2.3 From 3849faabb060d58fd7e056aa01445cf1ae3ab0f9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 11 Dec 2023 11:00:05 +0100 Subject: multi-line logging improvements (#597) * added ZEN_SCOPED_WARN and implemented multi-line logging * changed so file log also uses `fullformatter` for consistency and to get the multi-line support across the board --- src/zenserver/zenserver.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 2aeb6a4d5..418e3c325 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -565,12 +565,13 @@ ZenServer::Run() if (!m_TestMode) { - ZEN_INFO("__________ _________ __ "); - ZEN_INFO("\\____ /____ ____ / _____// |_ ___________ ____ "); - ZEN_INFO(" / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ "); - ZEN_INFO(" / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ "); - ZEN_INFO("/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >"); - ZEN_INFO(" \\/ \\/ \\/ \\/ \\/ "); + ZEN_INFO( + "__________ _________ __ \n" + "\\____ /____ ____ / _____// |_ ___________ ____ \n" + " / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ \n" + " / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ \n" + "/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >\n" + " \\/ \\/ \\/ \\/ \\/ \n"); } ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); -- cgit v1.2.3 From 37920b41048acffa30cf156d7d36bfc17ba15c0e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 11 Dec 2023 11:48:23 +0100 Subject: improved scrubbing of oplogs and filecas (#596) - Improvement: Scrub command now validates compressed buffer hashes in filecas storage (used for large chunks) - Improvement: Added --dry, --no-gc and --no-cas options to zen scrub command - Improvement: Implemented oplog scrubbing (previously was a no-op) - Improvement: Implemented support for running scrubbint at startup with --scrub= --- src/zenserver/zenserver.cpp | 68 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 4 deletions(-) (limited to 'src/zenserver/zenserver.cpp') diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 418e3c325..336f715f4 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -120,6 +120,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; const int ParentPid = ServerOptions.OwnerPid; + m_StartupScrubOptions = ServerOptions.ScrubOptions; if (ParentPid) { @@ -554,10 +555,6 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) void ZenServer::Run() { - // This is disabled for now, awaiting better scheduling - // - // ScrubStorage(); - if (m_ProcessMonitor.IsActive()) { CheckOwnerPid(); @@ -594,6 +591,69 @@ ZenServer::Run() OnReady(); + if (!m_StartupScrubOptions.empty()) + { + using namespace std::literals; + + ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions); + + bool DoScrub = true; + bool DoWait = false; + GcScheduler::TriggerScrubParams ScrubParams; + + ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) { + if (Token == "nocas"sv) + { + ScrubParams.SkipCas = true; + } + else if (Token == "nodelete"sv) + { + ScrubParams.SkipDelete = true; + } + else if (Token == "nogc"sv) + { + ScrubParams.SkipGc = true; + } + else if (Token == "no"sv) + { + DoScrub = false; + } + else if (Token == "wait"sv) + { + DoWait = true; + } + return true; + }); + + if (DoScrub) + { + m_GcScheduler.TriggerScrub(ScrubParams); + + if (DoWait) + { + auto State = m_GcScheduler.Status(); + + while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped)) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("waiting for Scrub/GC to complete..."); + + while (State == GcSchedulerStatus::kRunning) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("Scrub/GC completed"); + } + } + } + if (m_IsPowerCycle) { ZEN_INFO("Power cycle mode enabled -- shutting down"); -- cgit v1.2.3