From d02b0720813a62a4f1fe875e6e784843f5c2da46 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 15 Nov 2023 11:30:56 +0100 Subject: fix race contdition when signaling shutdown of process and waiting for completion (#539) --- src/zenserver/main.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 69cc2bbf5..44330d4c8 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -198,13 +198,15 @@ ZenEntryPoint::Run() ShutdownThread.reset(new std::thread{[&] { SetCurrentThreadName("shutdown_monitor"); - ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}'", ShutdownEventName); + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", + ShutdownEventName, + zen::GetCurrentProcessId()); if (ShutdownEvent->Wait()) { if (!IsApplicationExitRequested()) { - ZEN_INFO("shutdown signal received"); + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); Server.RequestExit(0); } } @@ -244,7 +246,7 @@ ZenEntryPoint::Run() } catch (std::exception& e) { - ZEN_CRITICAL("Caught exception in main: {}", e.what()); + ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what()); if (!IsApplicationExitRequested()) { RequestApplicationExit(1); -- cgit v1.2.3 From 372d3a8a1ff65e53d139a7ee1db022d8cfa55982 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 16 Nov 2023 17:20:52 +0100 Subject: add zenserver state snapshot support (#543) this introduces a --snapshot-dir command line option to zenserver which specifies a directory which will be propagated to the persistence root directory on start-up. This is most powerful with file systems which support block cloning, such as ReFS on Windows. This allows even very large state snapshots to be used repeatedly without having to worry about mutating the original dataset on disk. When using ReFS the state copy for even large state directories can be very fast since the duration is primarily proportional to the number of files in the tree rather than the size of the files being cloned. The storage requirements are also minimal as all data will be handled in a copy-on-write manner. --- src/zenserver/main.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 44330d4c8..1aa0aa4df 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -336,7 +336,7 @@ main(int argc, char* argv[]) ZenServerOptions ServerOptions; ParseCliOptions(argc, argv, ServerOptions); - if (ServerOptions.IsCleanStart) + if (ServerOptions.IsCleanStart || !ServerOptions.BaseSnapshotDir.empty()) { DeleteDirectories(ServerOptions.DataDir); } @@ -347,6 +347,11 @@ main(int argc, char* argv[]) std::filesystem::create_directories(ServerOptions.DataDir); } + if (!ServerOptions.BaseSnapshotDir.empty()) + { + CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true}); + } + #if ZEN_WITH_TRACE if (ServerOptions.TraceHost.size()) { -- cgit v1.2.3 From 12342f51038dba30ad8b490493596c72d9306994 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 22 Nov 2023 17:59:54 +0100 Subject: fix block cloning copy argument validation (#560) --- src/zenserver/main.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 1aa0aa4df..ff4df183e 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -349,6 +349,7 @@ main(int argc, char* argv[]) if (!ServerOptions.BaseSnapshotDir.empty()) { + ZEN_CONSOLE_INFO("copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir); CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true}); } -- cgit v1.2.3 From 4d95b578350ebfbbf6d54407c9403547b01cac4c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Nov 2023 14:32:19 +0100 Subject: optimized index snapshot reading/writing (#561) the previous implementation of in-memory index snapshots serialise data to memory before writing to disk and vice versa when reading. This leads to some memory spikes which end up pushing useful data out of system cache and also cause stalls on I/O operations. this change moves more code to a streaming serialisation approach which scales better from a memory usage perspective and also performs much better --- src/zenserver/main.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index ff4df183e..6901323e3 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -39,6 +39,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 # include +# include #endif #include @@ -295,6 +296,7 @@ test_main(int argc, char** argv) zen::zencore_forcelinktests(); zen::zenhttp_forcelinktests(); zen::zenstore_forcelinktests(); + zen::zenutil_forcelinktests(); zen::z$_forcelink(); zen::z$service_forcelink(); -- cgit v1.2.3 From 37920b41048acffa30cf156d7d36bfc17ba15c0e Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 11 Dec 2023 11:48:23 +0100 Subject: improved scrubbing of oplogs and filecas (#596) - Improvement: Scrub command now validates compressed buffer hashes in filecas storage (used for large chunks) - Improvement: Added --dry, --no-gc and --no-cas options to zen scrub command - Improvement: Implemented oplog scrubbing (previously was a no-op) - Improvement: Implemented support for running scrubbint at startup with --scrub= --- src/zenserver/main.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 6901323e3..868b5131c 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -338,9 +338,24 @@ main(int argc, char* argv[]) ZenServerOptions ServerOptions; ParseCliOptions(argc, argv, ServerOptions); - if (ServerOptions.IsCleanStart || !ServerOptions.BaseSnapshotDir.empty()) + std::string_view DeleteReason; + + if (ServerOptions.IsCleanStart) + { + DeleteReason = "clean start requested"sv; + } + else if (!ServerOptions.BaseSnapshotDir.empty()) + { + DeleteReason = "will initialize state from base snapshot"sv; + } + + if (!DeleteReason.empty()) { - DeleteDirectories(ServerOptions.DataDir); + if (std::filesystem::exists(ServerOptions.DataDir)) + { + ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); + DeleteDirectories(ServerOptions.DataDir); + } } if (!std::filesystem::exists(ServerOptions.DataDir)) -- cgit v1.2.3 From 7c0eaa5836d595fe4dbcea9d7c66f6444b29fddc Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 11 Dec 2023 11:49:26 +0100 Subject: added details to trace initialization (#588) this adds information on program name and command line to trace initialization --- src/zenserver/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/zenserver/main.cpp') diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 868b5131c..be2cdcc2d 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -373,15 +373,15 @@ main(int argc, char* argv[]) #if ZEN_WITH_TRACE if (ServerOptions.TraceHost.size()) { - TraceStart(ServerOptions.TraceHost.c_str(), TraceType::Network); + TraceStart("zenserver", ServerOptions.TraceHost.c_str(), TraceType::Network); } else if (ServerOptions.TraceFile.size()) { - TraceStart(ServerOptions.TraceFile.c_str(), TraceType::File); + TraceStart("zenserver", ServerOptions.TraceFile.c_str(), TraceType::File); } else { - TraceInit(); + TraceInit("zenserver"); } atexit(TraceShutdown); #endif // ZEN_WITH_TRACE -- cgit v1.2.3