Merge branch 'rpc-analyze' of https://github.ol.epicgames.net/ue-foundation/zen into rpc-analyze

10 files changed, 317 insertions, 106 deletions

diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp
index 75a333687..bcec74ce6 100644
--- a/src/zenserver/buildstore/httpbuildstore.cpp
+++ b/src/zenserver/buildstore/httpbuildstore.cpp
@@ -177,7 +177,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req)
 		const uint64_t	 BlobSize	 = Blob.GetSize();
 		const uint64_t	 MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0;
 		const uint64_t	 RangeSize	 = Min(Range.End - Range.Start + 1, MaxBlobSize);
-		if (Range.Start + RangeSize >= BlobSize)
+		if (Range.Start + RangeSize > BlobSize)
 		{
 			return ServerRequest.WriteResponse(HttpResponseCode::NoContent);
 		}
diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp
index 9f2e826d6..bb0c55618 100644
--- a/src/zenserver/cache/httpstructuredcache.cpp
+++ b/src/zenserver/cache/httpstructuredcache.cpp
@@ -1593,10 +1593,19 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext&  Co
 	ZEN_INFO("Replaying {} requests", RequestCount);
 	for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex)
 	{
-		Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>&) {
+		if (AbortFlag)
+		{
+			break;
+		}
+		Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>& AbortFlag) {
 			IoBuffer						Body;
 			zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body);
 
+			if (AbortFlag)
+			{
+				return;
+			}
+
 			if (Body)
 			{
 				uint32_t		 AcceptMagic = 0;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 055376b5c..1f9ae5fb6 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -16,6 +16,7 @@
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
 #include <zenutil/commandlineoptions.h>
+#include <zenutil/environmentoptions.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
@@ -429,6 +430,29 @@ MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>&
 };
 
 void
+ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult)
+{
+	using namespace std::literals;
+
+	EnvironmentOptions Options;
+	Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+
+	bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable;
+	Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv);
+
+	Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
+
+	Options.Parse(CmdLineResult);
+
+	if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable)
+	{
+		ServerOptions.SentryConfig.Disable = !EnvEnableSentry;
+	}
+}
+
+void
 ParseConfigFile(const std::filesystem::path& Path,
 				ZenServerOptions&			 ServerOptions,
 				const cxxopts::ParseResult&	 CmdLineResult,
@@ -441,9 +465,11 @@ ParseConfigFile(const std::filesystem::path& Path,
 	////// server
 	LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv);
 	LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv);
-	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv);
-	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv);
-	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv);
+	LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv);
+	LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv);
+	LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv);
+	LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv);
+	LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv);
 	LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv);
 	LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv);
 	LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv);
@@ -759,11 +785,15 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
 						  "Disable Sentry crash handler",
-						  cxxopts::value<bool>(ServerOptions.NoSentry)->default_value("false"));
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false"));
 	options.add_options()("sentry-allow-personal-info",
 						  "Allow personally identifiable information in sentry crash reports",
-						  cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false"));
-	options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryDsn));
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false"));
+	options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn));
+	options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment));
+	options.add_options()("sentry-debug",
+						  "Enable debug mode for Sentry",
+						  cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false"));
 	options.add_options()("detach",
 						  "Indicate whether zenserver should detach from parent process group",
 						  cxxopts::value<bool>(ServerOptions.Detach)->default_value("true"));
@@ -1336,6 +1366,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 
 		ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs);
 
+		ParseEnvVariables(ServerOptions, Result);
+
 		if (!ServerOptions.ConfigFile.empty())
 		{
 			ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile);
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 1a1793b8d..9753e3ae2 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -157,6 +157,15 @@ struct ZenWorkspacesConfig
 	bool AllowConfigurationChanges = false;
 };
 
+struct ZenSentryConfig
+{
+	bool		Disable	 = false;
+	bool		AllowPII = false;  // Allow personally identifiable information in sentry crash reports
+	std::string Dsn;
+	std::string Environment;
+	bool		Debug = false;	// Enable debug mode for Sentry
+};
+
 struct ZenServerOptions
 {
 	ZenUpstreamCacheConfig	 UpstreamCacheConfig;
@@ -169,31 +178,29 @@ struct ZenServerOptions
 	ZenBuildStoreConfig		 BuildStoreConfig;
 	ZenStatsConfig			 StatsConfig;
 	ZenWorkspacesConfig		 WorksSpacesConfig;
-	std::filesystem::path	 SystemRootDir;				// System root directory (used for machine level config)
-	std::filesystem::path	 DataDir;					// Root directory for state (used for testing)
-	std::filesystem::path	 ContentDir;				// Root directory for serving frontend content (experimental)
-	std::filesystem::path	 AbsLogFile;				// Absolute path to main log file
-	std::filesystem::path	 ConfigFile;				// Path to Lua config file
-	std::filesystem::path	 PluginsConfigFile;			// Path to plugins config file
-	std::filesystem::path	 BaseSnapshotDir;			// Path to server state snapshot (will be copied into data dir on start)
-	std::string				 ChildId;					// Id assigned by parent process (used for lifetime management)
-	std::string				 LogId;						// Id for tagging log output
-	std::string				 EncryptionKey;				// 256 bit AES encryption key
-	std::string				 EncryptionIV;				// 128 bit AES initialization vector
-	int						 BasePort		  = 8558;	// Service listen port (used for both UDP and TCP)
-	int						 OwnerPid		  = 0;		// Parent process id (zero for standalone)
-	bool					 InstallService	  = false;	// Flag used to initiate service install (temporary)
-	bool					 UninstallService = false;	// Flag used to initiate service uninstall (temporary)
-	bool					 IsDebug		  = false;
-	bool					 IsCleanStart	  = false;	// Indicates whether all state should be wiped on startup or not
-	bool					 IsPowerCycle	  = false;	// When true, the process shuts down immediately after initialization
-	bool					 IsTest			  = false;
-	bool					 IsDedicated	  = false;	// Indicates a dedicated/shared instance, with larger resource requirements
-	bool					 ShouldCrash	  = false;	// Option for testing crash handling
-	bool					 IsFirstRun		  = false;
-	bool					 NoSentry		  = false;
-	bool					 SentryAllowPII	  = false;	// Allow personally identifiable information in sentry crash reports
-	std::string				 SentryDsn;
+	ZenSentryConfig			 SentryConfig;
+	std::filesystem::path	 SystemRootDir;				  // System root directory (used for machine level config)
+	std::filesystem::path	 DataDir;					  // Root directory for state (used for testing)
+	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
+	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
+	std::filesystem::path	 ConfigFile;				  // Path to Lua config file
+	std::filesystem::path	 PluginsConfigFile;			  // Path to plugins config file
+	std::filesystem::path	 BaseSnapshotDir;			  // Path to server state snapshot (will be copied into data dir on start)
+	std::string				 ChildId;					  // Id assigned by parent process (used for lifetime management)
+	std::string				 LogId;						  // Id for tagging log output
+	std::string				 EncryptionKey;				  // 256 bit AES encryption key
+	std::string				 EncryptionIV;				  // 128 bit AES initialization vector
+	int						 BasePort			= 8558;	  // Service listen port (used for both UDP and TCP)
+	int						 OwnerPid			= 0;	  // Parent process id (zero for standalone)
+	bool					 InstallService		= false;  // Flag used to initiate service install (temporary)
+	bool					 UninstallService	= false;  // Flag used to initiate service uninstall (temporary)
+	bool					 IsDebug			= false;
+	bool					 IsCleanStart		= false;  // Indicates whether all state should be wiped on startup or not
+	bool					 IsPowerCycle		= false;  // When true, the process shuts down immediately after initialization
+	bool					 IsTest				= false;
+	bool					 IsDedicated		= false;  // Indicates a dedicated/shared instance, with larger resource requirements
+	bool					 ShouldCrash		= false;  // Option for testing crash handling
+	bool					 IsFirstRun			= false;
 	bool					 Detach				= true;	 // Whether zenserver should detach from existing process group (Mac/Linux)
 	bool					 ObjectStoreEnabled = false;
 	bool					 NoConsoleOutput	= false;  // Control default use of stdout for diagnostics
diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip
index eadfcf84c..5778fa3d2 100644
--- a/src/zenserver/frontend/html.zip
+++ b/src/zenserver/frontend/html.zip
diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js
index 472bb27ae..d1c13ccc7 100644
--- a/src/zenserver/frontend/html/pages/start.js
+++ b/src/zenserver/frontend/html/pages/start.js
@@ -41,6 +41,41 @@ export class Page extends ZenPage
 			action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id);
 		}
 
+		// cache
+		var section = this.add_section("z$");
+		columns = [
+			"namespace",
+			"dir",
+			"buckets",
+			"entries",
+			"size disk",
+			"size mem",
+			"actions",
+		]
+		var zcache_info = new Fetcher().resource("/z$/").json();
+		const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight);
+		for (const namespace of (await zcache_info)["Namespaces"])
+		{
+			new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => {
+				const row = cache_table.add_row(
+					"",
+					data["Configuration"]["RootDir"],
+					data["Buckets"].length,
+					data["EntryCount"],
+					Friendly.kib(data["StorageSize"].DiskSize),
+					Friendly.kib(data["StorageSize"].MemorySize)
+				);
+				var cell = row.get_cell(0);
+				cell.tag().text(namespace).on_click(() => this.view_zcache(namespace));
+				row.get_cell(1).tag().text(namespace);
+
+				cell = row.get_cell(-1);
+				const action_tb = new Toolbar(cell, true);
+				action_tb.left().add("view").on_click(() => this.view_zcache(namespace));
+				action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace));
+			});
+		}
+
 		// stats
 		section = this.add_section("stats");
 		columns = [
@@ -92,4 +127,23 @@ export class Page extends ZenPage
 			.option("Yes", () => drop())
 			.option("No");
 	}
+
+	view_zcache(namespace)
+	{
+		window.location = "?page=zcache&namespace=" + namespace;
+	}
+
+	drop_zcache(namespace)
+	{
+		const drop = async () => {
+			await new Fetcher().resource("z$", namespace).delete();
+			this.reload();
+		};
+
+		new Modal()
+			.title("Confirmation")
+			.message(`Drop zcache '${namespace}'?`)
+			.option("Yes", () => drop())
+			.option("No");
+	}
 }
diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js
new file mode 100644
index 000000000..974893b21
--- /dev/null
+++ b/src/zenserver/frontend/html/pages/zcache.js
@@ -0,0 +1,70 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+"use strict";
+
+import { ZenPage } from "./page.js"
+import { Fetcher } from "../util/fetcher.js"
+import { Friendly } from "../util/friendly.js"
+import { Modal } from "../util/modal.js"
+import { Table, PropTable, Toolbar } from "../util/widgets.js"
+
+////////////////////////////////////////////////////////////////////////////////
+export class Page extends ZenPage
+{
+	async main()
+	{
+		const namespace = this.get_param("namespace");
+
+		var info = new Fetcher().resource(`/z$/${namespace}/`).json();
+
+		this.set_title("cache - " + namespace);
+
+		var section = this.add_section("info");
+		var cfg_table = section.add_section("config").add_widget(PropTable);
+		var storage_table = section.add_section("storage").add_widget(PropTable);
+
+		info = await info;
+
+		cfg_table.add_object(info["Configuration"], true);
+
+		storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"]));
+		storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"]));
+		storage_table.add_property("entries", Friendly.sep(info["EntryCount"]));
+
+		var column_names = ["name", "disk", "mem", "entries", "actions"];
+		var bucket_table = this.add_section("buckets").add_widget(
+			Table,
+			column_names,
+			Table.Flag_BiasLeft
+		);
+		for (const bucket of info["Buckets"])
+		{
+			const row = bucket_table.add_row(bucket);
+			new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => {
+				row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"]));
+				row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"]));
+				row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"]));
+
+				const cell = row.get_cell(-1);
+				const action_tb = new Toolbar(cell, true);
+				action_tb.left().add("view")
+				action_tb.left().add("drop").on_click(() => this.drop_bucket(bucket));
+			});
+		}
+	}
+
+	drop_bucket(bucket)
+	{
+		const drop = async () => {
+			const namespace = this.get_param("namespace");
+			await new Fetcher().resource("z$", namespace, bucket).delete();
+			this.reload();
+		};
+
+		new Modal()
+			.title("Confirmation")
+			.message(`Drop bucket '${bucket}'?`)
+			.option("Yes", () => drop())
+			.option("No");
+	}
+}
diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp
index 868126533..b0d945814 100644
--- a/src/zenserver/main.cpp
+++ b/src/zenserver/main.cpp
@@ -96,15 +96,17 @@ ZenEntryPoint::Run()
 #if ZEN_USE_SENTRY
 	SentryIntegration Sentry;
 
-	if (m_ServerOptions.NoSentry == false)
+	if (m_ServerOptions.SentryConfig.Disable == false)
 	{
 		std::string SentryDatabasePath	 = (m_ServerOptions.DataDir / ".sentry-native").string();
 		std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string();
 
-		Sentry.Initialize(SentryDatabasePath,
-						  SentryAttachmentPath,
-						  m_ServerOptions.SentryDsn,
-						  m_ServerOptions.SentryAllowPII,
+		Sentry.Initialize({.DatabasePath	= SentryDatabasePath,
+						   .AttachmentsPath = SentryAttachmentPath,
+						   .Dsn				= m_ServerOptions.SentryConfig.Dsn,
+						   .Environment		= m_ServerOptions.SentryConfig.Environment,
+						   .AllowPII		= m_ServerOptions.SentryConfig.AllowPII,
+						   .Debug			= m_ServerOptions.SentryConfig.Debug},
 						  m_ServerOptions.CommandLine);
 	}
 #endif
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 6359b9db9..53e687983 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -1599,20 +1599,36 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo
 	std::atomic<bool> AbortFlag;
 	std::atomic<bool> PauseFlag;
 	ParallelWork	  Work(AbortFlag, PauseFlag);
-	for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
+	try
 	{
-		if (OptionalWorkerPool)
-		{
-			Work.ScheduleWork(*OptionalWorkerPool, [&, Index = OpIndex](std::atomic<bool>&) {
-				ZEN_MEMSCOPE(GetProjectstoreTag());
-				ValidateOne(Index);
-			});
-		}
-		else
+		for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++)
 		{
-			ValidateOne(OpIndex);
+			if (AbortFlag)
+			{
+				break;
+			}
+			if (OptionalWorkerPool)
+			{
+				Work.ScheduleWork(*OptionalWorkerPool, [&ValidateOne, Index = OpIndex](std::atomic<bool>& AbortFlag) {
+					ZEN_MEMSCOPE(GetProjectstoreTag());
+					if (AbortFlag)
+					{
+						return;
+					}
+					ValidateOne(Index);
+				});
+			}
+			else
+			{
+				ValidateOne(OpIndex);
+			}
 		}
 	}
+	catch (const std::exception& Ex)
+	{
+		AbortFlag.store(true);
+		ZEN_WARN("Failed validating oplogs in {}. Reason: '{}'", m_BasePath, Ex.what());
+	}
 	Work.Wait();
 
 	{
@@ -2110,67 +2126,81 @@ ProjectStore::Oplog::IterateChunks(std::span<Oid>																	  ChunkIds,
 	}
 	if (OptionalWorkerPool)
 	{
-		std::atomic_bool  Result = true;
 		std::atomic<bool> AbortFlag;
 		std::atomic<bool> PauseFlag;
 		ParallelWork	  Work(AbortFlag, PauseFlag);
-
-		for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
+		try
 		{
-			if (Result.load() == false)
+			for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++)
 			{
-				break;
-			}
-			Work.ScheduleWork(
-				*OptionalWorkerPool,
-				[this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result](
-					std::atomic<bool>&) {
-					if (Result.load() == false)
-					{
-						return;
-					}
-					size_t						 FileChunkIndex = FileChunkIndexes[ChunkIndex];
-					const std::filesystem::path& FilePath		= FileChunkPaths[ChunkIndex];
-					try
-					{
-						IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
-						if (!Payload)
+				if (AbortFlag)
+				{
+					break;
+				}
+				Work.ScheduleWork(
+					*OptionalWorkerPool,
+					[this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback](
+						std::atomic<bool>& AbortFlag) {
+						if (AbortFlag)
 						{
-							ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath);
+							return;
 						}
+						size_t						 FileChunkIndex = FileChunkIndexes[ChunkIndex];
+						const std::filesystem::path& FilePath		= FileChunkPaths[ChunkIndex];
+						try
+						{
+							IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath);
+							if (!Payload)
+							{
+								ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath);
+							}
 
-						if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+							if (!AsyncCallback(FileChunkIndex,
+											   Payload,
+											   IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0))
+							{
+								AbortFlag.store(true);
+							}
+						}
+						catch (const std::exception& Ex)
 						{
-							Result.store(false);
+							ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
+									 m_OuterProject->Identifier,
+									 m_OplogId,
+									 FileChunkIndex,
+									 FilePath,
+									 Ex.what());
 						}
-					}
-					catch (const std::exception& Ex)
-					{
-						ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'",
-								 m_OuterProject->Identifier,
-								 m_OplogId,
-								 FileChunkIndex,
-								 FilePath,
-								 Ex.what());
-					}
-				});
-		}
+					});
+			}
 
-		if (!CidChunkHashes.empty())
+			if (!CidChunkHashes.empty() && !AbortFlag)
+			{
+				m_CidStore.IterateChunks(
+					CidChunkHashes,
+					[&](size_t Index, const IoBuffer& Payload) {
+						size_t CidChunkIndex = CidChunkIndexes[Index];
+						if (AbortFlag)
+						{
+							return false;
+						}
+						return AsyncCallback(CidChunkIndex,
+											 Payload,
+											 IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
+					},
+					OptionalWorkerPool,
+					LargeSizeLimit);
+			}
+		}
+		catch (const std::exception& Ex)
 		{
-			m_CidStore.IterateChunks(
-				CidChunkHashes,
-				[&](size_t Index, const IoBuffer& Payload) {
-					size_t CidChunkIndex = CidChunkIndexes[Index];
-					return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0);
-				},
-				OptionalWorkerPool,
-				LargeSizeLimit);
+			AbortFlag.store(true);
+			ZEN_WARN("Failed iterating oplog chunks in {}. Reason: '{}'", m_BasePath, Ex.what());
 		}
 
 		Work.Wait();
 
-		return Result.load();
+		return !AbortFlag;
 	}
 	else
 	{
@@ -3894,19 +3924,26 @@ ProjectStore::Flush()
 	std::atomic<bool> AbortFlag;
 	std::atomic<bool> PauseFlag;
 	ParallelWork	  Work(AbortFlag, PauseFlag);
-
-	for (const Ref<Project>& Project : Projects)
+	try
 	{
-		Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) {
-			try
-			{
-				Project->Flush();
-			}
-			catch (const std::exception& Ex)
-			{
-				ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
-			}
-		});
+		for (const Ref<Project>& Project : Projects)
+		{
+			Work.ScheduleWork(WorkerPool, [this, Project](std::atomic<bool>&) {
+				try
+				{
+					Project->Flush();
+				}
+				catch (const std::exception& Ex)
+				{
+					ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what());
+				}
+			});
+		}
+	}
+	catch (const std::exception& Ex)
+	{
+		AbortFlag.store(true);
+		ZEN_WARN("Failed projects in {}. Reason: '{}'", m_ProjectBasePath, Ex.what());
 	}
 
 	Work.Wait();
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 1ad94ed63..27ec4c690 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 		return -1;
 	}
 
-	m_UseSentry				 = ServerOptions.NoSentry == false;
+	m_UseSentry				 = ServerOptions.SentryConfig.Disable == false;
 	m_ServerEntry			 = ServerEntry;
 	m_DebugOptionForcedCrash = ServerOptions.ShouldCrash;
 	m_IsPowerCycle			 = ServerOptions.IsPowerCycle;