Merge branch 'main' into zs/get-all-chunk-infoszs/get-all-chunk-infos

5 files changed, 80 insertions, 21 deletions

diff --git a/src/zenserver/cache/cachedisklayer.cpp b/src/zenserver/cache/cachedisklayer.cpp
index 2c344dd1d..700529443 100644
--- a/src/zenserver/cache/cachedisklayer.cpp
+++ b/src/zenserver/cache/cachedisklayer.cpp
@@ -209,6 +209,9 @@ namespace {
 			zen::Sleep(100);
 		} while (true);
 	}
+
+	uint64_t EstimateMemCachePayloadMemory(uint64_t PayloadSize) { return 8u + 32u + RoundUp(PayloadSize, 8u); }
+
 }  // namespace
 
 namespace fs = std::filesystem;
@@ -655,6 +658,9 @@ BucketManifestSerializer::WriteSidecarFile(const std::filesystem::path&
 
 //////////////////////////////////////////////////////////////////////////
 
+static const float IndexMinLoadFactor = 0.2f;
+static const float IndexMaxLoadFactor = 0.7f;
+
 ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
 											std::atomic_uint64_t&	   OuterCacheMemoryUsage,
 											std::string				   BucketName,
@@ -665,6 +671,9 @@ ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager&				   Gc,
 , m_Configuration(Config)
 , m_BucketId(Oid::Zero)
 {
+	m_Index.min_load_factor(IndexMinLoadFactor);
+	m_Index.max_load_factor(IndexMaxLoadFactor);
+
 	if (m_BucketName.starts_with(std::string_view("legacy")) || m_BucketName.ends_with(std::string_view("shadermap")))
 	{
 		const uint64_t LegacyOverrideSize = 16 * 1024 * 1024;
@@ -1289,14 +1298,26 @@ ZenCacheDiskLayer::CacheBucket::MemCacheTrim(GcClock::TimePoint ExpireTime)
 	GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count();
 
 	RwLock::ExclusiveLockScope _(m_IndexLock);
+	if (m_MemCachedPayloads.empty())
+	{
+		return;
+	}
 	for (const auto& Kv : m_Index)
 	{
-		if (m_AccessTimes[Kv.second] < ExpireTicks)
+		size_t		   Index   = Kv.second;
+		BucketPayload& Payload = m_Payloads[Index];
+		if (!Payload.MemCached)
+		{
+			continue;
+		}
+		if (m_AccessTimes[Index] < ExpireTicks)
 		{
-			BucketPayload& Payload = m_Payloads[Kv.second];
 			RemoveMemCachedData(Payload);
 		}
 	}
+	m_MemCachedPayloads.shrink_to_fit();
+	m_FreeMemCachedPayloads.shrink_to_fit();
+	m_FreeMetaDatas.shrink_to_fit();
 }
 
 void
@@ -1305,6 +1326,10 @@ ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint		TickStart,
 												 std::vector<uint64_t>& InOutUsageSlots)
 {
 	RwLock::SharedLockScope _(m_IndexLock);
+	if (m_MemCachedPayloads.empty())
+	{
+		return;
+	}
 	for (const auto& It : m_Index)
 	{
 		size_t		   Index   = It.second;
@@ -1929,10 +1954,9 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 
 	for (const auto& Entry : StructuredItemsWithUnknownAttachments)
 	{
-		const IoHash&		Key			 = Entry.first;
-		size_t				PayloadIndex = Entry.second;
-		BucketPayload&		Payload		 = Payloads[PayloadIndex];
-		const DiskLocation& Loc			 = Payload.Location;
+		const IoHash&		Key		= Entry.first;
+		BucketPayload&		Payload = Payloads[Entry.second];
+		const DiskLocation& Loc		= Payload.Location;
 		{
 			IoBuffer Buffer;
 			if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
@@ -1955,7 +1979,7 @@ ZenCacheDiskLayer::CacheBucket::GatherReferences(GcContext& GcCtx)
 #endif	// CALCULATE_BLOCKING_TIME
 				if (auto It = m_Index.find(Key); It != m_Index.end())
 				{
-					const BucketPayload& CachedPayload = Payloads[PayloadIndex];
+					const BucketPayload& CachedPayload = Payloads[It->second];
 					if (CachedPayload.MemCached)
 					{
 						Buffer = m_MemCachedPayloads[CachedPayload.MemCached];
@@ -2630,7 +2654,7 @@ ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffe
 		{
 			Payload.MemCached = MemCachedIndex(gsl::narrow<uint32_t>(m_MemCachedPayloads.size()));
 			m_MemCachedPayloads.push_back(MemCachedData);
-			AddMemCacheUsage(PayloadSize);
+			AddMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
 			m_MemoryWriteCount++;
 		}
 	}
@@ -2639,7 +2663,7 @@ ZenCacheDiskLayer::CacheBucket::SetMemCachedData(BucketPayload& Payload, IoBuffe
 		Payload.MemCached = m_FreeMemCachedPayloads.back();
 		m_FreeMemCachedPayloads.pop_back();
 		m_MemCachedPayloads[Payload.MemCached] = MemCachedData;
-		AddMemCacheUsage(PayloadSize);
+		AddMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
 		m_MemoryWriteCount++;
 	}
 }
@@ -2650,7 +2674,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveMemCachedData(BucketPayload& Payload)
 	if (Payload.MemCached)
 	{
 		size_t PayloadSize = m_MemCachedPayloads[Payload.MemCached].GetSize();
-		RemoveMemCacheUsage(PayloadSize);
+		RemoveMemCacheUsage(EstimateMemCachePayloadMemory(PayloadSize));
 		m_MemCachedPayloads[Payload.MemCached] = IoBuffer{};
 		m_FreeMemCachedPayloads.push_back(Payload.MemCached);
 		Payload.MemCached = {};
@@ -2844,7 +2868,8 @@ public:
 			m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys = std::make_unique<HashSet>(); });
 			auto __ = MakeGuard([&]() { m_Bucket.m_IndexLock.WithExclusiveLock([&]() { m_Bucket.m_UpdatedKeys.reset(); }); });
 
-			std::unordered_map<uint32_t, uint64_t> BlockUsage;
+			size_t					  InlineEntryCount = 0;
+			BlockStore::BlockUsageMap BlockUsage;
 			{
 				RwLock::SharedLockScope ___(m_Bucket.m_IndexLock);
 				for (const auto& Entry : m_Bucket.m_Index)
@@ -2857,15 +2882,17 @@ public:
 					{
 						continue;
 					}
+					InlineEntryCount++;
 					uint32_t BlockIndex = Loc.Location.BlockLocation.GetBlockIndex();
 					uint64_t ChunkSize	= RoundUp(Loc.Size(), m_Bucket.m_Configuration.PayloadAlignment);
 					if (auto It = BlockUsage.find(BlockIndex); It != BlockUsage.end())
 					{
-						It->second += ChunkSize;
+						It->second.EntryCount++;
+						It->second.DiskUsage += ChunkSize;
 					}
 					else
 					{
-						BlockUsage.insert_or_assign(BlockIndex, ChunkSize);
+						BlockUsage.insert_or_assign(BlockIndex, BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1});
 					}
 				}
 			}
@@ -2873,8 +2900,9 @@ public:
 			{
 				BlockStoreCompactState BlockCompactState;
 				std::vector<IoHash>	   BlockCompactStateKeys;
+				BlockCompactStateKeys.reserve(InlineEntryCount);
 
-				std::vector<uint32_t> BlocksToCompact =
+				BlockStore::BlockEntryCountMap BlocksToCompact =
 					m_Bucket.m_BlockStore.GetBlocksToCompact(BlockUsage, Ctx.Settings.CompactBlockUsageThresholdPercent);
 				BlockCompactState.IncludeBlocks(BlocksToCompact);
 
@@ -3149,7 +3177,7 @@ public:
 					uint32_t Size;
 				};
 				std::vector<std::vector<InlineEntry>> EntriesPerBlock;
-
+				size_t								  UpdateCount = 0;
 				{
 					RwLock::SharedLockScope IndexLock(m_CacheBucket.m_IndexLock);
 					for (const auto& Entry : m_CacheBucket.m_Index)
@@ -3174,6 +3202,7 @@ public:
 						{
 							continue;
 						}
+						UpdateCount++;
 						const IoHash& Key = Entry.first;
 						if (Loc.IsFlagSet(DiskLocation::kStandaloneFile))
 						{
@@ -3200,6 +3229,8 @@ public:
 					}
 				}
 
+				UpdateKeys.reserve(UpdateCount);
+
 				for (auto It : BlockIndexToEntriesPerBlockIndex)
 				{
 					uint32_t BlockIndex = It.first;
@@ -3652,6 +3683,8 @@ ZenCacheDiskLayer::CacheBucket::CompactState(std::vector<BucketPayload>&  Payloa
 		FirstReferenceIndex.reserve(EntryCount);
 	}
 	Index.reserve(EntryCount);
+	Index.min_load_factor(IndexMinLoadFactor);
+	Index.max_load_factor(IndexMaxLoadFactor);
 	for (auto It : m_Index)
 	{
 		PayloadIndex EntryIndex = PayloadIndex(Payloads.size());
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index fe92613f4..3fe0b0c63 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -9,6 +9,7 @@
 #include <zencore/except.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
+#include <zencore/logging.h>
 #include <zencore/string.h>
 #include <zenhttp/zenhttp.h>
 #include <zenutil/basicfile.h>
@@ -519,7 +520,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 						  cxxopts::value<bool>(ServerOptions.IsCleanStart)->default_value("false"));
 	options.add_options()("help", "Show command line help");
 	options.add_options()("t, test", "Enable test mode", cxxopts::value<bool>(ServerOptions.IsTest)->default_value("false"));
-	options.add_options()("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId));
 	options.add_options()("data-dir", "Specify persistence root", cxxopts::value<std::string>(DataDir));
 	options.add_options()("snapshot-dir",
 						  "Specify a snapshot of server state to mirror into the persistence root at startup",
@@ -528,7 +528,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("powercycle",
 						  "Exit immediately after initialization is complete",
 						  cxxopts::value<bool>(ServerOptions.IsPowerCycle));
-	options.add_options()("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile));
 	options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile));
 	options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile));
 	options.add_options()("no-sentry",
@@ -537,7 +536,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 	options.add_options()("sentry-allow-personal-info",
 						  "Allow personally identifiable information in sentry crash reports",
 						  cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false"));
-	options.add_options()("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"));
+
+	// clang-format off
+	options.add_options("logging")
+		("abslog",			"Path to log file",								cxxopts::value<std::string>(AbsLogFile))
+		("log-id",			"Specify id for adding context to log output",	cxxopts::value<std::string>(ServerOptions.LogId))
+		("quiet", 			"Disable console logging", 						cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false"))
+		("log-trace", 		"Change selected loggers to level TRACE", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace]))
+		("log-debug", 		"Change selected loggers to level DEBUG", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug]))
+		("log-info", 		"Change selected loggers to level INFO", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info]))
+		("log-warn", 		"Change selected loggers to level WARN", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn]))
+		("log-error", 		"Change selected loggers to level ERROR", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err]))
+		("log-critical", 	"Change selected loggers to level CRITICAL", 	cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical]))
+		("log-off", 		"Change selected loggers to level OFF", 		cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off]))
+	;
+	// clang-format on
 
 	options.add_option("security",
 					   "",
@@ -952,6 +965,12 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 			exit(0);
 		}
 
+		for (int i = 0; i < logging::level::LogLevelCount; ++i)
+		{
+			logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]);
+		}
+		logging::RefreshLogLevels();
+
 		ServerOptions.DataDir						  = MakeSafePath(DataDir);
 		ServerOptions.BaseSnapshotDir				  = MakeSafePath(BaseSnapshotDir);
 		ServerOptions.ContentDir					  = MakeSafePath(ContentDir);
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 8135bf8f0..11311f9d8 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/logbase.h>
 #include <zencore/zencore.h>
 #include <zenhttp/httpserver.h>
 #include <filesystem>
@@ -151,6 +152,7 @@ struct ZenServerOptions
 	bool					 SentryAllowPII		= false;  // Allow personally identifiable information in sentry crash reports
 	bool					 ObjectStoreEnabled = false;
 	bool					 NoConsoleOutput	= false;  // Control default use of stdout for diagnostics
+	std::string				 Loggers[zen::logging::level::LogLevelCount];
 #if ZEN_WITH_TRACE
 	std::string TraceHost;	// Host name or IP address to send trace data to
 	std::string TraceFile;	// Path of a file to write a trace
diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp
index e2d57b840..dc1675819 100644
--- a/src/zenserver/diag/logging.cpp
+++ b/src/zenserver/diag/logging.cpp
@@ -42,6 +42,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																	   /* max files */ 16,
 																	   /* rotate on open */ true);
 	auto HttpLogger = std::make_shared<spdlog::logger>("http_requests", HttpSink);
+	spdlog::apply_logger_env_levels(HttpLogger);
 	spdlog::register_logger(HttpLogger);
 
 	// Cache request logging
@@ -53,16 +54,19 @@ InitializeServerLogging(const ZenServerOptions& InOptions)
 																		/* max files */ 16,
 																		/* rotate on open */ false);
 	auto CacheLogger = std::make_shared<spdlog::logger>("z$", CacheSink);
+	spdlog::apply_logger_env_levels(CacheLogger);
 	spdlog::register_logger(CacheLogger);
 
 	// Jupiter - only log upstream HTTP traffic to file
 
 	auto JupiterLogger = std::make_shared<spdlog::logger>("jupiter", FileSink);
+	spdlog::apply_logger_env_levels(JupiterLogger);
 	spdlog::register_logger(JupiterLogger);
 
 	// Zen - only log upstream HTTP traffic to file
 
 	auto ZenClientLogger = std::make_shared<spdlog::logger>("zenclient", FileSink);
+	spdlog::apply_logger_env_levels(ZenClientLogger);
 	spdlog::register_logger(ZenClientLogger);
 
 	FinishInitializeLogging(LogOptions);
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 2430267c1..2aeb6a4d5 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -162,7 +162,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	// Ok so now we're configured, let's kick things off
 
 	m_Http				  = CreateHttpServer(ServerOptions.HttpServerConfig);
-	int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort);
+	int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir);
 
 	// Setup authentication manager
 	{
@@ -491,7 +491,7 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 						const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls);
 						if (Err)
 						{
-							ZEN_ERROR("resolve FAILED, reason '{}'", Err.message());
+							ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message());
 						}
 					}
 				}
@@ -643,6 +643,8 @@ ZenServer::Cleanup()
 
 		Flush();
 
+		ShutdownWorkerPools();
+
 		m_AdminService.reset();
 		m_VfsService.reset();
 		m_ObjStoreService.reset();
@@ -660,7 +662,6 @@ ZenServer::Cleanup()
 		m_AuthMgr.reset();
 		m_Http = {};
 		m_JobQueue.reset();
-		ShutdownWorkerPools();
 	}
 	catch (std::exception& Ex)
 	{