statsd metrics reporting (#496)

added support for reporting metrics via statsd style UDP messaging, which is supported by many monitoring solution providers

this change adds reporting only of three cache related metrics (hit/miss/put) but this should be extended to include more metrics after additional evaluation

9 files changed, 194 insertions, 14 deletions

diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 89123a70f..6fab14eee 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -16,6 +16,7 @@
 #include <zencore/timer.h>
 #include <zencore/trace.h>
 #include <zencore/workthreadpool.h>
+#include <zennet/statsdclient.h>
 #include <zenstore/scrubcontext.h>
 #include <zenutil/cache/cache.h>
 
@@ -663,7 +664,7 @@ ZenCacheStore::StorageSize() const
 }
 
 ZenCacheStore::CacheStoreStats
-ZenCacheStore::Stats()
+ZenCacheStore::Stats(bool IncludeNamespaceStats)
 {
 	ZenCacheStore::CacheStoreStats Result{.HitCount			  = m_HitCount,
 										  .MissCount		  = m_MissCount,
@@ -672,13 +673,32 @@ ZenCacheStore::Stats()
 										  .RejectedReadCount  = m_RejectedReadCount,
 										  .PutOps			  = m_PutOps.Snapshot(),
 										  .GetOps			  = m_GetOps.Snapshot()};
-	IterateNamespaces([&](std::string_view NamespaceName, ZenCacheNamespace& Store) {
-		Result.NamespaceStats.emplace_back(NamedNamespaceStats{.NamespaceName = std::string(NamespaceName), .Stats = Store.Stats()});
-	});
+
+	if (IncludeNamespaceStats)
+	{
+		IterateNamespaces([&](std::string_view NamespaceName, ZenCacheNamespace& Store) {
+			Result.NamespaceStats.emplace_back(NamedNamespaceStats{.NamespaceName = std::string(NamespaceName), .Stats = Store.Stats()});
+		});
+	}
+
 	return Result;
 }
 
 void
+ZenCacheStore::ReportMetrics(StatsDaemonClient& Statsd)
+{
+	const bool			   IncludeNamespaceStats = false;
+	const CacheStoreStats  Now					 = Stats(IncludeNamespaceStats);
+	const CacheStoreStats& Old					 = m_LastReportedMetrics;
+
+	Statsd.Meter("zen.cache_hits", Now.HitCount - Old.HitCount);
+	Statsd.Meter("zen.cache_misses", Now.MissCount - Old.MissCount);
+	Statsd.Meter("zen.cache_writes", Now.WriteCount - Old.WriteCount);
+
+	m_LastReportedMetrics = Now;
+}
+
+void
 ZenCacheStore::SetLoggingConfig(const Configuration::LogConfig& Loggingconfig)
 {
 	if (!Loggingconfig.EnableAccessLog && !Loggingconfig.EnableWriteLog)
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index 02d0e31c0..dacf482d8 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "cachedisklayer.h"
+#include "stats/statsreporter.h"
 
 #include <zencore/compactbinary.h>
 #include <zencore/iohash.h>
@@ -18,6 +19,8 @@
 
 namespace zen {
 
+class StatsDaemonClient;
+
 /******************************************************************************
 
   /$$$$$$$$                        /$$$$$$                   /$$
@@ -126,7 +129,7 @@ private:
 
   */
 
-class ZenCacheStore final : public RefCounted
+class ZenCacheStore final : public RefCounted, public StatsProvider
 {
 public:
 	static constexpr std::string_view DefaultNamespace =
@@ -161,11 +164,11 @@ public:
 
 	struct CacheStoreStats
 	{
-		uint64_t						 HitCount;
-		uint64_t						 MissCount;
-		uint64_t						 WriteCount;
-		uint64_t						 RejectedWriteCount;
-		uint64_t						 RejectedReadCount;
+		uint64_t						 HitCount			= 0;
+		uint64_t						 MissCount			= 0;
+		uint64_t						 WriteCount			= 0;
+		uint64_t						 RejectedWriteCount = 0;
+		uint64_t						 RejectedReadCount	= 0;
 		metrics::RequestStatsSnapshot	 PutOps;
 		metrics::RequestStatsSnapshot	 GetOps;
 		std::vector<NamedNamespaceStats> NamespaceStats;
@@ -199,7 +202,7 @@ public:
 									  const std::string_view ValueFilter) const;
 
 	GcStorageSize	StorageSize() const;
-	CacheStoreStats Stats();
+	CacheStoreStats Stats(bool IncludeNamespaceStats = true);
 
 	Configuration								 GetConfiguration() const { return m_Configuration; }
 	void										 SetLoggingConfig(const Configuration::LogConfig& Loggingconfig);
@@ -212,6 +215,9 @@ public:
 								 std::string_view																		  Bucket,
 								 std::function<void(const IoHash& Key, const CacheValueDetails::ValueDetails& Details)>&& Fn);
 
+	// StatsProvider
+	virtual void ReportMetrics(StatsDaemonClient& Statsd) override;
+
 private:
 	const ZenCacheNamespace* FindNamespace(std::string_view Namespace) const;
 	ZenCacheNamespace*		 GetNamespace(std::string_view Namespace);
@@ -219,6 +225,7 @@ private:
 
 	typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap;
 
+	CacheStoreStats									m_LastReportedMetrics;
 	const DiskWriteBlocker*							m_DiskWriteBlocker = nullptr;
 	mutable RwLock									m_NamespacesLock;
 	NamespaceMap									m_Namespaces;
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp
index 81b1dcfe2..ca438fe38 100644
--- a/src/zenserver/config.cpp
+++ b/src/zenserver/config.cpp
@@ -799,6 +799,11 @@ ParseConfigFile(const std::filesystem::path& Path,
 	LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceHost, "tracehost"sv);
 	LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceFile, "tracefile"sv);
 
+	////// stats
+	LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled);
+	LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost);
+	LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort);
+
 	////// cache
 	LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled);
 	LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log");
@@ -1299,6 +1304,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions)
 					   cxxopts::value<std::vector<std::string>>(BucketConfigs),
 					   "");
 
+	options.add_option("stats",
+					   "",
+					   "statsd",
+					   "",
+					   cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"),
+					   "Enable statsd reporter (localhost:8125)");
+
 	try
 	{
 		auto result = options.parse(argc, argv);
diff --git a/src/zenserver/config.h b/src/zenserver/config.h
index 7743e536f..a1e091665 100644
--- a/src/zenserver/config.h
+++ b/src/zenserver/config.h
@@ -96,6 +96,13 @@ struct ZenObjectStoreConfig
 	std::vector<BucketConfig> Buckets;
 };
 
+struct ZenStatsConfig
+{
+	bool		Enabled	   = false;
+	std::string StatsdHost = "localhost";
+	int			StatsdPort = 8125;
+};
+
 struct ZenStructuredCacheConfig
 {
 	bool	 Enabled				 = true;
@@ -116,6 +123,7 @@ struct ZenServerOptions
 	ZenObjectStoreConfig	 ObjectStoreConfig;
 	zen::HttpServerConfig	 HttpServerConfig;
 	ZenStructuredCacheConfig StructuredCacheConfig;
+	ZenStatsConfig			 StatsConfig;
 	std::filesystem::path	 DataDir;					  // Root directory for state (used for testing)
 	std::filesystem::path	 ContentDir;				  // Root directory for serving frontend content (experimental)
 	std::filesystem::path	 AbsLogFile;				  // Absolute path to main log file
diff --git a/src/zenserver/stats/statsreporter.cpp b/src/zenserver/stats/statsreporter.cpp
new file mode 100644
index 000000000..5d5ef4bfa
--- /dev/null
+++ b/src/zenserver/stats/statsreporter.cpp
@@ -0,0 +1,60 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "statsreporter.h"
+
+#include <zencore/logging.h>
+#include <zennet/statsdclient.h>
+
+namespace zen {
+
+StatsReporter::StatsReporter()
+{
+}
+
+StatsReporter::~StatsReporter()
+{
+}
+
+void
+StatsReporter::Initialize(const ZenStatsConfig& Config)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+
+	if (Config.Enabled)
+	{
+		ZEN_INFO("initializing stats reporter: {}:{}", Config.StatsdHost, Config.StatsdPort)
+		m_Statsd = CreateStatsDaemonClient(Config.StatsdHost, gsl::narrow<uint16_t>(Config.StatsdPort));
+		m_Statsd->SetMessageSize(1500, false);
+	}
+}
+
+void
+StatsReporter::Shutdown()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	m_Statsd.reset();
+}
+
+void
+StatsReporter::AddProvider(StatsProvider* Provider)
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	m_Providers.push_back(Provider);
+}
+
+void
+StatsReporter::ReportStats()
+{
+	RwLock::ExclusiveLockScope _(m_Lock);
+	if (m_Statsd)
+	{
+		for (StatsProvider* Provider : m_Providers)
+		{
+			Provider->ReportMetrics(*m_Statsd);
+		}
+
+		m_Statsd->Flush();
+	}
+}
+
+}  // namespace zen
diff --git a/src/zenserver/stats/statsreporter.h b/src/zenserver/stats/statsreporter.h
new file mode 100644
index 000000000..ed6ec6c55
--- /dev/null
+++ b/src/zenserver/stats/statsreporter.h
@@ -0,0 +1,39 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "config.h"
+
+#include <zencore/thread.h>
+
+namespace zen {
+
+class StatsDaemonClient;
+
+class StatsProvider
+{
+public:
+	virtual void ReportMetrics(StatsDaemonClient& Statsd) = 0;
+};
+
+class StatsReporter
+{
+public:
+	StatsReporter();
+	~StatsReporter();
+
+	StatsReporter& operator=(const StatsReporter&) = delete;
+	StatsReporter(const StatsReporter&)			   = delete;
+
+	void Initialize(const ZenStatsConfig& Config);
+	void Shutdown();
+	void AddProvider(StatsProvider* Provider);
+	void ReportStats();
+
+private:
+	RwLock							   m_Lock;
+	std::unique_ptr<StatsDaemonClient> m_Statsd;
+	std::vector<StatsProvider*>		   m_Providers;
+};
+
+}  // namespace zen
diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua
index 329435acb..123690be2 100644
--- a/src/zenserver/xmake.lua
+++ b/src/zenserver/xmake.lua
@@ -2,8 +2,12 @@
 
 target("zenserver")
     set_kind("binary")
-    add_deps("zencore", "zenhttp", "zenstore", "zenutil")
-    add_deps("zenvfs")
+    add_deps("zencore",
+             "zenhttp",
+             "zennet",
+             "zenstore",
+             "zenutil",
+             "zenvfs")
     add_headerfiles("**.h")
     add_files("**.cpp")
     add_files("zenserver.cpp", {unity_ignored  = true })
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index e5346abf0..d7fc2d069 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -41,7 +41,6 @@
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/format.h>
 #include <asio.hpp>
-#include <lua.hpp>
 ZEN_THIRD_PARTY_INCLUDES_END
 
 #include <exception>
@@ -197,7 +196,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 	m_Http->RegisterService(*m_AuthService);
 
 	m_Http->RegisterService(m_HealthService);
+
 	m_Http->RegisterService(m_StatsService);
+	m_StatsReporter.Initialize(ServerOptions.StatsConfig);
+	if (ServerOptions.StatsConfig.Enabled)
+	{
+		EnqueueStatsReportingTimer();
+	}
+
 	m_Http->RegisterService(m_StatusService);
 	m_StatusService.RegisterHandler("status", *this);
 
@@ -260,10 +266,12 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen
 		m_Http->RegisterService(*m_ObjStoreService);
 	}
 
+#if ZEN_WITH_VFS
 	m_VfsService = std::make_unique<VfsService>();
 	m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore));
 	m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore));
 	m_Http->RegisterService(*m_VfsService);
+#endif
 
 	ZEN_INFO("initializing GC, enabled '{}', interval {}, lightweight interval {}",
 			 ServerOptions.GcConfig.Enabled,
@@ -514,6 +522,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions)
 
 	m_Http->RegisterService(*m_StructuredCacheService);
 	m_Http->RegisterService(*m_UpstreamService);
+
+	m_StatsReporter.AddProvider(m_CacheStore.Get());
 }
 
 void
@@ -598,6 +608,8 @@ ZenServer::Cleanup()
 			m_JobQueue->Stop();
 		}
 
+		m_StatsReporter.Shutdown();
+
 		m_GcScheduler.Shutdown();
 		m_AdminService.reset();
 		m_VfsService.reset();
@@ -661,6 +673,20 @@ ZenServer::EnqueueSigIntTimer()
 }
 
 void
+ZenServer::EnqueueStatsReportingTimer()
+{
+	m_StatsReportingTimer.expires_after(std::chrono::milliseconds(500));
+	m_StatsReportingTimer.async_wait([this](const asio::error_code& Ec) {
+		if (!Ec)
+		{
+			m_StatsReporter.ReportStats();
+			EnqueueStatsReportingTimer();
+		}
+	});
+	EnsureIoRunner();
+}
+
+void
 ZenServer::CheckStateMarker()
 {
 	std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker";
diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h
index 25e45ccd0..de069be69 100644
--- a/src/zenserver/zenserver.h
+++ b/src/zenserver/zenserver.h
@@ -33,6 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include "objectstore/objectstore.h"
 #include "projectstore/httpprojectstore.h"
 #include "projectstore/projectstore.h"
+#include "stats/statsreporter.h"
 #include "upstream/upstream.h"
 #include "vfs/vfsservice.h"
 
@@ -75,6 +76,7 @@ public:
 	void EnqueueTimer();
 	void EnqueueStateMarkerTimer();
 	void EnqueueSigIntTimer();
+	void EnqueueStatsReportingTimer();
 	void CheckStateMarker();
 	void CheckSigInt();
 	void CheckOwnerPid();
@@ -96,6 +98,7 @@ private:
 	asio::steady_timer				m_PidCheckTimer{m_IoContext};
 	asio::steady_timer				m_StateMakerTimer{m_IoContext};
 	asio::steady_timer				m_SigIntTimer{m_IoContext};
+	asio::steady_timer				m_StatsReportingTimer{m_IoContext};
 	ProcessMonitor					m_ProcessMonitor;
 	NamedMutex						m_ServerMutex;
 
@@ -109,6 +112,7 @@ private:
 	inline void				SetNewState(ServerState NewState) { m_CurrentState = NewState; }
 	static std::string_view ToString(ServerState Value);
 
+	StatsReporter					 m_StatsReporter;
 	Ref<HttpServer>					 m_Http;
 	std::unique_ptr<AuthMgr>		 m_AuthMgr;
 	std::unique_ptr<HttpAuthService> m_AuthService;