1 files changed, 157 insertions, 25 deletions

diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index e347e2dfe..d6516fa1a 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -14,6 +14,7 @@
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
 #include "zenhttp/httpserver.h"
+#include "zenhttp/httpstats.h"
 
 #include <filesystem>
 #include <thread>
@@ -220,17 +221,20 @@ private:
 	StringBuilderBase& Builder;
 };
 
-HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
-: m_StatusService(StatusService)
+HttpObjectStoreService::HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatsService(StatsService)
+, m_StatusService(StatusService)
 , m_Cfg(std::move(Cfg))
 {
-	Inititalize();
+	Initialize();
+	m_StatsService.RegisterHandler("obj", *this);
 	m_StatusService.RegisterHandler("obj", *this);
 }
 
 HttpObjectStoreService::~HttpObjectStoreService()
 {
 	m_StatusService.UnregisterHandler("obj", *this);
+	m_StatsService.UnregisterHandler("obj", *this);
 }
 
 const char*
@@ -240,8 +244,10 @@ HttpObjectStoreService::BaseUri() const
 }
 
 void
-HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
+HttpObjectStoreService::HandleRequest(HttpServerRequest& Request)
 {
+	metrics::OperationTiming::Scope $(m_HttpRequests);
+
 	if (m_Router.HandleRequest(Request) == false)
 	{
 		ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri());
@@ -258,12 +264,36 @@ HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
 }
 
 void
-HttpObjectStoreService::Inititalize()
+HttpObjectStoreService::HandleStatsRequest(HttpServerRequest& Request)
+{
+	Request.WriteResponse(HttpResponseCode::OK, CollectStats());
+}
+
+CbObject
+HttpObjectStoreService::CollectStats()
+{
+	ZEN_TRACE_CPU("HttpObjectStoreService::Stats");
+	CbObjectWriter Cbo;
+
+	EmitSnapshot("requests", m_HttpRequests, Cbo);
+	Cbo << "total_bytes_served" << m_TotalBytesServed.load();
+
+	return Cbo.Save();
+}
+
+uint64_t
+HttpObjectStoreService::GetActivityCounter()
+{
+	return m_HttpRequests.Count();
+}
+
+void
+HttpObjectStoreService::Initialize()
 {
-	ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize");
+	ZEN_TRACE_CPU("HttpObjectStoreService::Initialize");
 
 	namespace fs = std::filesystem;
-	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
+	ZEN_LOG_INFO(LogObj, "Initializing Object Store in '{}'", m_Cfg.RootDirectory);
 
 	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
 	if (!IsDir(BucketsPath))
@@ -280,18 +310,28 @@ HttpObjectStoreService::Inititalize()
 						[](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidBucketCharactersSet); });
 
 	m_Router.RegisterRoute(
+		"",
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		[this](HttpRouterRequest& Request) { ListBuckets(Request); },
+		HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"bucket",
+		[this](HttpRouterRequest& Request) { CreateBucket(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 
 	m_Router.RegisterRoute(
 		"bucket",
-		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		[this](HttpRouterRequest& Request) { DeleteBucket(Request); },
 		HttpVerb::kDelete);
 
 	m_Router.RegisterRoute(
 		"bucket/{path}",
-		[this](zen::HttpRouterRequest& Request) {
+		[this](HttpRouterRequest& Request) {
 			const std::string_view Path		= Request.GetCapture(1);
 			const auto			   Sep		= Path.find_last_of('.');
 			const bool			   IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0;
@@ -309,7 +349,7 @@ HttpObjectStoreService::Inititalize()
 
 	m_Router.RegisterRoute(
 		"bucket/{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
+		[this](HttpRouterRequest& Request) { PutObject(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 }
 
@@ -317,7 +357,7 @@ std::filesystem::path
 HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 {
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
 										 std::end(m_Cfg.Buckets),
@@ -332,7 +372,99 @@ HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 }
 
 void
-HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::ListBuckets(HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
+
+	CbObjectWriter Response;
+	Response.BeginArray("buckets");
+	{
+		std::lock_guard _(m_BucketsMutex);
+
+		// Configured buckets
+		for (const ObjectStoreConfig::BucketConfig& Bucket : m_Cfg.Buckets)
+		{
+			Response.BeginObject();
+			Response << "name" << Bucket.Name;
+
+			const fs::path Dir = Bucket.Directory.empty() ? (m_Cfg.RootDirectory / Bucket.Name) : Bucket.Directory;
+			if (IsDir(Dir))
+			{
+				uint64_t TotalSize = 0;
+				uint64_t FileCount = 0;
+				for (const fs::directory_entry& Entry :
+					 fs::recursive_directory_iterator(Dir, fs::directory_options::skip_permission_denied))
+				{
+					if (Entry.is_regular_file())
+					{
+						TotalSize += Entry.file_size();
+						FileCount++;
+					}
+				}
+				Response << "size" << TotalSize;
+				Response << "object_count" << FileCount;
+			}
+			Response.EndObject();
+		}
+
+		// Dynamic buckets (on-disk directories not in config)
+		if (IsDir(BucketsPath))
+		{
+			for (const fs::directory_entry& Entry : fs::directory_iterator(BucketsPath))
+			{
+				if (!Entry.is_directory())
+				{
+					continue;
+				}
+				const std::string Name = Entry.path().filename().string();
+
+				// Skip if already listed as a configured bucket
+				bool IsConfigured = false;
+				for (const ObjectStoreConfig::BucketConfig& Bucket : m_Cfg.Buckets)
+				{
+					if (Bucket.Name == Name)
+					{
+						IsConfigured = true;
+						break;
+					}
+				}
+				if (IsConfigured)
+				{
+					continue;
+				}
+
+				Response.BeginObject();
+				Response << "name" << Name;
+
+				uint64_t TotalSize = 0;
+				uint64_t FileCount = 0;
+				for (const fs::directory_entry& FileEntry :
+					 fs::recursive_directory_iterator(Entry.path(), fs::directory_options::skip_permission_denied))
+				{
+					if (FileEntry.is_regular_file())
+					{
+						TotalSize += FileEntry.file_size();
+						FileCount++;
+					}
+				}
+				Response << "size" << TotalSize;
+				Response << "object_count" << FileCount;
+
+				Response.EndObject();
+			}
+		}
+	}
+	Response.EndArray();
+
+	Response << "total_bytes_served" << m_TotalBytesServed.load();
+
+	return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save());
+}
+
+void
+HttpObjectStoreService::CreateBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -346,7 +478,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		if (!IsDir(BucketPath))
 		{
 			CreateDirectories(BucketPath);
@@ -360,7 +492,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::ListBucket(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -431,7 +563,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 
 	if (IsDir(FullPath))
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		Traversal.TraverseFileSystem(FullPath, FileVisitor);
 	}
 	CbObject Result = FileVisitor.GetResult();
@@ -450,7 +582,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
 }
 
 void
-HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::DeleteBucket(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -464,7 +596,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 
 	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		DeleteDirectories(BucketPath);
 	}
 
@@ -473,7 +605,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path)
+HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_view Path)
 {
 	namespace fs = std::filesystem;
 
@@ -504,7 +636,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	zen::HttpRanges Ranges;
+	HttpRanges Ranges;
 	if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
 	{
 		// Only a single range is supported
@@ -513,7 +645,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	FileContents File;
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 		File = ReadFile(FilePath);
 	}
 
@@ -533,7 +665,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 
 	if (Ranges.empty())
 	{
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' ({}) [OK] (Served: {})",
@@ -548,7 +680,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 	{
 		const auto	   Range	   = Ranges[0];
 		const uint64_t RangeSize   = 1 + (Range.End - Range.Start);
-		const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+		const uint64_t TotalServed = m_TotalBytesServed.fetch_add(RangeSize) + RangeSize;
 
 		ZEN_LOG_DEBUG(LogObj,
 					  "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
@@ -572,7 +704,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st
 }
 
 void
-HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::PutObject(HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -597,7 +729,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 	const fs::path FileDirectory = FilePath.parent_path();
 
 	{
-		std::lock_guard _(BucketsMutex);
+		std::lock_guard _(m_BucketsMutex);
 
 		if (!IsDir(FileDirectory))
 		{