move all storage-related services into storage tree (#571)

* move all storage-related services into storage tree
* move config into config/
* also move admin service into storage since it mostly has storage related functionality
* header consolidation

1 files changed, 618 insertions, 0 deletions

diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
new file mode 100644
index 000000000..3f40bf616
--- /dev/null
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -0,0 +1,618 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "objectstore.h"
+
+#include <zencore/base64.h>
+#include <zencore/basicfile.h>
+#include <zencore/compactbinaryvalue.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/logging.h>
+#include <zencore/string.h>
+#include <zencore/trace.h>
+#include "zencore/compactbinary.h"
+#include "zencore/compactbinarybuilder.h"
+#include "zenhttp/httpcommon.h"
+#include "zenhttp/httpserver.h"
+
+#include <filesystem>
+#include <thread>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv);
+
+class CbXmlWriter
+{
+public:
+	explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder)
+	{
+		Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		Builder << LINE_TERMINATOR_ANSI;
+	}
+
+	void WriteField(CbFieldView Field)
+	{
+		using namespace std::literals;
+
+		bool					 SkipEndTag = false;
+		const std::u8string_view Tag		= Field.GetU8Name();
+
+		AppendBeginTag(Tag);
+
+		switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+		{
+			case CbFieldType::Null:
+				Builder << "Null"sv;
+				break;
+			case CbFieldType::Object:
+			case CbFieldType::UniformObject:
+				{
+					for (CbFieldView It : Field)
+					{
+						WriteField(It);
+					}
+				}
+				break;
+			case CbFieldType::Array:
+			case CbFieldType::UniformArray:
+				{
+					bool FirstField = true;
+					for (CbFieldView It : Field)
+					{
+						if (!FirstField)
+							AppendBeginTag(Tag);
+
+						WriteField(It);
+						AppendEndTag(Tag);
+						FirstField = false;
+					}
+					SkipEndTag = true;
+				}
+				break;
+			case CbFieldType::Binary:
+				AppendBase64String(Accessor.AsBinary());
+				break;
+			case CbFieldType::String:
+				Builder << Accessor.AsU8String();
+				break;
+			case CbFieldType::IntegerPositive:
+				Builder << Accessor.AsIntegerPositive();
+				break;
+			case CbFieldType::IntegerNegative:
+				Builder << Accessor.AsIntegerNegative();
+				break;
+			case CbFieldType::Float32:
+				{
+					const float Value = Accessor.AsFloat32();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.9g}", Value));
+					}
+					else
+					{
+						Builder << "Null"sv;
+					}
+				}
+				break;
+			case CbFieldType::Float64:
+				{
+					const double Value = Accessor.AsFloat64();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.17g}", Value));
+					}
+					else
+					{
+						Builder << "null"sv;
+					}
+				}
+				break;
+			case CbFieldType::BoolFalse:
+				Builder << "False"sv;
+				break;
+			case CbFieldType::BoolTrue:
+				Builder << "True"sv;
+				break;
+			case CbFieldType::ObjectAttachment:
+			case CbFieldType::BinaryAttachment:
+				{
+					Accessor.AsAttachment().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Hash:
+				{
+					Accessor.AsHash().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Uuid:
+				{
+					Accessor.AsUuid().ToString(Builder);
+				}
+				break;
+			case CbFieldType::DateTime:
+				Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+				break;
+			case CbFieldType::TimeSpan:
+				{
+					const TimeSpan Span(Accessor.AsTimeSpanTicks());
+					if (Span.GetDays() == 0)
+					{
+						Builder << Span.ToString("%h:%m:%s.%n");
+					}
+					else
+					{
+						Builder << Span.ToString("%d.%h:%m:%s.%n");
+					}
+					break;
+				}
+			case CbFieldType::ObjectId:
+				Accessor.AsObjectId().ToString(Builder);
+				break;
+			case CbFieldType::CustomById:
+				{
+					CbCustomById Custom = Accessor.AsCustomById();
+
+					AppendBeginTag(u8"Id"sv);
+					Builder << Custom.Id;
+					AppendEndTag(u8"Id"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			case CbFieldType::CustomByName:
+				{
+					CbCustomByName Custom = Accessor.AsCustomByName();
+
+					AppendBeginTag(u8"Name"sv);
+					Builder << Custom.Name;
+					AppendEndTag(u8"Name"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			default:
+				ZEN_ASSERT(false);
+				break;
+		}
+
+		if (!SkipEndTag)
+			AppendEndTag(Tag);
+	}
+
+private:
+	void AppendBeginTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << '<' << Tag << '>';
+		}
+	}
+
+	void AppendEndTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << "</"sv << Tag << '>';
+		}
+	}
+
+	void AppendBase64String(MemoryView Value)
+	{
+		Builder << '"';
+		ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+		const uint32_t EncodedSize	= Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+		const size_t   EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+		Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+	}
+
+private:
+	StringBuilderBase& Builder;
+};
+
+HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg)
+: m_StatusService(StatusService)
+, m_Cfg(std::move(Cfg))
+{
+	Inititalize();
+	m_StatusService.RegisterHandler("obj", *this);
+}
+
+HttpObjectStoreService::~HttpObjectStoreService()
+{
+	m_StatusService.UnregisterHandler("obj", *this);
+}
+
+const char*
+HttpObjectStoreService::BaseUri() const
+{
+	return "/obj/";
+}
+
+void
+HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
+{
+	if (m_Router.HandleRequest(Request) == false)
+	{
+		ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri());
+		return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv);
+	}
+}
+
+void
+HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request)
+{
+	CbObjectWriter Cbo;
+	Cbo << "ok" << true;
+	Request.WriteResponse(HttpResponseCode::OK, Cbo.Save());
+}
+
+void
+HttpObjectStoreService::Inititalize()
+{
+	ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize");
+
+	namespace fs = std::filesystem;
+	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
+
+	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
+	if (!IsDir(BucketsPath))
+	{
+		CreateDirectories(BucketsPath);
+	}
+
+	m_Router.RegisterRoute(
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		HttpVerb::kPost | HttpVerb::kPut);
+
+	m_Router.RegisterRoute(
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		HttpVerb::kDelete);
+
+	m_Router.RegisterRoute(
+		"bucket/{path}",
+		[this](zen::HttpRouterRequest& Request) {
+			const std::string_view Path		= Request.GetCapture(1);
+			const auto			   Sep		= Path.find_last_of('.');
+			const bool			   IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
+
+			if (IsObject)
+			{
+				GetObject(Request, Path);
+			}
+			else
+			{
+				ListBucket(Request, Path);
+			}
+		},
+		HttpVerb::kHead | HttpVerb::kGet);
+
+	m_Router.RegisterRoute(
+		"bucket/{bucket}/{path}",
+		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
+		HttpVerb::kPost | HttpVerb::kPut);
+}
+
+std::filesystem::path
+HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
+{
+	{
+		std::lock_guard _(BucketsMutex);
+
+		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
+										 std::end(m_Cfg.Buckets),
+										 [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; });
+			It != std::end(m_Cfg.Buckets))
+		{
+			return It->Directory.make_preferred();
+		}
+	}
+
+	return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred();
+}
+
+void
+HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
+	{
+		std::lock_guard _(BucketsMutex);
+		if (!IsDir(BucketPath))
+		{
+			CreateDirectories(BucketPath);
+			ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::Created);
+		}
+	}
+
+	ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path)
+{
+	namespace fs = std::filesystem;
+
+	const auto		  Sep = Path.find_first_of('/');
+	const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)};
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)};
+	if (BucketPrefix.empty())
+	{
+		const auto QueryParms = Request.ServerRequest().GetQueryParams();
+		if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false)
+		{
+			BucketPrefix = PrefixParam;
+		}
+	}
+	BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/'));
+	BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\'));
+
+	const fs::path BucketRoot		  = GetBucketDirectory(BucketName);
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
+	const fs::path FullPath			  = BucketRoot / RelativeBucketPath;
+
+	struct Visitor : FileSystemTraversal::TreeVisitor
+	{
+		Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path)
+		{
+			Writer.BeginObject("ListBucketResult"sv);
+			Writer << "Name"sv << BucketName;
+			std::string Tmp = Prefix.string();
+			std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
+			Writer << "Prefix"sv << Tmp;
+			Writer.BeginArray("Contents"sv);
+		}
+
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override
+		{
+			const fs::path FullPath		= Parent / fs::path(File);
+			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
+
+			std::string Key = RelativePath.string();
+			std::replace(Key.begin(), Key.end(), '\\', '/');
+
+			Writer.BeginObject();
+			Writer << "Key"sv << Key;
+			Writer << "Size"sv << FileSize;
+			Writer.EndObject();
+		}
+
+		bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; }
+
+		CbObject GetResult()
+		{
+			Writer.EndArray();
+			Writer.EndObject();
+			return Writer.Save();
+		}
+
+		CbObjectWriter Writer;
+		fs::path	   BucketPath;
+	};
+
+	Visitor				FileVisitor(BucketName, BucketRoot, RelativeBucketPath);
+	FileSystemTraversal Traversal;
+
+	if (IsDir(FullPath))
+	{
+		std::lock_guard _(BucketsMutex);
+		Traversal.TraverseFileSystem(FullPath, FileVisitor);
+	}
+	CbObject Result = FileVisitor.GetResult();
+
+	if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON)
+	{
+		ExtendableStringBuilder<1024> Sb;
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView());
+	}
+
+	ExtendableStringBuilder<1024> Xml;
+	CbXmlWriter					  XmlWriter(Xml);
+	XmlWriter.WriteField(Result.AsFieldView());
+
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView());
+}
+
+void
+HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
+	{
+		std::lock_guard _(BucketsMutex);
+		DeleteDirectories(BucketPath);
+	}
+
+	ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path)
+{
+	namespace fs = std::filesystem;
+
+	const auto		  Sep = Path.find_first_of('/');
+	const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)};
+	const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)};
+
+	const fs::path BucketDir = GetBucketDirectory(BucketName);
+
+	if (BucketDir.empty())
+	{
+		ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName);
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
+	}
+
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
+
+	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
+	{
+		ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName);
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
+	}
+
+	const fs::path FilePath = BucketDir / RelativeBucketPath;
+	if (!IsFile(FilePath))
+	{
+		ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath);
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
+	}
+
+	zen::HttpRanges Ranges;
+	if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1)
+	{
+		// Only a single range is supported
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	FileContents File;
+	{
+		std::lock_guard _(BucketsMutex);
+		File = ReadFile(FilePath);
+	}
+
+	if (File.ErrorCode)
+	{
+		ZEN_LOG_WARN(LogObj,
+					 "GET - '{}/{}' [FAILED] ('{}': {})",
+					 BucketName,
+					 FilePath,
+					 File.ErrorCode.category().name(),
+					 File.ErrorCode.value());
+
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	const IoBuffer& FileBuf = File.Data[0];
+
+	if (Ranges.empty())
+	{
+		const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size();
+
+		ZEN_LOG_DEBUG(LogObj,
+					  "GET - '{}/{}' ({}) [OK] (Served: {})",
+					  BucketName,
+					  RelativeBucketPath,
+					  NiceBytes(FileBuf.Size()),
+					  NiceBytes(TotalServed));
+
+		Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf);
+	}
+	else
+	{
+		const auto	   Range	   = Ranges[0];
+		const uint64_t RangeSize   = 1 + (Range.End - Range.Start);
+		const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize;
+
+		ZEN_LOG_DEBUG(LogObj,
+					  "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})",
+					  BucketName,
+					  RelativeBucketPath,
+					  Range.Start,
+					  Range.End,
+					  NiceBytes(RangeSize),
+					  NiceBytes(FileBuf.Size()),
+					  NiceBytes(TotalServed));
+
+		MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize);
+		if (RangeView.GetSize() != RangeSize)
+		{
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		}
+
+		IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize());
+		Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf);
+	}
+}
+
+void
+HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const std::string_view BucketName = Request.GetCapture(1);
+	const fs::path		   BucketDir  = GetBucketDirectory(BucketName);
+
+	if (BucketDir.empty())
+	{
+		ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName);
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
+	}
+
+	const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred();
+
+	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
+	{
+		ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName);
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
+	}
+
+	const fs::path FilePath		 = BucketDir / RelativeBucketPath;
+	const fs::path FileDirectory = FilePath.parent_path();
+
+	{
+		std::lock_guard _(BucketsMutex);
+
+		if (!IsDir(FileDirectory))
+		{
+			CreateDirectories(FileDirectory);
+		}
+
+		const IoBuffer FileBuf = Request.ServerRequest().ReadPayload();
+
+		if (FileBuf.Size() == 0)
+		{
+			ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		}
+
+		TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView());
+
+		ZEN_LOG_DEBUG(LogObj,
+					  "PUT - '{}' [OK] ({})",
+					  (fs::path(BucketName) / RelativeBucketPath).make_preferred(),
+					  NiceBytes(FileBuf.Size()));
+	}
+
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
+}
+
+}  // namespace zen