Make object store endpoint S3 compatible. (#535)

* Make object store endpoint S3 compatible.

* Removed XML pretty printing and set object store endpoint disabled by default.

1 files changed, 410 insertions, 42 deletions

diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp
index 3643e8011..76f2aba95 100644
--- a/src/zenserver/objectstore/objectstore.cpp
+++ b/src/zenserver/objectstore/objectstore.cpp
@@ -2,14 +2,18 @@
 
 #include <objectstore/objectstore.h>
 
+#include <zencore/base64.h>
+#include <zencore/compactbinaryvalue.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include "zencore/compactbinary.h"
 #include "zencore/compactbinarybuilder.h"
 #include "zenhttp/httpcommon.h"
 #include "zenhttp/httpserver.h"
 
+#include <filesystem>
 #include <thread>
 
 ZEN_THIRD_PARTY_INCLUDES_START
@@ -23,6 +27,198 @@ using namespace std::literals;
 
 ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv);
 
+class CbXmlWriter
+{
+public:
+	explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder)
+	{
+		Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+		Builder << LINE_TERMINATOR_ANSI;
+	}
+
+	void WriteField(CbFieldView Field)
+	{
+		using namespace std::literals;
+
+		bool					 SkipEndTag = false;
+		const std::u8string_view Tag		= Field.GetU8Name();
+
+		AppendBeginTag(Tag);
+
+		switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+		{
+			case CbFieldType::Null:
+				Builder << "Null"sv;
+				break;
+			case CbFieldType::Object:
+			case CbFieldType::UniformObject:
+				{
+					for (CbFieldView It : Field)
+					{
+						WriteField(It);
+					}
+				}
+				break;
+			case CbFieldType::Array:
+			case CbFieldType::UniformArray:
+				{
+					bool FirstField = true;
+					for (CbFieldView It : Field)
+					{
+						if (!FirstField)
+							AppendBeginTag(Tag);
+
+						WriteField(It);
+						AppendEndTag(Tag);
+						FirstField = false;
+					}
+					SkipEndTag = true;
+				}
+				break;
+			case CbFieldType::Binary:
+				AppendBase64String(Accessor.AsBinary());
+				break;
+			case CbFieldType::String:
+				Builder << Accessor.AsU8String();
+				break;
+			case CbFieldType::IntegerPositive:
+				Builder << Accessor.AsIntegerPositive();
+				break;
+			case CbFieldType::IntegerNegative:
+				Builder << Accessor.AsIntegerNegative();
+				break;
+			case CbFieldType::Float32:
+				{
+					const float Value = Accessor.AsFloat32();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.9g}", Value));
+					}
+					else
+					{
+						Builder << "Null"sv;
+					}
+				}
+				break;
+			case CbFieldType::Float64:
+				{
+					const double Value = Accessor.AsFloat64();
+					if (std::isfinite(Value))
+					{
+						Builder.Append(fmt::format("{:.17g}", Value));
+					}
+					else
+					{
+						Builder << "null"sv;
+					}
+				}
+				break;
+			case CbFieldType::BoolFalse:
+				Builder << "False"sv;
+				break;
+			case CbFieldType::BoolTrue:
+				Builder << "True"sv;
+				break;
+			case CbFieldType::ObjectAttachment:
+			case CbFieldType::BinaryAttachment:
+				{
+					Accessor.AsAttachment().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Hash:
+				{
+					Accessor.AsHash().ToHexString(Builder);
+				}
+				break;
+			case CbFieldType::Uuid:
+				{
+					Accessor.AsUuid().ToString(Builder);
+				}
+				break;
+			case CbFieldType::DateTime:
+				Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+				break;
+			case CbFieldType::TimeSpan:
+				{
+					const TimeSpan Span(Accessor.AsTimeSpanTicks());
+					if (Span.GetDays() == 0)
+					{
+						Builder << Span.ToString("%h:%m:%s.%n");
+					}
+					else
+					{
+						Builder << Span.ToString("%d.%h:%m:%s.%n");
+					}
+					break;
+				}
+			case CbFieldType::ObjectId:
+				Accessor.AsObjectId().ToString(Builder);
+				break;
+			case CbFieldType::CustomById:
+				{
+					CbCustomById Custom = Accessor.AsCustomById();
+
+					AppendBeginTag(u8"Id"sv);
+					Builder << Custom.Id;
+					AppendEndTag(u8"Id"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			case CbFieldType::CustomByName:
+				{
+					CbCustomByName Custom = Accessor.AsCustomByName();
+
+					AppendBeginTag(u8"Name"sv);
+					Builder << Custom.Name;
+					AppendEndTag(u8"Name"sv);
+
+					AppendBeginTag(u8"Data"sv);
+					AppendBase64String(Custom.Data);
+					AppendEndTag(u8"Data"sv);
+					break;
+				}
+			default:
+				ZEN_ASSERT(false);
+				break;
+		}
+
+		if (!SkipEndTag)
+			AppendEndTag(Tag);
+	}
+
+private:
+	void AppendBeginTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << '<' << Tag << '>';
+		}
+	}
+
+	void AppendEndTag(std::u8string_view Tag)
+	{
+		if (!Tag.empty())
+		{
+			Builder << "</"sv << Tag << '>';
+		}
+	}
+
+	void AppendBase64String(MemoryView Value)
+	{
+		Builder << '"';
+		ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024);
+		const uint32_t EncodedSize	= Base64::GetEncodedDataSize(uint32_t(Value.GetSize()));
+		const size_t   EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize));
+		Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex);
+	}
+
+private:
+	StringBuilderBase& Builder;
+};
+
 HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg))
 {
 	Inititalize();
@@ -51,64 +247,216 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request)
 void
 HttpObjectStoreService::Inititalize()
 {
+	namespace fs = std::filesystem;
 	ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory);
-	for (const auto& Bucket : m_Cfg.Buckets)
+
+	const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets";
+	if (!fs::exists(BucketsPath))
 	{
-		ZEN_LOG_INFO(LogObj, " - bucket '{}' -> '{}'", Bucket.Name, Bucket.Directory);
+		CreateDirectories(BucketsPath);
 	}
 
 	m_Router.RegisterRoute(
-		"distributionpoints/{bucket}",
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { CreateBucket(Request); },
+		HttpVerb::kPost | HttpVerb::kPut);
+
+	m_Router.RegisterRoute(
+		"bucket",
+		[this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); },
+		HttpVerb::kDelete);
+
+	m_Router.RegisterRoute(
+		"bucket/{path}",
 		[this](zen::HttpRouterRequest& Request) {
-			const std::string BucketName = Request.GetCapture(1);
+			const std::string Path	   = Request.GetCapture(1);
+			const auto		  Sep	   = Path.find_last_of('.');
+			const bool		  IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
 
-			ExtendableStringBuilder<1024> Json;
+			if (IsObject)
 			{
-				CbObjectWriter Writer;
-				Writer.BeginArray("distributions");
-				Writer << fmt::format("http://localhost:{}/obj/{}", m_Cfg.ServerPort, BucketName);
-				Writer.EndArray();
-				Writer.Save().ToJson(Json);
+				GetObject(Request, Path);
+			}
+			else
+			{
+				ListBucket(Request, Path);
 			}
-
-			Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Json.ToString());
 		},
 		HttpVerb::kGet);
 
 	m_Router.RegisterRoute(
-		"{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { GetBlob(Request); },
-		HttpVerb::kGet);
-
-	m_Router.RegisterRoute(
-		"{bucket}/{path}",
-		[this](zen::HttpRouterRequest& Request) { PutBlob(Request); },
+		"bucket/{bucket}/{path}",
+		[this](zen::HttpRouterRequest& Request) { PutObject(Request); },
 		HttpVerb::kPost | HttpVerb::kPut);
 }
 
 std::filesystem::path
 HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName)
 {
-	std::lock_guard _(BucketsMutex);
+	{
+		std::lock_guard _(BucketsMutex);
+
+		if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
+										 std::end(m_Cfg.Buckets),
+										 [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; });
+			It != std::end(m_Cfg.Buckets))
+		{
+			return It->Directory.make_preferred();
+		}
+	}
+
+	return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred();
+}
+
+void
+HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
 
-	if (const auto It = std::find_if(std::begin(m_Cfg.Buckets),
-									 std::end(m_Cfg.Buckets),
-									 [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; });
-		It != std::end(m_Cfg.Buckets))
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
 	{
-		return It->Directory;
+		std::lock_guard _(BucketsMutex);
+		if (!fs::exists(BucketPath))
+		{
+			CreateDirectories(BucketPath);
+			ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::Created);
+		}
 	}
 
-	return std::filesystem::path();
+	ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
 }
 
 void
-HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path)
 {
 	namespace fs = std::filesystem;
 
-	const std::string& BucketName = Request.GetCapture(1);
-	const fs::path	   BucketDir  = GetBucketDirectory(BucketName);
+	const auto		  Sep		 = Path.find_first_of('/');
+	const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+	if (BucketPrefix.empty())
+	{
+		const auto QueryParms = Request.ServerRequest().GetQueryParams();
+		if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false)
+		{
+			BucketPrefix = PrefixParam;
+		}
+	}
+
+	const fs::path BucketRoot		  = GetBucketDirectory(BucketName);
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
+	const fs::path FullPath			  = BucketRoot / RelativeBucketPath;
+
+	struct Visitor : FileSystemTraversal::TreeVisitor
+	{
+		Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path)
+		{
+			Writer.BeginObject("ListBucketResult"sv);
+			Writer << "Name"sv << BucketName;
+			std::string Tmp = Prefix.string();
+			std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
+			Writer << "Prefix"sv << Tmp;
+			Writer.BeginArray("Contents"sv);
+		}
+
+		void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override
+		{
+			const fs::path FullPath		= Parent / fs::path(File);
+			fs::path	   RelativePath = fs::relative(FullPath, BucketPath);
+
+			std::string Key = RelativePath.string();
+			std::replace(Key.begin(), Key.end(), '\\', '/');
+
+			Writer.BeginObject();
+			Writer << "Key"sv << Key;
+			Writer << "Size"sv << FileSize;
+			Writer.EndObject();
+		}
+
+		bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; }
+
+		CbObject GetResult()
+		{
+			Writer.EndArray();
+			Writer.EndObject();
+			return Writer.Save();
+		}
+
+		CbObjectWriter Writer;
+		fs::path	   BucketPath;
+	};
+
+	Visitor				FileVisitor(BucketName, BucketRoot, RelativeBucketPath);
+	FileSystemTraversal Traversal;
+
+	{
+		std::lock_guard _(BucketsMutex);
+		Traversal.TraverseFileSystem(FullPath, FileVisitor);
+	}
+	CbObject Result = FileVisitor.GetResult();
+
+	if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON)
+	{
+		ExtendableStringBuilder<1024> Sb;
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView());
+	}
+
+	ExtendableStringBuilder<1024> Xml;
+	CbXmlWriter					  XmlWriter(Xml);
+	XmlWriter.WriteField(Result.AsFieldView());
+
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView());
+}
+
+void
+HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request)
+{
+	namespace fs = std::filesystem;
+
+	const CbObject		   Params	  = Request.ServerRequest().ReadPayloadObject();
+	const std::string_view BucketName = Params["bucketname"].AsString();
+
+	if (BucketName.empty())
+	{
+		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+	}
+
+	const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName;
+	{
+		std::lock_guard _(BucketsMutex);
+		DeleteDirectories(BucketPath);
+	}
+
+	ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName);
+	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
+}
+
+void
+HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path)
+{
+	namespace fs = std::filesystem;
+
+	const auto		  Sep		 = Path.find_first_of('/');
+	const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep);
+	const std::string BucketPrefix =
+		Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1);
+
+	const fs::path BucketDir = GetBucketDirectory(BucketName);
 
 	if (BucketDir.empty())
 	{
@@ -116,7 +464,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	const fs::path RelativeBucketPath = Request.GetCapture(2);
+	const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred();
 
 	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
 	{
@@ -124,8 +472,8 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
 	}
 
-	fs::path FilePath = BucketDir / RelativeBucketPath;
-	if (fs::exists(FilePath) == false)
+	const fs::path FilePath = BucketDir / RelativeBucketPath;
+	if (!fs::exists(FilePath))
 	{
 		ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath);
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
@@ -138,7 +486,12 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
 	}
 
-	FileContents File = ReadFile(FilePath);
+	FileContents File;
+	{
+		std::lock_guard _(BucketsMutex);
+		File = ReadFile(FilePath);
+	}
+
 	if (File.ErrorCode)
 	{
 		ZEN_LOG_WARN(LogObj,
@@ -194,7 +547,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request)
 }
 
 void
-HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
+HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request)
 {
 	namespace fs = std::filesystem;
 
@@ -207,7 +560,7 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound);
 	}
 
-	const fs::path RelativeBucketPath = Request.GetCapture(2);
+	const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred();
 
 	if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with(".."))
 	{
@@ -215,17 +568,32 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request)
 		return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden);
 	}
 
-	fs::path	   FilePath = BucketDir / RelativeBucketPath;
-	const IoBuffer FileBuf	= Request.ServerRequest().ReadPayload();
+	const fs::path FilePath		 = BucketDir / RelativeBucketPath;
+	const fs::path FileDirectory = FilePath.parent_path();
 
-	if (FileBuf.Size() == 0)
 	{
-		ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [FAILED], empty file", BucketName, FilePath);
-		return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		std::lock_guard _(BucketsMutex);
+
+		if (!fs::exists(FileDirectory))
+		{
+			CreateDirectories(FileDirectory);
+		}
+
+		const IoBuffer FileBuf = Request.ServerRequest().ReadPayload();
+
+		if (FileBuf.Size() == 0)
+		{
+			ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath);
+			return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest);
+		}
+
+		WriteFile(FilePath, FileBuf);
+		ZEN_LOG_DEBUG(LogObj,
+					  "PUT - '{}' [OK] ({})",
+					  (fs::path(BucketName) / RelativeBucketPath).make_preferred(),
+					  NiceBytes(FileBuf.Size()));
 	}
 
-	WriteFile(FilePath, FileBuf);
-	ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [OK] ({})", BucketName, RelativeBucketPath, NiceBytes(FileBuf.Size()));
 	Request.ServerRequest().WriteResponse(HttpResponseCode::OK);
 }