diff options
Diffstat (limited to 'src/zenserver/objectstore/objectstore.cpp')
| -rw-r--r-- | src/zenserver/objectstore/objectstore.cpp | 452 |
1 files changed, 410 insertions, 42 deletions
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index 3643e8011..76f2aba95 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -2,14 +2,18 @@ #include <objectstore/objectstore.h> +#include <zencore/base64.h> +#include <zencore/compactbinaryvalue.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> +#include "zencore/compactbinary.h" #include "zencore/compactbinarybuilder.h" #include "zenhttp/httpcommon.h" #include "zenhttp/httpserver.h" +#include <filesystem> #include <thread> ZEN_THIRD_PARTY_INCLUDES_START @@ -23,6 +27,198 @@ using namespace std::literals; ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); +class CbXmlWriter +{ +public: + explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) + { + Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); + Builder << LINE_TERMINATOR_ANSI; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + bool SkipEndTag = false; + const std::u8string_view Tag = Field.GetU8Name(); + + AppendBeginTag(Tag); + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "Null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + for (CbFieldView It : Field) + { + WriteField(It); + } + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + bool FirstField = true; + for (CbFieldView It : Field) + { + if (!FirstField) + AppendBeginTag(Tag); + + WriteField(It); + AppendEndTag(Tag); + FirstField = false; + } + SkipEndTag = true; + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + Builder << Accessor.AsU8String(); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "Null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "False"sv; + break; + case CbFieldType::BoolTrue: + Builder << "True"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Accessor.AsAttachment().ToHexString(Builder); + } + break; + case CbFieldType::Hash: + { + Accessor.AsHash().ToHexString(Builder); + } + break; + case CbFieldType::Uuid: + { + Accessor.AsUuid().ToString(Builder); + } + break; + case CbFieldType::DateTime: + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + break; + } + case CbFieldType::ObjectId: + Accessor.AsObjectId().ToString(Builder); + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + + AppendBeginTag(u8"Id"sv); + Builder << Custom.Id; + AppendEndTag(u8"Id"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + + AppendBeginTag(u8"Name"sv); + Builder << Custom.Name; + AppendEndTag(u8"Name"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + default: + ZEN_ASSERT(false); + break; + } + + if (!SkipEndTag) + AppendEndTag(Tag); + } + +private: + void AppendBeginTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << '<' << Tag << '>'; + } + } + + void AppendEndTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << "</"sv << Tag << '>'; + } + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; +}; + HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg)) { Inititalize(); @@ -51,64 +247,216 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) void HttpObjectStoreService::Inititalize() { + namespace fs = std::filesystem; ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); - for (const auto& Bucket : m_Cfg.Buckets) + + const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; + if (!fs::exists(BucketsPath)) { - ZEN_LOG_INFO(LogObj, " - bucket '{}' -> '{}'", Bucket.Name, Bucket.Directory); + CreateDirectories(BucketsPath); } m_Router.RegisterRoute( - "distributionpoints/{bucket}", + "bucket", + [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "bucket/{path}", [this](zen::HttpRouterRequest& Request) { - const std::string BucketName = Request.GetCapture(1); + const std::string Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; - ExtendableStringBuilder<1024> Json; + if (IsObject) { - CbObjectWriter Writer; - Writer.BeginArray("distributions"); - Writer << fmt::format("http://localhost:{}/obj/{}", m_Cfg.ServerPort, BucketName); - Writer.EndArray(); - Writer.Save().ToJson(Json); + GetObject(Request, Path); + } + else + { + ListBucket(Request, Path); } - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Json.ToString()); }, HttpVerb::kGet); m_Router.RegisterRoute( - "{bucket}/{path}", - [this](zen::HttpRouterRequest& Request) { GetBlob(Request); }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "{bucket}/{path}", - [this](zen::HttpRouterRequest& Request) { PutBlob(Request); }, + "bucket/{bucket}/{path}", + [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, HttpVerb::kPost | HttpVerb::kPut); } std::filesystem::path HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) { - std::lock_guard _(BucketsMutex); + { + std::lock_guard _(BucketsMutex); + + if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), + std::end(m_Cfg.Buckets), + [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); + It != std::end(m_Cfg.Buckets)) + { + return It->Directory.make_preferred(); + } + } + + return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); +} + +void +HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } - if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), - std::end(m_Cfg.Buckets), - [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); - It != std::end(m_Cfg.Buckets)) + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { - return It->Directory; + std::lock_guard _(BucketsMutex); + if (!fs::exists(BucketPath)) + { + CreateDirectories(BucketPath); + ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); + } } - return std::filesystem::path(); + ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); } void -HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request) +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path) { namespace fs = std::filesystem; - const std::string& BucketName = Request.GetCapture(1); - const fs::path BucketDir = GetBucketDirectory(BucketName); + const auto Sep = Path.find_first_of('/'); + const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + if (BucketPrefix.empty()) + { + const auto QueryParms = Request.ServerRequest().GetQueryParams(); + if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) + { + BucketPrefix = PrefixParam; + } + } + + const fs::path BucketRoot = GetBucketDirectory(BucketName); + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + const fs::path FullPath = BucketRoot / RelativeBucketPath; + + struct Visitor : FileSystemTraversal::TreeVisitor + { + Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) + { + Writer.BeginObject("ListBucketResult"sv); + Writer << "Name"sv << BucketName; + std::string Tmp = Prefix.string(); + std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); + Writer << "Prefix"sv << Tmp; + Writer.BeginArray("Contents"sv); + } + + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override + { + const fs::path FullPath = Parent / fs::path(File); + fs::path RelativePath = fs::relative(FullPath, BucketPath); + + std::string Key = RelativePath.string(); + std::replace(Key.begin(), Key.end(), '\\', '/'); + + Writer.BeginObject(); + Writer << "Key"sv << Key; + Writer << "Size"sv << FileSize; + Writer.EndObject(); + } + + bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; } + + CbObject GetResult() + { + Writer.EndArray(); + Writer.EndObject(); + return Writer.Save(); + } + + CbObjectWriter Writer; + fs::path BucketPath; + }; + + Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); + FileSystemTraversal Traversal; + + { + std::lock_guard _(BucketsMutex); + Traversal.TraverseFileSystem(FullPath, FileVisitor); + } + CbObject Result = FileVisitor.GetResult(); + + if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) + { + ExtendableStringBuilder<1024> Sb; + return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); + } + + ExtendableStringBuilder<1024> Xml; + CbXmlWriter XmlWriter(Xml); + XmlWriter.WriteField(Result.AsFieldView()); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); +} + +void +HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + DeleteDirectories(BucketPath); + } + + ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); + const std::string BucketPrefix = + Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + + const fs::path BucketDir = GetBucketDirectory(BucketName); if (BucketDir.empty()) { @@ -116,7 +464,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } - const fs::path RelativeBucketPath = Request.GetCapture(2); + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) { @@ -124,8 +472,8 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); } - fs::path FilePath = BucketDir / RelativeBucketPath; - if (fs::exists(FilePath) == false) + const fs::path FilePath = BucketDir / RelativeBucketPath; + if (!fs::exists(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -138,7 +486,12 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } - FileContents File = ReadFile(FilePath); + FileContents File; + { + std::lock_guard _(BucketsMutex); + File = ReadFile(FilePath); + } + if (File.ErrorCode) { ZEN_LOG_WARN(LogObj, @@ -194,7 +547,7 @@ HttpObjectStoreService::GetBlob(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request) +HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { namespace fs = std::filesystem; @@ -207,7 +560,7 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } - const fs::path RelativeBucketPath = Request.GetCapture(2); + const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) { @@ -215,17 +568,32 @@ HttpObjectStoreService::PutBlob(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); } - fs::path FilePath = BucketDir / RelativeBucketPath; - const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); + const fs::path FilePath = BucketDir / RelativeBucketPath; + const fs::path FileDirectory = FilePath.parent_path(); - if (FileBuf.Size() == 0) { - ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [FAILED], empty file", BucketName, FilePath); - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + std::lock_guard _(BucketsMutex); + + if (!fs::exists(FileDirectory)) + { + CreateDirectories(FileDirectory); + } + + const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); + + if (FileBuf.Size() == 0) + { + ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + WriteFile(FilePath, FileBuf); + ZEN_LOG_DEBUG(LogObj, + "PUT - '{}' [OK] ({})", + (fs::path(BucketName) / RelativeBucketPath).make_preferred(), + NiceBytes(FileBuf.Size())); } - WriteFile(FilePath, FileBuf); - ZEN_LOG_DEBUG(LogObj, "PUT - '{}/{}' [OK] ({})", BucketName, RelativeBucketPath, NiceBytes(FileBuf.Size())); Request.ServerRequest().WriteResponse(HttpResponseCode::OK); } |