diff options
Diffstat (limited to 'src/zenserver/storage/objectstore/objectstore.cpp')
| -rw-r--r-- | src/zenserver/storage/objectstore/objectstore.cpp | 618 |
1 files changed, 618 insertions, 0 deletions
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp new file mode 100644 index 000000000..3f40bf616 --- /dev/null +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -0,0 +1,618 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "objectstore.h" + +#include <zencore/base64.h> +#include <zencore/basicfile.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zencore/trace.h> +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zenhttp/httpcommon.h" +#include "zenhttp/httpserver.h" + +#include <filesystem> +#include <thread> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); + +class CbXmlWriter +{ +public: + explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) + { + Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); + Builder << LINE_TERMINATOR_ANSI; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + bool SkipEndTag = false; + const std::u8string_view Tag = Field.GetU8Name(); + + AppendBeginTag(Tag); + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "Null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + for (CbFieldView It : Field) + { + WriteField(It); + } + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + bool FirstField = true; + for (CbFieldView It : Field) + { + if (!FirstField) + AppendBeginTag(Tag); + + WriteField(It); + AppendEndTag(Tag); + FirstField = false; + } + SkipEndTag = true; + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + Builder << Accessor.AsU8String(); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "Null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "False"sv; + break; + case CbFieldType::BoolTrue: + Builder << "True"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Accessor.AsAttachment().ToHexString(Builder); + } + break; + case CbFieldType::Hash: + { + Accessor.AsHash().ToHexString(Builder); + } + break; + case CbFieldType::Uuid: + { + Accessor.AsUuid().ToString(Builder); + } + break; + case CbFieldType::DateTime: + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + break; + } + case CbFieldType::ObjectId: + Accessor.AsObjectId().ToString(Builder); + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + + AppendBeginTag(u8"Id"sv); + Builder << Custom.Id; + AppendEndTag(u8"Id"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + + AppendBeginTag(u8"Name"sv); + Builder << Custom.Name; + AppendEndTag(u8"Name"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + default: + ZEN_ASSERT(false); + break; + } + + if (!SkipEndTag) + AppendEndTag(Tag); + } + +private: + void AppendBeginTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << '<' << Tag << '>'; + } + } + + void AppendEndTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << "</"sv << Tag << '>'; + } + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; +}; + +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) +{ + Inititalize(); + m_StatusService.RegisterHandler("obj", *this); +} + +HttpObjectStoreService::~HttpObjectStoreService() +{ + m_StatusService.UnregisterHandler("obj", *this); +} + +const char* +HttpObjectStoreService::BaseUri() const +{ + return "/obj/"; +} + +void +HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpObjectStoreService::Inititalize() +{ + ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); + + namespace fs = std::filesystem; + ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); + + const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; + if (!IsDir(BucketsPath)) + { + CreateDirectories(BucketsPath); + } + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "bucket/{path}", + [this](zen::HttpRouterRequest& Request) { + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + + if (IsObject) + { + GetObject(Request, Path); + } + else + { + ListBucket(Request, Path); + } + }, + HttpVerb::kHead | HttpVerb::kGet); + + m_Router.RegisterRoute( + "bucket/{bucket}/{path}", + [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, + HttpVerb::kPost | HttpVerb::kPut); +} + +std::filesystem::path +HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) +{ + { + std::lock_guard _(BucketsMutex); + + if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), + std::end(m_Cfg.Buckets), + [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); + It != std::end(m_Cfg.Buckets)) + { + return It->Directory.make_preferred(); + } + } + + return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); +} + +void +HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + if (!IsDir(BucketPath)) + { + CreateDirectories(BucketPath); + ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); + } + } + + ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + if (BucketPrefix.empty()) + { + const auto QueryParms = Request.ServerRequest().GetQueryParams(); + if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) + { + BucketPrefix = PrefixParam; + } + } + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\')); + + const fs::path BucketRoot = GetBucketDirectory(BucketName); + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + const fs::path FullPath = BucketRoot / RelativeBucketPath; + + struct Visitor : FileSystemTraversal::TreeVisitor + { + Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) + { + Writer.BeginObject("ListBucketResult"sv); + Writer << "Name"sv << BucketName; + std::string Tmp = Prefix.string(); + std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); + Writer << "Prefix"sv << Tmp; + Writer.BeginArray("Contents"sv); + } + + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override + { + const fs::path FullPath = Parent / fs::path(File); + fs::path RelativePath = fs::relative(FullPath, BucketPath); + + std::string Key = RelativePath.string(); + std::replace(Key.begin(), Key.end(), '\\', '/'); + + Writer.BeginObject(); + Writer << "Key"sv << Key; + Writer << "Size"sv << FileSize; + Writer.EndObject(); + } + + bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } + + CbObject GetResult() + { + Writer.EndArray(); + Writer.EndObject(); + return Writer.Save(); + } + + CbObjectWriter Writer; + fs::path BucketPath; + }; + + Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); + FileSystemTraversal Traversal; + + if (IsDir(FullPath)) + { + std::lock_guard _(BucketsMutex); + Traversal.TraverseFileSystem(FullPath, FileVisitor); + } + CbObject Result = FileVisitor.GetResult(); + + if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) + { + ExtendableStringBuilder<1024> Sb; + return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); + } + + ExtendableStringBuilder<1024> Xml; + CbXmlWriter XmlWriter(Xml); + XmlWriter.WriteField(Result.AsFieldView()); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); +} + +void +HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + DeleteDirectories(BucketPath); + } + + ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + if (!IsFile(FilePath)) + { + ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + zen::HttpRanges Ranges; + if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) + { + // Only a single range is supported + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + FileContents File; + { + std::lock_guard _(BucketsMutex); + File = ReadFile(FilePath); + } + + if (File.ErrorCode) + { + ZEN_LOG_WARN(LogObj, + "GET - '{}/{}' [FAILED] ('{}': {})", + BucketName, + FilePath, + File.ErrorCode.category().name(), + File.ErrorCode.value()); + + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const IoBuffer& FileBuf = File.Data[0]; + + if (Ranges.empty()) + { + const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' ({}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); + } + else + { + const auto Range = Ranges[0]; + const uint64_t RangeSize = 1 + (Range.End - Range.Start); + const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + Range.Start, + Range.End, + NiceBytes(RangeSize), + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); + if (RangeView.GetSize() != RangeSize) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); + Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); + } +} + +void +HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const std::string_view BucketName = Request.GetCapture(1); + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + const fs::path FileDirectory = FilePath.parent_path(); + + { + std::lock_guard _(BucketsMutex); + + if (!IsDir(FileDirectory)) + { + CreateDirectories(FileDirectory); + } + + const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); + + if (FileBuf.Size() == 0) + { + ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); + + ZEN_LOG_DEBUG(LogObj, + "PUT - '{}' [OK] ({})", + (fs::path(BucketName) / RelativeBucketPath).make_preferred(), + NiceBytes(FileBuf.Size())); + } + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +} // namespace zen |