diff options
Diffstat (limited to 'src/zenserver/objectstore/objectstore.cpp')
| -rw-r--r-- | src/zenserver/objectstore/objectstore.cpp | 618 |
1 files changed, 0 insertions, 618 deletions
diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp deleted file mode 100644 index b1e73c7df..000000000 --- a/src/zenserver/objectstore/objectstore.cpp +++ /dev/null @@ -1,618 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include <objectstore/objectstore.h> - -#include <zencore/base64.h> -#include <zencore/basicfile.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/string.h> -#include <zencore/trace.h> -#include "zencore/compactbinary.h" -#include "zencore/compactbinarybuilder.h" -#include "zenhttp/httpcommon.h" -#include "zenhttp/httpserver.h" - -#include <filesystem> -#include <thread> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <fmt/format.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -using namespace std::literals; - -ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); - -class CbXmlWriter -{ -public: - explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) - { - Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); - Builder << LINE_TERMINATOR_ANSI; - } - - void WriteField(CbFieldView Field) - { - using namespace std::literals; - - bool SkipEndTag = false; - const std::u8string_view Tag = Field.GetU8Name(); - - AppendBeginTag(Tag); - - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) - { - case CbFieldType::Null: - Builder << "Null"sv; - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - { - for (CbFieldView It : Field) - { - WriteField(It); - } - } - break; - case CbFieldType::Array: - case CbFieldType::UniformArray: - { - bool FirstField = true; - for (CbFieldView It : Field) - { - if (!FirstField) - AppendBeginTag(Tag); - - WriteField(It); - AppendEndTag(Tag); - FirstField = false; - } - SkipEndTag = true; - } - break; - case CbFieldType::Binary: - AppendBase64String(Accessor.AsBinary()); - break; - case CbFieldType::String: - Builder << Accessor.AsU8String(); - break; - case CbFieldType::IntegerPositive: - Builder << Accessor.AsIntegerPositive(); - break; - case CbFieldType::IntegerNegative: - Builder << Accessor.AsIntegerNegative(); - break; - case CbFieldType::Float32: - { - const float Value = Accessor.AsFloat32(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.9g}", Value)); - } - else - { - Builder << "Null"sv; - } - } - break; - case CbFieldType::Float64: - { - const double Value = Accessor.AsFloat64(); - if (std::isfinite(Value)) - { - Builder.Append(fmt::format("{:.17g}", Value)); - } - else - { - Builder << "null"sv; - } - } - break; - case CbFieldType::BoolFalse: - Builder << "False"sv; - break; - case CbFieldType::BoolTrue: - Builder << "True"sv; - break; - case CbFieldType::ObjectAttachment: - case CbFieldType::BinaryAttachment: - { - Accessor.AsAttachment().ToHexString(Builder); - } - break; - case CbFieldType::Hash: - { - Accessor.AsHash().ToHexString(Builder); - } - break; - case CbFieldType::Uuid: - { - Accessor.AsUuid().ToString(Builder); - } - break; - case CbFieldType::DateTime: - Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); - break; - case CbFieldType::TimeSpan: - { - const TimeSpan Span(Accessor.AsTimeSpanTicks()); - if (Span.GetDays() == 0) - { - Builder << Span.ToString("%h:%m:%s.%n"); - } - else - { - Builder << Span.ToString("%d.%h:%m:%s.%n"); - } - break; - } - case CbFieldType::ObjectId: - Accessor.AsObjectId().ToString(Builder); - break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); - - AppendBeginTag(u8"Id"sv); - Builder << Custom.Id; - AppendEndTag(u8"Id"sv); - - AppendBeginTag(u8"Data"sv); - AppendBase64String(Custom.Data); - AppendEndTag(u8"Data"sv); - break; - } - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); - - AppendBeginTag(u8"Name"sv); - Builder << Custom.Name; - AppendEndTag(u8"Name"sv); - - AppendBeginTag(u8"Data"sv); - AppendBase64String(Custom.Data); - AppendEndTag(u8"Data"sv); - break; - } - default: - ZEN_ASSERT(false); - break; - } - - if (!SkipEndTag) - AppendEndTag(Tag); - } - -private: - void AppendBeginTag(std::u8string_view Tag) - { - if (!Tag.empty()) - { - Builder << '<' << Tag << '>'; - } - } - - void AppendEndTag(std::u8string_view Tag) - { - if (!Tag.empty()) - { - Builder << "</"sv << Tag << '>'; - } - } - - void AppendBase64String(MemoryView Value) - { - Builder << '"'; - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - } - -private: - StringBuilderBase& Builder; -}; - -HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) -: m_StatusService(StatusService) -, m_Cfg(std::move(Cfg)) -{ - Inititalize(); - m_StatusService.RegisterHandler("obj", *this); -} - -HttpObjectStoreService::~HttpObjectStoreService() -{ - m_StatusService.UnregisterHandler("obj", *this); -} - -const char* -HttpObjectStoreService::BaseUri() const -{ - return "/obj/"; -} - -void -HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) -{ - if (m_Router.HandleRequest(Request) == false) - { - ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); - return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); - } -} - -void -HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) -{ - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -void -HttpObjectStoreService::Inititalize() -{ - ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); - - namespace fs = std::filesystem; - ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); - - const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!IsDir(BucketsPath)) - { - CreateDirectories(BucketsPath); - } - - m_Router.RegisterRoute( - "bucket", - [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, - HttpVerb::kPost | HttpVerb::kPut); - - m_Router.RegisterRoute( - "bucket", - [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, - HttpVerb::kDelete); - - m_Router.RegisterRoute( - "bucket/{path}", - [this](zen::HttpRouterRequest& Request) { - const std::string_view Path = Request.GetCapture(1); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; - - if (IsObject) - { - GetObject(Request, Path); - } - else - { - ListBucket(Request, Path); - } - }, - HttpVerb::kHead | HttpVerb::kGet); - - m_Router.RegisterRoute( - "bucket/{bucket}/{path}", - [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, - HttpVerb::kPost | HttpVerb::kPut); -} - -std::filesystem::path -HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) -{ - { - std::lock_guard _(BucketsMutex); - - if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), - std::end(m_Cfg.Buckets), - [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); - It != std::end(m_Cfg.Buckets)) - { - return It->Directory.make_preferred(); - } - } - - return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); -} - -void -HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const CbObject Params = Request.ServerRequest().ReadPayloadObject(); - const std::string_view BucketName = Params["bucketname"].AsString(); - - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; - { - std::lock_guard _(BucketsMutex); - if (!IsDir(BucketPath)) - { - CreateDirectories(BucketPath); - ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); - } - } - - ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -void -HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) -{ - namespace fs = std::filesystem; - - const auto Sep = Path.find_first_of('/'); - const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; - if (BucketPrefix.empty()) - { - const auto QueryParms = Request.ServerRequest().GetQueryParams(); - if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) - { - BucketPrefix = PrefixParam; - } - } - BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); - BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\')); - - const fs::path BucketRoot = GetBucketDirectory(BucketName); - const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); - const fs::path FullPath = BucketRoot / RelativeBucketPath; - - struct Visitor : FileSystemTraversal::TreeVisitor - { - Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) - { - Writer.BeginObject("ListBucketResult"sv); - Writer << "Name"sv << BucketName; - std::string Tmp = Prefix.string(); - std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); - Writer << "Prefix"sv << Tmp; - Writer.BeginArray("Contents"sv); - } - - void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override - { - const fs::path FullPath = Parent / fs::path(File); - fs::path RelativePath = fs::relative(FullPath, BucketPath); - - std::string Key = RelativePath.string(); - std::replace(Key.begin(), Key.end(), '\\', '/'); - - Writer.BeginObject(); - Writer << "Key"sv << Key; - Writer << "Size"sv << FileSize; - Writer.EndObject(); - } - - bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } - - CbObject GetResult() - { - Writer.EndArray(); - Writer.EndObject(); - return Writer.Save(); - } - - CbObjectWriter Writer; - fs::path BucketPath; - }; - - Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); - FileSystemTraversal Traversal; - - if (IsDir(FullPath)) - { - std::lock_guard _(BucketsMutex); - Traversal.TraverseFileSystem(FullPath, FileVisitor); - } - CbObject Result = FileVisitor.GetResult(); - - if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) - { - ExtendableStringBuilder<1024> Sb; - return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); - } - - ExtendableStringBuilder<1024> Xml; - CbXmlWriter XmlWriter(Xml); - XmlWriter.WriteField(Result.AsFieldView()); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); -} - -void -HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const CbObject Params = Request.ServerRequest().ReadPayloadObject(); - const std::string_view BucketName = Params["bucketname"].AsString(); - - if (BucketName.empty()) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; - { - std::lock_guard _(BucketsMutex); - DeleteDirectories(BucketPath); - } - - ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -void -HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) -{ - namespace fs = std::filesystem; - - const auto Sep = Path.find_first_of('/'); - const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; - const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; - - const fs::path BucketDir = GetBucketDirectory(BucketName); - - if (BucketDir.empty()) - { - ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); - - if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) - { - ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); - } - - const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!IsFile(FilePath)) - { - ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - zen::HttpRanges Ranges; - if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) - { - // Only a single range is supported - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - FileContents File; - { - std::lock_guard _(BucketsMutex); - File = ReadFile(FilePath); - } - - if (File.ErrorCode) - { - ZEN_LOG_WARN(LogObj, - "GET - '{}/{}' [FAILED] ('{}': {})", - BucketName, - FilePath, - File.ErrorCode.category().name(), - File.ErrorCode.value()); - - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - const IoBuffer& FileBuf = File.Data[0]; - - if (Ranges.empty()) - { - const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); - - ZEN_LOG_DEBUG(LogObj, - "GET - '{}/{}' ({}) [OK] (Served: {})", - BucketName, - RelativeBucketPath, - NiceBytes(FileBuf.Size()), - NiceBytes(TotalServed)); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); - } - else - { - const auto Range = Ranges[0]; - const uint64_t RangeSize = 1 + (Range.End - Range.Start); - const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; - - ZEN_LOG_DEBUG(LogObj, - "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", - BucketName, - RelativeBucketPath, - Range.Start, - Range.End, - NiceBytes(RangeSize), - NiceBytes(FileBuf.Size()), - NiceBytes(TotalServed)); - - MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); - if (RangeView.GetSize() != RangeSize) - { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); - Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); - } -} - -void -HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) -{ - namespace fs = std::filesystem; - - const std::string_view BucketName = Request.GetCapture(1); - const fs::path BucketDir = GetBucketDirectory(BucketName); - - if (BucketDir.empty()) - { - ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); - } - - const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); - - if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) - { - ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName); - return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); - } - - const fs::path FilePath = BucketDir / RelativeBucketPath; - const fs::path FileDirectory = FilePath.parent_path(); - - { - std::lock_guard _(BucketsMutex); - - if (!IsDir(FileDirectory)) - { - CreateDirectories(FileDirectory); - } - - const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); - - if (FileBuf.Size() == 0) - { - ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } - - TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); - - ZEN_LOG_DEBUG(LogObj, - "PUT - '{}' [OK] ({})", - (fs::path(BucketName) / RelativeBucketPath).make_preferred(), - NiceBytes(FileBuf.Size())); - } - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK); -} - -} // namespace zen |