diff options
Diffstat (limited to 'src/zenserver')
75 files changed, 9314 insertions, 12256 deletions
diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 8b3f5a785..97522e892 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -17,12 +17,11 @@ # include <mimalloc.h> #endif -#include <zenstore/cidstore.h> #include <zenstore/gc.h> #include <zenstore/cache/structuredcachestore.h> +#include <zenutil/workerpools.h> #include "config.h" -#include "projectstore/projectstore.h" #include <chrono> @@ -38,34 +37,46 @@ struct DirStats DirStats GetStatsForDirectory(std::filesystem::path Dir) { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) return {}; - FileSystemTraversal Traversal; - - struct StatsTraversal : public FileSystemTraversal::TreeVisitor + struct StatsTraversal : public GetDirectoryContentVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - ZEN_UNUSED(Parent, File); - ++TotalFileCount; - TotalBytes += FileSize; + ZEN_UNUSED(RelativeRoot); + + uint64_t FileCount = Content.FileNames.size(); + uint64_t DirCount = Content.DirectoryNames.size(); + uint64_t FilesSize = 0; + for (uint64_t FileSize : Content.FileSizes) + { + FilesSize += FileSize; + } + TotalBytes += FilesSize; + TotalFileCount += FileCount; + TotalDirCount += DirCount; } - virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override + + std::atomic_uint64_t TotalBytes = 0; + std::atomic_uint64_t TotalFileCount = 0; + std::atomic_uint64_t TotalDirCount = 0; + + DirStats GetStats() { - ++TotalDirCount; - return true; + return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()}; } + } DirTraverser; - uint64_t TotalBytes = 0; - uint64_t TotalFileCount = 0; - uint64_t TotalDirCount = 0; + Latch PendingWorkCount(1); - DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; } - }; - - StatsTraversal DirTraverser; - Traversal.TraverseFileSystem(Dir, DirTraverser); + GetDirectoryContent(Dir, + DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes, + DirTraverser, + GetSmallWorkerPool(EWorkloadType::Background), + PendingWorkCount); + PendingWorkCount.CountDown(); + PendingWorkCount.Wait(); return DirTraverser.GetStats(); } @@ -90,15 +101,13 @@ GetStatsForStateDirectory(std::filesystem::path StateDir) HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, - CidStore* CidStore, - ProjectStore* ProjectStore, + std::function<void()>&& FlushFunction, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) , m_BackgroundJobQueue(BackgroundJobQueue) , m_CacheStore(CacheStore) -, m_CidStore(CidStore) -, m_ProjectStore(ProjectStore) +, m_FlushFunction(std::move(FlushFunction)) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { @@ -159,8 +168,20 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { if (!State.CurrentOp.empty()) { - Obj.AddString("CurrentOp"sv, State.CurrentOp); - Obj.AddInteger("CurrentOpPercentComplete"sv, State.CurrentOpPercentComplete); + Obj.AddString( + "CurrentOp"sv, + State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails)); + Obj.AddString("Op"sv, State.CurrentOp); + if (!State.CurrentOpDetails.empty()) + { + Obj.AddString("Details"sv, State.CurrentOpDetails); + } + Obj.AddInteger("TotalCount"sv, gsl::narrow<uint64_t>(State.TotalCount)); + Obj.AddInteger("RemainingCount"sv, gsl::narrow<uint64_t>(State.RemainingCount)); + Obj.AddInteger("CurrentOpPercentComplete"sv, + State.TotalCount > 0 + ? gsl::narrow<uint32_t>((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount) + : 0); } if (!State.Messages.empty()) { @@ -171,6 +192,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } Obj.EndArray(); } + if (!State.AbortReason.empty()) + { + Obj.AddString("AbortReason"sv, State.AbortReason); + } }; auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { @@ -210,10 +235,12 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Aborted"sv); + WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Obj.AddInteger("ReturnCode", CurrentState->ReturnCode); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; @@ -276,15 +303,18 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "Interval" << ToTimeSpan(State.Config.Interval); Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval); - Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1) ? "1" : "2"); + Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1_Deprecated) ? "1" : "2"); Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent; Response << "Verbose" << State.Config.Verbose; + Response << "SingleThreaded" << State.Config.SingleThreaded; + Response << "AttachmentPassCount" << State.Config.AttachmentPassCount; } Response.EndObject(); Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; @@ -312,6 +342,11 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); } + if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max) + { + Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin; + Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax; + } } Response.EndObject(); Response.BeginObject("LightweightGC"); @@ -364,6 +399,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } } + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt<uint64_t>(Param)) @@ -384,7 +427,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, if (auto Param = Params.GetValue("forceusegcv1"); Param.empty() == false) { - GcParams.ForceGCVersion = GcVersion::kV1; + GcParams.ForceGCVersion = GcVersion::kV1_Deprecated; } if (auto Param = Params.GetValue("forceusegcv2"); Param.empty() == false) @@ -405,6 +448,36 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, GcParams.Verbose = Param == "true"sv; } + if (auto Param = Params.GetValue("singlethreaded"); Param.empty() == false) + { + GcParams.SingleThreaded = Param == "true"sv; + } + + if (auto Param = Params.GetValue("referencehashlow"); Param.empty() == false) + { + GcParams.AttachmentRangeMin = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("referencehashhigh"); Param.empty() == false) + { + GcParams.AttachmentRangeMax = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("storecacheattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreCacheAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("storeprojectattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreProjectAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("enablevalidation"); Param.empty() == false) + { + GcParams.EnableValidation = Param == "true"sv; + } + const bool Started = m_GcScheduler.TriggerGc(GcParams); CbObjectWriter Response; @@ -521,31 +594,35 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - TraceType Type = TraceType::None; - std::string HostOrPath; - if (auto Param = Params.GetValue("file"); Param.empty() == false) + TraceOptions TraceOptions; + + if (!IsTracing()) { - Type = TraceType::File; - HostOrPath = Param; + TraceInit("zenserver"); } - if (auto Param = Params.GetValue("host"); Param.empty() == false) + + if (auto Channels = Params.GetValue("channels"); Channels.empty() == false) { - Type = TraceType::Network; - HostOrPath = Param; + TraceOptions.Channels = Channels; } - if (Type == TraceType::None) + + if (auto File = Params.GetValue("file"); File.empty() == false) { - return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Invalid trace type, use `file` or `host`"sv); + TraceOptions.File = File; + } + else if (auto Host = Params.GetValue("host"); Host.empty() == false) + { + TraceOptions.Host = Host; } - if (IsTracing()) + else { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, - "Tracing is already enabled"sv); + "Invalid trace type, use `file` or `host`"sv); } - TraceStart("zenserver", HostOrPath.c_str(), Type); + + TraceConfigure(TraceOptions); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); }, HttpVerb::kPost); @@ -598,7 +675,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, EmitStats("cas", Stats.CasStats); EmitStats("project", Stats.ProjectStats); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what()); } @@ -617,7 +694,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Obj.EndArray(); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what()); } @@ -703,18 +780,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, "flush", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); - if (m_CidStore) - { - m_CidStore->Flush(); - } - if (m_CacheStore) - { - m_CacheStore->Flush(); - } - if (m_ProjectStore) - { - m_ProjectStore->Flush(); - } + m_FlushFunction(); HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index 563c4f536..9a49f5120 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -4,14 +4,13 @@ #include <zencore/compactbinary.h> #include <zenhttp/httpserver.h> +#include <functional> namespace zen { class GcScheduler; class JobQueue; class ZenCacheStore; -class CidStore; -class ProjectStore; struct ZenServerOptions; class HttpAdminService : public zen::HttpService @@ -26,8 +25,7 @@ public: HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, - CidStore* CidStore, - ProjectStore* ProjectStore, + std::function<void()>&& FlushFunction, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions); ~HttpAdminService(); @@ -40,8 +38,7 @@ private: GcScheduler& m_GcScheduler; JobQueue& m_BackgroundJobQueue; ZenCacheStore* m_CacheStore; - CidStore* m_CidStore; - ProjectStore* m_ProjectStore; + std::function<void()> m_FlushFunction; LogPaths m_LogPaths; const ZenServerOptions& m_ServerOptions; }; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..bce993f17 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -0,0 +1,561 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenhttp/packageformat.h> +#include <zenstore/buildstore/buildstore.h> +#include <zenutil/workerpools.h> + +#include <numeric> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize > BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector<IoHash> BlobHashes; + std::vector<IoBuffer> MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst)); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector<IoBuffer> BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector<CbAttachment> Attachments; + tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector<BuildStore::BlobExistsResult> BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasMetadata); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); + + Cbo << "count" << StorageStats.EntryCount; + Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount; + Cbo.BeginObject("blobs"); + { + Cbo << "count" << StorageStats.BlobCount; + Cbo << "bytes" << StorageStats.BlobBytes; + } + Cbo.EndObject(); // blobs + + Cbo.BeginObject("metadata"); + { + Cbo << "count" << StorageStats.MetadataCount; + Cbo << "bytes" << StorageStats.MetadataByteCount; + } + Cbo.EndObject(); // metadata + } + Cbo.EndObject(); // size + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h new file mode 100644 index 000000000..50cb5db12 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> + +#include <filesystem> + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index c62b5325e..dd5bf05cb 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -5,11 +5,14 @@ #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/compress.h> #include <zencore/enumflags.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/memory/llm.h> +#include <zencore/parallelwork.h> #include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/timer.h> @@ -17,14 +20,14 @@ #include <zencore/workthreadpool.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> +#include <zenhttp/packageformat.h> +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenstore/cache/cache.h> #include <zenstore/cache/structuredcachestore.h> #include <zenstore/gc.h> -#include <zenutil/cache/cache.h> -#include <zenutil/cache/cacherequests.h> -#include <zenutil/cache/rpcrecording.h> -#include <zenutil/packageformat.h> +#include <zenutil/rpcrecording.h> +#include <zenutil/workerpools.h> -#include "upstream/jupiter.h" #include "upstream/upstreamcache.h" #include "upstream/zen.h" #include "zenstore/cidstore.h" @@ -36,16 +39,20 @@ #include <queue> #include <thread> -#include <cpr/cpr.h> #include <gsl/gsl-lite.hpp> -#if ZEN_WITH_TESTS -# include <zencore/testing.h> -# include <zencore/testutils.h> -#endif - namespace zen { +const FLLMTag& +GetCacheHttpTag() +{ + static FLLMTag CacheHttpTag("http", FLLMTag("cache")); + + return CacheHttpTag; +} + +extern const FLLMTag& GetCacheRpcTag(); + using namespace std::literals; ////////////////////////////////////////////////////////////////////////// @@ -63,184 +70,6 @@ namespace { static constinit std::string_view HttpZCacheUtilStopRecording = "exec$/stop-recording"sv; static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv; static constinit std::string_view HttpZCacheDetailsPrefix = "details$"sv; - - constinit AsciiSet ValidNamespaceNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; - constinit AsciiSet ValidBucketNameCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; - - std::optional<std::string> GetValidNamespaceName(std::string_view Name) - { - if (Name.empty()) - { - ZEN_WARN("Namespace is invalid, empty namespace is not allowed"); - return {}; - } - - if (Name.length() > 64) - { - ZEN_WARN("Namespace '{}' is invalid, length exceeds 64 characters", Name); - return {}; - } - - if (!AsciiSet::HasOnly(Name, ValidNamespaceNameCharactersSet)) - { - ZEN_WARN("Namespace '{}' is invalid, invalid characters detected", Name); - return {}; - } - - return ToLower(Name); - } - - std::optional<std::string> GetValidBucketName(std::string_view Name) - { - if (Name.empty()) - { - ZEN_WARN("Bucket name is invalid, empty bucket name is not allowed"); - return {}; - } - - if (!AsciiSet::HasOnly(Name, ValidBucketNameCharactersSet)) - { - ZEN_WARN("Bucket name '{}' is invalid, invalid characters detected", Name); - return {}; - } - - return ToLower(Name); - } - - std::optional<IoHash> GetValidIoHash(std::string_view Hash) - { - if (Hash.length() != IoHash::StringLength) - { - return {}; - } - - IoHash KeyHash; - if (!ParseHexBytes(Hash.data(), Hash.size(), KeyHash.Hash)) - { - return {}; - } - return KeyHash; - } - - struct HttpRequestData - { - std::optional<std::string> Namespace; - std::optional<std::string> Bucket; - std::optional<IoHash> HashKey; - std::optional<IoHash> ValueContentId; - }; - - bool HttpRequestParseRelativeUri(std::string_view Key, HttpRequestData& Data) - { - std::vector<std::string_view> Tokens; - uint32_t TokenCount = ForEachStrTok(Key, '/', [&](const std::string_view& Token) { - Tokens.push_back(Token); - return true; - }); - - switch (TokenCount) - { - case 0: - return true; - case 1: - Data.Namespace = GetValidNamespaceName(Tokens[0]); - return Data.Namespace.has_value(); - case 2: - { - std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); - if (PossibleHashKey.has_value()) - { - // Legacy bucket/key request - Data.Bucket = GetValidBucketName(Tokens[0]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = PossibleHashKey; - Data.Namespace = ZenCacheStore::DefaultNamespace; - return true; - } - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - return true; - } - case 3: - { - std::optional<IoHash> PossibleHashKey = GetValidIoHash(Tokens[1]); - if (PossibleHashKey.has_value()) - { - // Legacy bucket/key/valueid request - Data.Bucket = GetValidBucketName(Tokens[0]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = PossibleHashKey; - Data.ValueContentId = GetValidIoHash(Tokens[2]); - if (!Data.ValueContentId.has_value()) - { - return false; - } - Data.Namespace = ZenCacheStore::DefaultNamespace; - return true; - } - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - Data.HashKey = GetValidIoHash(Tokens[2]); - if (!Data.HashKey) - { - return false; - } - return true; - } - case 4: - { - Data.Namespace = GetValidNamespaceName(Tokens[0]); - if (!Data.Namespace.has_value()) - { - return false; - } - - Data.Bucket = GetValidBucketName(Tokens[1]); - if (!Data.Bucket.has_value()) - { - return false; - } - - Data.HashKey = GetValidIoHash(Tokens[2]); - if (!Data.HashKey.has_value()) - { - return false; - } - - Data.ValueContentId = GetValidIoHash(Tokens[3]); - if (!Data.ValueContentId.has_value()) - { - return false; - } - return true; - } - default: - return false; - } - } - } // namespace ////////////////////////////////////////////////////////////////////////// @@ -250,7 +79,8 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, - const DiskWriteBlocker* InDiskWriteBlocker) + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache) : m_Log(logging::Get("cache")) , m_CacheStore(InCacheStore) , m_StatsService(StatsService) @@ -258,6 +88,7 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach , m_CidStore(InCidStore) , m_UpstreamCache(UpstreamCache) , m_DiskWriteBlocker(InDiskWriteBlocker) +, m_OpenProcessCache(InOpenProcessCache) , m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) { m_StatsService.RegisterHandler("z$", *this); @@ -290,24 +121,6 @@ HttpStructuredCacheService::Flush() } void -HttpStructuredCacheService::ScrubStorage(ScrubContext& Ctx) -{ - if (m_LastScrubTime == Ctx.ScrubTimestamp()) - { - return; - } - - ZenCacheStore::Info Info = m_CacheStore.GetInfo(); - - ZEN_INFO("scrubbing '{}'", Info.BasePath); - - m_LastScrubTime = Ctx.ScrubTimestamp(); - - m_CidStore.ScrubStorage(Ctx); - m_CacheStore.ScrubStorage(Ctx); -} - -void HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) { std::string_view Key = Request.RelativeUri(); @@ -538,18 +351,38 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) { ZEN_TRACE_CPU("z$::Http::HandleRequest"); + ZEN_MEMSCOPE(GetCacheHttpTag()); + metrics::OperationTiming::Scope $(m_HttpRequests); - std::string_view Key = Request.RelativeUri(); - if (Key == HttpZCacheRPCPrefix) + const std::string_view Key = Request.RelativeUri(); + + std::string_view UriNamespace; + + if (Key.ends_with(HttpZCacheRPCPrefix)) { - return HandleRpcRequest(Request); + const size_t RpcOffset = Key.length() - HttpZCacheRPCPrefix.length(); + + if (RpcOffset) + { + std::string_view KeyPrefix = Key.substr(0, RpcOffset); + + if (KeyPrefix.back() == '/') + { + KeyPrefix.remove_suffix(1); + + UriNamespace = KeyPrefix; + } + } + + return HandleRpcRequest(Request, UriNamespace); } if (Key == HttpZCacheUtilStartRecording) { - HttpServerRequest::QueryParams Params = Request.GetQueryParams(); - std::string RecordPath = cpr::util::urlDecode(std::string(Params.GetValue("path"))); + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); { RwLock::ExclusiveLockScope _(m_RequestRecordingLock); @@ -586,9 +419,11 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) m_RequestRecorder.reset(); } - HttpServerRequest::QueryParams Params = Request.GetQueryParams(); - std::string RecordPath = cpr::util::urlDecode(std::string(Params.GetValue("path"))); - uint32_t ThreadCount = std::thread::hardware_concurrency(); + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); + + uint32_t ThreadCount = std::thread::hardware_concurrency(); if (auto Param = Params.GetValue("thread_count"); Param.empty() == false) { if (auto Value = ParseInt<uint64_t>(Param)) @@ -602,7 +437,7 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) std::unique_ptr<cache::IRpcRequestReplayer> Replayer(cache::MakeDiskRequestReplayer(RecordPath, false)); ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount); - ZEN_INFO("cache RPC replay STARTED"); + ZEN_INFO("cache RPC replay COMPLETED"); Request.WriteResponse(HttpResponseCode::OK); return; @@ -614,8 +449,8 @@ HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) return; } - HttpRequestData RequestData; - if (!HttpRequestParseRelativeUri(Key, RequestData)) + HttpCacheRequestData RequestData; + if (!HttpCacheRequestParseRelativeUri(Key, ZenCacheStore::DefaultNamespace, RequestData)) { m_CacheStats.BadRequestCount++; return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL @@ -740,7 +575,6 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque ResponseWriter.AddInteger("MemCacheTargetFootprintBytes"sv, Info->Config.DiskLayerConfig.MemCacheTargetFootprintBytes); ResponseWriter.AddInteger("MemCacheTrimIntervalSeconds"sv, Info->Config.DiskLayerConfig.MemCacheTrimIntervalSeconds); ResponseWriter.AddInteger("MemCacheMaxAgeSeconds"sv, Info->Config.DiskLayerConfig.MemCacheMaxAgeSeconds); - ResponseWriter.AddBool("EnableReferenceCaching"sv, Info->Config.DiskLayerConfig.BucketConfig.EnableReferenceCaching); } ResponseWriter.EndObject(); @@ -764,6 +598,82 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); + if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) + { + ResponseWriter.BeginObject("BucketSizes"); + + ResponseWriter.BeginArray("Buckets"); + + std::vector<std::string> BucketNames; + if (Buckets == "*") // Get all - empty FieldFilter equal getting all fields + { + BucketNames = Info.value().BucketNames; + } + else + { + ForEachStrTok(Buckets, ',', [&](std::string_view BucketName) { + BucketNames.push_back(std::string(BucketName)); + return true; + }); + } + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + std::vector<IoHash> AllAttachments; + for (const std::string& BucketName : BucketNames) + { + ResponseWriter.BeginObject(); + ResponseWriter << "Name" << BucketName; + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + AllAttachments.insert(AllAttachments.end(), ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + } + ResponseWriter.EndObject(); + } + + ResponseWriter.EndArray(); + + ResponseWriter.BeginObject("Attachments"); + std::sort(AllAttachments.begin(), AllAttachments.end()); + auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end()); + AllAttachments.erase(NewEnd, AllAttachments.end()); + + uint64_t AttachmentsSize = 0; + + m_CidStore.IterateChunks( + AllAttachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "Count" << AllAttachments.size(); + ResponseWriter << "Size" << AttachmentsSize; + + ResponseWriter.EndObject(); + + ResponseWriter.EndObject(); + } + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); } break; @@ -814,6 +724,46 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); + if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") + { + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + uint64_t AttachmentsSize = 0; + + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + m_CidStore.IterateChunks( + ContentStats.Attachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "AttachmentsSize" << AttachmentsSize; + } + } + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); } break; @@ -863,7 +813,8 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); - bool Success = false; + bool Success = false; + uint32_t MissingCount = 0; ZenCacheValue ClientResultValue; if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) { @@ -885,37 +836,60 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { if (ContentType == ZenContentType::kCbObject) { - CbPackage Package; - uint32_t MissingCount = 0; - - CbObjectView CacheRecord(ClientResultValue.Value.Data()); - CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { - if (SkipData) - { - if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) - { - MissingCount++; - } - } - else - { - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + CbPackage Package; + CbValidateError ValidateError = CbValidateError::None; + if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError); + ValidateError == CbValidateError::None) + { + CbObjectView CacheRecord(ClientResultValue.Value.Data()); + CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + if (SkipData) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); - Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + { + MissingCount++; + } } else { - MissingCount++; + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + } + else + { + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); + MissingCount++; + } + } + else + { + MissingCount++; + } } - } - }); + }); - Success = MissingCount == 0 || PartialRecord; + Success = MissingCount == 0 || PartialRecord; + } + else + { + ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(ValidateError)); + Success = false; + } if (Success) { - Package.SetObject(LoadCompactBinaryObject(ClientResultValue.Value)); + CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value)); + + Package.SetObject(std::move(PackageObject)); BinaryWriter MemStream; Package.Save(MemStream); @@ -954,7 +928,9 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con else { // kCbPackage handled SkipData when constructing the ClientResultValue, kcbObject ignores SkipData - return Request.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); + return Request.WriteResponse((MissingCount == 0) ? HttpResponseCode::OK : HttpResponseCode::PartialContent, + ClientResultValue.Value.GetContentType(), + ClientResultValue.Value); } } else if (!HasUpstream || !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) @@ -1014,8 +990,14 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (Success && StoreLocal) { - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}); - m_CacheStats.WriteCount++; + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = + m_CacheStore + .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } } } else if (AcceptType == ZenContentType::kCbPackage) @@ -1023,17 +1005,20 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con CbPackage Package; if (Package.TryLoad(ClientResultValue.Value)) { - CbObject CacheRecord = Package.GetObject(); - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector<const CbAttachment*> AttachmentsToStoreLocally; - std::vector<IoHash> ReferencedAttachments; - AttachmentsToStoreLocally.reserve(NumAttachments); + CbObject CacheRecord = Package.GetObject(); + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector<IoHash> ReferencedAttachments; + std::vector<IoBuffer> WriteAttachmentBuffers; + WriteAttachmentBuffers.reserve(NumAttachments); + std::vector<IoHash> WriteRawHashes; + WriteRawHashes.reserve(NumAttachments); CacheRecord.IterateAttachments([this, &Package, &Ref, - &AttachmentsToStoreLocally, + &WriteAttachmentBuffers, + &WriteRawHashes, &ReferencedAttachments, &Count, QueryLocal, @@ -1047,7 +1032,9 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con { if (StoreLocal) { - AttachmentsToStoreLocally.emplace_back(Attachment); + const CompressedBuffer& Chunk = Attachment->AsCompressedBinary(); + WriteAttachmentBuffers.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Attachment->GetHash()); } Count.Valid++; } @@ -1095,20 +1082,34 @@ HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, con if (StoreLocal) { - m_CacheStore - .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments); - m_CacheStats.WriteCount++; - } - - for (const CbAttachment* Attachment : AttachmentsToStoreLocally) - { - ZEN_ASSERT_SLOW(StoreLocal); - CompressedBuffer Chunk = Attachment->AsCompressedBinary(); - CidStore::InsertResult InsertResult = - m_CidStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); - if (InsertResult.New) + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + CacheValue, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) { - Count.New++; + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) + { + std::vector<CidStore::InsertResult> InsertResults = + m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& Result : InsertResults) + { + if (Result.New) + { + Count.New++; + } + } + } + + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; } } @@ -1200,6 +1201,27 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con return Request.WriteResponse(HttpResponseCode::InsufficientStorage); } + auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) { + ZEN_UNUSED(PutResult); + + HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError; + switch (PutResult.Status) + { + case zen::PutStatus::Conflict: + ResponseCode = HttpResponseCode::Conflict; + break; + case zen::PutStatus::Invalid: + ResponseCode = HttpResponseCode::BadRequest; + break; + } + + if (PutResult.Details) + { + Request.WriteResponse(ResponseCode, PutResult.Details); + } + return Request.WriteResponse(ResponseCode); + }; + const HttpContentType ContentType = Request.RequestContentType(); Body.SetContentType(ContentType); @@ -1228,12 +1250,20 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con { RawHash = IoHash::HashBuffer(SharedBuffer(Body)); } - m_CacheStore.Put(RequestContext, - Ref.Namespace, - Ref.BucketSegment, - Ref.HashKey, - {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, - {}); + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) @@ -1282,7 +1312,21 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con TotalCount++; }); - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, {.Value = Body}, ReferencedAttachments); + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}", @@ -1326,23 +1370,27 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con CbObject CacheRecord = Package.GetObject(); - AttachmentCount Count; - size_t NumAttachments = Package.GetAttachments().size(); - std::vector<IoHash> ValidAttachments; - std::vector<IoHash> ReferencedAttachments; - std::vector<const CbAttachment*> AttachmentsToStoreLocally; + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector<IoHash> ValidAttachments; + std::vector<IoHash> ReferencedAttachments; ValidAttachments.reserve(NumAttachments); - AttachmentsToStoreLocally.reserve(NumAttachments); + std::vector<IoBuffer> WriteAttachmentBuffers; + std::vector<IoHash> WriteRawHashes; + WriteAttachmentBuffers.reserve(NumAttachments); + WriteRawHashes.reserve(NumAttachments); CacheRecord.IterateAttachments( - [this, &Ref, &Package, &AttachmentsToStoreLocally, &ValidAttachments, &ReferencedAttachments, &Count](CbFieldView HashView) { + [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( + CbFieldView HashView) { const IoHash Hash = HashView.AsHash(); ReferencedAttachments.push_back(Hash); if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) { if (Attachment->IsCompressedBinary()) { - AttachmentsToStoreLocally.emplace_back(Attachment); + WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Hash); ValidAttachments.emplace_back(Hash); Count.Valid++; } @@ -1371,20 +1419,32 @@ HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, con return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); } + const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); + ZenCacheValue CacheValue; CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); CacheValue.Value.SetContentType(ZenContentType::kCbObject); - m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = + m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } m_CacheStats.WriteCount++; - for (const CbAttachment* Attachment : AttachmentsToStoreLocally) + if (!WriteAttachmentBuffers.empty()) { - CompressedBuffer Chunk = Attachment->AsCompressedBinary(); - CidStore::InsertResult InsertResult = m_CidStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), Attachment->GetHash()); - if (InsertResult.New) + std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& InsertResult : InsertResults) { - Count.New++; + if (InsertResult.New) + { + Count.New++; + } } + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; } ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total) in {}", @@ -1580,15 +1640,26 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co WorkerThreadPool WorkerPool(ThreadCount); uint64_t RequestCount = Replayer.GetRequestCount(); Stopwatch Timer; - auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); - Latch JobLatch(RequestCount); + auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); + std::atomic<bool> AbortFlag; + std::atomic<bool> PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); ZEN_INFO("Replaying {} requests", RequestCount); for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) { - WorkerPool.ScheduleWork([this, &Context, &JobLatch, &Replayer, RequestIndex]() { + if (AbortFlag) + { + break; + } + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>& AbortFlag) { IoBuffer Body; zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); + if (AbortFlag) + { + return; + } + if (Body) { uint32_t AcceptMagic = 0; @@ -1596,6 +1667,7 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co int TargetPid = 0; CbPackage RpcResult; if (m_RpcHandler.HandleRpcRequest(Context, + /* UriNamespace */ {}, RequestInfo.ContentType, std::move(Body), AcceptMagic, @@ -1628,21 +1700,22 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co } } } - JobLatch.CountDown(); }); } - while (!JobLatch.Wait(10000)) - { + Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); ZEN_INFO("Replayed {} of {} requests, elapsed {}", - RequestCount - JobLatch.Remaining(), + RequestCount - PendingWork, RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - } + }); } void -HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request) +HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace) { + ZEN_MEMSCOPE(GetCacheRpcTag()); + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest"); const bool HasUpstream = m_UpstreamCache.IsActive(); @@ -1663,85 +1736,70 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request) return Request.WriteResponse(HttpResponseCode::BadRequest); } - auto HandleRpc = - [this, RequestContext, Body = Request.ReadPayload(), ContentType, AcceptType](HttpServerRequest& AsyncRequest) mutable { - uint64_t RequestIndex = ~0ull; - - if (m_RequestRecordingEnabled) + auto HandleRpc = [this, + RequestContext, + Body = Request.ReadPayload(), + ContentType, + AcceptType, + UriNamespaceString = std::string{UriNamespace}](HttpServerRequest& AsyncRequest) mutable { + if (m_RequestRecordingEnabled) + { + RwLock::SharedLockScope _(m_RequestRecordingLock); + if (m_RequestRecorder) { - RwLock::SharedLockScope _(m_RequestRecordingLock); - if (m_RequestRecorder) - { - RequestIndex = m_RequestRecorder->RecordRequest( - {.ContentType = ContentType, .AcceptType = AcceptType, .SessionId = RequestContext.SessionId}, - Body); - } + m_RequestRecorder->RecordRequest( + {.ContentType = ContentType, .AcceptType = AcceptType, .SessionId = RequestContext.SessionId}, + Body); } + } - uint32_t AcceptMagic = 0; - RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; - int TargetProcessId = 0; - CbPackage RpcResult; + uint32_t AcceptMagic = 0; + RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; + int TargetProcessId = 0; + CbPackage RpcResult; - CacheRpcHandler::RpcResponseCode ResultCode = m_RpcHandler.HandleRpcRequest(RequestContext, - ContentType, - std::move(Body), - AcceptMagic, - AcceptFlags, - TargetProcessId, - RpcResult); + CacheRpcHandler::RpcResponseCode ResultCode = m_RpcHandler.HandleRpcRequest(RequestContext, + UriNamespaceString, + ContentType, + std::move(Body), + /* out */ AcceptMagic, + /* out */ AcceptFlags, + /* out */ TargetProcessId, + /* out */ RpcResult); - HttpResponseCode HttpResultCode = HttpResponseCode(int(ResultCode)); + HttpResponseCode HttpResultCode = HttpResponseCode(int(ResultCode)); - if (!IsHttpSuccessCode(HttpResultCode)) - { - return AsyncRequest.WriteResponse(HttpResultCode); - } + if (!IsHttpSuccessCode(HttpResultCode)) + { + return AsyncRequest.WriteResponse(HttpResultCode); + } - if (AcceptMagic == kCbPkgMagic) + if (AcceptMagic == kCbPkgMagic) + { + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) { - void* TargetProcessHandle = nullptr; - FormatFlags Flags = FormatFlags::kDefault; - if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) - { - Flags |= FormatFlags::kAllowLocalReferences; - if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) - { - Flags |= FormatFlags::kDenyPartialLocalReferences; - } - TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(RequestContext.SessionId, TargetProcessId); - } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); - if (RequestIndex != ~0ull) + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) { - RwLock::SharedLockScope _(m_RequestRecordingLock); - if (m_RequestRecorder) - { - m_RequestRecorder->RecordResponse(RequestIndex, HttpContentType::kCbPackage, RpcResponseBuffer); - } + Flags |= FormatFlags::kDenyPartialLocalReferences; } - AsyncRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(RequestContext.SessionId, TargetProcessId); } - else - { - BinaryWriter MemStream; - RpcResult.Save(MemStream); + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); + AsyncRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + else + { + BinaryWriter MemStream; + RpcResult.Save(MemStream); - if (RequestIndex != ~0ull) - { - RwLock::SharedLockScope _(m_RequestRecordingLock); - if (m_RequestRecorder) - { - m_RequestRecorder->RecordResponse(RequestIndex, - HttpContentType::kCbPackage, - IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); - } - } - AsyncRequest.WriteResponse(HttpResponseCode::OK, - HttpContentType::kCbPackage, - IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); - } - }; + AsyncRequest.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + } + }; if (HasUpstream) { @@ -1766,6 +1824,8 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request) void HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) { + ZEN_MEMSCOPE(GetCacheHttpTag()); + CbObjectWriter Cbo; EmitSnapshot("requests", m_HttpRequests, Cbo); @@ -1789,8 +1849,8 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; - const CidStoreSize CidSize = m_CidStore.TotalSize(); - const GcStorageSize CacheSize = m_CacheStore.StorageSize(); + const CidStoreSize CidSize = m_CidStore.TotalSize(); + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; @@ -1989,106 +2049,4 @@ HttpStructuredCacheService::AreDiskWritesAllowed() const return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); } -#if ZEN_WITH_TESTS - -TEST_CASE("z$service.parse.relative.Uri") -{ - HttpRequestData RootRequest; - CHECK(HttpRequestParseRelativeUri("", RootRequest)); - CHECK(!RootRequest.Namespace.has_value()); - CHECK(!RootRequest.Bucket.has_value()); - CHECK(!RootRequest.HashKey.has_value()); - CHECK(!RootRequest.ValueContentId.has_value()); - - RootRequest = {}; - CHECK(HttpRequestParseRelativeUri("/", RootRequest)); - CHECK(!RootRequest.Namespace.has_value()); - CHECK(!RootRequest.Bucket.has_value()); - CHECK(!RootRequest.HashKey.has_value()); - CHECK(!RootRequest.ValueContentId.has_value()); - - HttpRequestData LegacyBucketRequestBecomesNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("test", LegacyBucketRequestBecomesNamespaceRequest)); - CHECK(LegacyBucketRequestBecomesNamespaceRequest.Namespace == "test"sv); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.Bucket.has_value()); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.HashKey.has_value()); - CHECK(!LegacyBucketRequestBecomesNamespaceRequest.ValueContentId.has_value()); - - HttpRequestData LegacyHashKeyRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", LegacyHashKeyRequest)); - CHECK(LegacyHashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); - CHECK(LegacyHashKeyRequest.Bucket == "test"sv); - CHECK(LegacyHashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); - CHECK(!LegacyHashKeyRequest.ValueContentId.has_value()); - - HttpRequestData LegacyValueContentIdRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - LegacyValueContentIdRequest)); - CHECK(LegacyValueContentIdRequest.Namespace == ZenCacheStore::DefaultNamespace); - CHECK(LegacyValueContentIdRequest.Bucket == "test"sv); - CHECK(LegacyValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); - CHECK(LegacyValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); - - HttpRequestData V2DefaultNamespaceRequest; - CHECK(HttpRequestParseRelativeUri("ue4.ddc", V2DefaultNamespaceRequest)); - CHECK(V2DefaultNamespaceRequest.Namespace == "ue4.ddc"); - CHECK(!V2DefaultNamespaceRequest.Bucket.has_value()); - CHECK(!V2DefaultNamespaceRequest.HashKey.has_value()); - CHECK(!V2DefaultNamespaceRequest.ValueContentId.has_value()); - - HttpRequestData V2NamespaceRequest; - CHECK(HttpRequestParseRelativeUri("nicenamespace", V2NamespaceRequest)); - CHECK(V2NamespaceRequest.Namespace == "nicenamespace"sv); - CHECK(!V2NamespaceRequest.Bucket.has_value()); - CHECK(!V2NamespaceRequest.HashKey.has_value()); - CHECK(!V2NamespaceRequest.ValueContentId.has_value()); - - HttpRequestData V2BucketRequestWithDefaultNamespace; - CHECK(HttpRequestParseRelativeUri("ue4.ddc/test", V2BucketRequestWithDefaultNamespace)); - CHECK(V2BucketRequestWithDefaultNamespace.Namespace == "ue4.ddc"); - CHECK(V2BucketRequestWithDefaultNamespace.Bucket == "test"sv); - CHECK(!V2BucketRequestWithDefaultNamespace.HashKey.has_value()); - CHECK(!V2BucketRequestWithDefaultNamespace.ValueContentId.has_value()); - - HttpRequestData V2BucketRequestWithNamespace; - CHECK(HttpRequestParseRelativeUri("nicenamespace/test", V2BucketRequestWithNamespace)); - CHECK(V2BucketRequestWithNamespace.Namespace == "nicenamespace"sv); - CHECK(V2BucketRequestWithNamespace.Bucket == "test"sv); - CHECK(!V2BucketRequestWithNamespace.HashKey.has_value()); - CHECK(!V2BucketRequestWithNamespace.ValueContentId.has_value()); - - HttpRequestData V2HashKeyRequest; - CHECK(HttpRequestParseRelativeUri("test/0123456789abcdef12340123456789abcdef1234", V2HashKeyRequest)); - CHECK(V2HashKeyRequest.Namespace == ZenCacheStore::DefaultNamespace); - CHECK(V2HashKeyRequest.Bucket == "test"); - CHECK(V2HashKeyRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); - CHECK(!V2HashKeyRequest.ValueContentId.has_value()); - - HttpRequestData V2ValueContentIdRequest; - CHECK( - HttpRequestParseRelativeUri("nicenamespace/test/0123456789abcdef12340123456789abcdef1234/56789abcdef12345678956789abcdef123456789", - V2ValueContentIdRequest)); - CHECK(V2ValueContentIdRequest.Namespace == "nicenamespace"sv); - CHECK(V2ValueContentIdRequest.Bucket == "test"sv); - CHECK(V2ValueContentIdRequest.HashKey == IoHash::FromHexString("0123456789abcdef12340123456789abcdef1234"sv)); - CHECK(V2ValueContentIdRequest.ValueContentId == IoHash::FromHexString("56789abcdef12345678956789abcdef123456789"sv)); - - HttpRequestData Invalid; - CHECK(!HttpRequestParseRelativeUri("bad\2_namespace", Invalid)); - CHECK(!HttpRequestParseRelativeUri("nice/\2\1bucket", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789a", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/pppppppp89abcdef12340123456789abcdef1234", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/56789abcd", Invalid)); - CHECK(!HttpRequestParseRelativeUri("namespace/bucket/0123456789abcdef12340123456789abcdef1234/ppppppppdef12345678956789abcdef123456789", - Invalid)); -} - -#endif - -void -z$service_forcelink() -{ -} - } // namespace zen diff --git a/src/zenserver/cache/httpstructuredcache.h b/src/zenserver/cache/httpstructuredcache.h index da4bdd63c..a157148c9 100644 --- a/src/zenserver/cache/httpstructuredcache.h +++ b/src/zenserver/cache/httpstructuredcache.h @@ -6,8 +6,8 @@ #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> #include <zenhttp/httpstatus.h> +#include <zenstore/cache/cache.h> #include <zenstore/cache/cacherpc.h> -#include <zenutil/cache/cache.h> #include <zenutil/openprocesscache.h> #include <memory> @@ -75,14 +75,14 @@ public: HttpStatsService& StatsService, HttpStatusService& StatusService, UpstreamCache& UpstreamCache, - const DiskWriteBlocker* InDiskWriteBlocker); + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache); ~HttpStructuredCacheService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; void Flush(); - void ScrubStorage(ScrubContext& Ctx); private: struct CacheRef @@ -99,7 +99,7 @@ private: void HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); - void HandleRpcRequest(HttpServerRequest& Request); + void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace); void HandleDetailsRequest(HttpServerRequest& Request); void HandleCacheRequest(HttpServerRequest& Request); @@ -117,12 +117,11 @@ private: HttpStatusService& m_StatusService; CidStore& m_CidStore; UpstreamCache& m_UpstreamCache; - uint64_t m_LastScrubTime = 0; metrics::OperationTiming m_HttpRequests; metrics::OperationTiming m_UpstreamGetRequestTiming; CacheStats m_CacheStats; const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - OpenProcessCache m_OpenProcessCache; + OpenProcessCache& m_OpenProcessCache; CacheRpcHandler m_RpcHandler; void ReplayRequestRecorder(const CacheRequestContext& Context, cache::IRpcRequestReplayer& Replayer, uint32_t ThreadCount); @@ -136,6 +135,4 @@ private: std::unique_ptr<cache::IRpcRequestRecorder> m_RequestRecorder; }; -void z$service_forcelink(); - } // namespace zen diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 012925b51..0cf5a9ca3 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -5,7 +5,9 @@ #include "config/luaconfig.h" #include "diag/logging.h" +#include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryutil.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/crypto.h> #include <zencore/except.h> @@ -14,71 +16,27 @@ #include <zencore/logging.h> #include <zencore/string.h> #include <zenhttp/zenhttp.h> -#include <zenutil/basicfile.h> +#include <zenutil/commandlineoptions.h> +#include <zenutil/environmentoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> +#include <fmt/ranges.h> #include <zencore/logging.h> #include <cxxopts.hpp> +#include <json11.hpp> #include <sol/sol.hpp> ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_WINDOWS # include <conio.h> #else -# include <pwd.h> # include <unistd.h> #endif #include <unordered_map> #include <unordered_set> -#if ZEN_PLATFORM_WINDOWS - -// Used for getting My Documents for default data directory -# include <ShlObj.h> -# pragma comment(lib, "shell32.lib") -# pragma comment(lib, "ole32.lib") - -namespace zen { - -std::filesystem::path -PickDefaultSystemRootDirectory() -{ - // Pick sensible default - PWSTR ProgramDataDir = nullptr; - HRESULT hRes = SHGetKnownFolderPath(FOLDERID_ProgramData, 0, NULL, &ProgramDataDir); - - if (SUCCEEDED(hRes)) - { - std::filesystem::path FinalPath(ProgramDataDir); - FinalPath /= L"Epic\\Zen"; - ::CoTaskMemFree(ProgramDataDir); - - return FinalPath; - } - - return L""; -} - -} // namespace zen - -#else - -namespace zen { - -std::filesystem::path -PickDefaultSystemRootDirectory() -{ - int UserId = getuid(); - const passwd* Passwd = getpwuid(UserId); - return std::filesystem::path(Passwd->pw_dir) / ".zen"; -} - -} // namespace zen - -#endif - namespace zen { std::filesystem::path @@ -109,26 +67,25 @@ ReadAllCentralManifests(const std::filesystem::path& SystemRoot) std::vector<CbObject> Manifests; DirectoryContent Content; - GetDirectoryContent(SystemRoot / "States", DirectoryContent::IncludeFilesFlag, Content); + GetDirectoryContent(SystemRoot / "States", DirectoryContentFlags::IncludeFiles, Content); for (std::filesystem::path& File : Content.Files) { try { - FileContents FileData = ReadFile(File); - IoBuffer DataBuffer = FileData.Flatten(); - CbValidateError ValidateError = ValidateCompactBinary(DataBuffer, CbValidateMode::All); - - if (ValidateError == CbValidateError::None) + FileContents FileData = ReadFile(File); + CbValidateError ValidateError; + if (CbObject Manifest = ValidateAndReadCompactBinaryObject(FileData.Flatten(), ValidateError); + ValidateError == CbValidateError::None) { - Manifests.push_back(LoadCompactBinaryObject(DataBuffer)); + Manifests.emplace_back(std::move(Manifest)); } else { ZEN_WARN("failed to load manifest '{}': {}", File, ToString(ValidateError)); } } - catch (std::exception& Ex) + catch (const std::exception& Ex) { ZEN_WARN("failed to load manifest '{}': {}", File, Ex.what()); } @@ -142,26 +99,38 @@ ValidateOptions(ZenServerOptions& ServerOptions) { if (ServerOptions.EncryptionKey.empty() == false) { - const auto Key = zen::AesKey256Bit::FromString(ServerOptions.EncryptionKey); + const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); if (Key.IsValid() == false) { - throw zen::OptionParseException("Invalid AES encryption key"); + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); } } if (ServerOptions.EncryptionIV.empty() == false) { - const auto IV = zen::AesIV128Bit::FromString(ServerOptions.EncryptionIV); + const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); if (IV.IsValid() == false) { - throw zen::OptionParseException("Invalid AES initialization vector"); + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); } } if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) { - throw zen::OptionParseException("Dedicated server can not be used with forced local server address"); + throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); + } + if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) + { + throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", + ServerOptions.GcConfig.AttachmentPassCount, + ZenGcConfig::GcMaxAttachmentPassCount), + {}); + } + if (ServerOptions.GcConfig.UseGCV2 == false) + { + ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); + ServerOptions.GcConfig.UseGCV2 = true; } } @@ -214,32 +183,11 @@ ParseBucketConfigs(std::span<std::string> Buckets) return Cfg; } -static std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - class CachePolicyOption : public LuaConfig::OptionValue { public: CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} - virtual void Print(std::string_view, zen::StringBuilderBase& StringBuilder) override + virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override { switch (Value) { @@ -286,7 +234,7 @@ class ZenAuthConfigOption : public LuaConfig::OptionValue { public: ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override { if (Value.OpenIdProviders.empty()) { @@ -329,7 +277,7 @@ class ZenObjectStoreConfigOption : public LuaConfig::OptionValue { public: ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} - virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override { if (Value.Buckets.empty()) { @@ -360,7 +308,7 @@ public: std::string Name = Bucket.value().get_or("name", std::string("Default")); std::string Directory = Bucket.value().get_or("directory", std::string()); - Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)}); + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); } } } @@ -368,30 +316,160 @@ public: ZenObjectStoreConfig& Value; }; +class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue +{ +public: + ZenStructuredCacheBucketsConfigOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const std::pair<std::string, ZenStructuredCacheBucketConfig>& Bucket : Value) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Bucket.first); + const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; + + Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); + Writer.BeginContainer("memlayer"); + { + Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); + } + Writer.EndContainer(); + + Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) + { + ZenStructuredCacheBucketConfig BucketConfig; + std::string Name = Kv.first.as<std::string>(); + if (Name.empty()) + { + throw OptionParseException("Cache bucket option must have a name.", {}); + } + + const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); + if (MaxBlockSize == 0) + { + throw OptionParseException( + fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MaxBlockSize = MaxBlockSize; + + if (sol::optional<sol::table> Memlayer = Bucket.value().get_or("memlayer", sol::table())) + { + const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + if (MemCacheSizeThreshold == 0) + { + throw OptionParseException( + fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + } + + const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); + if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) + { + throw OptionParseException( + fmt::format( + "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", + Name), + {}); + } + BucketConfig.PayloadAlignment = PayloadAlignment; + + const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); + if (LargeObjectThreshold == 0) + { + throw OptionParseException( + fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + + BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); + + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); + } + } + } + } + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value; +}; + std::shared_ptr<LuaConfig::OptionValue> -MakeOption(zen::UpstreamCachePolicy& Value) +MakeOption(UpstreamCachePolicy& Value) { return std::make_shared<CachePolicyOption>(Value); }; std::shared_ptr<LuaConfig::OptionValue> -MakeOption(zen::ZenAuthConfig& Value) +MakeOption(ZenAuthConfig& Value) { return std::make_shared<ZenAuthConfigOption>(Value); }; std::shared_ptr<LuaConfig::OptionValue> -MakeOption(zen::ZenObjectStoreConfig& Value) +MakeOption(ZenObjectStoreConfig& Value) { return std::make_shared<ZenObjectStoreConfigOption>(Value); }; +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) +{ + return std::make_shared<ZenStructuredCacheBucketsConfigOption>(Value); +}; + +void +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +{ + using namespace std::literals; + + EnvironmentOptions Options; + Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; + Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + + Options.Parse(CmdLineResult); + + if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) + { + ServerOptions.SentryConfig.Disable = !EnvEnableSentry; + } +} + void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult, std::string_view OutputConfigFile) { + ZEN_TRACE_CPU("ParseConfigFile"); + using namespace std::literals; LuaConfig::Options LuaOptions; @@ -399,20 +477,29 @@ ParseConfigFile(const std::filesystem::path& Path, ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); - LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv); + LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); + LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); ////// objectstore LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); @@ -431,33 +518,47 @@ ParseConfigFile(const std::filesystem::path& Path, #if ZEN_WITH_TRACE ////// trace - LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceHost, "tracehost"sv); - LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceFile, "tracefile"sv); + LuaOptions.AddOption("trace.channels"sv, ServerOptions.TraceOptions.Channels, "trace"sv); + LuaOptions.AddOption("trace.host"sv, ServerOptions.TraceOptions.Host, "tracehost"sv); + LuaOptions.AddOption("trace.file"sv, ServerOptions.TraceOptions.File, "tracefile"sv); #endif ////// stats - LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled); + LuaOptions.AddOption("stats.enable"sv, ServerOptions.StatsConfig.Enabled, "statsd"sv); LuaOptions.AddOption("stats.host"sv, ServerOptions.StatsConfig.StatsdHost); LuaOptions.AddOption("stats.port"sv, ServerOptions.StatsConfig.StatsdPort); ////// cache LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); - LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"); - LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"); - LuaOptions.AddOption("cache.referencecache"sv, - ServerOptions.StructuredCacheConfig.EnableReferenceCaching, - "cache-reference-cache-enabled"); + LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); + LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, - "cache-memlayer-sizethreshold"); + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-memlayer-sizethreshold"sv); LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, - "cache-memlayer-targetfootprint"); + "cache-memlayer-targetfootprint"sv); LuaOptions.AddOption("cache.memlayer.triminterval"sv, ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, - "cache-memlayer-triminterval"); - LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"); + "cache-memlayer-triminterval"sv); + LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); + + LuaOptions.AddOption("cache.bucket.maxblocksize"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + "cache-bucket-maxblocksize"sv); + LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-bucket-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.bucket.payloadalignment"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + "cache-bucket-payloadalignment"sv); + LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + "cache-bucket-largeobjectthreshold"sv); + LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, + "cache-bucket-limit-overwrites"sv); ////// cache.upstream LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); @@ -500,6 +601,7 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); + ////// gc LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); @@ -511,22 +613,42 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, "gc-low-diskspace-threshold"sv); - LuaOptions.AddOption("gc.lightweightntervalseconds"sv, + LuaOptions.AddOption("gc.lightweightintervalseconds"sv, ServerOptions.GcConfig.LightweightIntervalSeconds, "gc-lightweight-interval-seconds"sv); LuaOptions.AddOption("gc.compactblockthreshold"sv, ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, "gc-compactblock-threshold"sv); LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); + LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); + LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); + LuaOptions.AddOption("gc.projectstore.attachment.store"sv, + ServerOptions.GcConfig.StoreProjectAttachmentMetaData, + "gc-projectstore-attachment-store"); + LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); + LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); - ////// gc LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); + LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, + ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, + "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); ////// security LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); + ////// workspaces + LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); + LuaOptions.AddOption("workspaces.allowconfigchanges"sv, + ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, + "workspaces-allow-changes"sv); + + LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); + LuaOptions.Parse(Path, CmdLineResult); // These have special command line processing so we make sure we export them if they were configured on command line @@ -538,27 +660,106 @@ ParseConfigFile(const std::filesystem::path& Path, { LuaOptions.Touch("server.objectstore.buckets"sv); } + if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) + { + LuaOptions.Touch("cache.buckets"sv); + } if (!OutputConfigFile.empty()) { - std::filesystem::path WritePath(MakeSafePath(OutputConfigFile)); - zen::ExtendableStringBuilder<512> ConfigStringBuilder; + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); + ExtendableStringBuilder<512> ConfigStringBuilder; LuaOptions.Print(ConfigStringBuilder, CmdLineResult); - zen::BasicFile Output; - Output.Open(WritePath, zen::BasicFile::Mode::kTruncate); + BasicFile Output; + Output.Open(WritePath, BasicFile::Mode::kTruncate); Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); } } void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); + return; + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); + continue; + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump()); + continue; + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + +void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) { -#if ZEN_WITH_HTTPSYS - const char* DefaultHttp = "httpsys"; -#else const char* DefaultHttp = "asio"; + +#if ZEN_WITH_HTTPSYS + if (!windows::IsRunningOnWine()) + { + DefaultHttp = "httpsys"; + } #endif + for (int i = 0; i < argc; ++i) + { + if (i) + { + ServerOptions.CommandLine.push_back(' '); + } + + ServerOptions.CommandLine += argv[i]; + } + // Note to those adding future options; std::filesystem::path-type options // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> // stream operator to convert argv value into the options type. std::fs::path @@ -569,6 +770,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) std::string ContentDir; std::string AbsLogFile; std::string ConfigFile; + std::string PluginsConfigFile; std::string OutputConfigFile; std::string BaseSnapshotDir; @@ -596,26 +798,39 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Exit immediately after initialization is complete", cxxopts::value<bool>(ServerOptions.IsPowerCycle)); options.add_options()("config", "Path to Lua config file", cxxopts::value<std::string>(ConfigFile)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile)); options.add_options()("write-config", "Path to output Lua config file", cxxopts::value<std::string>(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", - cxxopts::value<bool>(ServerOptions.NoSentry)->default_value("false")); + cxxopts::value<bool>(ServerOptions.SentryConfig.Disable)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value<bool>(ServerOptions.SentryAllowPII)->default_value("false")); + cxxopts::value<bool>(ServerOptions.SentryConfig.AllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value<std::string>(ServerOptions.SentryConfig.Dsn)); + options.add_options()("sentry-environment", "Sentry environment", cxxopts::value<std::string>(ServerOptions.SentryConfig.Environment)); + options.add_options()("sentry-debug", + "Enable debug mode for Sentry", + cxxopts::value<bool>(ServerOptions.SentryConfig.Debug)->default_value("false")); + options.add_options()("detach", + "Indicate whether zenserver should detach from parent process group", + cxxopts::value<bool>(ServerOptions.Detach)->default_value("true")); + options.add_options()("malloc", + "Configure memory allocator subsystem", + cxxopts::value(ServerOptions.MemoryOptions)->default_value("mimalloc")); // clang-format off options.add_options("logging") - ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) - ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) - ("quiet", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) + ("abslog", "Path to log file", cxxopts::value<std::string>(AbsLogFile)) + ("log-id", "Specify id for adding context to log output", cxxopts::value<std::string>(ServerOptions.LogId)) + ("quiet", "Configure console logger output to level WARN", cxxopts::value<bool>(ServerOptions.QuietConsole)->default_value("false")) + ("noconsole", "Disable console logging", cxxopts::value<bool>(ServerOptions.NoConsoleOutput)->default_value("false")) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value<std::string>(ServerOptions.Loggers[logging::level::Off])) ; // clang-format on @@ -721,18 +936,28 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "<httpsys request logging>"); #if ZEN_WITH_TRACE + // We only have this in options for command line help purposes - we parse these argument separately earlier using + // GetTraceOptionsFromCommandline() + + options.add_option("ue-trace", + "", + "trace", + "Specify which trace channels should be enabled", + cxxopts::value<std::string>(ServerOptions.TraceOptions.Channels)->default_value(""), + ""); + options.add_option("ue-trace", "", "tracehost", "Hostname to send the trace to", - cxxopts::value<std::string>(ServerOptions.TraceHost)->default_value(""), + cxxopts::value<std::string>(ServerOptions.TraceOptions.Host)->default_value(""), ""); options.add_option("ue-trace", "", "tracefile", "Path to write a trace to", - cxxopts::value<std::string>(ServerOptions.TraceFile)->default_value(""), + cxxopts::value<std::string>(ServerOptions.TraceOptions.File)->default_value(""), ""); #endif // ZEN_WITH_TRACE @@ -856,19 +1081,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), ""); - options.add_option("cache", - "", - "cache-reference-cache-enabled", - "Whether caching of references is enabled", - cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.EnableReferenceCaching)->default_value("false"), - ""); - options.add_option( "cache", "", "cache-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", - cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold)->default_value("1024"), + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching. " + "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), ""); options.add_option("cache", @@ -892,6 +1111,64 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), ""); + options.add_option("cache", + "", + "cache-bucket-maxblocksize", + "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), + ""); + + options.add_option("cache", + "", + "cache-bucket-payloadalignment", + "Payload alignement for cache bucket blocks. Default set to 16.", + cxxopts::value<uint32_t>(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-largeobjectthreshold", + "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-bucket-limit-overwrites", + "Whether to require policy flag pattern before allowing overwrites in cache bucket", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-cache-attachment-store", + "Enable storing attachments referenced by a cache record in block store meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-attachment-store", + "Enable storing attachments referenced by project oplogs in meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-validation", + "Enable validation of references after full GC.", + cxxopts::value<bool>(ServerOptions.GcConfig.EnableValidation)->default_value("true"), + ""); + options.add_option("gc", "", "gc-enabled", @@ -903,7 +1180,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "", "gc-v2", "Use V2 of GC implementation or not.", - cxxopts::value<bool>(ServerOptions.GcConfig.UseGCV2)->default_value("false"), + cxxopts::value<bool>(ServerOptions.GcConfig.UseGCV2)->default_value("true"), ""); options.add_option("gc", @@ -943,6 +1220,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_option("gc", "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + + options.add_option("gc", + "", "disk-reserve-size", "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), @@ -984,6 +1268,21 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<bool>(ServerOptions.GcConfig.Verbose)->default_value("false"), ""); + options.add_option("gc", + "", + "gc-single-threaded", + "Force GC to run single threaded.", + cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-attachment-passes", + "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " + "which includes all the attachments.", + cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), + ""); + options.add_option("objectstore", "", "objectstore-enabled", @@ -999,6 +1298,19 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<std::vector<std::string>>(BucketConfigs), ""); + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit", + "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", + cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), + ""); + options.add_option("stats", "", "statsd", @@ -1006,6 +1318,19 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"), "Enable statsd reporter (localhost:8125)"); + options.add_option("workspaces", + "", + "workspaces-enabled", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), + "Enable workspaces support with folder sharing"); + + options.add_option("workspaces", + "", + "workspaces-allow-changes", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), + "Allow adding/modifying/deleting of workspace and shares via http endpoint"); try { cxxopts::ParseResult Result; @@ -1014,9 +1339,9 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) { Result = options.parse(argc, argv); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { - throw zen::OptionParseException(Ex.what()); + throw OptionParseException(Ex.what(), options.help()); } if (Result.count("help")) @@ -1032,34 +1357,66 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) exit(0); } + if (!ServerOptions.HasTraceCommandlineOptions) + { + // Apply any Lua settings if we don't have them set from the command line + TraceConfigure(ServerOptions.TraceOptions); + } + + ZEN_TRACE_CPU("ConfigParse"); + + if (ServerOptions.QuietConsole) + { + bool HasExplicitConsoleLevel = false; + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + if (ServerOptions.Loggers[i].find("console") != std::string::npos) + { + HasExplicitConsoleLevel = true; + break; + } + } + + if (!HasExplicitConsoleLevel) + { + std::string& WarnLoggers = ServerOptions.Loggers[logging::level::Warn]; + if (!WarnLoggers.empty()) + { + WarnLoggers += ","; + } + WarnLoggers += "console"; + } + } + for (int i = 0; i < logging::level::LogLevelCount; ++i) { logging::ConfigureLogLevels(logging::level::LogLevel(i), ServerOptions.Loggers[i]); } logging::RefreshLogLevels(); - ServerOptions.SystemRootDir = MakeSafePath(SystemRootDir); - ServerOptions.DataDir = MakeSafePath(DataDir); - ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir); - ServerOptions.ContentDir = MakeSafePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafePath(AbsLogFile); - ServerOptions.ConfigFile = MakeSafePath(ConfigFile); + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); if (!BaseSnapshotDir.empty()) { if (DataDir.empty()) - throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); + throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); - if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) - throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); + if (!IsDir(ServerOptions.BaseSnapshotDir)) + throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", BaseSnapshotDir)); } if (OpenIdProviderUrl.empty() == false) { if (OpenIdClientId.empty()) { - throw zen::OptionParseException("Invalid OpenID client ID"); + throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); } ServerOptions.AuthConfig.OpenIdProviders.push_back( @@ -1068,6 +1425,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); + ParseEnvVariables(ServerOptions, Result); + if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); @@ -1077,12 +1436,17 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile); } + if (!ServerOptions.PluginsConfigFile.empty()) + { + ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort); + } + ValidateOptions(ServerOptions); } - catch (zen::OptionParseException& e) + catch (const OptionParseException& e) { - ZEN_CONSOLE_ERROR("Error parsing zenserver arguments: {}\n\n{}", e.what(), options.help()); - + ZEN_CONSOLE("{}\n", options.help()); + ZEN_CONSOLE_ERROR("Invalid zenserver arguments: {}", e.what()); throw; } diff --git a/src/zenserver/config.h b/src/zenserver/config.h index b5314b600..8380e72e7 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -3,6 +3,7 @@ #pragma once #include <zencore/logbase.h> +#include <zencore/trace.h> #include <zencore/zencore.h> #include <zenhttp/httpserver.h> #include <filesystem> @@ -59,11 +60,17 @@ struct ZenProjectStoreEvictionPolicy int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; }; +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + struct ZenGcConfig { // ZenCasEvictionPolicy Cas; ZenCacheEvictionPolicy Cache; ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; int32_t MonitorIntervalSeconds = 30; int32_t IntervalSeconds = 0; bool CollectSmallObjects = true; @@ -75,6 +82,12 @@ struct ZenGcConfig bool UseGCV2 = false; uint32_t CompactBlockUsageThresholdPercent = 90; bool Verbose = false; + bool SingleThreaded = false; + static constexpr uint16_t GcMaxAttachmentPassCount = 256; + uint16_t AttachmentPassCount = 1; + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; + bool EnableValidation = true; }; struct ZenOpenIdProviderConfig @@ -107,16 +120,52 @@ struct ZenStatsConfig int StatsdPort = 8125; }; +struct ZenStructuredCacheBucketConfig +{ + uint64_t MaxBlockSize = 1ull << 30; + uint32_t PayloadAlignment = 1u << 4; + uint64_t MemCacheSizeThreshold = 1 * 1024; + uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; +}; + struct ZenStructuredCacheConfig { - bool Enabled = true; - bool WriteLogEnabled = false; - bool AccessLogEnabled = false; - bool EnableReferenceCaching = false; - uint64_t MemCacheSizeThreshold = 1 * 1024; - uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; - uint64_t MemTrimIntervalSeconds = 60; - uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); + bool Enabled = true; + bool WriteLogEnabled = false; + bool AccessLogEnabled = false; + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs; + ZenStructuredCacheBucketConfig BucketConfig; + uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; + uint64_t MemTrimIntervalSeconds = 60; + uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); +}; + +struct ZenProjectStoreConfig +{ + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; +}; + +struct ZenBuildStoreConfig +{ + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB +}; + +struct ZenWorkspacesConfig +{ + bool Enabled = false; + bool AllowConfigurationChanges = false; +}; + +struct ZenSentryConfig +{ + bool Disable = false; + bool AllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string Dsn; + std::string Environment; + bool Debug = false; // Enable debug mode for Sentry }; struct ZenServerOptions @@ -127,12 +176,17 @@ struct ZenServerOptions ZenObjectStoreConfig ObjectStoreConfig; zen::HttpServerConfig HttpServerConfig; ZenStructuredCacheConfig StructuredCacheConfig; + ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; + ZenWorkspacesConfig WorksSpacesConfig; + ZenSentryConfig SentryConfig; std::filesystem::path SystemRootDir; // System root directory (used for machine level config) std::filesystem::path DataDir; // Root directory for state (used for testing) std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) std::filesystem::path AbsLogFile; // Absolute path to main log file std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) std::string ChildId; // Id assigned by parent process (used for lifetime management) std::string LogId; // Id for tagging log output @@ -149,16 +203,18 @@ struct ZenServerOptions bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; - bool NoSentry = false; - bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports + bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics + bool QuietConsole = false; // Configure console logger output to level WARN std::string Loggers[zen::logging::level::LogLevelCount]; std::string ScrubOptions; #if ZEN_WITH_TRACE - std::string TraceHost; // Host name or IP address to send trace data to - std::string TraceFile; // Path of a file to write a trace + bool HasTraceCommandlineOptions = false; + TraceOptions TraceOptions; #endif + std::string MemoryOptions; // Memory allocation options + std::string CommandLine; }; void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions); diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp index cdc808cf6..2c54de29e 100644 --- a/src/zenserver/config/luaconfig.cpp +++ b/src/zenserver/config/luaconfig.cpp @@ -4,27 +4,6 @@ namespace zen::LuaConfig { -std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - void EscapeBackslash(std::string& InOutString) { @@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object) std::string Str = Object.as<std::string>(); if (!Str.empty()) { - Value = MakeSafePath(Str); + Value = MakeSafeAbsolutePath(Str); } } @@ -280,7 +259,7 @@ Options::Parse(const std::filesystem::path& Path, const cxxopts::ParseResult& Cm config(); } - catch (std::exception& e) + catch (const std::exception& e) { throw std::runtime_error(fmt::format("failed to load config script ('{}'): {}", Path, e.what()).c_str()); } diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h index 76b3088a3..ce7013a9a 100644 --- a/src/zenserver/config/luaconfig.h +++ b/src/zenserver/config/luaconfig.h @@ -4,10 +4,10 @@ #include <zenbase/concepts.h> #include <zencore/fmtutils.h> +#include <zenutil/commandlineoptions.h> ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> -#include <cxxopts.hpp> #include <sol/sol.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen::LuaConfig { -std::string MakeSafePath(const std::string_view Path); -void EscapeBackslash(std::string& InOutString); +void EscapeBackslash(std::string& InOutString); class OptionValue { diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp index 1d62b4d17..8abf6e8a3 100644 --- a/src/zenserver/diag/diagsvcs.cpp +++ b/src/zenserver/diag/diagsvcs.cpp @@ -7,17 +7,25 @@ #include <zencore/config.h> #include <zencore/filesystem.h> #include <zencore/logging.h> +#include <zencore/memory/llm.h> #include <zencore/string.h> #include <fstream> #include <sstream> ZEN_THIRD_PARTY_INCLUDES_START #include <spdlog/logger.h> -#include <json11.hpp> ZEN_THIRD_PARTY_INCLUDES_END namespace zen { +const FLLMTag& +GetHealthTag() +{ + static FLLMTag CacheHttpTag("health"); + + return CacheHttpTag; +} + using namespace std::literals; static bool @@ -37,7 +45,7 @@ ReadLogFile(const std::string& Path, StringBuilderBase& Out) return true; } - catch (std::exception&) + catch (const std::exception&) { Out.Reset(); return false; @@ -46,6 +54,8 @@ ReadLogFile(const std::string& Path, StringBuilderBase& Out) HttpHealthService::HttpHealthService() { + ZEN_MEMSCOPE(GetHealthTag()); + m_Router.RegisterRoute( "", [](HttpRouterRequest& RoutedReq) { @@ -116,6 +126,7 @@ HttpHealthService::HttpHealthService() void HttpHealthService::SetHealthInfo(HealthServiceInfo&& Info) { + ZEN_MEMSCOPE(GetHealthTag()); RwLock::ExclusiveLockScope _(m_InfoLock); m_HealthInfo = std::move(Info); } @@ -129,6 +140,7 @@ HttpHealthService::BaseUri() const void HttpHealthService::HandleRequest(HttpServerRequest& Request) { + ZEN_MEMSCOPE(GetHealthTag()); if (!m_Router.HandleRequest(Request)) { Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, u8"OK!"sv); diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index dc1675819..34d9b05b7 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -6,6 +6,7 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> +#include <zencore/memory/llm.h> #include <zencore/session.h> #include <zencore/string.h> #include <zenutil/logging.h> @@ -20,10 +21,13 @@ namespace zen { void InitializeServerLogging(const ZenServerOptions& InOptions) { + ZEN_MEMSCOPE(ELLMTag::Logging); + const LoggingOptions LogOptions = {.IsDebug = InOptions.IsDebug, .IsVerbose = false, .IsTest = InOptions.IsTest, .NoConsoleOutput = InOptions.NoConsoleOutput, + .QuietConsole = InOptions.QuietConsole, .AbsLogFile = InOptions.AbsLogFile, .LogId = InOptions.LogId}; @@ -37,7 +41,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions) std::filesystem::path HttpLogPath = InOptions.DataDir / "logs" / "http.log"; zen::CreateDirectories(HttpLogPath.parent_path()); - auto HttpSink = std::make_shared<zen::logging::RotatingFileSink>(zen::PathToUtf8(HttpLogPath), + auto HttpSink = std::make_shared<zen::logging::RotatingFileSink>(HttpLogPath, /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); @@ -49,7 +53,7 @@ InitializeServerLogging(const ZenServerOptions& InOptions) std::filesystem::path CacheLogPath = InOptions.DataDir / "logs" / "z$.log"; zen::CreateDirectories(CacheLogPath.parent_path()); - auto CacheSink = std::make_shared<zen::logging::RotatingFileSink>(zen::PathToUtf8(CacheLogPath), + auto CacheSink = std::make_shared<zen::logging::RotatingFileSink>(CacheLogPath, /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ false); @@ -73,12 +77,17 @@ InitializeServerLogging(const ZenServerOptions& InOptions) const zen::Oid ServerSessionId = zen::GetSessionId(); - spdlog::apply_all([&](auto Logger) { Logger->info("server session id: {}", ServerSessionId); }); + spdlog::apply_all([&](auto Logger) { + ZEN_MEMSCOPE(ELLMTag::Logging); + Logger->info("server session id: {}", ServerSessionId); + }); } void ShutdownServerLogging() { + ZEN_MEMSCOPE(ELLMTag::Logging); + zen::ShutdownLogging(); } diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 9bc408711..2b157581f 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -2,11 +2,13 @@ #include "frontend.h" +#include <zencore/compactbinarybuilder.h> #include <zencore/endian.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> +#include <zencore/trace.h> ZEN_THIRD_PARTY_INCLUDES_START #if ZEN_PLATFORM_WINDOWS @@ -14,19 +16,30 @@ ZEN_THIRD_PARTY_INCLUDES_START #endif ZEN_THIRD_PARTY_INCLUDES_END +#if !defined(ZEN_EMBED_HTML_ZIP) +# define ZEN_EMBED_HTML_ZIP ZEN_BUILD_RELEASE +#endif + +#if ZEN_EMBED_HTML_ZIP static unsigned char gHtmlZipData[] = { -#include <html.zip.h> +# include <html.zip.h> }; +#endif namespace zen { //////////////////////////////////////////////////////////////////////////////// -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService) +: m_Directory(Directory) +, m_StatusService(StatusService) { + ZEN_TRACE_CPU("HttpFrontendService::HttpFrontendService"); std::filesystem::path SelfPath = GetRunningExecutablePath(); - // Locate a .zip file appended onto the end of this binary +#if ZEN_EMBED_HTML_ZIP + // Load an embedded Zip archive IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1); m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer)); +#endif if (m_Directory.empty() && !m_ZipFs) { @@ -42,7 +55,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { break; } - if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode)) + if (IsFile(ParentPath / "xmake.lua", ErrorCode)) { if (ErrorCode) { @@ -51,7 +64,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html"; - if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + if (IsDir(HtmlDir, ErrorCode)) { m_Directory = HtmlDir; } @@ -73,10 +86,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { ZEN_INFO("front-end is NOT AVAILABLE"); } + m_StatusService.RegisterHandler("dashboard", *this); } HttpFrontendService::~HttpFrontendService() { + m_StatusService.UnregisterHandler("dashboard", *this); } const char* @@ -87,6 +102,14 @@ HttpFrontendService::BaseUri() const //////////////////////////////////////////////////////////////////////////////// void +HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { using namespace std::literals; @@ -125,7 +148,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) // The given content directory overrides any zip-fs discovered in the binary if (!m_Directory.empty()) { - FileContents File = ReadFile(m_Directory / Uri); + auto FullPath = m_Directory / std::filesystem::path(Uri).make_preferred(); + FileContents File = ReadFile(FullPath); if (!File.ErrorCode) { diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 6eac20620..84ffaac42 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -3,23 +3,26 @@ #pragma once #include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> #include "zipfs.h" #include <filesystem> namespace zen { -class HttpFrontendService final : public zen::HttpService +class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpFrontendService(std::filesystem::path Directory); + HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService); virtual ~HttpFrontendService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: ZipFs m_ZipFs; std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip Binary files differindex fa2f2febf..bb3d61198 100644 --- a/src/zenserver/frontend/html.zip +++ b/src/zenserver/frontend/html.zip diff --git a/src/zenserver/frontend/html/favicon.ico b/src/zenserver/frontend/html/favicon.ico Binary files differnew file mode 100644 index 000000000..1cfa301a2 --- /dev/null +++ b/src/zenserver/frontend/html/favicon.ico diff --git a/src/zenserver/frontend/html/index.html b/src/zenserver/frontend/html/index.html index 96b69a643..6a736e914 100644 --- a/src/zenserver/frontend/html/index.html +++ b/src/zenserver/frontend/html/index.html @@ -1,60 +1,15 @@ +<!-- Copyright Epic Games, Inc. All Rights Reserved. --> <!DOCTYPE html> <html> <head> - <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" integrity="sha384-F3w7mX95PdgyTmZZMECAngseQB83DfGTowi0iMjiWaeVhAn4FJkqJByhZMI3AhiU" crossorigin="anonymous"> - <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.min.js" integrity="sha384-skAcpIdS7UcVUC05LJ9Dxay8AXcDYfBJqt1CJ85S/CFujBsIzCIv+l9liuYLaMQ/" crossorigin="anonymous"></script> - <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/font/bootstrap-icons.css"> - <style type="text/css"> - body { - background-color: #fafafa; + <script> + if (window.location.pathname == "/dashboard") + { + window.location.pathname = "/dashboard/"; } - </style> - <script type="text/javascript"> - const getCacheStats = () => { - const opts = { headers: { "Accept": "application/json" } }; - const queryString = window.location.search; - fetch("/stats/z$" + queryString, opts) - .then(response => { - if (!response.ok) { - throw Error(response.statusText); - } - return response.json(); - }) - .then(json => { - document.getElementById("status").innerHTML = "connected" - document.getElementById("stats").innerHTML = JSON.stringify(json, null, 4); - }) - .catch(error => { - document.getElementById("status").innerHTML = "disconnected" - document.getElementById("stats").innerHTML = "" - console.log(error); - }) - .finally(() => { - window.setTimeout(getCacheStats, 1000); - }); - }; - getCacheStats(); </script> + <link rel="shortcut icon" href="favicon.ico"> + <link rel="stylesheet" type="text/css" href="zen.css" /> + <script type="module" src="zen.js"></script> </head> -<body> - <div class="container"> - <div class="row"> - <div class="text-center mt-5"> - <pre> -__________ _________ __ -\____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ - / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ -/_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > - \/ \/ \/ \/ \/ - </pre> - <pre id="status"/> - </div> - </div> - <div class="row"> - <pre class="mb-0">Z$:</pre> - <pre id="stats"></pre> - </div> - </div> -</body> </html> diff --git a/src/zenserver/frontend/html/indexer/cache.js b/src/zenserver/frontend/html/indexer/cache.js new file mode 100644 index 000000000..b90194855 --- /dev/null +++ b/src/zenserver/frontend/html/indexer/cache.js @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +//////////////////////////////////////////////////////////////////////////////// +export class Cache +{ + constructor(db_name, ...store_names) + { + this._db_name = db_name; + this._store_names = store_names; + this._version = 2; + this._db = this._open(); + } + + put(store_name, key, value) + { + const executor = async (resolve, reject) => { + const db = await this._db; + const transaction = db.transaction(store_name, "readwrite"); + const store = transaction.objectStore(store_name); + const request = store.put(value, key); + request.onerror = (evt) => reject(Error("put transaction error")); + request.onsuccess = (evt) => resolve(true); + }; + return new Promise(executor); + } + + get(store_name, key) + { + const executor = async (resolve, reject) => { + const db = await this._db; + const transaction = db.transaction(store_name, "readonly"); + const store = transaction.objectStore(store_name); + const request = store.get(key); + request.onerror = (evt) => reject(Error("get transaction error")); + request.onsuccess = (evt) => { + if (request.result) + resolve(request.result); + else + resolve(false); + }; + }; + return new Promise(executor); + } + + _open() + { + const executor = (resolve, reject) => { + const request = indexedDB.open(this._db_name, this._version); + request.onerror = (evt) => reject(Error("Failed to open IndexedDb")); + request.onsuccess = (evt) => resolve(evt.target.result); + request.onupgradeneeded = (evt) => { + const db = evt.target.result; + + for (const store_name of db.objectStoreNames) + db.deleteObjectStore(store_name) + + for (const store_name of this._store_names) + db.createObjectStore(store_name); + }; + }; + return new Promise(executor); + } +} diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js new file mode 100644 index 000000000..688bc71b0 --- /dev/null +++ b/src/zenserver/frontend/html/indexer/indexer.js @@ -0,0 +1,207 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { Cache } from "./cache.js" +import { Message } from "./worker.js" +import { Fetcher } from "../util/fetcher.js" + +//////////////////////////////////////////////////////////////////////////////// +class Indexer +{ + constructor(pages) + { + this._pages = pages; + } + + lookup_id(entry_id) + { + const bin_search = function(page) { + var l = 0; + var r = page.length; + while (l < r) + { + const mid = l + ((r - l) >> 1); + const d = entry_id - page[mid][0]; + if (d < 0n) r = mid; + else if (d > 0n) l = mid + 1; + else return mid; + } + + return -1; + }; + + for (const page of this._pages) + { + const index = bin_search(page); + if (index >= 0) + return page[index][1]; + } + + return ""; + } + + *search(needle) + { + var needleLwr = needle.toLowerCase(); + for (const page of this._pages) + for (const [_, name] of page) + if (name.toLowerCase().indexOf(needleLwr) >= 0) + yield name; + } + + *enum_names() + { + for (const page of this._pages) + for (const [_, name] of page) + yield name; + } + + *enum_all() + { + for (const page of this._pages) + for (const [_, name, size, raw_size] of page) + yield [name, size|0n, raw_size|0n]; + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +async function save(progress_cb, oplog_info, pages) +{ + const project_id = oplog_info["project"]; + const cache = new Cache(project_id, "pages"); + + const page_count = pages.length; + const puts = new Array(page_count); + for (var i = 0; i < page_count; ++i) + puts[i] = cache.put("pages", i, pages[i]); + + var okay = true + for (var i = 0; i < page_count; ++i) + { + okay &= await puts[i]; + progress_cb("saving", i + 1, page_count); + } + if (!okay) + return false; + + cache.put("pages", "$", { + "page_count" : pages.length, + "total_size" : oplog_info["totalsize"], + "op_count" : oplog_info["opcount"], + "timestamp" : (Date.now() / 1000) | 0, + }); + + return true +} + +//////////////////////////////////////////////////////////////////////////////// +async function build(progress_cb, oplog_info, max_workers=6, page_size=48 << 10) +{ + const project_id = oplog_info["project"]; + const oplog = oplog_info["id"]; + const init_msg = Message.create(Message.Init, project_id, oplog); + + const worker_n = Math.min(navigator.hardwareConcurrency / 2, max_workers); + const stride = page_size * worker_n; + const end = oplog_info["opcount"]; + var entry_count = 0; + + const pages = new Array(); + + const executor = function(index, resolve, reject) { + const worker = new Worker("indexer/worker.js", { type: "module" }); + worker.onerror = (evt) => reject(Error("Worker error")); + worker.onmessage = (evt) => { + const [msg_id, ...params] = evt.data; + switch (msg_id) + { + case Message.MapDone: + resolve(); + worker.terminate(); + break; + + case Message.MapPage: + const [page] = params; + pages.push(page); + entry_count += page.length; + progress_cb("parsing", entry_count, end); + break; + } + } + worker.postMessage(init_msg); + + const start = page_size * index; + const map_msg = Message.create(Message.Map, start, end, page_size, stride); + worker.postMessage(map_msg); + }; + + const workers = [] + for (var i = 0; i < worker_n; ++i) + { + const worker = new Promise((...args) => executor(i, ...args)); + workers.push(worker); + } + + for (const worker of workers) + await worker; + + return pages; +} + +//////////////////////////////////////////////////////////////////////////////// +async function load(progress_cb, oplog_info) +{ + const project_id = oplog_info["project"]; + const cache = new Cache(project_id, "pages"); + const meta = await cache.get("pages", "$"); + + var hit = false; + if (meta) + { + const yesterday = (Date.now() / 1000) - (24 * 60 * 60); + hit = true; + hit &= (meta["total_size"] == oplog_info["totalsize"]); + hit &= (meta["op_count"] == oplog_info["opcount"]); + hit &= (meta["timestamp"] >= yesterday); + } + if (!hit) + return null; + + const page_count = meta["page_count"]; + const gets = new Array(page_count); + const pages = new Array(page_count); + for (var i = 0; i < page_count; ++i) + gets[i] = cache.get("pages", i); + + progress_cb("loading", 0, page_count); + for (var i = 0; i < page_count; ++i) + { + pages[i] = await gets[i]; + progress_cb("loading", i + 1, page_count); + } + + return pages; +} + +//////////////////////////////////////////////////////////////////////////////// +export async function create_indexer(project_id, oplog, progress_cb) +{ + if (!window.Worker) + throw Error("browser does not support web workers"); + + const oplog_info = await new Fetcher() + .resource("prj", project_id, "oplog", oplog) + .json(); + + var pages = await load(progress_cb, oplog_info); + if (!pages) + { + pages = await build(progress_cb, oplog_info); + await save(progress_cb, oplog_info, pages); + } + + return new Indexer(pages); +} diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js new file mode 100644 index 000000000..c0cbb7e11 --- /dev/null +++ b/src/zenserver/frontend/html/indexer/worker.js @@ -0,0 +1,173 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { Fetcher } from "../util/fetcher.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Message +{ + static None = 0; // + static Init = 1; // project_id, oplog + static Map = 2; // start, end, page_size, stride + static MapPage = 3; // page + static MapDone = 4; // + + static create(msg, ...args) { return [msg, ...args]; } +} + + + +//////////////////////////////////////////////////////////////////////////////// +async function map_id_to_key(project_id, oplog, start, end, page_size, stride) +{ + if (start >= end) + return postMessage(Message.create(Message.MapDone)); + + const uri = "/prj/" + project_id + "/oplog/" + oplog + "/entries"; + + const fetch_page = async function(index) { + const cbo = new Fetcher() + .resource(uri) + .param("start", index) + .param("count", page_size) + .param("fieldfilter", "packagedata,bulkdata,key") + .cbo() + + const entry_count = Math.min(page_size, -(index - end)); + return [await cbo, entry_count]; + }; + + var fetch = fetch_page(start); + while (fetch !== undefined) + { + performance.mark("fetch"); + + const [cbo, entry_count] = await fetch; + start += stride; + fetch = (start < end) ? fetch_page(start) : undefined; + + var entries = (await cbo).as_object().find("entries"); + if (entries == undefined) + break; + + entries = entries.as_array(); + if (entries.num() == 0) + break; + + performance.mark("build"); + var count = 0; + var result = new Array(entry_count); + for (var entry of entries) + { + if (!entry.is_object()) + continue + entry = entry.as_object(); + + var key = undefined; + var pkg_data = undefined; + var bulk_data = undefined; + for (const field of entry) + { + if (field.is_named("key")) key = field; + else if (field.is_named("packagedata")) pkg_data = field; + else if (field.is_named("bulkdata")) bulk_data = field; + } + + if (key == undefined) + continue; + + var id = 0n; + var size = 0n; + var raw_size = 0n; + + if (pkg_data) + { + for (const item of pkg_data.as_array()) + { + var found = 0, pkg_id = undefined; + for (const field of item.as_object()) + { + if (!id && field.is_named("id")) pkg_id = field.as_value(); + else if (field.is_named("size")) size += field.as_value(); + else if (field.is_named("rawsize")) raw_size += field.as_value(); + else continue; + if (found++ >= 3) + break; + } + + if (pkg_id === undefined) + continue; + + pkg_id = pkg_id.subarray(0, 8); + for (var i = 7; i >= 0; --i) + { + id <<= 8n; + id |= BigInt(pkg_id[i]); + } + } + } + + if (bulk_data) + { + for (const item of bulk_data.as_array()) + { + var found = 0; + for (const field of item.as_object()) + { + if (field.is_named("size")) size += field.as_value(); + else if (field.is_named("rawsize")) raw_size += field.as_value(); + else continue; + if (found++ >= 2) + break; + } + } + } + + result[count] = [id, key.as_value(), size, raw_size]; + count++; + } + + if (count == 0) + continue; + + if (count != result.length) + result = result.slice(0, count); + + performance.mark("sort"); + result.sort(function(l, r) { return Number(l[0] - r[0]); }); + + const msg = Message.create(Message.MapPage, result); + postMessage(msg); + } + + postMessage(Message.create(Message.MapDone)); +} + +//////////////////////////////////////////////////////////////////////////////// +function worker_scope() +{ + var project_id; + var oplog; + + return (evt) => { + const [msg_id, ...params] = evt.data; + switch (msg_id) + { + case Message.Init: + [project_id, oplog] = params; + break; + + case Message.Map: + var [start, end, page_size, stride] = params; + map_id_to_key(project_id, oplog, start, end, page_size, stride); + break; + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +if (typeof DedicatedWorkerGlobalScope != "undefined" && self instanceof DedicatedWorkerGlobalScope) +{ + onmessage = worker_scope(); +} diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js new file mode 100644 index 000000000..08589b090 --- /dev/null +++ b/src/zenserver/frontend/html/pages/entry.js @@ -0,0 +1,295 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Table, PropTable, Toolbar, ProgressBar } from "../util/widgets.js" +import { create_indexer } from "../indexer/indexer.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + main() + { + this.set_title("oplog entry"); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const opkey = this.get_param("opkey"); + + this._entry = new Fetcher() + .resource("prj", project, "oplog", oplog, "entries") + .param("opkey", opkey) + .cbo(); + + this._indexer = this.load_indexer(project, oplog); + + this._build_page(); + } + + async load_indexer(project, oplog, loaded_cb) + { + const progress_bar = this.add_widget(ProgressBar); + progress_bar.set_progress("indexing"); + const indexer = await create_indexer(project, oplog, (...args) => { + progress_bar.set_progress(...args); + }); + progress_bar.destroy(); + return indexer; + } + + async _build_deps(section, tree) + { + const indexer = await this._indexer; + + for (const dep_name in tree) + { + const dep_section = section.add_section(dep_name); + const table = dep_section.add_widget(Table, ["name", "id"], Table.Flag_PackRight); + for (const dep_id of tree[dep_name]) + { + const cell_values = ["", dep_id.toString(16).padStart(16, "0")]; + const row = table.add_row(...cell_values); + + var opkey = indexer.lookup_id(dep_id); + row.get_cell(0).text(opkey).on_click((k) => this.view_opkey(k), opkey); + } + } + } + + _find_iohash_field(container, name) + { + const found_field = container.find(name); + if (found_field != undefined) + { + var found_value = found_field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + return ret; + } + } + return null; + } + + async _build_meta(section, entry) + { + var tree = {} + + for (const field of entry) + { + var visibleKey = undefined; + const name = field.get_name(); + if (name == "CookPackageArtifacts") + { + visibleKey = name; + } + else if (name.startsWith("meta.")) + { + visibleKey = name.slice(5); + } + + if (visibleKey != undefined) + { + var found_value = field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + tree[visibleKey] = ret; + } + } + + } + + if (Object.keys(tree).length == 0) + return; + + const sub_section = section.add_section("meta"); + + const table = sub_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + for (const key in tree) + { + const row = table.add_row(key); + const value = tree[key]; + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + ); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); + } + } + + async _build_page() + { + var entry = await this._entry; + entry = entry.as_object().find("entry").as_object(); + + const name = entry.find("key").as_value(); + var section = this.add_section(name); + + // tree + { + var tree = entry.find("$tree"); + if (tree == undefined) + tree = this._convert_legacy_to_tree(entry); + + if (tree == undefined) + return this._display_unsupported(section, entry); + + delete tree["$id"]; + + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("deps"); + this._build_deps(sub_section, tree); + } + } + + // meta + { + this._build_meta(section, entry); + } + + // data + { + const sub_section = section.add_section("data"); + const table = sub_section.add_widget( + Table, + ["name", "size", "rawsize", "actions"], Table.Flag_PackRight + ); + table.id("datatable"); + for (const field_name of ["packagedata", "bulkdata"]) + { + var pkg_data = entry.find(field_name); + if (pkg_data == undefined) + continue; + + for (const item of pkg_data.as_array()) + { + var io_hash, size, raw_size, file_name; + for (const field of item.as_object()) + { + if (field.is_named("data")) io_hash = field.as_value(); + else if (field.is_named("filename")) file_name = field.as_value(); + else if (field.is_named("size")) size = field.as_value(); + else if (field.is_named("rawsize")) raw_size = field.as_value(); + } + + if (io_hash instanceof Uint8Array) + { + var ret = ""; + for (var x of io_hash) + ret += x.toString(16).padStart(2, "0"); + io_hash = ret; + } + + size = (size !== undefined) ? Friendly.kib(size) : ""; + raw_size = (raw_size !== undefined) ? Friendly.kib(raw_size) : ""; + + const row = table.add_row(file_name, size, raw_size); + + var base_name = file_name.split("/").pop().split("\\").pop(); + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, io_hash].join("/") + ); + link.first_child().attr("download", `${io_hash}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, io_hash); + } + } + } + + // props + { + const object = entry.to_js_object(); + var sub_section = section.add_section("props"); + sub_section.add_widget(PropTable).add_object(object); + } + } + + _display_unsupported(section, entry) + { + const replacer = (key, value) => + typeof value === "bigint" ? { $bigint: value.toString() } : value; + + const object = entry.to_js_object(); + const text = JSON.stringify(object, replacer, " "); + section.tag("pre").text(text); + } + + _convert_legacy_to_tree(entry) + { + const raw_pkgst_entry = entry.find("packagestoreentry"); + if (raw_pkgst_entry == undefined) //if there is no packagestorentry then don't show the fancy webpage, just show the raw json + return; + + const tree = {}; + + const pkg_data = entry.find("packagedata"); + if (pkg_data) + { + var id = 0n; + for (var item of pkg_data.as_array()) + { + var pkg_id = item.as_object().find("id"); + if (pkg_id == undefined) + continue; + + pkg_id = pkg_id.as_value().subarray(0, 8); + for (var i = 7; i >= 0; --i) + { + id <<= 8n; + id |= BigInt(pkg_id[i]); + } + break; + } + tree["$id"] = id; + } + + const pkgst_entry = raw_pkgst_entry.as_object(); + + for (const field of pkgst_entry) + { + const field_name = field.get_name(); + if (!field_name.endsWith("importedpackageids")) + continue; + + var dep_name = field_name.slice(0, -18); + if (dep_name.length == 0) + dep_name = "imported"; + + var out = tree[dep_name] = []; + for (var item of field.as_array()) + out.push(item.as_value(BigInt)); + } + + return tree; + } + + view_opkey(opkey) + { + const params = this._params; + params.set("opkey", opkey); + window.location.search = params; + } +} diff --git a/src/zenserver/frontend/html/pages/map.js b/src/zenserver/frontend/html/pages/map.js new file mode 100644 index 000000000..58046b255 --- /dev/null +++ b/src/zenserver/frontend/html/pages/map.js @@ -0,0 +1,166 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Friendly } from "../util/friendly.js" +import { ProgressBar } from "../util/widgets.js" +import { create_indexer } from "../indexer/indexer.js" + + +//////////////////////////////////////////////////////////////////////////////// +function squarify(weights, callback, area_threshold=-1) +{ + const rect = [1.0, 1.0]; + for (var start = 0; start < weights.length;) + { + const ri = +(rect[0] >= rect[1]); + + const length = rect[ri]; + var end = start; + var area = 0; + var prev_rd = Infinity; + for (; end < weights.length; ++end) + { + const w = (area + weights[end]) / length; + const r = weights[end] / (w * w); + const rd = Math.abs(1.0 - r); + if (prev_rd < rd) + break; + prev_rd = rd; + area += weights[end]; + } + const v = area / length; + + const tl = [1.0 - rect[0], 1.0 - rect[1]]; + const wh = [undefined, undefined]; + for (var i = start; i < end; ++i) + { + wh[ri ^ 0] = weights[i] / v; + wh[ri ^ 1] = v; + callback(i, tl[0], tl[1], wh[0], wh[1], ri); + tl[ri] += wh[ri]; + } + + start = end; + rect[ri ^ 1] -= v; + + if (rect[0] * rect[1] < area_threshold) + break; + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + main() + { + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + this._indexer = this._load_indexer(project, oplog); + + this.set_title("map"); + + const section = this.add_section(project + " - " + oplog); + this._build(section); + } + + async _load_indexer(project, oplog) + { + const progress_bar = this.add_widget(ProgressBar); + progress_bar.set_progress("indexing"); + var indexer = create_indexer(project, oplog, (...args) => { + progress_bar.set_progress(...args); + }); + indexer = await indexer; + progress_bar.destroy(); + return indexer; + } + + async _build(section) + { + const indexer = await this._indexer; + + var prefix = this.get_param("path", "/"); + if (!prefix.endsWith("/")) + prefix += "/"; + + var total_size = 0; + var branch_size = 0; + const new_nodes = new Object(); + for (var [name, size] of indexer.enum_all()) + { + total_size += size; + if (!name.startsWith(prefix)) + continue; + + branch_size += size; + + name = name.substr(prefix.length); + const slash = name.indexOf("/"); + if (slash != -1) + name = name.substr(0, slash + 1); + + if (new_nodes[name] !== undefined) + new_nodes[name] += size; + else + new_nodes[name] = size; + } + + const sorted_keys = Object.keys(new_nodes).sort((l, r) => { + return new_nodes[r] - new_nodes[l]; + }); + const nodes = new Array(); + for (const name of sorted_keys) + nodes.push(new_nodes[name] / branch_size); + + var stats = Friendly.kib(branch_size); + stats += " / "; + stats += Friendly.kib(total_size); + stats += " ("; + stats += 0|((branch_size * 100) / total_size); + stats += "%)"; + section.tag().text(prefix + " : " + stats); + const treemap = section.tag().id("treemap"); + const canvas = treemap.tag("canvas").inner(); + + const width = canvas.offsetWidth; + var height = window.visualViewport.height; + height -= treemap.inner().getBoundingClientRect().top + window.scrollY; + height -= 50; + + canvas.width = canvas.offsetWidth; + canvas.height = height; + const context = canvas.getContext("2d"); + context.textBaseline = "top"; + context.imageSmoothingEnabled = false; + context.font = "13px sans-serif"; + context.strokeStyle = "#666666"; + + const palette = [ + "#8dd3c7", "#ffffb3", "#bebada", "#fb8072", "#80b1d3", "#fdb462", + "#b3de69", "#fccde5", "#d9d9d9", "#bc80bd", "#ccebc5", + ]; + + const callback = (i, x, y, w, h, d) => { + const r = function(u,v) { return Math.floor(u * (v - 1e-7)); }; + x = r(x, width); + y = r(y, height); + w = r(w, width); + h = r(h, height); + context.save(); + context.beginPath(); + context.rect(x, y, w, h); + context.clip(); + context.fillStyle = palette[(i * 0x493) % palette.length]; + context.fill(); + context.stroke(); + context.fillStyle = "#000000"; + context.fillText(sorted_keys[i], x + 4, y + 4); + context.restore(); + }; + squarify(nodes, callback, 0.01); + } +} diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js new file mode 100644 index 000000000..bef5bacce --- /dev/null +++ b/src/zenserver/frontend/html/pages/oplog.js @@ -0,0 +1,179 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Table, Toolbar, ProgressBar } from "../util/widgets.js" +import { create_indexer } from "../indexer/indexer.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + constructor(...args) + { + super(...args); + + this._index_start = Number(this.get_param("start", 0)) || 0; + this._index_count = Number(this.get_param("count", 50)) || 0; + } + + async main() + { + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + + var oplog_info = new Fetcher() + .resource("prj", project, "oplog", oplog) + .json(); + + this._indexer = this._load_indexer(project, oplog); + + this.set_title("oplog - " + oplog); + + var section = this.add_section(project + " - " + oplog); + + oplog_info = await oplog_info; + this._index_max = oplog_info["opcount"]; + this._build_nav(section, oplog_info); + + this._entry_table = section.add_widget(Table, ["key"]); + await this._build_table(this._index_start); + } + + async _load_indexer(project, oplog) + { + const progress_bar = this.add_widget(ProgressBar); + progress_bar.set_progress("indexing"); + var indexer = create_indexer(project, oplog, (...args) => { + progress_bar.set_progress(...args); + }); + indexer = await indexer; + progress_bar.destroy(); + return indexer; + } + + _build_nav(section, oplog_info) + { + const nav = section.add_widget(Toolbar); + const left = nav.left(); + left.add("|<") .on_click(() => this._on_next_prev(-10e10)); + left.add("<<").on_click(() => this._on_next_prev(-10)); + left.add("prev") .on_click(() => this._on_next_prev( -1)); + left.add("next") .on_click(() => this._on_next_prev( 1)); + left.add(">>").on_click(() => this._on_next_prev( 10)); + left.add(">|") .on_click(() => this._on_next_prev( 10e10)); + + left.sep(); + for (var count of [10, 25, 50, 100]) + { + var handler = (n) => this._on_change_count(n); + left.add(count).on_click(handler, count); + } + + left.sep(); + left.add("tree").link("", { + "page" : "tree", + "project" : this.get_param("project"), + "oplog" : this.get_param("oplog"), + }); + + const right = nav.right(); + right.add(Friendly.sep(oplog_info["opcount"])); + right.add("(" + Friendly.kib(oplog_info["totalsize"]) + ")"); + right.sep(); + + var search_input = right.add("search:", "label").tag("input") + search_input.on("change", (x) => this._search(x.inner().value), search_input); + } + + async _build_table(index) + { + this._index_count = Math.max(this._index_count, 1); + index = Math.min(index, this._index_max - this._index_count); + index = Math.max(index, 0); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + + var entries = new Fetcher() + .resource("prj", project, "oplog", oplog, "entries") + .param("start", index) + .param("count", this.set_param("count", this._index_count)) + .json(); + + entries = (await entries)["entries"]; + if (entries == undefined) + return; + + if (entries.length == 0) + return; + + this._entry_table.clear(index); + for (const entry of entries) + { + var row = this._entry_table.add_row(entry["key"]); + + row.get_cell(0).link("", { + "page" : "entry", + "project" : project, + "oplog" : oplog, + "opkey" : entry["key"], + }); + } + + this.set_param("start", index); + this._index_start = index; + } + + _on_change_count(value) + { + this._index_count = parseInt(value); + this._build_table(this._index_start); + } + + _on_next_prev(direction) + { + var index = this._index_start + (this._index_count * direction); + index = Math.max(0, index); + this._build_table(index); + } + + async _search(needle) + { + if (needle.length == 0) + { + this._build_table(this._index_start); + return; + } + needle = needle.trim(); + + this._entry_table.clear(this._index_start); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + + const indexer = await this._indexer; + + var added = 0; + const truncate_at = this.get_param("searchmax") || 250; + for (var name of indexer.search(needle)) + { + var row = this._entry_table.add_row(name); + + row.get_cell(0).link("", { + "page" : "entry", + "project" : project, + "oplog" : oplog, + "opkey" : name, + }); + + if (++added >= truncate_at) + { + this._entry_table.add_row("...truncated"); + break; + } + } + } +} diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js new file mode 100644 index 000000000..9a9541904 --- /dev/null +++ b/src/zenserver/frontend/html/pages/page.js @@ -0,0 +1,128 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { WidgetHost } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class PageBase extends WidgetHost +{ + constructor(parent, params) + { + super(parent) + this._params = params; + } + + set_title(name) + { + var value = document.title; + if (name.length && value.length) + name = value + " - " + name; + document.title = name; + } + + get_param(name, fallback=undefined) + { + var ret = this._params.get(name); + if (ret != undefined) + return ret; + + if (fallback != undefined) + this.set_param(name, fallback); + + return fallback; + } + + set_param(name, value, update=true) + { + this._params.set(name, value); + if (!update) + return value; + + const url = new URL(window.location); + for (var [key, xfer] of this._params) + url.searchParams.set(key, xfer); + history.replaceState(null, "", url); + + return value; + } + + reload() + { + window.location.reload(); + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class ZenPage extends PageBase +{ + constructor(parent, ...args) + { + super(parent, ...args); + super.set_title("zen"); + this.add_branding(parent); + this.generate_crumbs(); + } + + add_branding(parent) + { + var root = parent.tag().id("branding"); + + const zen_store = root.tag("pre").id("logo").text( + "_________ _______ __\n" + + "\\____ /___ ___ / ___// |__ ___ ______ ____\n" + + " / __/ __ \\ / \\ \\___ \\\\_ __// \\\\_ \\/ __ \\\n" + + " / \\ __// | \\/ \\| | ( - )| |\\/\\ __/\n" + + "/______/\\___/\\__|__/\\______/|__| \\___/ |__| \\___|" + ); + zen_store.tag().id("go_home").on_click(() => window.location.search = ""); + + root.tag("img").attr("src", "favicon.ico").id("ue_logo"); + + /* + _________ _______ __ + \____ /___ ___ / ___// |__ ___ ______ ____ + / __/ __ \ / \ \___ \\_ __// \\_ \/ __ \ + / \ __// | \/ \| | ( - )| |\/\ __/ + /______/\___/\__|__/\______/|__| \___/ |__| \___| + */ + } + + set_title(...args) + { + super.set_title(...args); + } + + generate_crumbs() + { + const auto_name = this.get_param("page") || "start"; + if (auto_name == "start") + return; + + const crumbs = this.tag().id("crumbs"); + const new_crumb = function(name, search=undefined) { + crumbs.tag(); + var crumb = crumbs.tag().text(name); + if (search != undefined) + crumb.on_click((x) => window.location.search = x, search); + }; + + new_crumb("home", ""); + + var project = this.get_param("project"); + if (project != undefined) + { + var oplog = this.get_param("oplog"); + if (oplog != undefined) + { + new_crumb("project", `?page=project&project=${project}`); + if (this.get_param("opkey")) + new_crumb("oplog", `?page=oplog&project=${project}&oplog=${oplog}`); + } + } + + new_crumb(auto_name.toLowerCase()); + } +} diff --git a/src/zenserver/frontend/html/pages/project.js b/src/zenserver/frontend/html/pages/project.js new file mode 100644 index 000000000..42ae30c8c --- /dev/null +++ b/src/zenserver/frontend/html/pages/project.js @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" +import { Table, PropTable, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + // info + var section = this.add_section("info"); + + const project = this.get_param("project"); + + this.set_title("project - " + project); + + var info = await new Fetcher().resource("prj", project).json(); + var prop_table = section.add_widget(PropTable); + for (const key in info) + { + if (key == "oplogs") + continue; + + prop_table.add_property(key, info[key]); + } + + // oplog + section = this.add_section("oplogs"); + + var oplog_table = section.add_widget( + Table, + ["name", "marker", "size", "ops", "expired", "actions"], + Table.Flag_PackRight + ) + + var count = 0; + for (const oplog of info["oplogs"]) + { + const name = oplog["id"]; + + var info = new Fetcher().resource("prj", project, "oplog", name).json(); + + var row = oplog_table.add_row(name); + + var cell = row.get_cell(0); + this.as_link(cell, "oplog", name) + + cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true).left(); + this.as_link(action_tb.add("list"), "oplog", name); + this.as_link(action_tb.add("tree"), "tree", name); + action_tb.add("drop").on_click((x) => this.drop_oplog(x), name); + + info = await info; + row.get_cell(1).text(info["markerpath"]); + row.get_cell(2).text(Friendly.kib(info["totalsize"])); + row.get_cell(3).text(Friendly.sep(info["opcount"])); + row.get_cell(4).text(info["expired"]); + } + } + + as_link(component, page, oplog_id) + { + component.link("", { + "page" : page, + "project" : this.get_param("project"), + "oplog" : oplog_id, + }); + } + + drop_oplog(oplog_id) + { + const drop = async () => { + await new Fetcher() + .resource("prj", this.get_param("project"), "oplog", oplog_id) + .delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop oplog '${oplog_id}'?`) + .option("Yes", () => drop()) + .option("No"); + } +} diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js new file mode 100644 index 000000000..d1c13ccc7 --- /dev/null +++ b/src/zenserver/frontend/html/pages/start.js @@ -0,0 +1,149 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" +import { Table, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + var section = this.add_section("projects"); + + // project list + var columns = [ + "name", + "project_dir", + "engine_dir", + "actions", + ]; + var table = section.add_widget(Table, columns); + + for (const project of await new Fetcher().resource("/prj/list").json()) + { + var row = table.add_row( + "", + project.ProjectRootDir, + project.EngineRootDir, + ); + + var cell = row.get_cell(0); + cell.tag().text(project.Id).on_click((x) => this.view_project(x), project.Id); + + var cell = row.get_cell(-1); + var action_tb = new Toolbar(cell, true); + action_tb.left().add("view").on_click((x) => this.view_project(x), project.Id); + action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id); + } + + // cache + var section = this.add_section("z$"); + columns = [ + "namespace", + "dir", + "buckets", + "entries", + "size disk", + "size mem", + "actions", + ] + var zcache_info = new Fetcher().resource("/z$/").json(); + const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight); + for (const namespace of (await zcache_info)["Namespaces"]) + { + new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => { + const row = cache_table.add_row( + "", + data["Configuration"]["RootDir"], + data["Buckets"].length, + data["EntryCount"], + Friendly.kib(data["StorageSize"].DiskSize), + Friendly.kib(data["StorageSize"].MemorySize) + ); + var cell = row.get_cell(0); + cell.tag().text(namespace).on_click(() => this.view_zcache(namespace)); + row.get_cell(1).tag().text(namespace); + + cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view").on_click(() => this.view_zcache(namespace)); + action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace)); + }); + } + + // stats + section = this.add_section("stats"); + columns = [ + "name", + "req count", + "size disk", + "size mem", + "cid total", + ]; + const stats_table = section.add_widget(Table, columns, Table.Flag_PackRight); + var providers = new Fetcher().resource("stats").json(); + for (var provider of (await providers)["providers"]) + { + var stats = await new Fetcher().resource("stats", provider).json(); + var values = [""]; + try { + values.push(stats.requests.count); + const size_stat = (stats.store || stats.cache).size; + values.push(Friendly.kib(size_stat.disk)); + values.push(Friendly.kib(size_stat.memory)); + values.push(stats.cid.size.total); + } + catch {} + row = stats_table.add_row(...values); + row.get_cell(0).tag().text(provider).on_click((x) => this.view_stat(x), provider); + } + } + + view_stat(provider) + { + window.location = "?page=stat&provider=" + provider; + } + + view_project(project_id) + { + window.location = "?page=project&project=" + project_id; + } + + drop_project(project_id) + { + const drop = async () => { + await new Fetcher().resource("prj", project_id).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop project '${project_id}'?`) + .option("Yes", () => drop()) + .option("No"); + } + + view_zcache(namespace) + { + window.location = "?page=zcache&namespace=" + namespace; + } + + drop_zcache(namespace) + { + const drop = async () => { + await new Fetcher().resource("z$", namespace).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop zcache '${namespace}'?`) + .option("Yes", () => drop()) + .option("No"); + } +} diff --git a/src/zenserver/frontend/html/pages/stat.js b/src/zenserver/frontend/html/pages/stat.js new file mode 100644 index 000000000..c7902d5ed --- /dev/null +++ b/src/zenserver/frontend/html/pages/stat.js @@ -0,0 +1,153 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { PropTable, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +class TemporalStat +{ + constructor(data, as_bytes) + { + this._data = data; + this._as_bytes = as_bytes; + } + + toString() + { + const columns = [ + /* count */ {}, + /* rate */ {}, + /* t */ {}, {}, + ]; + const data = this._data; + for (var key in data) + { + var out = columns[0]; + if (key.startsWith("rate_")) out = columns[1]; + else if (key.startsWith("t_p")) out = columns[3]; + else if (key.startsWith("t_")) out = columns[2]; + out[key] = data[key]; + } + + var friendly = this._as_bytes ? Friendly.kib : Friendly.sep; + + var content = ""; + for (var i = 0; i < columns.length; ++i) + { + content += "<pre>"; + const column = columns[i]; + for (var key in column) + { + var value = column[key]; + if (i) + { + value = Friendly.sep(value, 2); + key = key.padStart(9); + content += key + ": " + value; + } + else + content += friendly(value); + content += "\n"; + } + content += "</pre>"; + } + + return content; + } +} + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + const provider = this.get_param("provider", "z$"); + var stats = new Fetcher() + .resource("stats", provider) + .param("cidstorestats", "true") + .param("cachestorestats", "true") + .json(); + + this.set_title("stat - " + provider); + const section = this.add_section(provider); + + var toolbar = section.add_widget(Toolbar); + var tb_right = toolbar.right(); + tb_right.add("filter:"); + tb_right.add("-none-").on_click((x) => this.update_filter("")); + for (var preset of ["read.", "write.", ".request", ".bytes"]) + tb_right.add(preset).on_click((x) => this.update_filter(x), preset); + this._filter_input = tb_right.add("", "label").tag("input"); + this._filter_input.on("change", (x) => this.update_filter(x.inner().value), this._filter_input); + + this._table = section.add_widget(PropTable); + + this._stats = stats = await stats; + this._condense(stats); + + var first = undefined; + for (var name in stats) + { + first = first || name; + toolbar.left().add(name).on_click((x) => this.view_category(x), name); + } + + var filter = this.get_param("filter"); + + first = this.get_param("view", first); + this.view_category(first); + + if (filter) + this.update_filter(filter); + } + + view_category(name) + { + const friendly = (this.get_param("raw") == undefined); + this._table.clear(); + this._table.add_object(this._stats[name], friendly, 3); + this.set_param("view", name); + this.update_filter(""); + } + + update_filter(needle) + { + this._filter_input.attr("value", needle); + + this.set_param("filter", needle); + if (!needle) + return this._table.filter(); + + var needles = needle.split(" "); + this._table.filter(...needles); + } + + _condense(stats) + { + const impl = function(node) + { + for (var name in node) + { + const candidate = node[name]; + if (!(candidate instanceof Object)) + continue; + + if (candidate["rate_mean"] != undefined) + { + const as_bytes = (name.indexOf("bytes") >= 0); + node[name] = new TemporalStat(candidate, as_bytes); + continue; + } + + impl(candidate); + } + } + + for (var name in stats) + impl(stats[name]); + } +} diff --git a/src/zenserver/frontend/html/pages/test.js b/src/zenserver/frontend/html/pages/test.js new file mode 100644 index 000000000..2a84ff163 --- /dev/null +++ b/src/zenserver/frontend/html/pages/test.js @@ -0,0 +1,147 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Table, PropTable, Toolbar, ProgressBar } from "../util/widgets.js" +import { Modal, } from "../util/modal.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + main() + { + var gen_word = (function() { + var s = 0x314251; + var r = function(a, b) { + s = (s * 0x493) & 0x7fffffff; + return ((s >> 3) % (b - a)) + a; + }; + return function(a=5, b=10) { + const co = "aeioubcdfghjklmnpqrstvwxyz"; + var ret = ""; + for (var i = 0, n = r(a,b); i < n; ++i) + ret += co[r(0, co.length)]; + return ret; + }; + })(); + var gen_para = function(a=5, b=10, s=" ") { + var ret = gen_word(2, 9); + for (var i = 0; i < ((ret.length * 0x493) % (b - a)) + b; ++i) + ret += s + gen_word(2, 9); + return ret; + } + + this.set_title("test"); + + // swatches + const swatches = this.tag() + .style("position", "absolute") + .style("top", "3.5em") + .style("left", "3.5em") + for (var suffix of ["g0", "g1", "g2", "g3", "g4", + "p0", "p1", "p2", "p3", "p4", + "ln", "er"]) + { + swatches.tag() + .style("float", "left") + .style("width", "2em") + .style("height", "2em") + .style("background-color", `var(--theme_${suffix})`) + .text(suffix); + } + + // section + var section0 = this.add_section("section"); + var section1 = section0.add_section("sub-section"); + var section2 = section1.add_section("sub-sub-section"); + + // table + const cols = [gen_word(), gen_word(), gen_word(), gen_word()]; + var tables = [ + section0.add_widget(Table, cols), + section1.add_widget(Table, cols, Table.Flag_EvenSpacing, 5), + section2.add_widget(Table, cols, Table.Flag_EvenSpacing, -1), + ]; + + for (const table of tables) + { + table.add_row(gen_word()); + table.add_row(gen_word(), gen_word(), gen_word(), gen_word()); + table.add_row(gen_word(), gen_word(), gen_para(15, 25), gen_word(), gen_word(), gen_word(), gen_word(), gen_word()); + } + + // spacing tests + { + const spacing_section = section0.add_section("spacing"); + const flags = { + "EvenSpacing" : Table.Flag_EvenSpacing, + "EvenSpacing|BiasLeft" : Table.Flag_EvenSpacing | Table.Flag_BiasLeft, + "PackRight" : Table.Flag_PackRight, + }; + for (const flag_name in flags) + { + const flag = flags[flag_name]; + const another_table = spacing_section.add_widget( + Table, + [flag_name, gen_word(), gen_word(), gen_word(), gen_word()], + flag, + ); + for (var i = 0; i < 3; ++i) + another_table.add_row(gen_para(1, 5), gen_para(1, 3), gen_word(), gen_word(), gen_word()); + } + } + + // prop-table + var pt_section = section0.add_section("prop-table") + var prop_table = pt_section.add_widget(PropTable); + for (var i = 0; i < 7; ++i) + prop_table.add_property(gen_word(), gen_para(1, 20, "/")); + + // misc + const misc_section = section0.add_section("misc").add_section("misc"); + misc_section.tag().text("just text"); + misc_section.tag().text("this is a link").link(); + misc_section.tag().text("MODAL DIALOG").on_click((e) => { + new Modal() + .title("modal") + .message("here is a message what I wrote") + .option("press me!", () => { alert("hi"); }) + .option("cancel", () => void(0)); + }); + + // toolbar + pt_section.add_section("toolbar"); + var toolbar = pt_section.add_widget(Toolbar); + for (const side of [toolbar.left(), toolbar.right()]) + { + side.add("tb_item0"); + side.add("tb_item1"); + side.sep(); + side.add("tb_item2"); + } + + var tb_item_clicked = function(arg0, arg1) { + alert(arg0 + " != " + arg1); + }; + var row = prop_table.add_property("toolbar", ""); + toolbar = new Toolbar(row.get_cell(-1), true); + toolbar.left() .add("tbitem0").on_click(tb_item_clicked, 11, -22); + toolbar.left() .add("tbitem1").on_click(tb_item_clicked, 22, -33); + toolbar.right().add("tbitem2").on_click(tb_item_clicked, 33, -55); + toolbar.right().add("tbitem3").on_click(tb_item_clicked, 44, -88); + + // progress bar + const progress_bar = this.add_widget(ProgressBar); + setInterval(function() { + var count = 0 + return () => { + count = (count + 1) % 100; + progress_bar.set_progress("testing", count, 100); + }; + }(), 49.3); + + // error + throw Error("deliberate error"); + } +} diff --git a/src/zenserver/frontend/html/pages/tree.js b/src/zenserver/frontend/html/pages/tree.js new file mode 100644 index 000000000..08a578492 --- /dev/null +++ b/src/zenserver/frontend/html/pages/tree.js @@ -0,0 +1,177 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Friendly } from "../util/friendly.js" +import { ProgressBar } from "../util/widgets.js" +import { create_indexer } from "../indexer/indexer.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + main() + { + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const sort_by = this.get_param("sort", -1); + + this._indexer = this._load_indexer(project, oplog); + + this.set_title("tree - " + oplog); + const section = this.add_section(project + " - " + oplog); + + this._create_tree(section); + this._expand(this._root); + this._activate_sort_header(-1, sort_by); + } + + _create_tree(section) + { + const list = section.tag().id("tree_root").tag("ul"); + const root = list.tag("li"); + root.attr("part", "/"); + const header = root.tag(); + header.tag().text("/"); + this._sort_headers = [ + header.tag().text("size").on_click(() => this._change_sort(0)), + header.tag().text("rawsize").on_click(() => this._change_sort(1)), + header.tag().text("count").on_click(() => this._change_sort(2)), + ]; + this._root = root; + } + + async _load_indexer(project, oplog) + { + const progress_bar = this.add_widget(ProgressBar); + progress_bar.set_progress("indexing"); + var indexer = create_indexer(project, oplog, (...args) => { + progress_bar.set_progress(...args); + }); + indexer = await indexer; + progress_bar.destroy(); + return indexer; + } + + async _expand(node) + { + var prefix = ""; + for (var item = node;; item = item.parent()) + { + if (item.is("div")) break; + if (!item.is("li")) continue; + prefix = item.attr("part") + prefix; + } + + const indexer = await this._indexer; + + const new_nodes = new Object(); + for (var [name, size, raw_size] of indexer.enum_all()) + { + if (!name.startsWith(prefix)) + continue; + + name = name.substr(prefix.length); + const slash = name.indexOf("/"); + if (slash != -1) + name = name.substr(0, slash + 1); + + if (new_nodes[name] !== undefined) + { + new_nodes[name][0] += size; + new_nodes[name][1] += raw_size; + new_nodes[name][2] += 1; + } + else + new_nodes[name] = [size, raw_size, 1]; + } + + var sort_by = this.get_param("sort", -1)|0; + sort_by = Math.min(Math.max(sort_by, -1), 3); + + const sorted_keys = Object.keys(new_nodes).sort((l, r) => { + const is_node_l = l.endsWith("/"); + const any_nodes = is_node_l + r.endsWith("/"); + if (any_nodes == 1) return is_node_l ? -1 : 1; + if (sort_by >= 0) return Number(new_nodes[r][sort_by] - new_nodes[l][sort_by]); + return r < l; + }) + + const list = node.tag("ul"); + for (const name of sorted_keys) + { + const item = list.tag("li").attr("part", name); + const info = item.tag(); + const label = info.tag().text(name); + + for (var i = 0; i < 2; ++i) + { + const size = Friendly.kib(new_nodes[name][i]); + info.tag().text(size); + } + + if (name.endsWith("/")) + { + const count = Friendly.sep(new_nodes[name][2]); + info.tag().text(count); + label.on_click((x) => this.expand_collapse(x), item); + continue; + } + + item.attr("leaf", ""); + label.link("", { + "page" : "entry", + "project" : this.get_param("project"), + "oplog" : this.get_param("oplog"), + "opkey" : prefix + name, + }); + info.tag(); + } + + node.attr("expanded", "") + } + + _collapse(node) + { + node.first_child().next_sibling().destroy(); + node.attr("expanded", null); + } + + expand_collapse(node) + { + if (node.attr("expanded") === null) + return this._expand(node); + return this._collapse(node); + } + + _activate_sort_header(current, next) + { + const impl = (index, is_on) => { + if (index >= 0 && index < this._sort_headers.length) + this._sort_headers[index].attr("active", is_on ? "" : null); + }; + impl(current, false); + impl(next, true); + } + + _change_sort(sort_by) + { + const current = this.get_param("sort"); + if (current == sort_by) + sort_by = -1; + + this._activate_sort_header(current, sort_by); + + this.set_param("sort", sort_by); + + for (var node = this._root.first_child(); node;) + { + const next = node.next_sibling(); + if (node.is("ul")) + node.destroy(); + node = next; + } + + this._expand(this._root); + } +} diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js new file mode 100644 index 000000000..974893b21 --- /dev/null +++ b/src/zenserver/frontend/html/pages/zcache.js @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" +import { Table, PropTable, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + const namespace = this.get_param("namespace"); + + var info = new Fetcher().resource(`/z$/${namespace}/`).json(); + + this.set_title("cache - " + namespace); + + var section = this.add_section("info"); + var cfg_table = section.add_section("config").add_widget(PropTable); + var storage_table = section.add_section("storage").add_widget(PropTable); + + info = await info; + + cfg_table.add_object(info["Configuration"], true); + + storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"])); + storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"])); + storage_table.add_property("entries", Friendly.sep(info["EntryCount"])); + + var column_names = ["name", "disk", "mem", "entries", "actions"]; + var bucket_table = this.add_section("buckets").add_widget( + Table, + column_names, + Table.Flag_BiasLeft + ); + for (const bucket of info["Buckets"]) + { + const row = bucket_table.add_row(bucket); + new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => { + row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"])); + row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"])); + row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"])); + + const cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view") + action_tb.left().add("drop").on_click(() => this.drop_bucket(bucket)); + }); + } + } + + drop_bucket(bucket) + { + const drop = async () => { + const namespace = this.get_param("namespace"); + await new Fetcher().resource("z$", namespace, bucket).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop bucket '${bucket}'?`) + .option("Yes", () => drop()) + .option("No"); + } +} diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js new file mode 100644 index 000000000..90e4249f6 --- /dev/null +++ b/src/zenserver/frontend/html/util/compactbinary.js @@ -0,0 +1,464 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +//////////////////////////////////////////////////////////////////////////////// +class VarInt +{ +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.measure = function(data_view) +{ + var value = data_view[0]; + var ret = 1; + for (; value & 0x80; value <<= 1, ++ret); + return ret; +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.read_uint = function(data_view, return_type=Number) +{ + const length = VarInt.measure(data_view); + var value = return_type(data_view[0] & (0xff >> length)); + for (var i = 1; i < length; ++i) + { + value <<= return_type(8); + value |= return_type(data_view[i]); + } + return [value, length]; +} + +//////////////////////////////////////////////////////////////////////////////// +VarInt.read_int = function(data_view, return_type=Number) +{ + var [value, length] = VarInt.read_uint(data_view, return_type); + value = -(value & return_type(1)) ^ (value >> return_type(1)); + return [value, length]; +} + + + +//////////////////////////////////////////////////////////////////////////////// +function cb_assert(expr_result) +{ + if (Boolean(expr_result) == false) + throw Error("compactbinary error"); +} + + + +//////////////////////////////////////////////////////////////////////////////// +const CbFieldType = { + None : 0x00, + Null : 0x01, + Object : 0x02, + UniformObject : 0x03, + Array : 0x04, + UniformArray : 0x05, + Binary : 0x06, + String : 0x07, + IntegerPositive : 0x08, + IntegerNegative : 0x09, + Float32 : 0x0a, + Float64 : 0x0b, + BoolFalse : 0x0c, + BoolTrue : 0x0d, + ObjectAttachment : 0x0e, + BinaryAttachment : 0x0f, + Hash : 0x10, + Uuid : 0x11, + DateTime : 0x12, + TimeSpan : 0x13, + ObjectId : 0x14, + CustomById : 0x1e, + CustomByName : 0x1f, + Reserved : 0x20, + HasFieldType : 0x40, + HasFieldName : 0x80, +} + +//////////////////////////////////////////////////////////////////////////////// +class CbFieldTypeOps +{ + static SerializedTypeMask = 0b10111111; + static TypeMask = 0b00111111; + static ObjectMask = 0b00111110; + static ObjectBase = 0b00000010; + static ArrayMask = 0b00111110; + static ArrayBase = 0b00000100; + static IntegerMask = 0b00111110; + static IntegerBase = 0b00001000; + static FloatMask = 0b00111100; + static FloatBase = 0b00001000; + static BoolMask = 0b00111110; + static BoolBase = 0b00001100; + static AttachmentMask = 0b00111110; + static AttachmentBase = 0b00001110; + + static get_type(type) { return type & CbFieldTypeOps.TypeMask; } + static get_serialized_type(type) { return type & CbFieldTypeOps.SerializedTypeMask; } + static has_field_type(type) { return (type & CbFieldType.HasFieldType) != 0; } + static has_field_name(type) { return (type & CbFieldType.HasFieldName) != 0; } + static is_none(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.None; } + static is_null(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Null; } + static is_object(type) { return (type & CbFieldTypeOps.ObjectMask) == CbFieldTypeOps.ObjectBase; } + static is_array(type) { return (type & CbFieldTypeOps.ArrayMask) == CbFieldTypeOps.ArrayBase; } + static is_binary(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Binary; } + static is_string(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.String; } + static is_integer(type) { return (type & CbFieldTypeOps.IntegerMask) == CbFieldTypeOps.IntegerBase; } + static is_float(type) { return (type & CbFieldTypeOps.FloatMask) == CbFieldTypeOps.FloatBase; } + static is_bool(type) { return (type & CbFieldTypeOps.BoolMask) == CbFieldTypeOps.BoolBase; } + static is_object_attachment(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.ObjectAttachment; } + static is_binary_attachment(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.BinaryAttachment; } + static is_attachment(type) { return (type & CbFieldTypeOps.AttachmentMask) == CbFieldTypeOps.AttachmentBase; } + static is_uuid(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.Uuid; } + static is_object_id(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.ObjectId; } + static is_custom_by_id(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.CustomById; } + static is_custom_by_name(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.CustomByName; } + static is_date_time(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.DateTime; } + static is_time_span(type) { return CbFieldTypeOps.get_type(type) == CbFieldType.TimeSpan; } + static is_hash(type) { var t = CbFieldTypeOps.get_type(type); return t >= CbFieldType.ObjectAttachment && t <= CbFieldType.Hash; } + static may_contain_attachments(type){ var t = CbFieldTypeOps.get_type(type); return is_object(t) || is_array(t) || is_attachement(t); } +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbFieldView +{ + constructor() + { + this._type = CbFieldType.None; + this._name = undefined; + this._data_view = undefined; + } +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._from_field = function(field) +{ + this._type = field._type; + this._name = field._name; + this._data_view = field._data_view; + return this; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._from_data = function(data_view, type=CbFieldType.HasFieldType) +{ + if (CbFieldTypeOps.has_field_type(type)) + { + type = data_view[0] | CbFieldType.HasFieldType; + data_view = data_view.subarray(1); + } + + if (CbFieldTypeOps.has_field_name(type)) + { + const [n, varint_len] = VarInt.read_uint(data_view); + this._name = data_view.subarray(varint_len, n + varint_len); + data_view = data_view.subarray(n + varint_len); + } + + this._type = type; + this._data_view = data_view; + return this; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView._iterate = function*(data_view, uniform_type) +{ + while (data_view.length > 0) + { + const field = new CbFieldView()._from_data(data_view, uniform_type); + yield field; + + const field_size = field.get_payload_size(); + cb_assert(field_size <= data_view.length); + data_view = field.get_payload().subarray(field_size); + } +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_type = function() +{ + return this._type; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_name = function() +{ + return new TextDecoder().decode(this._name); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.is_named = function(rhs) +{ + if (!this._name) return false; + if (rhs.length != this._name.length) return false; + for (var i = 0; i < rhs.length; ++i) + if (rhs.charCodeAt(i) != this._name[i]) + return false; + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_payload = function() +{ + return this._data_view; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.get_payload_size = function() +{ + switch (CbFieldTypeOps.get_type(this.get_type())) + { + case CbFieldType.None: + case CbFieldType.Null: + return 0; + case CbFieldType.Object: + case CbFieldType.UniformObject: + case CbFieldType.Array: + case CbFieldType.UniformArray: + case CbFieldType.Binary: + case CbFieldType.String: + const [value, varint_len] = VarInt.read_uint(this._data_view); + return value + varint_len; + case CbFieldType.IntegerPositive: + case CbFieldType.IntegerNegative: + return VarInt.measure(this._data_view); + case CbFieldType.Float32: + return 4; + case CbFieldType.Float64: + return 8; + case CbFieldType.BoolFalse: + case CbFieldType.BoolTrue: + return 0; + case CbFieldType.ObjectAttachment: + case CbFieldType.BinaryAttachment: + case CbFieldType.Hash: + return 20; + case CbFieldType.Uuid: + return 16; + case CbFieldType.ObjectId: + return 12; + case CbFieldType.DateTime: + case CbFieldType.TimeSpan: + return 8; + } + return 0; +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype._is = function(func) { return func(this.get_type()); } +CbFieldView.prototype.is_null = function() { return this._is(CbFieldTypeOps.is_null); } +CbFieldView.prototype.is_object = function() { return this._is(CbFieldTypeOps.is_object); } +CbFieldView.prototype.is_array = function() { return this._is(CbFieldTypeOps.is_array); } +CbFieldView.prototype.is_binary = function() { return this._is(CbFieldTypeOps.is_binary); } +CbFieldView.prototype.is_string = function() { return this._is(CbFieldTypeOps.is_string); } +CbFieldView.prototype.is_integer = function() { return this._is(CbFieldTypeOps.is_integer); } +CbFieldView.prototype.is_float = function() { return this._is(CbFieldTypeOps.is_float); } +CbFieldView.prototype.is_bool = function() { return this._is(CbFieldTypeOps.is_bool); } +CbFieldView.prototype.is_object_attachment = function() { return this._is(CbFieldTypeOps.is_object_attachment); } +CbFieldView.prototype.is_binary_attachment = function() { return this._is(CbFieldTypeOps.is_binary_attachment); } +CbFieldView.prototype.is_attachment = function() { return this._is(CbFieldTypeOps.is_attachment); } +CbFieldView.prototype.is_hash = function() { return this._is(CbFieldTypeOps.is_hash); } +CbFieldView.prototype.is_uuid = function() { return this._is(CbFieldTypeOps.is_uuid); } +CbFieldView.prototype.is_object_id = function() { return this._is(CbFieldTypeOps.is_object_id); } +CbFieldView.prototype.is_date_time = function() { return this._is(CbFieldTypeOps.is_date_time); } +CbFieldView.prototype.is_time_span = function() { return this._is(CbFieldTypeOps.is_time_span); } + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_object = function() +{ + cb_assert(CbFieldTypeOps.is_object(this.get_type())); + return new CbObjectView()._from_field(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_array = function() +{ + cb_assert(CbFieldTypeOps.is_array(this.get_type())); + return new CbArrayView()._from_field(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.as_value = function(int_type=BigInt) +{ + switch (CbFieldTypeOps.get_type(this.get_type())) + { + case CbFieldType.None: return undefined; + case CbFieldType.Null: return null; + + case CbFieldType.Object: + case CbFieldType.UniformObject: return this.as_object(); + + case CbFieldType.Array: + case CbFieldType.UniformArray: return this.as_array(); + + case CbFieldType.Binary: { + const [n, vn] = VarInt.read_uint(this._data_view); + return this._data_view.subarray(vn, n + vn); + } + + case CbFieldType.String: { + const [n, vn] = VarInt.read_uint(this._data_view); + return new TextDecoder().decode(this._data_view.subarray(vn, n + vn)); + } + + case CbFieldType.IntegerPositive: return VarInt.read_uint(this._data_view, int_type)[0]; + case CbFieldType.IntegerNegative: return VarInt.read_int(this._data_view, int_type)[0]; + + case CbFieldType.Float32: return new DataView(this._data_view.subarray(0, 4)).getFloat32(0, false); + case CbFieldType.Float64: return new DataView(this._data_view.subarray(0, 8)).getFloat64(0, false); + case CbFieldType.BoolFalse: return false; + case CbFieldType.BoolTrue: return true; + + case CbFieldType.ObjectAttachment: + case CbFieldType.BinaryAttachment: + case CbFieldType.Hash: return this._data_view.subarray(0, 20); + + case CbFieldType.Uuid: return this._data_view.subarray(0, 16); + case CbFieldType.ObjectId: return this._data_view.subarray(0, 12); + + case CbFieldType.DateTime: + case CbFieldType.TimeSpan: return this._data_view.subarray(0, 8); + } + + cb_assert(false); +} + +//////////////////////////////////////////////////////////////////////////////// +CbFieldView.prototype.clone = function() +{ + const ret = new CbFieldView() + ret._type = this._type; + ret._name = ret._name; + ret._data_view = new Uint8Array(this._data_view); + return ret; +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbObjectView extends CbFieldView +{ +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype[Symbol.iterator] = function() +{ + var data_view = this.get_payload(); + + const [payload_size, varint_len] = VarInt.read_uint(data_view); + if (payload_size == 0) + return {}; + data_view = data_view.subarray(varint_len, payload_size + varint_len); + + var uniform_type = CbFieldType.HasFieldType; + if (CbFieldTypeOps.get_type(this.get_type()) == CbFieldType.UniformObject) + { + uniform_type = data_view[0]; + data_view = data_view.subarray(1); + } + + return CbFieldView._iterate(data_view, uniform_type); +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype.to_js_object = function() +{ + const impl = function(node) + { + if (node.is_object()) + { + const ret = {}; + for (var item of node.as_object()) + ret[item.get_name()] = impl(item); + return ret; + } + + if (node.is_array()) + { + const ret = []; + for (var item of node.as_array()) + ret.push(impl(item)); + return ret; + } + + if (node.is_string()) return node.as_value(); + if (node.is_integer()) return node.as_value(); + if (node.is_float()) return node.as_value(); + + var ret = node.as_value(); + if (ret instanceof Uint8Array) + { + ret = ""; + for (var x of node.as_value()) + ret += x.toString(16).padStart(2, "0"); + } + return ret; + }; + + return impl(this); +} + +//////////////////////////////////////////////////////////////////////////////// +CbObjectView.prototype.find = function(name) +{ + for (const field of this) + if (field.is_named(name)) + return field; +} + + + +//////////////////////////////////////////////////////////////////////////////// +class CbArrayView extends CbFieldView +{ +} + +//////////////////////////////////////////////////////////////////////////////// +CbArrayView.prototype[Symbol.iterator] = function() +{ + var data_view = this.get_payload(); + + const [payload_size, varint_len] = VarInt.read_uint(data_view); + data_view = data_view.subarray(varint_len, payload_size + varint_len); + + const item_count_bytes = VarInt.measure(data_view); + if (item_count_bytes >= payload_size) + return {}; + data_view = data_view.subarray(item_count_bytes); + + var uniform_type = CbFieldType.HasFieldType; + if (CbFieldTypeOps.get_type(this.get_type()) == CbFieldType.UniformArray) + { + uniform_type = data_view[0]; + data_view = data_view.subarray(1); + } + + return CbFieldView._iterate(data_view, uniform_type); +} + +//////////////////////////////////////////////////////////////////////////////// +CbArrayView.prototype.num = function() +{ + var data_view = this._data_view; + const [n, n_len] = VarInt.read_uint(data_view); + data_view = data_view.subarray(n_len); + return VarInt.read_uint(data_view)[0]; +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class CbObject extends CbFieldView +{ + constructor(uint8_array) + { + super(); + this._from_data(uint8_array); + } +} diff --git a/src/zenserver/frontend/html/util/component.js b/src/zenserver/frontend/html/util/component.js new file mode 100644 index 000000000..205aa038e --- /dev/null +++ b/src/zenserver/frontend/html/util/component.js @@ -0,0 +1,161 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +//////////////////////////////////////////////////////////////////////////////// +class ComponentBase +{ + constructor(element) + { + if (element instanceof ComponentBase) + element = element._element; + + this._element = element; + } + + inner() + { + return this._element; + } + + parent() + { + const e = this._element.parentElement; + return e ? this.new_component(e) : null; + } + + first_child() + { + const e = this._element.firstElementChild; + return e ? this.new_component(e) : null; + } + + next_sibling() + { + const e = this._element.nextElementSibling; + return e ? this.new_component(e) : null; + } + + destroy() + { + this._element.parentNode.removeChild(this._element); + } +} + +//////////////////////////////////////////////////////////////////////////////// +class ComponentDom extends ComponentBase +{ + is(tag) + { + return this._element.tagName == tag.toUpperCase(); + } + + tag(tag="div") + { + var element = document.createElement(tag); + this._element.appendChild(element); + return this.new_component(element); + } + + retag(new_tag) + { + if (this._element.tagName == new_tag.toUpperCase()) + return this; + + var element = document.createElement(new_tag); + element.innerHTML = this._element.innerHTML; + this._element.parentNode.replaceChild(element, this._element); + this._element = element; + return this; + } + + text(value) + { + value = (value == undefined) ? "undefined" : value.toString(); + this._element.innerHTML = (value != "") ? value : ""; + return this; + } + + id(value) + { + this._element.id = value; + return this; + } + + classify(value) + { + this._element.classList.add(value); + return this; + } + + style(key, value) + { + this._element.style[key] = value; + return this; + } + + attr(key, value=undefined) + { + if (value === undefined) + return this._element.getAttribute(key); + else if (value === null) + this._element.removeAttribute(key); + else + this._element.setAttribute(key, value); + return this; + } +} + +//////////////////////////////////////////////////////////////////////////////// +class ComponentInteract extends ComponentDom +{ + link(resource=undefined, query_params={}) + { + if (resource != undefined) + { + var href = resource; + var sep = "?"; + for (const key in query_params) + { + href += sep + key + "=" + query_params[key]; + sep = "&"; + } + } + else + href = "javascript:void(0);"; + + var text = this._element.innerHTML; + this._element.innerHTML = ""; + this.tag("a").text(text).attr("href", href); + return this; + } + + on(what, func, ...args) + { + const thunk = (src) => { + if (src.target != this._element) + return; + + func(...args); + src.stopPropagation(); + }; + + this._element.addEventListener(what, thunk); + return this; + } + + on_click(func, ...args) + { + this.classify("zen_action"); + return this.on("click", func, ...args); + } +} + +//////////////////////////////////////////////////////////////////////////////// +export class Component extends ComponentInteract +{ + new_component(...args) + { + return new Component(...args); + } +} diff --git a/src/zenserver/frontend/html/util/fetcher.js b/src/zenserver/frontend/html/util/fetcher.js new file mode 100644 index 000000000..45f597404 --- /dev/null +++ b/src/zenserver/frontend/html/util/fetcher.js @@ -0,0 +1,76 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { CbObject } from "./compactbinary.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Fetcher +{ + constructor() + { + this._resource = ""; + this._query = {}; + } + + resource(...parts) + { + var value = parts.join("/"); + if (!value.startsWith("/")) + value= "/" + value; + this._resource = value; + return this; + } + + param(name, value) + { + this._query[name] = value; + return this; + } + + async json() + { + const response = await this._get("application/json"); + return response ? (await response.json()) : {}; + } + + async cbo() + { + const response = await this._get("application/x-ue-cb"); + if (!response) + return null; + + const buffer = await response.arrayBuffer(); + const data = new Uint8Array(buffer); + return new CbObject(data); + } + + async delete() + { + const resource = this._build_uri(); + const response = await fetch(resource, { "method" : "DELETE" }); + } + + _build_uri() + { + var suffix = ""; + for (var key in this._query) + { + suffix += suffix ? "&" : "?"; + suffix += key + "=" + this._query[key]; + } + return this._resource + suffix; + } + + async _get(accept="*") + { + const resource = this._build_uri(); + const response = await fetch(resource, { + "method" : "GET", + "headers" : { "Accept": accept }, + }); + + if (response.status >= 200 && response.status <= 299) + return response; + } +} diff --git a/src/zenserver/frontend/html/util/friendly.js b/src/zenserver/frontend/html/util/friendly.js new file mode 100644 index 000000000..a15252faf --- /dev/null +++ b/src/zenserver/frontend/html/util/friendly.js @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +//////////////////////////////////////////////////////////////////////////////// +export class Friendly +{ + static sep(value, prec=0) + { + return (+Number(value)).toLocaleString("en", { + style: "decimal", + minimumFractionDigits : prec, + maximumFractionDigits : prec, + }); + } + + static k(x, p=0) { return Friendly.sep((BigInt(x) + 999n) / BigInt(Math.pow(10, 3))|0n, p) + "K"; } + static m(x, p=1) { return Friendly.sep( BigInt(x) / BigInt(Math.pow(10, 6)), p) + "M"; } + static g(x, p=2) { return Friendly.sep( BigInt(x) / BigInt(Math.pow(10, 9)), p) + "G"; } + static kib(x, p=0) { return Friendly.sep((BigInt(x) + 1023n) / (1n << 10n)|0n, p) + " KiB"; } + static mib(x, p=1) { return Friendly.sep( BigInt(x) / (1n << 20n), p) + " MiB"; } + static gib(x, p=2) { return Friendly.sep( BigInt(x) / (1n << 30n), p) + " GiB"; } +} diff --git a/src/zenserver/frontend/html/util/modal.js b/src/zenserver/frontend/html/util/modal.js new file mode 100644 index 000000000..a28b013d1 --- /dev/null +++ b/src/zenserver/frontend/html/util/modal.js @@ -0,0 +1,46 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { Component } from "./component.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Modal +{ + constructor() + { + const body = new Component(document.body); + this._root = body.tag().classify("zen_modal"); + + const bg = this._root.tag().classify("zen_modal_bg"); + bg.on("click", () => this._root.destroy()); + + const rect = this._root.tag(); + this._title = rect.tag().classify("zen_modal_title"); + this._content = rect.tag().classify("zen_modal_message"); + this._buttons = rect.tag().classify("zen_modal_buttons"); + } + + title(value) + { + this._title.text(value); + return this; + } + + message(value) + { + this._content.text(value); + return this; + } + + option(name, func, ...args) + { + const thunk = () => { + this._root.destroy(); + if (func) + func(...args); + }; + this._buttons.tag().text(name).on("click", thunk); + return this; + } +} diff --git a/src/zenserver/frontend/html/util/widgets.js b/src/zenserver/frontend/html/util/widgets.js new file mode 100644 index 000000000..32a3f4d28 --- /dev/null +++ b/src/zenserver/frontend/html/util/widgets.js @@ -0,0 +1,295 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { Component } from "./component.js" +import { Friendly } from "../util/friendly.js" + +//////////////////////////////////////////////////////////////////////////////// +class Widget extends Component +{ +} + + + +//////////////////////////////////////////////////////////////////////////////// +class TableCell extends Widget +{ + constructor(element, row) + { + super(element); + this._row = row; + } + + get_table() { return this.get_row().get_table(); } + get_row() { return this._row; } +} + +//////////////////////////////////////////////////////////////////////////////// +class TableRow extends Widget +{ + constructor(element, table, index, cells) + { + super(element); + this._table = table; + this._index = index; + this._cells = cells; + } + + *[Symbol.iterator]() + { + for (var cell of this._cells) + yield cell; + } + + get_table() { return this._table; } + get_index() { return this._index; } + get_cell(index) { return this._cells.at(index); } +} + +//////////////////////////////////////////////////////////////////////////////// +export class Table extends Widget +{ + static Flag_EvenSpacing = 1 << 0; + static Flag_PackRight = 1 << 1; + static Flag_BiasLeft = 1 << 2; + static Flag_FitLeft = 1 << 3; + + constructor(parent, column_names, flags=Table.Flag_EvenSpacing, index_base=0) + { + var root = parent.tag().classify("zen_table"); + super(root); + + const column_width = 0 | (100 / column_names.length); + + var column_style; + if (flags & Table.Flag_FitLeft) column_style = "max-content"; + else if (column_names.length == 1) column_style = "1fr"; + else if (flags & Table.Flag_BiasLeft) column_style = `minmax(${column_width * 2}%, 1fr)`; + else column_style = `minmax(${column_width}%, 1fr)`; + for (var i = 1; i < column_names.length; ++i) + { + const style = (flags & Table.Flag_PackRight) ? " auto" : " 1fr"; + column_style += style; + } + + if (index_base >= 0) + { + column_names = ["#", ...column_names]; + column_style = "max-content " + column_style; + } + + root.style("gridTemplateColumns", column_style); + + this._add_row(column_names, false); + + this._index = index_base; + this._num_columns = column_names.length; + this._rows = []; + } + + *[Symbol.iterator]() + { + for (var row of this._rows) + yield row; + } + + get_row(index) + { + return this._rows.at(index); + } + + _add_row(cells, indexed=true) + { + var index = -1; + if (indexed && this._index >= 0) + { + index = this._index++; + cells = [index, ...cells]; + } + + cells = cells.slice(0, this._num_columns); + while (cells.length < this._num_columns) + cells.push(""); + + var ret = []; + var row = this.tag(); + row = new TableRow(row, this, index, ret); + for (const cell of cells) + { + var leaf = row.tag().text(cell); + ret.push(new TableCell(leaf, row)); + } + + if (this._index >= 0) + ret.shift(); + + return row; + } + + add_row(...args) + { + var row = this._add_row(args); + this._rows.push(row); + return row; + } + + clear(index=0) + { + const elem = this._element; + elem.replaceChildren(elem.firstElementChild); + this._index = (this._index >= 0) ? index : -1; + this._rows = []; + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class PropTable extends Table +{ + constructor(parent) + { + super(parent, ["prop", "value"], Table.Flag_FitLeft, -1); + this.classify("zen_proptable"); + } + + add_property(key, value) + { + return this.add_row(key, value); + } + + add_object(object, friendly=false, prec=2) + { + const impl = (node, prefix="") => { + for (const key in node) + { + var value = node[key]; + if (value instanceof Object && + (value.constructor.name == "Object" || + value.constructor.name == "Array")) + { + impl(value, prefix + key + "."); + continue; + } + + if (friendly && ((typeof value == "number") || (typeof value == "bigint"))) + { + if (key.indexOf("memory") >= 0) value = Friendly.kib(value); + else if (key.indexOf("disk") >= 0) value = Friendly.kib(value); + else if (value > 100000) value = Friendly.k(value); + else if (value % 1) value = Friendly.sep(value, 3); + else value = Friendly.sep(value, 0); + } + + this.add_property(prefix + key, value); + } + }; + + return impl(object); + } + + filter(...needles) + { + for (var row of this) + row.retag("div"); + + if (needles.length == 0) + return; + + for (var row of this) + { + var hide = false; + var cell = row.get_cell(0); + for (var needle of needles) + hide = hide || (cell.inner().innerHTML.indexOf(needle) < 0); + + if (hide) + row.retag("hidden"); + } + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class Toolbar extends Widget +{ + static Side = class extends Widget + { + add(name, tag="div") { return this.tag(tag).text(name); } + sep() { return this.tag().text("|").classify("zen_toolbar_sep"); } + } + + constructor(parent, inline=false) + { + var root = parent.tag().classify("zen_toolbar"); + super(root); + + if (inline) + root.classify("zen_toolbar_inline"); + + this._left = new Toolbar.Side(root.tag()); + this._right = new Toolbar.Side(root.tag()); + } + + left() { return this._left; } + right() { return this._right; } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class ProgressBar extends Widget +{ + constructor(parent) + { + const root = parent.tag().classify("zen_progressbar"); + super(root); + this._label = root.tag(); + root.tag(); // bg + this._bar = root.tag(); + } + + set_progress(what, count=0, end=1) + { + const percent = (((count * 100) / end) | 0).toString() + "%"; + this._bar.style("width", percent); + this._label.text(`${what}... ${count}/${end} (${percent})`); + } +} + + + +//////////////////////////////////////////////////////////////////////////////// +export class WidgetHost +{ + constructor(parent, depth=1) + { + this._parent = parent; + this._depth = depth; + } + + add_section(name) + { + var node = this._parent.tag(); + if (this._depth == 1) + node.classify("zen_sector"); + + node.tag("h" + this._depth).text(name); + return new WidgetHost(node, this._depth + 1); + } + + add_widget(type, ...args) + { + if (!(type.prototype instanceof Widget)) + throw Error("Incorrect widget type"); + + return new type(this._parent, ...args); + } + + tag(...args) + { + return this._parent.tag(...args); + } +} diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css new file mode 100644 index 000000000..532b71571 --- /dev/null +++ b/src/zenserver/frontend/html/zen.css @@ -0,0 +1,489 @@ +/* Copyright Epic Games, Inc. All Rights Reserved. */ + +/* theme -------------------------------------------------------------------- */ + +@media (prefers-color-scheme: light) { + :root { + --theme_g0: #000; + --theme_g4: #fff; + --theme_g1: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 45%); + --theme_g2: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 80%); + --theme_g3: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 96%); + + --theme_p0: #069; + --theme_p4: hsl(210deg 40% 94%); + --theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%); + --theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%); + --theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%); + + --theme_ln: var(--theme_p0); + --theme_er: #fcc; + } +} + +@media (prefers-color-scheme: dark) { + :root { + --theme_g0: #ddd; + --theme_g4: #222; + --theme_g1: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 35%); + --theme_g2: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 65%); + --theme_g3: color-mix(in oklab, var(--theme_g0), var(--theme_g4) 88%); + + --theme_p0: #479; + --theme_p4: #333; + --theme_p1: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 35%); + --theme_p2: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 60%); + --theme_p3: color-mix(in oklab, var(--theme_p0), var(--theme_p4) 85%); + + --theme_ln: #feb; + --theme_er: #622; + } +} + +/* page --------------------------------------------------------------------- */ + +body, input { + font-family: consolas, monospace; + font-size: 11pt; +} + +body { + overflow-y: scroll; + margin: 0; + background-color: var(--theme_g4); + color: var(--theme_g0); +} + +pre { + margin: 0; +} + +input { + color: var(--theme_g0); + background-color: var(--theme_g3); + border: 1px solid var(--theme_g2); +} + +* { + box-sizing: border-box; +} + +#container { + max-width: 130em; + min-width: 80em; + margin: auto; + + > div { + margin: 0.0em 2.2em 0.0em 2.2em; + padding-top: 1.0em; + padding-bottom: 1.5em; + } +} + +/* links -------------------------------------------------------------------- */ + +a { + text-decoration: unset; +} + +.zen_action, a { + cursor: pointer; + color: var(--theme_ln); + + &:hover { + text-decoration: underline var(--theme_ln); + } +} + +/* sector ------------------------------------------------------------------- */ + +.zen_sector { + h1, h2, h3 { + white-space: nowrap; + } + + h1 { + font-size: 1.5em; + width: 100%; + border-bottom: 1px solid var(--theme_g2); + } + + h2 { + font-size: 1.25em; + margin-bottom: 0.5em; + } + + h3 { + font-size: 1.1em; + margin: 0em; + padding: 0.4em; + background-color: var(--theme_p4); + border-left: 5px solid var(--theme_p2); + font-weight: normal; + } + + margin-bottom: 3em; + > *:not(h1) { + margin-left: 2em; + } +} + + +/* table -------------------------------------------------------------------- */ + +.zen_table { + display: grid; + border: 1px solid var(--theme_g2); + border-left-style: none; + margin-bottom: 1.2em; + + > div { + display: contents; + } + + > div:nth-of-type(odd) { + background-color: var(--theme_g3); + } + + > div:first-of-type { + font-weight: bold; + background-color: var(--theme_p3); + } + + > div:hover { + background-color: var(--theme_p4); + } + + > hidden { + visibility: hidden; + display: none; + } + + > div > div { + padding: 0.3em; + padding-left: 0.75em; + padding-right: 0.75em; + align-content: center; + border-left: 1px solid var(--theme_g2); + overflow: auto; + overflow-wrap: break-word; + background-color: inherit; + } +} + +/* toolbar ------------------------------------------------------------------ */ + +.zen_toolbar { + display: flex; + margin-top: 0.5em; + margin-bottom: 0.6em; + + > div { + display: flex; + align-items: center; + } + + > div > .zen_toolbar_sep { + color: var(--theme_g2); + } + + > div:last-child { + margin-left: auto; + } + + > div > div { + padding-right: 0.7em; + } + + > div:last-child > :last-child { + padding-right: 0; + } + + &.zen_toolbar_inline { + margin: unset; + } +} + + +/* modal -------------------------------------------------------------------- */ + +.zen_modal { + position: fixed; + z-index: 1; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + justify-content: center; + align-items: center; + backdrop-filter: blur(5px); + + .zen_modal_bg { + position: absolute; + z-index: -1; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: var(--theme_g0); + opacity: 0.4; + } + + > div { + border-radius: 0.5em; + background-color: var(--theme_g4); + opacity: 1.0; + width: 35em; + padding: 0em 2em 2em 2em; + } + + > div > div { + text-align: center; + } + + .zen_modal_title { + font-size: 1.2em; + border-bottom: 1px solid var(--theme_g2); + padding: 1.2em 0em 0.5em 0em; + color: var(--theme_g1); + } + + .zen_modal_buttons { + display: flex; + justify-content: center; + padding-bottom: 0em; + + > div { + margin: 0em 1em 0em 1em; + padding: 1em; + align-content: center; + border-radius: 0.3em; + background-color: var(--theme_p3); + width: 6em; + cursor: pointer; + } + + > div:hover { + background-color: var(--theme_p4); + } + } + + .zen_modal_message { + padding: 2em; + min-height: 8em; + align-content: center; + } +} + +/* progress bar ------------------------------------------------------------- */ + +.zen_progressbar { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 0.5em; + + > div:first-of-type { + /* label */ + padding: 0.3em; + padding-top: 0.8em; + background-color: var(--theme_p4); + width: max-content; + font-size: 0.8em; + } + + > div:last-of-type { + /* bar */ + position: absolute; + top: 0; + left: 0; + width: 0%; + height: 100%; + background-color: var(--theme_p1); + } + + > div:nth-of-type(2) { + /* bg */ + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + background-color: var(--theme_p3); + } +} + +/* crumbs ------------------------------------------------------------------- */ + +#crumbs { + display: flex; + position: relative; + top: -1em; + + > div { + padding-right: 0.5em; + } + + > div:nth-child(odd)::after { + content: ":"; + font-weight: bolder; + color: var(--theme_p2); + } +} + +/* branding ----------------------------------------------------------------- */ + +#branding { + font-size: 10pt; + font-weight: bolder; + margin-bottom: 2.6em; + position: relative; + + #logo { + width: min-content; + margin: auto; + user-select: none; + position: relative; + + #go_home { + width: 100%; + height: 100%; + position: absolute; + top: 0; + left: 0; + } + } + + #logo:hover { + filter: drop-shadow(0 0.15em 0.1em var(--theme_p2)); + } + + #ue_logo { + position: absolute; + top: 1em; + right: 0; + width: 5em; + margin: auto; + } +} + +/* error -------------------------------------------------------------------- */ + +#error { + position: fixed; + bottom: 0; + z-index: 1; + color: var(--theme_g0); + background-color: var(--theme_er); + padding: 1.0em 2em 2em 2em; + width: 100%; + border-top: 1px solid var(--theme_g0); + display: flex; + + > div:nth-child(1) { + font-size: 2.5em; + font-weight: bolder; + font-family: serif; + transform: rotate(-13deg); + color: var(--theme_p0); + } + + > div:nth-child(2) { + margin-left: 2em; + } + + > div:nth-child(2) > pre:nth-child(2) { + margin-top: 0.5em; + font-size: 0.8em; + color: var(--theme_g1); + } +} + +/* stats -------------------------------------------------------------------- */ + +#stat .zen_proptable pre { + float: left; + min-width: 15%; +} + +/* entry -------------------------------------------------------------------- */ + +#entry { + #datatable > div { + > div:nth-child(3), > div:nth-child(4) { + text-align: right; + } + } +} + +/* tree --------------------------------------------------------------------- */ + +#tree { + #tree_root > ul { + margin-left: 0em; + } + ul { + list-style-type: none; + padding-left: 0; + margin-left: 1em; + } + li > div { + display: flex; + border-bottom: 1px solid transparent; + padding-left: 0.3em; + padding-right: 0.3em; + } + li > div > div[active] { + text-transform: uppercase; + } + li > div > div:nth-last-child(3) { + margin-left: auto; + } + li > div > div:nth-last-child(-n + 3) { + font-size: 0.8em; + width: 10em; + text-align: right; + } + li > div > div:nth-last-child(1) { + width: 6em; + } + li > div:hover { + background-color: var(--theme_p4); + border-bottom: 1px solid var(--theme_g2); + } + li a { + font-weight: bolder; + } + li::marker { + content: "+"; + color: var(--theme_g1); + } + li[expanded]::marker { + content: "-"; + } + li[leaf]::marker { + content: "|"; + } + li:last-child::marker { + content: "\\"; + } +} + +/* map ---------------------------------------------------------------------- */ + +html:has(#map) { + height: 100%; + body, #container, #map { + height: 100%; + } +} +#map { + #treemap { + position: relative; + canvas { + width: 100%; + } + } +} diff --git a/src/zenserver/frontend/html/zen.js b/src/zenserver/frontend/html/zen.js new file mode 100644 index 000000000..3e5514173 --- /dev/null +++ b/src/zenserver/frontend/html/zen.js @@ -0,0 +1,43 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { Component } from "./util/component.js" + +//////////////////////////////////////////////////////////////////////////////// +function display_error(message, stack) +{ + const pane = new Component(document.body).tag().id("error"); + pane.tag().text("!"); + const content = pane.tag(); + content.tag("pre").text(message); + content.tag("pre").text(stack); +} + +window.addEventListener("error", function(evt) { + const reason = evt.error; + display_error(reason.message, reason.stack); +}); + +window.addEventListener("unhandledrejection", function(evt) { + const reason = evt.reason; + display_error(reason.message, reason.stack); +}); + +//////////////////////////////////////////////////////////////////////////////// +async function main() +{ + const body = new Component(document.body); + const root = body.tag().id("container").tag(); + + const params = new URLSearchParams(window.location.search); + var page = params.get("page") || "start"; + page = page.replaceAll(".", ""); + page = page.replaceAll("/", ""); + page = page.replaceAll("\\", ""); + root.id(page); + const module = await import(`./pages/${page}.js`); + new module.Page(root, params).main(); +} + +main(); diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index be2cdcc2d..a91c95ffb 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -5,10 +5,12 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/config.h> +#include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/scopeguard.h> +#include <zencore/sentryintegration.h> #include <zencore/session.h> #include <zencore/string.h> #include <zencore/thread.h> @@ -16,20 +18,20 @@ #include <zencore/trace.h> #include <zenhttp/httpserver.h> +#include <zencore/memory/fmalloc.h> +#include <zencore/memory/llm.h> +#include <zencore/memory/memory.h> +#include <zencore/memory/memorytrace.h> +#include <zencore/memory/newdelete.h> + +#include <zenutil/service.h> + #include "config.h" #include "diag/logging.h" -#include "sentryintegration.h" - -#if ZEN_USE_MIMALLOC -ZEN_THIRD_PARTY_INCLUDES_START -# include <mimalloc-new-delete.h> -# include <mimalloc.h> -ZEN_THIRD_PARTY_INCLUDES_END -#endif #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> -# include "windows/service.h" +# include <zenutil/windows/windowsservice.h> #endif ////////////////////////////////////////////////////////////////////////// @@ -59,6 +61,14 @@ SignalCallbackHandler(int SigNum) namespace zen { +static const FLLMTag& +GetZenserverTag() +{ + static FLLMTag _("zenserver"); + + return _; +} + using namespace std::literals; //////////////////////////////////////////////////////////////////////////////// @@ -83,20 +93,26 @@ ZenEntryPoint::ZenEntryPoint(ZenServerOptions& ServerOptions) : m_ServerOptions( int ZenEntryPoint::Run() { + ZEN_INFO("ZenEntryPoint::Run()"); zen::SetCurrentThreadName("main"); #if ZEN_USE_SENTRY SentryIntegration Sentry; - if (m_ServerOptions.NoSentry == false) + if (m_ServerOptions.SentryConfig.Disable == false) { - std::string SentryDatabasePath = PathToUtf8(m_ServerOptions.DataDir / ".sentry-native"); - std::string SentryAttachmentPath = PathToUtf8(m_ServerOptions.AbsLogFile); - - Sentry.Initialize(SentryDatabasePath, SentryAttachmentPath, m_ServerOptions.SentryAllowPII); + std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); + std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); + + Sentry.Initialize({.DatabasePath = SentryDatabasePath, + .AttachmentsPath = SentryAttachmentPath, + .Dsn = m_ServerOptions.SentryConfig.Dsn, + .Environment = m_ServerOptions.SentryConfig.Environment, + .AllowPII = m_ServerOptions.SentryConfig.AllowPII, + .Debug = m_ServerOptions.SentryConfig.Debug}, + m_ServerOptions.CommandLine); } #endif - try { // Mutual exclusion and synchronization @@ -104,25 +120,73 @@ ZenEntryPoint::Run() ServerState.Initialize(); ServerState.Sweep(); - ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(m_ServerOptions.BasePort); + auto NotifyReady = [&] { + if (!m_ServerOptions.ChildId.empty()) + { + NamedEvent ParentEvent{m_ServerOptions.ChildId}; + ParentEvent.Set(); + } + }; - if (Entry) + uint32_t AttachSponsorProcessRetriesLeft = 3; + ZenServerState::ZenServerEntry* Entry = ServerState.Lookup(m_ServerOptions.BasePort); + while (Entry) { if (m_ServerOptions.OwnerPid) { + std::error_code Ec; + if (!IsProcessRunning(m_ServerOptions.OwnerPid, Ec)) + { + if (Ec) + { + ZEN_WARN(ZEN_APP_NAME + " exiting, sponsor owner pid {} can not be checked for running state, reason: '{}'. Will not add sponsor " + "to process " + "listening to port {} (pid: {})", + m_ServerOptions.OwnerPid, + Ec.message(), + m_ServerOptions.BasePort, + Entry->Pid.load()); + } + else + { + ZEN_WARN(ZEN_APP_NAME + " exiting, sponsor owner pid {} is no longer running, will not add sponsor to process listening to port " + "{} (pid: {})", + m_ServerOptions.OwnerPid, + m_ServerOptions.BasePort, + Entry->Pid.load()); + } + std::exit(1); + } ZEN_INFO( "Looks like there is already a process listening to this port {} (pid: {}), attaching owner pid {} to running instance", m_ServerOptions.BasePort, Entry->Pid.load(), m_ServerOptions.OwnerPid); - Entry->AddSponsorProcess(m_ServerOptions.OwnerPid); - - std::exit(0); + // Sponsor processes are checked every second, so 2 second wait time should be enough + if (Entry->AddSponsorProcess(m_ServerOptions.OwnerPid, 2000)) + { + NotifyReady(); + std::exit(0); + } + if (AttachSponsorProcessRetriesLeft-- > 0) + { + Entry = ServerState.Lookup(m_ServerOptions.BasePort); + } + else + { + ZEN_WARN(ZEN_APP_NAME " exiting, failed to add sponsor owner pid {} to process listening to port {} (pid: {})", + m_ServerOptions.OwnerPid, + m_ServerOptions.BasePort, + Entry->Pid.load()); + std::exit(1); + } } else { - ZEN_WARN("Exiting since there is already a process listening to port {} (pid: {})", + ZEN_WARN(ZEN_APP_NAME " exiting, there is already a process listening to port {} (pid: {})", m_ServerOptions.BasePort, Entry->Pid.load()); std::exit(1); @@ -133,26 +197,39 @@ ZenEntryPoint::Run() std::filesystem::path LockFilePath = m_ServerOptions.DataDir / ".lock"; - bool IsReady = false; - - auto MakeLockData = [&] { - CbObjectWriter Cbo; - Cbo << "pid" << GetCurrentProcessId() << "data" << PathToUtf8(m_ServerOptions.DataDir) << "port" << m_ServerOptions.BasePort - << "session_id" << GetSessionId() << "ready" << IsReady; - return Cbo.Save(); + auto MakeLockData = [&](bool IsReady) { + return MakeLockFilePayload({.Pid = GetCurrentProcessId(), + .SessionId = GetSessionId(), + .EffectiveListenPort = gsl::narrow<uint16_t>(m_ServerOptions.BasePort), + .Ready = IsReady, + .DataDir = m_ServerOptions.DataDir, + .ExecutablePath = GetRunningExecutablePath()}); }; - m_LockFile.Create(LockFilePath, MakeLockData(), Ec); + m_LockFile.Create(LockFilePath, MakeLockData(false), Ec); if (Ec) { - ZEN_WARN("ERROR: Unable to grab lock at '{}' (error: '{}')", LockFilePath, Ec.message()); + ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); + Sleep(100); - std::exit(99); + m_LockFile.Create(LockFilePath, MakeLockData(false), Ec); + if (Ec) + { + ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); + Sleep(500); + if (Ec) + { + ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message()); + std::exit(99); + } + } } InitializeServerLogging(m_ServerOptions); + ZEN_INFO("Command line: {}", m_ServerOptions.CommandLine); + #if ZEN_USE_SENTRY Sentry.LogStartupInformation(); #endif @@ -167,7 +244,8 @@ ZenEntryPoint::Run() if (m_ServerOptions.OwnerPid) { - Entry->AddSponsorProcess(m_ServerOptions.OwnerPid); + // We are adding a sponsor process to our own entry, can't wait for pick since the code is not run until later + Entry->AddSponsorProcess(m_ServerOptions.OwnerPid, 0); } ZenServer Server; @@ -179,6 +257,11 @@ ZenEntryPoint::Run() auto ServerCleanup = MakeGuard([&Server] { Server.Cleanup(); }); int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); + if (EffectiveBasePort == -1) + { + // Server.Initialize has already logged what the issue is - just exit with failure code here. + std::exit(1); + } Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); if (EffectiveBasePort != m_ServerOptions.BasePort) @@ -205,11 +288,8 @@ ZenEntryPoint::Run() if (ShutdownEvent->Wait()) { - if (!IsApplicationExitRequested()) - { - ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); - Server.RequestExit(0); - } + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); + Server.RequestExit(0); } else { @@ -218,6 +298,8 @@ ZenEntryPoint::Run() }}); auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { + ReportServiceStatus(ServiceStatus::Stopping); + if (ShutdownEvent) { ShutdownEvent->Set(); @@ -232,30 +314,38 @@ ZenEntryPoint::Run() // to be able to communicate readiness with the parent Server.SetIsReadyFunc([&] { - IsReady = true; - - m_LockFile.Update(MakeLockData(), Ec); - - if (!m_ServerOptions.ChildId.empty()) - { - NamedEvent ParentEvent{m_ServerOptions.ChildId}; - ParentEvent.Set(); - } + m_LockFile.Update(MakeLockData(true), Ec); + ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); }); Server.Run(); } - catch (std::exception& e) + catch (const AssertException& AssertEx) { - ZEN_CRITICAL("Caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what()); - if (!IsApplicationExitRequested()) - { - RequestApplicationExit(1); - } + ZEN_CRITICAL(ZEN_APP_NAME " caught assert exception in main for process {}: {}", + zen::GetCurrentProcessId(), + AssertEx.FullDescription()); + RequestApplicationExit(1); + } + catch (const std::system_error& e) + { + ZEN_CRITICAL(ZEN_APP_NAME " caught system error exception in main for process {}: {} ({})", + zen::GetCurrentProcessId(), + e.what(), + e.code().value()); + RequestApplicationExit(1); + } + catch (const std::exception& e) + { + ZEN_CRITICAL(ZEN_APP_NAME " caught exception in main for process {}: {}", zen::GetCurrentProcessId(), e.what()); + RequestApplicationExit(1); } ShutdownServerLogging(); + ReportServiceStatus(ServiceStatus::Stopped); + return ApplicationExitCode(); } @@ -293,12 +383,7 @@ ZenWindowsService::Run() int test_main(int argc, char** argv) { - zen::zencore_forcelinktests(); - zen::zenhttp_forcelinktests(); - zen::zenstore_forcelinktests(); - zen::zenutil_forcelinktests(); - zen::z$_forcelink(); - zen::z$service_forcelink(); + zen::zenserver_forcelinktests(); zen::logging::InitializeLogging(); zen::logging::SetLogLevel(zen::logging::level::Debug); @@ -314,10 +399,6 @@ main(int argc, char* argv[]) { using namespace zen; -#if ZEN_USE_MIMALLOC - mi_version(); -#endif - if (argc >= 2) { if (argv[1] == "test"sv) @@ -332,12 +413,37 @@ main(int argc, char* argv[]) } signal(SIGINT, utils::SignalCallbackHandler); + signal(SIGTERM, utils::SignalCallbackHandler); + +#if ZEN_PLATFORM_LINUX + IgnoreChildSignals(); +#endif try { ZenServerOptions ServerOptions; + + { +#if ZEN_WITH_TRACE + TraceInit("zenserver"); + ServerOptions.HasTraceCommandlineOptions = GetTraceOptionsFromCommandline(ServerOptions.TraceOptions); + if (ServerOptions.HasTraceCommandlineOptions) + { + TraceConfigure(ServerOptions.TraceOptions); + } +#endif // ZEN_WITH_TRACE + } + ParseCliOptions(argc, argv, ServerOptions); + if (ServerOptions.Detach) + { +#if ZEN_PLATFORM_LINUX | ZEN_PLATFORM_MAC + // Detach ourselves from any parent process + setsid(); +#endif + } + std::string_view DeleteReason; if (ServerOptions.IsCleanStart) @@ -351,40 +457,26 @@ main(int argc, char* argv[]) if (!DeleteReason.empty()) { - if (std::filesystem::exists(ServerOptions.DataDir)) + if (IsDir(ServerOptions.DataDir)) { - ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); + ZEN_CONSOLE_INFO("Deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } - if (!std::filesystem::exists(ServerOptions.DataDir)) + if (!IsDir(ServerOptions.DataDir)) { ServerOptions.IsFirstRun = true; - std::filesystem::create_directories(ServerOptions.DataDir); + CreateDirectories(ServerOptions.DataDir); } if (!ServerOptions.BaseSnapshotDir.empty()) { - ZEN_CONSOLE_INFO("copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir); + ZEN_CONSOLE_INFO("Copying snapshot from '{}' into '{}", ServerOptions.BaseSnapshotDir, ServerOptions.DataDir); CopyTree(ServerOptions.BaseSnapshotDir, ServerOptions.DataDir, {.EnableClone = true}); } -#if ZEN_WITH_TRACE - if (ServerOptions.TraceHost.size()) - { - TraceStart("zenserver", ServerOptions.TraceHost.c_str(), TraceType::Network); - } - else if (ServerOptions.TraceFile.size()) - { - TraceStart("zenserver", ServerOptions.TraceFile.c_str(), TraceType::File); - } - else - { - TraceInit("zenserver"); - } - atexit(TraceShutdown); -#endif // ZEN_WITH_TRACE + ZEN_MEMSCOPE(GetZenserverTag()); #if ZEN_PLATFORM_WINDOWS if (ServerOptions.InstallService) @@ -413,9 +505,20 @@ main(int argc, char* argv[]) return App.Run(); #endif } - catch (std::exception& Ex) + catch (const OptionParseException& ParseEx) + { + // The parsing error already outputs all the details so no need to output the command line here + fprintf(stderr, ZEN_APP_NAME " ERROR: %s\n", ParseEx.what()); + return 1; + } + catch (const AssertException& AssertEx) + { + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught assert exception in main: '%s'", AssertEx.FullDescription().c_str()); + return 1; + } + catch (const std::exception& Ex) { - fprintf(stderr, "ERROR: Caught exception in main: '%s'", Ex.what()); + fprintf(stderr, ZEN_APP_NAME " ERROR: Caught exception in main: '%s'", Ex.what()); return 1; } diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index 47ef5c8b3..b1e73c7df 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -3,11 +3,13 @@ #include <objectstore/objectstore.h> #include <zencore/base64.h> +#include <zencore/basicfile.h> #include <zencore/compactbinaryvalue.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> #include <zencore/string.h> +#include <zencore/trace.h> #include "zencore/compactbinary.h" #include "zencore/compactbinarybuilder.h" #include "zenhttp/httpcommon.h" @@ -18,7 +20,6 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <fmt/format.h> -#include <json11.hpp> ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -219,13 +220,17 @@ private: StringBuilderBase& Builder; }; -HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg)) +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) { Inititalize(); + m_StatusService.RegisterHandler("obj", *this); } HttpObjectStoreService::~HttpObjectStoreService() { + m_StatusService.UnregisterHandler("obj", *this); } const char* @@ -245,13 +250,23 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) } void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpObjectStoreService::Inititalize() { + ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); + namespace fs = std::filesystem; ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!fs::exists(BucketsPath)) + if (!IsDir(BucketsPath)) { CreateDirectories(BucketsPath); } @@ -269,9 +284,9 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "bucket/{path}", [this](zen::HttpRouterRequest& Request) { - const std::string Path = Request.GetCapture(1); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; if (IsObject) { @@ -324,7 +339,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { std::lock_guard _(BucketsMutex); - if (!fs::exists(BucketPath)) + if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); @@ -337,18 +352,18 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path) +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; - const auto Sep = Path.find_first_of('/'); - const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; if (BucketName.empty()) { return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } - std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; if (BucketPrefix.empty()) { const auto QueryParms = Request.ServerRequest().GetQueryParams(); @@ -376,7 +391,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Writer.BeginArray("Contents"sv); } - void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize) override + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { const fs::path FullPath = Parent / fs::path(File); fs::path RelativePath = fs::relative(FullPath, BucketPath); @@ -390,7 +405,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Writer.EndObject(); } - bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return false; } + bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } CbObject GetResult() { @@ -406,6 +421,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); FileSystemTraversal Traversal; + if (IsDir(FullPath)) { std::lock_guard _(BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); @@ -449,14 +465,13 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path) +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; - const auto Sep = Path.find_first_of('/'); - const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); - const std::string BucketPrefix = - Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; const fs::path BucketDir = GetBucketDirectory(BucketName); @@ -475,7 +490,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st } const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!fs::exists(FilePath)) + if (!IsFile(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -524,7 +539,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st else { const auto Range = Ranges[0]; - const uint64_t RangeSize = Range.End - Range.Start; + const uint64_t RangeSize = 1 + (Range.End - Range.Start); const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; ZEN_LOG_DEBUG(LogObj, @@ -553,8 +568,8 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { namespace fs = std::filesystem; - const std::string& BucketName = Request.GetCapture(1); - const fs::path BucketDir = GetBucketDirectory(BucketName); + const std::string_view BucketName = Request.GetCapture(1); + const fs::path BucketDir = GetBucketDirectory(BucketName); if (BucketDir.empty()) { @@ -576,7 +591,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { std::lock_guard _(BucketsMutex); - if (!fs::exists(FileDirectory)) + if (!IsDir(FileDirectory)) { CreateDirectories(FileDirectory); } @@ -589,7 +604,8 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } - WriteFile(FilePath, FileBuf); + TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); + ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [OK] ({})", (fs::path(BucketName) / RelativeBucketPath).make_preferred(), diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index c905ceab3..44e50e208 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -3,6 +3,7 @@ #pragma once #include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> #include <atomic> #include <filesystem> #include <mutex> @@ -23,24 +24,26 @@ struct ObjectStoreConfig std::vector<BucketConfig> Buckets; }; -class HttpObjectStoreService final : public zen::HttpService +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpObjectStoreService(ObjectStoreConfig Cfg); + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); virtual ~HttpObjectStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: void Inititalize(); std::filesystem::path GetBucketDirectory(std::string_view BucketName); void CreateBucket(zen::HttpRouterRequest& Request); - void ListBucket(zen::HttpRouterRequest& Request, const std::string& Path); + void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); void DeleteBucket(zen::HttpRouterRequest& Request); - void GetObject(zen::HttpRouterRequest& Request, const std::string& Path); + void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); void PutObject(zen::HttpRouterRequest& Request); + HttpStatusService& m_StatusService; ObjectStoreConfig m_Cfg; std::mutex BucketsMutex; HttpRequestRouter m_Router; diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp deleted file mode 100644 index 8029d02de..000000000 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "fileremoteprojectstore.h" - -#include <zencore/compress.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/timer.h> - -namespace zen { - -using namespace std::literals; - -class LocalExportProjectStore : public RemoteProjectStore -{ -public: - LocalExportProjectStore(std::string_view Name, - std::string_view OptionalBaseName, - const std::filesystem::path& FolderPath, - bool ForceDisableBlocks, - bool ForceEnableTempBlocks) - : m_Name(Name) - , m_OptionalBaseName(OptionalBaseName) - , m_OutputPath(FolderPath) - { - if (ForceDisableBlocks) - { - m_EnableBlocks = false; - } - if (ForceEnableTempBlocks) - { - m_UseTempBlocks = true; - } - } - - virtual RemoteStoreInfo GetInfo() const override - { - return { - .CreateBlocks = m_EnableBlocks, - .UseTempBlockFiles = m_UseTempBlocks, - .Description = - fmt::format("[file] {}/{}{}{}"sv, m_OutputPath, m_Name, m_OptionalBaseName.empty() ? "" : " Base: ", m_OptionalBaseName)}; - } - - virtual SaveResult SaveContainer(const IoBuffer& Payload) override - { - Stopwatch Timer; - SaveResult Result; - - { - CbObject ContainerObject = LoadCompactBinaryObject(Payload); - - ContainerObject.IterateAttachments([&](CbFieldView FieldView) { - IoHash AttachmentHash = FieldView.AsBinaryAttachment(); - std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash); - if (!std::filesystem::exists(AttachmentPath)) - { - Result.Needs.insert(AttachmentHash); - } - }); - } - - std::filesystem::path ContainerPath = m_OutputPath; - ContainerPath.append(m_Name); - - try - { - CreateDirectories(m_OutputPath); - BasicFile ContainerFile; - ContainerFile.Open(ContainerPath, BasicFile::Mode::kTruncate); - std::error_code Ec; - ContainerFile.WriteAll(Payload, Ec); - if (Ec) - { - throw std::system_error(Ec, Ec.message()); - } - Result.RawHash = IoHash::HashBuffer(Payload); - } - catch (std::exception& Ex) - { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed saving oplog container to '{}'. Reason: {}", ContainerPath, Ex.what()); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override - { - Stopwatch Timer; - SaveAttachmentResult Result; - std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::exists(ChunkPath)) - { - try - { - CreateDirectories(ChunkPath.parent_path()); - - BasicFile ChunkFile; - ChunkFile.Open(ChunkPath, BasicFile::Mode::kTruncate); - size_t Offset = 0; - for (const SharedBuffer& Segment : Payload.GetSegments()) - { - ChunkFile.Write(Segment.GetView(), Offset); - Offset += Segment.GetSize(); - } - } - catch (std::exception& Ex) - { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed saving oplog attachment to '{}'. Reason: {}", ChunkPath, Ex.what()); - } - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override - { - Stopwatch Timer; - - for (const SharedBuffer& Chunk : Chunks) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer()); - SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash()); - if (ChunkResult.ErrorCode) - { - ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return SaveAttachmentsResult{ChunkResult}; - } - } - SaveAttachmentsResult Result; - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual FinalizeResult FinalizeContainer(const IoHash&) override { return {}; } - - virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Name); } - virtual LoadContainerResult LoadBaseContainer() override - { - if (m_OptionalBaseName.empty()) - { - return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}}; - } - return LoadContainer(m_OptionalBaseName); - } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override - { - Stopwatch Timer; - LoadAttachmentResult Result; - std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::is_regular_file(ChunkPath)) - { - Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); - Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - { - BasicFile ChunkFile; - ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); - Result.Bytes = ChunkFile.ReadAll(); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override - { - Stopwatch Timer; - LoadAttachmentsResult Result; - for (const IoHash& Hash : RawHashes) - { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash); - if (ChunkResult.ErrorCode) - { - ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return LoadAttachmentsResult{ChunkResult}; - } - ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000))); - Result.Chunks.emplace_back( - std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))}); - } - return Result; - } - -private: - LoadContainerResult LoadContainer(const std::string& Name) - { - Stopwatch Timer; - LoadContainerResult Result; - std::filesystem::path SourcePath = m_OutputPath; - SourcePath.append(Name); - if (!std::filesystem::is_regular_file(SourcePath)) - { - Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); - Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - IoBuffer ContainerPayload; - { - BasicFile ContainerFile; - ContainerFile.Open(SourcePath, BasicFile::Mode::kRead); - ContainerPayload = ContainerFile.ReadAll(); - } - Result.ContainerObject = LoadCompactBinaryObject(ContainerPayload); - if (!Result.ContainerObject) - { - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The file {} is not formatted as a compact binary object", SourcePath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - std::filesystem::path GetAttachmentPath(const IoHash& RawHash) const - { - ExtendablePathBuilder<128> ShardedPath; - ShardedPath.Append(m_OutputPath.c_str()); - ExtendableStringBuilder<64> HashString; - RawHash.ToHexString(HashString); - const char* str = HashString.c_str(); - ShardedPath.AppendSeparator(); - ShardedPath.AppendAsciiRange(str, str + 3); - - ShardedPath.AppendSeparator(); - ShardedPath.AppendAsciiRange(str + 3, str + 5); - - ShardedPath.AppendSeparator(); - ShardedPath.AppendAsciiRange(str + 5, str + 40); - - return ShardedPath.ToPath(); - } - - const std::string m_Name; - const std::string m_OptionalBaseName; - const std::filesystem::path m_OutputPath; - bool m_EnableBlocks = true; - bool m_UseTempBlocks = false; -}; - -std::shared_ptr<RemoteProjectStore> -CreateFileRemoteStore(const FileRemoteStoreOptions& Options) -{ - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<LocalExportProjectStore>(Options.Name, - Options.OptionalBaseName, - std::filesystem::path(Options.FolderPath), - Options.ForceDisableBlocks, - Options.ForceEnableTempBlocks); - return RemoteStore; -} - -} // namespace zen diff --git a/src/zenserver/projectstore/fileremoteprojectstore.h b/src/zenserver/projectstore/fileremoteprojectstore.h deleted file mode 100644 index 8da9692d5..000000000 --- a/src/zenserver/projectstore/fileremoteprojectstore.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "remoteprojectstore.h" - -namespace zen { - -struct FileRemoteStoreOptions : RemoteStoreOptions -{ - std::filesystem::path FolderPath; - std::string Name; - std::string OptionalBaseName; - bool ForceDisableBlocks = false; - bool ForceEnableTempBlocks = false; -}; - -std::shared_ptr<RemoteProjectStore> CreateFileRemoteStore(const FileRemoteStoreOptions& Options); - -} // namespace zen diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 0ba49cf8a..1c6b5d6b0 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -2,34 +2,37 @@ #include "httpprojectstore.h" -#include "projectstore.h" - #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> +#include <zencore/compactbinaryvalidation.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> +#include <zencore/memory/llm.h> +#include <zencore/scopeguard.h> #include <zencore/stream.h> #include <zencore/trace.h> +#include <zenhttp/packageformat.h> +#include <zenremotestore/projectstore/buildsremoteprojectstore.h> +#include <zenremotestore/projectstore/fileremoteprojectstore.h> +#include <zenremotestore/projectstore/jupiterremoteprojectstore.h> +#include <zenremotestore/projectstore/remoteprojectstore.h> +#include <zenremotestore/projectstore/zenremoteprojectstore.h> +#include <zenstore/oplogreferencedset.h> +#include <zenstore/projectstore.h> +#include <zenstore/zenstore.h> +#include <zenutil/openprocesscache.h> +#include <zenutil/workerpools.h> namespace zen { -Oid -OpKeyStringAsOId(std::string_view OpKey) +const FLLMTag& +GetProjectHttpTag() { - using namespace std::literals; - - CbObjectWriter Writer; - Writer << "key"sv << OpKey; - - XXH3_128Stream KeyHasher; - Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); - XXH3_128 KeyHash = KeyHasher.GetHash(); - - Oid OpId; - memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits)); + static FLLMTag _("http", FLLMTag("project")); - return OpId; + return _; } void @@ -50,15 +53,15 @@ CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter) } void -CSVWriteOp(CidStore& CidStore, - std::string_view ProjectId, - std::string_view OplogId, - bool Details, - bool AttachmentDetails, - int LSN, - const Oid& Key, - CbObjectView Op, - StringBuilderBase& CSVWriter) +CSVWriteOp(CidStore& CidStore, + std::string_view ProjectId, + std::string_view OplogId, + bool Details, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + StringBuilderBase& CSVWriter) { StringBuilder<32> KeyStringBuilder; Key.ToString(KeyStringBuilder); @@ -70,8 +73,8 @@ CSVWriteOp(CidStore& CidStore, const IoHash AttachmentHash = FieldView.AsAttachment(); IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); CSVWriter << "\r\n" - << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << AttachmentHash.ToHexString() << ", " - << gsl::narrow<uint64_t>(Attachment.GetSize()); + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << AttachmentHash.ToHexString() + << ", " << gsl::narrow<uint64_t>(Attachment.GetSize()); }); } else if (Details) @@ -85,8 +88,8 @@ CSVWriteOp(CidStore& CidStore, AttachmentsSize += Attachment.GetSize(); }); CSVWriter << "\r\n" - << ProjectId << ", " << OplogId << ", " << LSN << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Op.GetSize()) << ", " - << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize); + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Op.GetSize()) + << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize); } else { @@ -98,21 +101,21 @@ CSVWriteOp(CidStore& CidStore, namespace { - void CbWriteOp(CidStore& CidStore, - bool Details, - bool OpDetails, - bool AttachmentDetails, - int LSN, - const Oid& Key, - CbObjectView Op, - CbObjectWriter& CbWriter) + void CbWriteOp(CidStore& CidStore, + bool Details, + bool OpDetails, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + CbObjectWriter& CbWriter) { CbWriter.BeginObject(); { CbWriter.AddObjectId("key", Key); if (Details) { - CbWriter.AddInteger("lsn", LSN); + CbWriter.AddInteger("lsn", LSN.Number); CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Op.GetSize())); } if (AttachmentDetails) @@ -174,7 +177,9 @@ namespace { { Cbo.BeginArray("ops"); { - Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) { + Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo); }); } @@ -208,8 +213,8 @@ namespace { { for (const std::string& OpLogId : OpLogs) { - ProjectStore::Oplog* Oplog = Project.OpenOplog(OpLogId); - if (Oplog != nullptr) + Ref<ProjectStore::Oplog> Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (Oplog) { CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo); } @@ -234,20 +239,287 @@ namespace { Cbo.EndObject(); } + struct CreateRemoteStoreResult + { + std::shared_ptr<RemoteProjectStore> Store; + std::string Description; + }; + + CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, + AuthMgr& AuthManager, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + const std::filesystem::path& TempFilePath) + { + ZEN_MEMSCOPE(GetProjectHttpTag()); + + using namespace std::literals; + + std::shared_ptr<RemoteProjectStore> RemoteStore; + + if (CbObjectView File = Params["file"sv].AsObjectView(); File) + { + std::filesystem::path FolderPath(File["path"sv].AsString()); + if (FolderPath.empty()) + { + return {nullptr, "Missing file path"}; + } + std::string_view Name(File["name"sv].AsString()); + if (Name.empty()) + { + return {nullptr, "Missing file name"}; + } + std::string_view OptionalBaseName(File["basename"sv].AsString()); + bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); + bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); + + FileRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + FolderPath, + std::string(Name), + std::string(OptionalBaseName), + ForceDisableBlocks, + ForceEnableTempBlocks}; + RemoteStore = CreateFileRemoteStore(Options); + } + + if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) + { + std::string_view CloudServiceUrl = Cloud["url"sv].AsString(); + if (CloudServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(CloudServiceUrl); + std::string_view Namespace = Cloud["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Cloud["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Cloud["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Cloud["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Cloud["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view KeyParam = Cloud["key"sv].AsString(); + if (KeyParam.empty()) + { + return {nullptr, "Missing key"}; + } + if (KeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid key"}; + } + IoHash Key = IoHash::FromHexString(KeyParam); + if (Key == IoHash::Zero) + { + return {nullptr, "Invalid key string"}; + } + IoHash BaseKey = IoHash::Zero; + std::string_view BaseKeyParam = Cloud["basekey"sv].AsString(); + if (!BaseKeyParam.empty()) + { + if (BaseKeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid base key"}; + } + BaseKey = IoHash::FromHexString(BaseKeyParam); + if (BaseKey == IoHash::Zero) + { + return {nullptr, "Invalid base key string"}; + } + } + + bool ForceDisableBlocks = Cloud["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); + + JupiterRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + Key, + BaseKey, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2}; + RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) + { + std::string_view Url = Zen["url"sv].AsString(); + std::string_view Project = Zen["project"sv].AsString(); + if (Project.empty()) + { + return {nullptr, "Missing project"}; + } + std::string_view Oplog = Zen["oplog"sv].AsString(); + if (Oplog.empty()) + { + return {nullptr, "Missing oplog"}; + } + ZenRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + std::string(Url), + std::string(Project), + std::string(Oplog)}; + RemoteStore = CreateZenRemoteStore(Options, TempFilePath); + } + + if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) + { + std::string_view BuildsServiceUrl = Builds["url"sv].AsString(); + if (BuildsServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(BuildsServiceUrl); + std::string_view Namespace = Builds["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Builds["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Builds["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Builds["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Builds["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); + if (BuildIdParam.empty()) + { + return {nullptr, "Missing build id"}; + } + if (BuildIdParam.length() != Oid::StringLength) + { + return {nullptr, "Invalid build id"}; + } + Oid BuildId = Oid::FromHexString(BuildIdParam); + if (BuildId == Oid::Zero) + { + return {nullptr, "Invalid build id string"}; + } + + bool ForceDisableBlocks = Builds["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Builds["assumehttp2"sv].AsBool(false); + + MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); + IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); + + BuildsRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + BuildId, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2, + MetaData}; + RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (!RemoteStore) + { + return {nullptr, "Unknown remote store type"}; + } + + return {std::move(RemoteStore), ""}; + } + + std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result) + { + if (Result.ErrorCode == 0) + { + return {HttpResponseCode::OK, Result.Text}; + } + return {static_cast<HttpResponseCode>(Result.ErrorCode), + Result.Reason.empty() ? Result.Text + : Result.Text.empty() ? Result.Reason + : fmt::format("{}: {}", Result.Reason, Result.Text)}; + } + } // namespace ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr) +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_AuthMgr(AuthMgr) +, m_OpenProcessCache(InOpenProcessCache) +, m_JobQueue(InJobQueue) { - using namespace std::literals; + ZEN_MEMSCOPE(GetProjectHttpTag()); - m_StatsService.RegisterHandler("prj", *this); + using namespace std::literals; m_Router.AddPattern("project", "([[:alnum:]_.]+)"); m_Router.AddPattern("log", "([[:alnum:]_.]+)"); @@ -306,6 +578,11 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpVerb::kPost); m_Router.RegisterRoute( + "{project}/oplog/{log}/validate", + [this](HttpRouterRequest& Req) { HandleOplogValidateRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( "{project}/oplog/{log}/{op}", [this](HttpRouterRequest& Req) { HandleOpLogOpRequest(Req); }, HttpVerb::kGet); @@ -362,11 +639,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, "details\\$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); } HttpProjectService::~HttpProjectService() { m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); } const char* @@ -380,6 +661,8 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) { m_ProjectStats.RequestCount++; + ZEN_MEMSCOPE(GetProjectHttpTag()); + metrics::OperationTiming::Scope $(m_HttpRequests); if (m_Router.HandleRequest(Request) == false) @@ -460,6 +743,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) } void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) { ZEN_TRACE_CPU("ProjectService::ProjectList"); @@ -485,7 +777,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) } Project->TouchProject(); - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -558,7 +850,7 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) { const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - IoBuffer FoundChunk = FoundLog->FindChunk(RequestedChunk.ChunkId); + IoBuffer FoundChunk = FoundLog->FindChunk(Project->RootDir, RequestedChunk.ChunkId, nullptr); if (FoundChunk) { if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1)) @@ -585,22 +877,24 @@ HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) ResponsePtr += sizeof(ResponseHdr); for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) { - // const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; - const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); - ResponseChunkEntry ResponseChunk; - ResponseChunk.CorrelationId = ChunkIndex; + const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); + ResponseChunkEntry ResponseChunk; + ResponseChunk.CorrelationId = RequestedChunk.CorrelationId; if (FoundChunk) { ResponseChunk.ChunkSize = FoundChunk.Size(); + m_ProjectStats.ChunkHitCount++; } else { ResponseChunk.ChunkSize = uint64_t(-1); + m_ProjectStats.ChunkMissCount++; } memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk)); ResponsePtr += sizeof(ResponseChunk); } - m_ProjectStats.ChunkHitCount += RequestHdr.ChunkCount; + std::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); } @@ -620,31 +914,57 @@ HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); - const bool FilterClient = Params.GetValue("filter"sv) == "client"sv; - - CbObject ResponsePayload; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetProjectFiles(ProjectId, OplogId, FilterClient, ResponsePayload); - if (Result.first == HttpResponseCode::OK) + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) { - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } } else { - if (Result.first == HttpResponseCode::BadRequest) + const bool FilterClient = Params.GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) { - m_ProjectStats.BadRequestCount++; + WantedFieldNames.insert("serverpath"); } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); } - if (Result.second.empty()) + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) { - return HttpReq.WriteResponse(Result.first); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectFiles(Log(), *Project, *FoundLog, WantedFieldNames); + + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void @@ -657,29 +977,54 @@ HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req) const auto& ProjectId = Req.GetCapture(1); const auto& OplogId = Req.GetCapture(2); - CbObject ResponsePayload; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetProjectChunkInfos(ProjectId, OplogId, ResponsePayload); - if (Result.first == HttpResponseCode::OK) + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) { - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } } else { - if (Result.first == HttpResponseCode::BadRequest) - { - m_ProjectStats.BadRequestCount++; - } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); + WantedFieldNames.insert("id"); + WantedFieldNames.insert("rawhash"); + WantedFieldNames.insert("rawsize"); } - if (Result.second.empty()) + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) { - return HttpReq.WriteResponse(Result.first); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectChunkInfos(Log(), *Project, *FoundLog, WantedFieldNames); + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void @@ -693,35 +1038,48 @@ HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) const auto& OplogId = Req.GetCapture(2); const auto& ChunkId = Req.GetCapture(3); - CbObject ResponsePayload; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunkInfo(ProjectId, OplogId, ChunkId, ResponsePayload); - if (Result.first == HttpResponseCode::OK) + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) { - m_ProjectStats.ChunkHitCount++; - return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info request for unknown project '{}'", ProjectId)); } - else if (Result.first == HttpResponseCode::NotFound) + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) { - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown oplog '{}/{}'", ProjectId, OplogId)); } - else + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) { - if (Result.first == HttpResponseCode::BadRequest) - { - m_ProjectStats.BadRequestCount++; - } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)); + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); + if (ResponsePayload) + { + m_ProjectStats.ChunkHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); } - if (Result.second.empty()) + else { - return HttpReq.WriteResponse(Result.first); + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown chunk '{}/{}/{}'", ProjectId, OplogId, ChunkId)); } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void @@ -766,39 +1124,65 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) } } - HttpContentType AcceptType = HttpReq.AcceptContentType(); - - IoBuffer Chunk; - std::pair<HttpResponseCode, std::string> Result = - m_ProjectStore->GetChunkRange(ProjectId, OplogId, ChunkId, Offset, Size, AcceptType, Chunk); - if (Result.first == HttpResponseCode::OK) + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) { - m_ProjectStats.ChunkHitCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Chunk.GetContentType())); - return HttpReq.WriteResponse(HttpResponseCode::OK, Chunk.GetContentType(), Chunk); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); } - else if (Result.first == HttpResponseCode::NotFound) + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) { - m_ProjectStats.ChunkMissCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); } - else + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) { - if (Result.first == HttpResponseCode::BadRequest) - { - m_ProjectStats.BadRequestCount++; - } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, ChunkId)); } - if (Result.second.empty()) - { - return HttpReq.WriteResponse(Result.first); + + const Oid Obj = Oid::FromHexString(ChunkId); + + HttpContentType AcceptType = HttpReq.AcceptContentType(); + + ProjectStore::GetChunkRangeResult Result = + ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); + + switch (Result.Error) + { + case ProjectStore::GetChunkRangeResult::EError::Ok: + m_ProjectStats.ChunkHitCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Result.ContentType)); + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::NotFound: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::MalformedContent: + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + case ProjectStore::GetChunkRangeResult::EError::OutOfRange: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' OUT OF RANGE", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + default: + ZEN_ASSERT(false); + break; } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void @@ -814,40 +1198,104 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) HttpContentType AcceptType = HttpReq.AcceptContentType(); HttpContentType RequestType = HttpReq.RequestContentType(); + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (Cid.length() != IoHash::StringLength) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, Cid)); + } + + const IoHash Hash = IoHash::FromHexString(Cid); + switch (HttpReq.RequestVerb()) { case HttpVerb::kGet: { - IoBuffer Value; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value); - - if (Result.first == HttpResponseCode::OK) + IoBuffer Value = m_ProjectStore->GetChunk(*Project, *FoundLog, Hash); + if (Value) { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (DecompressedBuffer) + { + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } + else + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); + } + } m_ProjectStats.ChunkHitCount++; return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); } - else if (Result.first == HttpResponseCode::NotFound) + else { m_ProjectStats.ChunkMissCount++; ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - else - { - if (Result.first == HttpResponseCode::BadRequest) - { - m_ProjectStats.BadRequestCount++; - } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); - } - if (Result.second.empty()) - { - return HttpReq.WriteResponse(Result.first); - } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } case HttpVerb::kPost: { @@ -855,30 +1303,23 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } - std::pair<HttpResponseCode, std::string> Result = - m_ProjectStore->PutChunk(ProjectId, OplogId, Cid, RequestType, HttpReq.ReadPayload()); - if (Result.first == HttpResponseCode::OK || Result.first == HttpResponseCode::Created) - { - m_ProjectStats.ChunkWriteCount++; - return HttpReq.WriteResponse(Result.first); - } - else + if (RequestType != HttpContentType::kCompressedBinary) { - if (Result.first == HttpResponseCode::BadRequest) - { - m_ProjectStats.BadRequestCount++; - } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); - } - if (Result.second.empty()) - { - return HttpReq.WriteResponse(Result.first); + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for chunk id '{}/{}'/'{}' as unexpected content type: '{}'", + ProjectId, + OplogId, + Cid, + ToString(RequestType))); } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); + IoBuffer Payload = HttpReq.ReadPayload(); + Payload.SetContentType(RequestType); + bool IsNew = m_ProjectStore->PutChunk(*Project, *FoundLog, Hash, std::move(Payload)); + + m_ProjectStats.ChunkWriteCount++; + return HttpReq.WriteResponse(IsNew ? HttpResponseCode::Created : HttpResponseCode::OK); } break; } @@ -903,7 +1344,7 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) } Project->TouchProject(); - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -914,35 +1355,44 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) // chunks are not present on this server. This list is then returned in // the "need" list in the response - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject RequestObject = LoadCompactBinaryObject(Payload); - - std::vector<IoHash> NeedList; - - for (auto Entry : RequestObject["have"sv]) + CbValidateError ValidateResult; + if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) { - const IoHash FileHash = Entry.AsHash(); + std::vector<IoHash> NeedList; - if (!m_CidStore.ContainsChunk(FileHash)) { - ZEN_DEBUG("prep - NEED: {}", FileHash); + eastl::fixed_vector<IoHash, 16> ChunkList; + CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); + ChunkList.reserve(HaveList.Num()); + for (auto& Entry : HaveList) + { + ChunkList.push_back(Entry.AsHash()); + } - NeedList.push_back(FileHash); + NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); } - } - CbObjectWriter Cbo; - Cbo.BeginArray("need"); + CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : NeedList) + { + ZEN_DEBUG("prep - NEED: {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); + CbObject Response = Cbo.Save(); - for (const IoHash& Hash : NeedList) + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); + } + else { - Cbo << Hash; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult))); } - - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } void @@ -981,7 +1431,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) } Project->TouchProject(); - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -1036,7 +1486,9 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) { - if (CbObject Core = LoadCompactBinaryObject(Payload)) + CbValidateError ValidateResult; + if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Core) { Package.SetObject(Core); } @@ -1045,7 +1497,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) std::filesystem::path BadPackagePath = Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); - ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath); + ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath); WriteFile(BadPackagePath, Payload); @@ -1088,33 +1540,158 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); } - // Write core to oplog + eastl::fixed_vector<IoHash, 16> ReferencedChunks; + Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); }); - size_t AttachmentCount = Package.GetAttachments().size(); - const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Package); + // Write core to oplog - if (OpLsn == ProjectStore::Oplog::kInvalidOp) + size_t AttachmentCount = Package.GetAttachments().size(); + const ProjectStore::LogSequenceNumber OpLsn = Oplog.AppendNewOplogEntry(Package); + if (!OpLsn) { m_ProjectStats.BadRequestCount++; return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } m_ProjectStats.ChunkWriteCount += AttachmentCount; + // Once we stored the op, we no longer need to retain any chunks this op references + if (!ReferencedChunks.empty()) + { + FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks))); + } + m_ProjectStats.OpWriteCount++; - ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn, NiceBytes(Payload.Size()), Core["key"sv].AsString()); + ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn.Number, NiceBytes(Payload.Size()), Core["key"sv].AsString()); HttpReq.WriteResponse(HttpResponseCode::Created); } void +HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpValidate"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ZenContentType::kText, fmt::format("Project '{}' not found", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + ZenContentType::kText, + fmt::format("Oplog '{}' not found in project '{}'", OplogId, ProjectId)); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + std::atomic_bool CancelFlag = false; + ProjectStore::Oplog::ValidationResult Result = Oplog.Validate(Project->RootDir, CancelFlag, &GetSmallWorkerPool(EWorkloadType::Burst)); + tsl::robin_map<Oid, std::string, Oid::Hasher> KeyNameLookup; + KeyNameLookup.reserve(Result.OpKeys.size()); + for (const auto& It : Result.OpKeys) + { + KeyNameLookup.insert_or_assign(It.first, It.second); + } + CbObjectWriter Writer; + Writer << "HasMissingData" << !Result.IsEmpty(); + Writer << "OpCount" << Result.OpCount; + Writer << "LSNLow" << Result.LSNLow.Number; + Writer << "LSNHigh" << Result.LSNHigh.Number; + if (!Result.MissingFiles.empty()) + { + Writer.BeginArray("MissingFiles"); + for (const auto& MissingFile : Result.MissingFiles) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingFile.first; + Writer << "KeyName" << KeyNameLookup[MissingFile.first]; + Writer << "Id" << MissingFile.second.Id; + Writer << "Hash" << MissingFile.second.Hash; + Writer << "ServerPath" << MissingFile.second.ServerPath; + Writer << "ClientPath" << MissingFile.second.ClientPath; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingChunks.empty()) + { + Writer.BeginArray("MissingChunks"); + for (const auto& MissingChunk : Result.MissingChunks) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingChunk.first; + Writer << "KeyName" << KeyNameLookup[MissingChunk.first]; + Writer << "Id" << MissingChunk.second.Id; + Writer << "Hash" << MissingChunk.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingMetas.empty()) + { + Writer.BeginArray("MissingMetas"); + for (const auto& MissingMeta : Result.MissingMetas) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Id" << MissingMeta.second.Id; + Writer << "Hash" << MissingMeta.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingAttachments.empty()) + { + Writer.BeginArray("MissingAttachments"); + for (const auto& MissingMeta : Result.MissingAttachments) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Hash" << MissingMeta.second; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + CbObject Response = Writer.Save(); + HttpReq.WriteResponse(HttpResponseCode::OK, Response); +} + +void HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) { ZEN_TRACE_CPU("ProjectService::OplogOp"); HttpServerRequest& HttpReq = Req.ServerRequest(); - const std::string& ProjectId = Req.GetCapture(1); - const std::string& OplogId = Req.GetCapture(2); - const std::string& OpIdString = Req.GetCapture(3); + const std::string_view ProjectId = Req.GetCapture(1); + const std::string_view OplogId = Req.GetCapture(2); + const std::string_view OpIdString = Req.GetCapture(3); Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); if (!Project) @@ -1123,7 +1700,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) } Project->TouchProject(); - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -1134,7 +1711,7 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString)) { - if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(OpId.value())) + if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(ProjectStore::LogSequenceNumber(OpId.value()))) { CbObject& Op = MaybeOp.value(); if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage) @@ -1145,25 +1722,23 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) Op.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsAttachment(); IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); - - // We force this for now as content type is not consistently tracked (will - // be fixed in CidStore refactor) - Payload.SetContentType(ZenContentType::kCompressedBinary); - if (Payload) { switch (Payload.GetContentType()) { case ZenContentType::kCbObject: - if (CbObject Object = LoadCompactBinaryObject(Payload)) { - Package.AddAttachment(CbAttachment(Object)); - } - else - { - // Error - malformed object - - ZEN_WARN("malformed object returned for {}", AttachmentHash); + CbValidateError ValidateResult; + if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Object) + { + Package.AddAttachment(CbAttachment(Object)); + } + else + { + // Error - malformed object + ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult)); + } } break; @@ -1225,14 +1800,13 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) { case HttpVerb::kGet: { - ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> OplogIt = Project->ReadOplog(OplogId); if (!OplogIt) { return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("oplog {} not found in project {}", OplogId, ProjectId)); } - Project->TouchOplog(OplogId); ProjectStore::Oplog& Log = *OplogIt; @@ -1240,7 +1814,6 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str() << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount" << Log.OplogCount() << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log); - HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save()); m_ProjectStats.OpLogReadCount++; @@ -1259,7 +1832,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) OplogMarkerPath = Params["gcpath"sv].AsString(); } - ProjectStore::Oplog* OplogIt = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); if (!OplogIt) { if (!Project->NewOplog(OplogId, OplogMarkerPath)) @@ -1296,7 +1869,7 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) OplogMarkerPath = Params["gcpath"sv].AsString(); } - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); if (!FoundLog) { if (!Project->NewOplog(OplogId, OplogMarkerPath)) @@ -1326,10 +1899,17 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) { ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId); - Project->DeleteOplog(OplogId); - - m_ProjectStats.OpLogDeleteCount++; - return HttpReq.WriteResponse(HttpResponseCode::OK); + if (Project->DeleteOplog(OplogId)) + { + m_ProjectStats.OpLogDeleteCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::Locked, + HttpContentType::kText, + fmt::format("oplog {}/{} is in use", ProjectId, OplogId)); + } } break; @@ -1338,6 +1918,28 @@ HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) } } +std::optional<OplogReferencedSet> +LoadReferencedSet(ProjectStore::Project& Project, ProjectStore::Oplog& Log) +{ + using namespace std::literals; + + Oid ReferencedSetOplogId = OpKeyStringAsOid(OplogReferencedSet::ReferencedSetOplogKey); + std::optional<CbObject> ReferencedSetOp = Log.GetOpByKey(ReferencedSetOplogId); + if (!ReferencedSetOp) + { + return std::optional<OplogReferencedSet>(); + } + // We expect only a single file in the "files" array; get the chunk for the first file + CbFieldView FileField = *(*ReferencedSetOp)["files"sv].AsArrayView().CreateViewIterator(); + Oid ChunkId = FileField.AsObjectView()["id"sv].AsObjectId(); + if (ChunkId == Oid::Zero) + { + return std::optional<OplogReferencedSet>(); + } + + return OplogReferencedSet::LoadFromChunk(Log.FindChunk(Project.RootDir, ChunkId, nullptr)); +} + void HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) { @@ -1357,7 +1959,7 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) } Project->TouchProject(); - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -1368,16 +1970,44 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) if (FoundLog->OplogCount() > 0) { + std::unordered_set<std::string> FieldNamesFilter; + auto FilterObject = [&FieldNamesFilter](CbObjectView& Object) -> CbObject { + CbObject RewrittenOp = RewriteCbObject(Object, [&FieldNamesFilter](CbObjectWriter&, CbFieldView Field) -> bool { + if (FieldNamesFilter.contains(std::string(Field.GetName()))) + { + return false; + } + + return true; + }); + + return RewrittenOp; + }; + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldfilter")); !FieldFilter.empty()) + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + FieldNamesFilter.insert(std::string(FieldName)); + return true; + }); + } if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty()) { - Oid OpKeyId = OpKeyStringAsOId(OpKey); + Oid OpKeyId = OpKeyStringAsOid(OpKey); std::optional<CbObject> Op = FoundLog->GetOpByKey(OpKeyId); if (Op.has_value()) { - Response << "entry"sv << Op.value(); + if (FieldNamesFilter.empty()) + { + Response << "entry"sv << Op.value(); + } + else + { + Response << "entry"sv << FilterObject(Op.value()); + } } else { @@ -1386,15 +2016,85 @@ HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) } else { + ProjectStore::Oplog::Paging EntryPaging; + if (std::string_view Param = Params.GetValue("start"); !Param.empty()) + { + if (auto Value = ParseInt<int32>(Param)) + { + EntryPaging.Start = *Value; + } + } + if (std::string_view Param = Params.GetValue("count"); !Param.empty()) + { + if (auto Value = ParseInt<int32>(Param)) + { + EntryPaging.Count = *Value; + } + } + + std::optional<OplogReferencedSet> MaybeReferencedSet; + if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true") + { + MaybeReferencedSet = LoadReferencedSet(*Project, *FoundLog); + } Response.BeginArray("entries"sv); - FoundLog->IterateOplog([&Response](CbObjectView Op) { Response << Op; }); + bool ShouldFilterFields = !FieldNamesFilter.empty(); + + if (MaybeReferencedSet) + { + const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value(); + FoundLog->IterateOplogWithKey( + [this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](ProjectStore::LogSequenceNumber /* LSN */, + const Oid& Key, + CbObjectView Op) { + if (!ReferencedSet.Contains(Key)) + { + if (!OplogReferencedSet::IsNonPackage(Op["key"].AsString())) + { + return; + } + } + + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } + else + { + FoundLog->IterateOplog( + [this, &Response, &FilterObject, ShouldFilterFields](CbObjectView Op) { + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } Response.EndArray(); } } - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(Response.Save().GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + } } void @@ -1404,8 +2104,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) using namespace std::literals; - HttpServerRequest& HttpReq = Req.ServerRequest(); - const std::string ProjectId = Req.GetCapture(1); + HttpServerRequest& HttpReq = Req.ServerRequest(); + const std::string_view ProjectId = Req.GetCapture(1); switch (HttpReq.RequestVerb()) { @@ -1416,27 +2116,37 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Params = LoadCompactBinaryObject(Payload); - std::string_view Root = Params["root"sv].AsString(); // Workspace root (i.e `D:/UE5/`) - std::string_view EngineRoot = Params["engine"sv].AsString(); // Engine root (i.e `D:/UE5/Engine`) - std::string_view ProjectRoot = Params["project"sv].AsString(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::string_view ProjectFilePath = - Params["projectfile"sv].AsString(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; - m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - m_ProjectStats.ProjectWriteCount++; - HttpReq.WriteResponse(HttpResponseCode::Created); + m_ProjectStats.ProjectWriteCount++; + HttpReq.WriteResponse(HttpResponseCode::Created); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } } break; @@ -1446,43 +2156,52 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) { return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } - - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Params = LoadCompactBinaryObject(Payload); - std::string_view Root = Params["root"sv].AsString(); // Workspace root (i.e `D:/UE5/`) - std::string_view EngineRoot = Params["engine"sv].AsString(); // Engine root (i.e `D:/UE5/Engine`) - std::string_view ProjectRoot = Params["project"sv].AsString(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) - std::string_view ProjectFilePath = - Params["projectfile"sv].AsString(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) - - if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) { - m_ProjectStats.ProjectWriteCount++; - ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - HttpReq.WriteResponse(HttpResponseCode::OK); + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) + { + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::Created); + } } else { - const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; - m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - - m_ProjectStats.ProjectWriteCount++; - ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", - ProjectId, - Root, - EngineRoot, - ProjectRoot, - ProjectFilePath, - ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); - - HttpReq.WriteResponse(HttpResponseCode::Created); + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); } } break; @@ -1571,29 +2290,113 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req) } IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Response; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->WriteOplog(ProjectId, OplogId, std::move(Payload), Response); - if (Result.first == HttpResponseCode::OK) + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) { - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown project '{}'", ProjectId)); } - else + Project->TouchProject(); + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); + if (!Oplog) { - if (Result.first == HttpResponseCode::BadRequest) + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbValidateError ValidateResult; + if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) + { + RwLock AttachmentsLock; + tsl::robin_set<IoHash, IoHash::Hasher> Attachments; + + auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); }; + auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + if (BlockHash != IoHash::Zero) + { + Attachments.insert(BlockHash); + } + else + { + Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); + } + }; + auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + Attachments.insert(RawHash); + }; + + auto OnChunkedAttachment = [](const ChunkedInfo&) {}; + + auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; + + // Make sure we retain any attachments we download before writing the oplog + Oplog->EnableUpdateCapture(); + auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); + + RemoteProjectStore::Result Result = SaveOplogContainer(*Oplog, + ContainerObject, + OnReferencedAttachments, + HasAttachment, + OnNeedBlock, + OnNeedAttachment, + OnChunkedAttachment, + nullptr); + + if (Result.ErrorCode == 0) { - m_ProjectStats.BadRequestCount++; + if (Attachments.empty()) + { + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : Attachments) + { + ZEN_DEBUG("Need attachment {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); // "need" + + CbObject ResponsePayload = Cbo.Save(); + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + } + else + { + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + Result.ErrorCode, + Result.Reason); + + if (Result.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode), HttpContentType::kText, Result.Reason); + } } - ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", - ToString(HttpReq.RequestVerb()), - HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); } - if (Result.second.empty()) + else { - return HttpReq.WriteResponse(Result.first); + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid payload: '{}'", ToString(ValidateResult))); } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void @@ -1604,42 +2407,114 @@ HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) HttpServerRequest& HttpReq = Req.ServerRequest(); const auto& ProjectId = Req.GetCapture(1); const auto& OplogId = Req.GetCapture(2); + + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + if (HttpReq.AcceptContentType() != HttpContentType::kCbObject) { m_ProjectStats.BadRequestCount++; return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type"); } - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Response; - std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->ReadOplog(ProjectId, OplogId, HttpReq.GetQueryParams(), Response); - if (Result.first == HttpResponseCode::OK) + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) { - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown project '{}'", ProjectId)); } - else + Project->TouchProject(); + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + size_t MaxBlockSize = RemoteStoreOptions::DefaultMaxBlockSize; + if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false) { - if (Result.first == HttpResponseCode::BadRequest) + if (auto Value = ParseInt<size_t>(Param)) { - m_ProjectStats.BadRequestCount++; + MaxBlockSize = Value.value(); } + } + size_t MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize; + if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + MaxChunkEmbedSize = Value.value(); + } + } + size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock; + if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + MaxChunksPerBlock = Value.value(); + } + } + + size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit; + if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + ChunkFileSizeLimit = Value.value(); + } + } + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( + m_CidStore, + *Project, + *Oplog, + WorkerPool, + MaxBlockSize, + MaxChunkEmbedSize, + MaxChunksPerBlock, + ChunkFileSizeLimit, + /* BuildBlocks */ false, + /* IgnoreMissingAttachments */ false, + /* AllowChunking*/ false, + [](CompressedBuffer&&, ChunkBlockDescription&&) {}, + [](const IoHash&, TGetAttachmentBufferFunc&&) {}, + [](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, + /* EmbedLooseFiles*/ false); + + if (ContainerResult.ErrorCode == 0) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerResult.ContainerObject); + } + else + { ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", ToString(HttpReq.RequestVerb()), HttpReq.QueryString(), - static_cast<int>(Result.first), - Result.second); - } - if (Result.second.empty()) - { - return HttpReq.WriteResponse(Result.first); + ContainerResult.ErrorCode, + ContainerResult.Reason); + + if (ContainerResult.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode), HttpContentType::kText, ContainerResult.Reason); + } } - return HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); } void HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) { ZEN_TRACE_CPU("ProjectService::Rpc"); + using namespace std::literals; HttpServerRequest& HttpReq = Req.ServerRequest(); @@ -1647,10 +2522,558 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) const auto& OplogId = Req.GetCapture(2); IoBuffer Payload = HttpReq.ReadPayload(); - bool OkRequest = m_ProjectStore->Rpc(HttpReq, ProjectId, OplogId, std::move(Payload), m_AuthMgr); - if (!OkRequest) + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + CbPackage Package; + CbObject Cb; + switch (PayloadContentType) { - m_ProjectStats.BadRequestCount++; + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + } + } + break; + case HttpContentType::kCbObject: + { + CbValidateError ValidateResult; + if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult != CbValidateError::None || !Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult))); + } + break; + } + case HttpContentType::kCbPackage: + try + { + Package = ParsePackageMessage(Payload); + Cb = Package.GetObject(); + } + catch (const std::invalid_argument& ex) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Failed to parse package request, reason: '{}'", ex.what())); + } + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected package message format"); + } + break; + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + std::string_view Method = Cb["method"sv].AsString(); + + bool VerifyPathOnDisk = Method != "getchunks"sv; + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, VerifyPathOnDisk); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + uint32_t MethodHash = HashStringDjb2(Method); + + switch (MethodHash) + { + case HashStringDjb2("import"sv): + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool CleanOplog = Params["clean"].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ChunkStore = &m_CidStore, + ActualRemoteStore = std::move(RemoteStore), + Oplog, + Force, + IgnoreMissingAttachments, + CleanOplog](JobContext& Context) { + Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", + Oplog->GetOuterProjectIdentifier(), + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description)); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = LoadOplog(m_CidStore, + *ActualRemoteStore, + *Oplog, + NetworkWorkerPool, + WorkerPool, + Force, + IgnoreMissingAttachments, + CleanOplog, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("export"sv): + { + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + size_t MaxChunksPerBlock = Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock); + size_t ChunkFileSizeLimit = Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Export oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ActualRemoteStore = std::move(RemoteStore), + Project, + Oplog, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments](JobContext& Context) { + Context.ReportMessage(fmt::format("Saving oplog '{}/{}' to {}, maxblocksize {}, maxchunkembedsize {}", + Project->Identifier, + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description, + NiceBytes(MaxBlockSize), + NiceBytes(MaxChunkEmbedSize))); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = SaveOplog(m_CidStore, + *ActualRemoteStore, + *Project, + *Oplog, + NetworkWorkerPool, + WorkerPool, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("getchunks"sv): + { + RpcAcceptOptions AcceptFlags = static_cast<RpcAcceptOptions>(Cb["AcceptFlags"sv].AsUInt16(0u)); + int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); + + std::vector<ProjectStore::ChunkRequest> Requests = m_ProjectStore->ParseChunksRequests(*Project, *Oplog, Cb); + std::vector<ProjectStore::ChunkResult> Results = + Requests.empty() ? std::vector<ProjectStore::ChunkResult>{} : m_ProjectStore->GetChunks(*Project, *Oplog, Requests); + CbPackage Response = m_ProjectStore->WriteChunksRequestResponse(*Project, *Oplog, std::move(Requests), std::move(Results)); + + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(HttpReq.SessionId(), TargetProcessId); + } + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(Response, Flags, TargetProcessHandle); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + case HashStringDjb2("putchunks"sv): + { + ZEN_TRACE_CPU("Store::Rpc::putchunks"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObject Object = Package.GetObject(); + const bool UsingTempFiles = Object["usingtmpfiles"].AsBool(false); + + std::span<const CbAttachment> Attachments = Package.GetAttachments(); + if (!Attachments.empty()) + { + std::vector<IoBuffer> WriteAttachmentBuffers; + std::vector<IoHash> WriteRawHashes; + + WriteAttachmentBuffers.reserve(Attachments.size()); + WriteRawHashes.reserve(Attachments.size()); + + for (const CbAttachment& Attachment : Attachments) + { + IoHash RawHash = Attachment.GetHash(); + const CompressedBuffer& Compressed = Attachment.AsCompressedBinary(); + IoBuffer AttachmentPayload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + if (UsingTempFiles) + { + AttachmentPayload.SetDeleteOnClose(true); + } + WriteAttachmentBuffers.push_back(std::move(AttachmentPayload)); + WriteRawHashes.push_back(RawHash); + } + + Oplog->CaptureAddedAttachments(WriteRawHashes); + m_CidStore.AddChunks(WriteAttachmentBuffers, + WriteRawHashes, + UsingTempFiles ? CidStore::InsertMode::kMayBeMovedInPlace : CidStore::InsertMode::kCopyOnly); + } + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + case HashStringDjb2("snapshot"sv): + { + ZEN_TRACE_CPU("Store::Rpc::snapshot"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + // Snapshot all referenced files. This brings the content of all + // files into the CID store + + uint32_t OpCount = 0; + uint64_t InlinedBytes = 0; + uint64_t InlinedFiles = 0; + uint64_t TotalBytes = 0; + uint64_t TotalFiles = 0; + + std::vector<CbObject> NewOps; + struct AddedChunk + { + IoBuffer Buffer; + uint64_t RawSize = 0; + }; + tsl::robin_map<IoHash, AddedChunk, IoHash::Hasher> AddedChunks; + + Oplog->IterateOplog( + [&](CbObjectView Op) { + bool OpRewritten = false; + bool AllOk = true; + + CbWriter FilesArrayWriter; + FilesArrayWriter.BeginArray("files"sv); + + for (CbFieldView& Field : Op["files"sv]) + { + bool CopyField = true; + + if (CbObjectView View = Field.AsObjectView()) + { + const IoHash DataHash = View["data"sv].AsHash(); + + if (DataHash == IoHash::Zero) + { + std::string_view ServerPath = View["serverpath"sv].AsString(); + std::filesystem::path FilePath = Project->RootDir / ServerPath; + BasicFile DataFile; + std::error_code Ec; + DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); + + if (Ec) + { + // Error... + + ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); + + AllOk = false; + } + else + { + // Read file contents into memory, compress and keep in map of chunks to add to Cid store + IoBuffer FileIoBuffer = DataFile.ReadAll(); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); + const uint64_t RawSize = Compressed.DecodeRawSize(); + const IoHash RawHash = Compressed.DecodeRawHash(); + if (!AddedChunks.contains(RawHash)) + { + const std::filesystem::path TempChunkPath = Oplog->TempPath() / RawHash.ToHexString(); + BasicFile ChunkTempFile; + ChunkTempFile.Open(TempChunkPath, BasicFile::Mode::kTruncateDelete); + ChunkTempFile.Write(Compressed.GetCompressed(), 0, Ec); + if (Ec) + { + Oid ChunkId = View["id"sv].AsObjectId(); + ZEN_ERROR("unable to write external file as compressed chunk '{}', id {}: {}", + FilePath, + ChunkId, + Ec.message()); + AllOk = false; + } + else + { + void* FileHandle = ChunkTempFile.Detach(); + IoBuffer ChunkBuffer(IoBuffer::File, + FileHandle, + 0, + Compressed.GetCompressed().GetSize(), + /*IsWholeFile*/ true); + ChunkBuffer.SetDeleteOnClose(true); + AddedChunks.insert_or_assign( + RawHash, + AddedChunk{.Buffer = std::move(ChunkBuffer), .RawSize = RawSize}); + } + } + + TotalBytes += RawSize; + ++TotalFiles; + + // Rewrite file array entry with new data reference + CbObjectWriter Writer(View.GetSize()); + RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { + if (Field.GetName() == "data"sv) + { + // omit this field as we will write it explicitly ourselves + return true; + } + return false; + }); + Writer.AddBinaryAttachment("data"sv, RawHash); + + CbObject RewrittenOp = Writer.Save(); + FilesArrayWriter.AddObject(std::move(RewrittenOp)); + CopyField = false; + } + } + } + + if (CopyField) + { + FilesArrayWriter.AddField(Field); + } + else + { + OpRewritten = true; + } + } + + if (OpRewritten && AllOk) + { + FilesArrayWriter.EndArray(); + CbArray FilesArray = FilesArrayWriter.Save().AsArray(); + + CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { + if (Field.GetName() == "files"sv) + { + NewWriter.AddArray("files"sv, FilesArray); + + return true; + } + + return false; + }); + + NewOps.push_back(std::move(RewrittenOp)); + } + + OpCount++; + }, + ProjectStore::Oplog::Paging{}); + + CbObjectWriter ResponseObj; + + // Persist rewritten oplog entries + if (!NewOps.empty()) + { + ResponseObj.BeginArray("rewritten_ops"); + + for (CbObject& NewOp : NewOps) + { + ProjectStore::LogSequenceNumber NewLsn = Oplog->AppendNewOplogEntry(std::move(NewOp)); + + ZEN_DEBUG("appended rewritten op at LSN: {}", NewLsn.Number); + + ResponseObj.AddInteger(NewLsn.Number); + } + + ResponseObj.EndArray(); + } + + // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the + // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // unreferenced chunks. + for (auto It : AddedChunks) + { + const IoHash& RawHash = It.first; + AddedChunk& Chunk = It.second; + CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); + if (Result.New) + { + InlinedBytes += Chunk.RawSize; + ++InlinedFiles; + } + } + + ResponseObj << "inlined_bytes" << InlinedBytes << "inlined_files" << InlinedFiles; + ResponseObj << "total_bytes" << TotalBytes << "total_files" << TotalFiles; + + ZEN_INFO("oplog '{}/{}': rewrote {} oplog entries (out of {})", ProjectId, OplogId, NewOps.size(), OpCount); + + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); + } + case HashStringDjb2("appendops"sv): + { + ZEN_TRACE_CPU("Store::Rpc::appendops"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbArrayView OpsArray = Cb["ops"sv].AsArrayView(); + + size_t OpsBufferSize = 0; + for (CbFieldView OpView : OpsArray) + { + OpsBufferSize += OpView.GetSize(); + } + UniqueBuffer OpsBuffers = UniqueBuffer::Alloc(OpsBufferSize); + MutableMemoryView OpsBuffersMemory = OpsBuffers.GetMutableView(); + + std::vector<CbObjectView> Ops; + Ops.reserve(OpsArray.Num()); + for (CbFieldView OpView : OpsArray) + { + OpView.CopyTo(OpsBuffersMemory); + Ops.push_back(CbObjectView(OpsBuffersMemory.GetData())); + OpsBuffersMemory.MidInline(OpView.GetSize()); + } + + std::vector<ProjectStore::LogSequenceNumber> LSNs = Oplog->AppendNewOplogEntries(Ops); + ZEN_ASSERT(LSNs.size() == Ops.size()); + + std::vector<IoHash> MissingAttachments; + for (size_t OpIndex = 0; OpIndex < Ops.size(); OpIndex++) + { + if (LSNs[OpIndex]) + { + CbObjectView Op = Ops[OpIndex]; + Op.IterateAttachments([this, &MissingAttachments](CbFieldView AttachmentView) { + const IoHash Cid = AttachmentView.AsAttachment(); + if (!m_CidStore.ContainsChunk(Cid)) + { + MissingAttachments.push_back(Cid); + } + }); + } + } + + CbPackage ResponsePackage; + + { + CbObjectWriter ResponseObj; + ResponseObj.BeginArray("written_ops"); + + for (ProjectStore::LogSequenceNumber NewLsn : LSNs) + { + ZEN_DEBUG("appended written op at LSN: {}", NewLsn.Number); + ResponseObj.AddInteger(NewLsn.Number); + } + ResponseObj.EndArray(); + + if (!MissingAttachments.empty()) + { + ResponseObj.BeginArray("need"); + + for (const IoHash& Cid : MissingAttachments) + { + ResponseObj.AddHash(Cid); + } + ResponseObj.EndArray(); + } + ResponsePackage.SetObject(ResponseObj.Save()); + } + + std::vector<IoBuffer> ResponseBuffers = FormatPackageMessage(ResponsePackage); + + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, ResponseBuffers); + } + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Unknown rpc method '{}'", Method)); } } void @@ -1676,7 +3099,9 @@ HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) { + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); }); }); @@ -1730,10 +3155,11 @@ HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) CSVHeader(Details, AttachmentDetails, CSVWriter); FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } @@ -1773,7 +3199,7 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -1786,10 +3212,11 @@ HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) ExtendableStringBuilder<4096> CSVWriter; CSVHeader(Details, AttachmentDetails, CSVWriter); - Oplog.IterateOplogWithKey( - [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](int LSN, const Oid& Key, CbObjectView Op) { - CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); - }); + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); } else @@ -1828,7 +3255,7 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - ProjectStore::Oplog* FoundLog = FoundProject->OpenOplog(OplogId); + Ref<ProjectStore::Oplog> FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); if (!FoundLog) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); @@ -1846,13 +3273,13 @@ HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) ProjectStore::Project& Project = *FoundProject.Get(); ProjectStore::Oplog& Oplog = *FoundLog; - int LSN = Oplog.GetOpIndexByKey(ObjId); - if (LSN == -1) + std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId); + if (!Op.has_value()) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); } - std::optional<CbObject> Op = Oplog.GetOpByIndex(LSN); - if (!Op.has_value()) + ProjectStore::LogSequenceNumber LSN = Oplog.GetOpIndexByKey(ObjId); + if (!LSN) { return HttpReq.WriteResponse(HttpResponseCode::NotFound); } diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 9990ee264..f0a0bcfa1 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -3,13 +3,16 @@ #pragma once #include <zencore/stats.h> -#include <zenhttp/auth/authmgr.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> #include <zenstore/cidstore.h> namespace zen { +class AuthMgr; +class JobQueue; +class OpenProcessCache; class ProjectStore; ////////////////////////////////////////////////////////////////////////// @@ -31,16 +34,23 @@ class ProjectStore; // refs: // -class HttpProjectService : public HttpService, public IHttpStatsProvider +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue); ~HttpProjectService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct ProjectStats @@ -70,6 +80,7 @@ private: void HandleChunkByCidRequest(HttpRouterRequest& Req); void HandleOplogOpPrepRequest(HttpRouterRequest& Req); void HandleOplogOpNewRequest(HttpRouterRequest& Req); + void HandleOplogValidateRequest(HttpRouterRequest& Req); void HandleOpLogOpRequest(HttpRouterRequest& Req); void HandleOpLogRequest(HttpRouterRequest& Req); void HandleOpLogEntriesRequest(HttpRouterRequest& Req); @@ -88,8 +99,11 @@ private: CidStore& m_CidStore; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; AuthMgr& m_AuthMgr; + OpenProcessCache& m_OpenProcessCache; + JobQueue& m_JobQueue; ProjectStats m_ProjectStats; metrics::OperationTiming m_HttpRequests; }; diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp deleted file mode 100644 index fbc3c8e16..000000000 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "jupiterremoteprojectstore.h" - -#include <zencore/compress.h> -#include <zencore/fmtutils.h> - -#include <upstream/jupiter.h> -#include <zenhttp/auth/authmgr.h> - -namespace zen { - -using namespace std::literals; - -class JupiterRemoteStore : public RemoteProjectStore -{ -public: - JupiterRemoteStore(Ref<CloudCacheClient>&& CloudClient, - std::string_view Namespace, - std::string_view Bucket, - const IoHash& Key, - const IoHash& OptionalBaseKey, - bool ForceDisableBlocks, - bool ForceDisableTempBlocks, - const std::filesystem::path& TempFilePath) - : m_CloudClient(std::move(CloudClient)) - , m_Namespace(Namespace) - , m_Bucket(Bucket) - , m_Key(Key) - , m_OptionalBaseKey(OptionalBaseKey) - , m_TempFilePath(TempFilePath) - { - if (ForceDisableBlocks) - { - m_EnableBlocks = false; - } - if (ForceDisableTempBlocks) - { - m_UseTempBlocks = false; - } - } - - virtual RemoteStoreInfo GetInfo() const override - { - return {.CreateBlocks = m_EnableBlocks, - .UseTempBlockFiles = m_UseTempBlocks, - .Description = fmt::format("[cloud] {} as {}/{}/{}{}"sv, - m_CloudClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_Key, - m_OptionalBaseKey == IoHash::Zero ? "" : fmt::format(" Base {}", m_OptionalBaseKey))}; - } - - virtual SaveResult SaveContainer(const IoBuffer& Payload) override - { - const int32_t MaxAttempts = 3; - PutRefResult PutResult; - { - CloudCacheSession Session(m_CloudClient.Get()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++) - { - PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject); - if (!PutResult.Success) - { - Sleep(100 * (Attempt + 1)); - } - } - } - - SaveResult Result{ConvertResult(PutResult), {PutResult.Needs.begin(), PutResult.Needs.end()}, PutResult.RawHash}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_Key, - Result.Reason); - } - return Result; - } - - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override - { - const int32_t MaxAttempts = 3; - CloudCacheResult PutResult; - { - CloudCacheSession Session(m_CloudClient.Get()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !PutResult.Success; Attempt++) - { - PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); - if (!PutResult.Success) - { - Sleep(100 * (Attempt + 1)); - } - } - } - - SaveAttachmentResult Result{ConvertResult(PutResult)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - RawHash, - Result.Reason); - } - return Result; - } - - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override - { - SaveAttachmentsResult Result; - for (const SharedBuffer& Chunk : Chunks) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(Chunk.AsIoBuffer()); - SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash()); - if (ChunkResult.ErrorCode) - { - return SaveAttachmentsResult{ChunkResult}; - } - } - return Result; - } - - virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override - { - const int32_t MaxAttempts = 3; - FinalizeRefResult FinalizeRefResult; - { - CloudCacheSession Session(m_CloudClient.Get()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !FinalizeRefResult.Success; Attempt++) - { - FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash); - if (!FinalizeRefResult.Success) - { - Sleep(100 * (Attempt + 1)); - } - } - } - FinalizeResult Result{ConvertResult(FinalizeRefResult), {FinalizeRefResult.Needs.begin(), FinalizeRefResult.Needs.end()}}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed finalizing oplog container to {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - m_Bucket, - m_Key, - Result.Reason); - } - return Result; - } - - virtual LoadContainerResult LoadContainer() override { return LoadContainer(m_Key); } - - virtual LoadContainerResult LoadBaseContainer() override - { - if (m_OptionalBaseKey == IoHash::Zero) - { - return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}}; - } - return LoadContainer(m_OptionalBaseKey); - } - - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override - { - const int32_t MaxAttempts = 3; - CloudCacheResult GetResult; - { - CloudCacheSession Session(m_CloudClient.Get()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++) - { - GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); - if (!GetResult.Success) - { - Sleep(100 * (Attempt + 1)); - } - } - } - LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; - if (GetResult.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - RawHash, - Result.Reason); - } - return Result; - } - - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override - { - LoadAttachmentsResult Result; - for (const IoHash& Hash : RawHashes) - { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash); - if (ChunkResult.ErrorCode) - { - return LoadAttachmentsResult{ChunkResult}; - } - ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(ChunkResult.ElapsedSeconds * 1000))); - Result.Chunks.emplace_back( - std::pair<IoHash, CompressedBuffer>{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))}); - } - return Result; - } - -private: - LoadContainerResult LoadContainer(const IoHash& Key) - { - const int32_t MaxAttempts = 3; - CloudCacheResult GetResult; - { - CloudCacheSession Session(m_CloudClient.Get()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !GetResult.Success; Attempt++) - { - GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject); - if (!GetResult.Success) - { - Sleep(100 * (Attempt + 1)); - } - } - } - - if (GetResult.ErrorCode || !GetResult.Success) - { - LoadContainerResult Result{ConvertResult(GetResult)}; - Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}/{}. Reason: '{}'", - m_CloudClient->ServiceUrl(), - m_Namespace, - m_Bucket, - Key, - Result.Reason); - return Result; - } - - CbObject ContainerObject = LoadCompactBinaryObject(GetResult.Response); - if (!ContainerObject) - { - return LoadContainerResult{ - RemoteProjectStore::Result{.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError), - .ElapsedSeconds = GetResult.ElapsedSeconds, - .Reason = fmt::format("The ref {}/{}/{}/{} is not formatted as a compact binary object"sv, - m_CloudClient->ServiceUrl(), - m_Namespace, - m_Bucket, - Key)}, - {}}; - } - return LoadContainerResult{ConvertResult(GetResult), std::move(ContainerObject)}; - } - - static Result ConvertResult(const CloudCacheResult& Response) - { - std::string Text; - int32_t ErrorCode = 0; - if (Response.ErrorCode != 0) - { - ErrorCode = Response.ErrorCode; - } - else if (!Response.Success) - { - ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - if (Response.Response.GetContentType() == ZenContentType::kText) - { - Text = - std::string(reinterpret_cast<const std::string::value_type*>(Response.Response.GetData()), Response.Response.GetSize()); - } - } - return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.ElapsedSeconds, .Reason = Response.Reason, .Text = Text}; - } - - Ref<CloudCacheClient> m_CloudClient; - const std::string m_Namespace; - const std::string m_Bucket; - const IoHash m_Key; - const IoHash m_OptionalBaseKey; - std::filesystem::path m_TempFilePath; - bool m_EnableBlocks = true; - bool m_UseTempBlocks = true; -}; - -std::shared_ptr<RemoteProjectStore> -CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::filesystem::path& TempFilePath) -{ - std::string Url = Options.Url; - if (Url.find("://"sv) == std::string::npos) - { - // Assume https URL - Url = fmt::format("https://{}"sv, Url); - } - CloudCacheClientOptions ClientOptions{.Name = "Remote store"sv, - .ServiceUrl = Url, - .ConnectTimeout = std::chrono::milliseconds(2000), - .Timeout = std::chrono::milliseconds(1800000), - .AssumeHttp2 = Options.AssumeHttp2}; - // 1) Access token as parameter in request - // 2) Environment variable (different win vs linux/mac) - // 3) openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider - - std::unique_ptr<CloudCacheTokenProvider> TokenProvider; - if (!Options.AccessToken.empty()) - { - TokenProvider = CloudCacheTokenProvider::CreateFromCallback([AccessToken = "Bearer " + Options.AccessToken]() { - return CloudCacheAccessToken{.Value = AccessToken, .ExpireTime = GcClock::TimePoint::max()}; - }); - } - else - { - TokenProvider = - CloudCacheTokenProvider::CreateFromCallback([&AuthManager = Options.AuthManager, OpenIdProvider = Options.OpenIdProvider]() { - AuthMgr::OpenIdAccessToken Token = AuthManager.GetOpenIdAccessToken(OpenIdProvider.empty() ? "Default" : OpenIdProvider); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); - } - - Ref<CloudCacheClient> CloudClient(new CloudCacheClient(ClientOptions, std::move(TokenProvider))); - - std::shared_ptr<RemoteProjectStore> RemoteStore = std::make_shared<JupiterRemoteStore>(std::move(CloudClient), - Options.Namespace, - Options.Bucket, - Options.Key, - Options.OptionalBaseKey, - Options.ForceDisableBlocks, - Options.ForceDisableTempBlocks, - TempFilePath); - return RemoteStore; -} - -} // namespace zen diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h deleted file mode 100644 index 27f3d9b73..000000000 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "remoteprojectstore.h" - -namespace zen { - -class AuthMgr; - -struct JupiterRemoteStoreOptions : RemoteStoreOptions -{ - std::string Url; - std::string Namespace; - std::string Bucket; - IoHash Key; - IoHash OptionalBaseKey; - std::string OpenIdProvider; - std::string AccessToken; - AuthMgr& AuthManager; - bool ForceDisableBlocks = false; - bool ForceDisableTempBlocks = false; - bool AssumeHttp2 = false; -}; - -std::shared_ptr<RemoteProjectStore> CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath); - -} // namespace zen diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp deleted file mode 100644 index 038a6db47..000000000 --- a/src/zenserver/projectstore/projectstore.cpp +++ /dev/null @@ -1,4162 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "projectstore.h" - -#include <zencore/assertfmt.h> -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compactbinaryutil.h> -#include <zencore/compactbinaryvalidation.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/jobqueue.h> -#include <zencore/logging.h> -#include <zencore/scopeguard.h> -#include <zencore/stream.h> -#include <zencore/timer.h> -#include <zencore/trace.h> -#include <zenstore/caslog.h> -#include <zenstore/cidstore.h> -#include <zenstore/scrubcontext.h> -#include <zenutil/cache/rpcrecording.h> -#include <zenutil/packageformat.h> - -#include "fileremoteprojectstore.h" -#include "jupiterremoteprojectstore.h" -#include "remoteprojectstore.h" -#include "zenremoteprojectstore.h" - -ZEN_THIRD_PARTY_INCLUDES_START -#include <cpr/cpr.h> -#include <tsl/robin_set.h> -#include <xxh3.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_WITH_TESTS -# include <zencore/testing.h> -# include <zencore/testutils.h> -#endif // ZEN_WITH_TESTS - -namespace zen { - -namespace { - bool PrepareDirectoryDelete(const std::filesystem::path& Dir, std::filesystem::path& OutDeleteDir) - { - int DropIndex = 0; - do - { - if (!std::filesystem::exists(Dir)) - { - return true; - } - - std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); - std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) - { - DropIndex++; - continue; - } - - std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); - if (!Ec) - { - OutDeleteDir = DroppedBucketPath; - return true; - } - if (Ec && !std::filesystem::exists(DroppedBucketPath)) - { - // We can't move our folder, probably because it is busy, bail.. - return false; - } - Sleep(100); - } while (true); - } - - struct CreateRemoteStoreResult - { - std::shared_ptr<RemoteProjectStore> Store; - std::string Description; - }; - CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, - AuthMgr& AuthManager, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - const std::filesystem::path& TempFilePath) - { - using namespace std::literals; - - std::shared_ptr<RemoteProjectStore> RemoteStore; - - if (CbObjectView File = Params["file"sv].AsObjectView(); File) - { - std::filesystem::path FolderPath(File["path"sv].AsString()); - if (FolderPath.empty()) - { - return {nullptr, "Missing file path"}; - } - std::string_view Name(File["name"sv].AsString()); - if (Name.empty()) - { - return {nullptr, "Missing file name"}; - } - std::string_view OptionalBaseName(File["basename"sv].AsString()); - bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); - bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); - - FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - FolderPath, - std::string(Name), - std::string(OptionalBaseName), - ForceDisableBlocks, - ForceEnableTempBlocks}; - RemoteStore = CreateFileRemoteStore(Options); - } - - if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) - { - std::string_view CloudServiceUrl = Cloud["url"sv].AsString(); - if (CloudServiceUrl.empty()) - { - return {nullptr, "Missing service url"}; - } - - std::string Url = cpr::util::urlDecode(std::string(CloudServiceUrl)); - std::string_view Namespace = Cloud["namespace"sv].AsString(); - if (Namespace.empty()) - { - return {nullptr, "Missing namespace"}; - } - std::string_view Bucket = Cloud["bucket"sv].AsString(); - if (Bucket.empty()) - { - return {nullptr, "Missing bucket"}; - } - std::string_view OpenIdProvider = Cloud["openid-provider"sv].AsString(); - std::string AccessToken = std::string(Cloud["access-token"sv].AsString()); - if (AccessToken.empty()) - { - std::string_view AccessTokenEnvVariable = Cloud["access-token-env"].AsString(); - if (!AccessTokenEnvVariable.empty()) - { - AccessToken = GetEnvVariable(AccessTokenEnvVariable); - } - } - std::string_view KeyParam = Cloud["key"sv].AsString(); - if (KeyParam.empty()) - { - return {nullptr, "Missing key"}; - } - if (KeyParam.length() != IoHash::StringLength) - { - return {nullptr, "Invalid key"}; - } - IoHash Key = IoHash::FromHexString(KeyParam); - if (Key == IoHash::Zero) - { - return {nullptr, "Invalid key string"}; - } - IoHash BaseKey = IoHash::Zero; - std::string_view BaseKeyParam = Cloud["basekey"sv].AsString(); - if (!BaseKeyParam.empty()) - { - if (BaseKeyParam.length() != IoHash::StringLength) - { - return {nullptr, "Invalid base key"}; - } - BaseKey = IoHash::FromHexString(BaseKeyParam); - if (BaseKey == IoHash::Zero) - { - return {nullptr, "Invalid base key string"}; - } - } - - bool ForceDisableBlocks = Cloud["disableblocks"sv].AsBool(false); - bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); - bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); - - JupiterRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, - std::string(Namespace), - std::string(Bucket), - Key, - BaseKey, - std::string(OpenIdProvider), - AccessToken, - AuthManager, - ForceDisableBlocks, - ForceDisableTempBlocks, - AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath); - } - - if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) - { - std::string_view Url = Zen["url"sv].AsString(); - std::string_view Project = Zen["project"sv].AsString(); - if (Project.empty()) - { - return {nullptr, "Missing project"}; - } - std::string_view Oplog = Zen["oplog"sv].AsString(); - if (Oplog.empty()) - { - return {nullptr, "Missing oplog"}; - } - ZenRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - std::string(Url), - std::string(Project), - std::string(Oplog)}; - RemoteStore = CreateZenRemoteStore(Options); - } - - if (!RemoteStore) - { - return {nullptr, "Unknown remote store type"}; - } - - return {std::move(RemoteStore), ""}; - } - - std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result) - { - if (Result.ErrorCode == 0) - { - return {HttpResponseCode::OK, Result.Text}; - } - return {static_cast<HttpResponseCode>(Result.ErrorCode), - Result.Reason.empty() ? Result.Text - : Result.Text.empty() ? Result.Reason - : fmt::format("{}. Reason: '{}'", Result.Text, Result.Reason)}; - } - -} // namespace - -////////////////////////////////////////////////////////////////////////// - -struct ProjectStore::OplogStorage : public RefCounted -{ - OplogStorage(ProjectStore::Oplog* OwnerOplog, std::filesystem::path BasePath) : m_OwnerOplog(OwnerOplog), m_OplogStoragePath(BasePath) - { - } - - ~OplogStorage() - { - ZEN_INFO("closing oplog storage at {}", m_OplogStoragePath); - Flush(); - } - - [[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); } - [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath) - { - return std::filesystem::exists(BasePath / "ops.zlog") && std::filesystem::exists(BasePath / "ops.zops"); - } - - static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); } - - uint64_t OpBlobsSize() const { return OpBlobsSize(m_OplogStoragePath); } - static uint64_t OpBlobsSize(const std::filesystem::path& BasePath) - { - using namespace std::literals; - if (Exists(BasePath)) - { - return std::filesystem::file_size(BasePath / "ops.zlog"sv) + std::filesystem::file_size(BasePath / "ops.zops"sv); - } - return 0; - } - - void Open(bool IsCreate) - { - ZEN_TRACE_CPU("Store::OplogStorage::Open"); - - using namespace std::literals; - - ZEN_INFO("initializing oplog storage at '{}'", m_OplogStoragePath); - - if (IsCreate) - { - DeleteDirectories(m_OplogStoragePath); - CreateDirectories(m_OplogStoragePath); - } - - m_Oplog.Open(m_OplogStoragePath / "ops.zlog"sv, IsCreate ? CasLogFile::Mode::kTruncate : CasLogFile::Mode::kWrite); - m_Oplog.Initialize(); - - m_OpBlobs.Open(m_OplogStoragePath / "ops.zops"sv, IsCreate ? BasicFile::Mode::kTruncate : BasicFile::Mode::kWrite); - - ZEN_ASSERT(IsPow2(m_OpsAlign)); - ZEN_ASSERT(!(m_NextOpsOffset & (m_OpsAlign - 1))); - } - - void ReplayLog(std::function<void(CbObjectView, const OplogEntry&)>&& Handler) - { - ZEN_TRACE_CPU("Store::OplogStorage::ReplayLog"); - - // This could use memory mapping or do something clever but for now it just reads the file sequentially - - ZEN_INFO("replaying log for '{}'", m_OplogStoragePath); - - Stopwatch Timer; - - uint64_t InvalidEntries = 0; - uint64_t TombstoneEntries = 0; - - std::vector<OplogEntry> OpLogEntries; - std::vector<size_t> OplogOrder; - { - tsl::robin_map<Oid, size_t, Oid::Hasher> LatestKeys; - const uint64_t SkipEntryCount = 0; - - m_Oplog.Replay( - [&](const OplogEntry& LogEntry) { - if (LogEntry.IsTombstone()) - { - if (auto It = LatestKeys.find(LogEntry.OpKeyHash); It == LatestKeys.end()) - { - ZEN_SCOPED_WARN("found tombstone referencing unknown key {}", LogEntry.OpKeyHash); - } - } - else - { - if (LogEntry.OpCoreSize == 0) - { - ++InvalidEntries; - return; - } - - const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign; - m_NextOpsOffset = - Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); - } - - if (auto It = LatestKeys.find(LogEntry.OpKeyHash); It != LatestKeys.end()) - { - OplogEntry& Entry = OpLogEntries[It->second]; - - if (LogEntry.IsTombstone() && Entry.IsTombstone()) - { - ZEN_SCOPED_WARN("found double tombstone - '{}'", LogEntry.OpKeyHash); - } - - Entry = LogEntry; - } - else - { - const size_t OpIndex = OpLogEntries.size(); - LatestKeys[LogEntry.OpKeyHash] = OpIndex; - OplogOrder.push_back(OpIndex); - OpLogEntries.push_back(LogEntry); - } - }, - SkipEntryCount); - } - - std::sort(OplogOrder.begin(), OplogOrder.end(), [&](size_t Lhs, size_t Rhs) { - const OplogEntry& LhsEntry = OpLogEntries[Lhs]; - const OplogEntry& RhsEntry = OpLogEntries[Rhs]; - return LhsEntry.OpCoreOffset < RhsEntry.OpCoreOffset; - }); - - BasicFileBuffer OpBlobsBuffer(m_OpBlobs, 65536); - - for (size_t OplogOrderIndex : OplogOrder) - { - const OplogEntry& LogEntry = OpLogEntries[OplogOrderIndex]; - - if (LogEntry.IsTombstone()) - { - TombstoneEntries++; - } - else - { - // Verify checksum, ignore op data if incorrect - - auto VerifyAndHandleOp = [&](MemoryView OpBufferView) { - const uint32_t OpCoreHash = uint32_t(XXH3_64bits(OpBufferView.GetData(), LogEntry.OpCoreSize) & 0xffffFFFF); - - if (OpCoreHash == LogEntry.OpCoreHash) - { - Handler(CbObjectView(OpBufferView.GetData()), LogEntry); - } - else - { - ZEN_WARN("skipping oplog entry with bad checksum!"); - InvalidEntries++; - } - }; - - const uint64_t OpFileOffset = LogEntry.OpCoreOffset * m_OpsAlign; - const MemoryView OpBufferView = OpBlobsBuffer.MakeView(LogEntry.OpCoreSize, OpFileOffset); - if (OpBufferView.GetSize() == LogEntry.OpCoreSize) - { - VerifyAndHandleOp(OpBufferView); - } - else - { - IoBuffer OpBuffer(LogEntry.OpCoreSize); - OpBlobsBuffer.Read((void*)OpBuffer.Data(), LogEntry.OpCoreSize, OpFileOffset); - - VerifyAndHandleOp(OpBuffer); - } - } - } - - if (InvalidEntries) - { - ZEN_WARN("ignored {} invalid oplog entries", InvalidEntries); - } - - ZEN_INFO("oplog replay completed in {} - Max LSN# {}, Next offset: {}, {} tombstones", - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - m_MaxLsn.load(), - m_NextOpsOffset.load(), - TombstoneEntries); - } - - void ReplayLogEntries(const std::span<OplogEntryAddress> Entries, std::function<void(CbObjectView)>&& Handler) - { - ZEN_TRACE_CPU("Store::OplogStorage::ReplayLogEntries"); - - BasicFileBuffer OpBlobsBuffer(m_OpBlobs, 65536); - - for (const OplogEntryAddress& Entry : Entries) - { - const uint64_t OpFileOffset = Entry.Offset * m_OpsAlign; - MemoryView OpBufferView = OpBlobsBuffer.MakeView(Entry.Size, OpFileOffset); - if (OpBufferView.GetSize() == Entry.Size) - { - Handler(CbObjectView(OpBufferView.GetData())); - continue; - } - IoBuffer OpBuffer(Entry.Size); - OpBlobsBuffer.Read((void*)OpBuffer.Data(), Entry.Size, OpFileOffset); - Handler(CbObjectView(OpBuffer.Data())); - } - } - - CbObject GetOp(const OplogEntryAddress& Entry) - { - ZEN_TRACE_CPU("Store::OplogStorage::GetOp"); - - IoBuffer OpBuffer(Entry.Size); - - const uint64_t OpFileOffset = Entry.Offset * m_OpsAlign; - m_OpBlobs.Read((void*)OpBuffer.Data(), Entry.Size, OpFileOffset); - - return CbObject(SharedBuffer(std::move(OpBuffer))); - } - - OplogEntry AppendOp(SharedBuffer Buffer, uint32_t OpCoreHash, Oid KeyHash) - { - ZEN_TRACE_CPU("Store::OplogStorage::AppendOp"); - - using namespace std::literals; - - uint64_t WriteSize = Buffer.GetSize(); - - RwLock::ExclusiveLockScope Lock(m_RwLock); - const uint64_t WriteOffset = m_NextOpsOffset; - const uint32_t OpLsn = ++m_MaxLsn; - m_NextOpsOffset = RoundUp(WriteOffset + WriteSize, m_OpsAlign); - Lock.ReleaseNow(); - - ZEN_ASSERT(IsMultipleOf(WriteOffset, m_OpsAlign)); - - OplogEntry Entry = {.OpLsn = OpLsn, - .OpCoreOffset = gsl::narrow_cast<uint32_t>(WriteOffset / m_OpsAlign), - .OpCoreSize = uint32_t(WriteSize), - .OpCoreHash = OpCoreHash, - .OpKeyHash = KeyHash}; - - m_Oplog.Append(Entry); - m_OpBlobs.Write(Buffer.GetData(), WriteSize, WriteOffset); - - return Entry; - } - - void AppendTombstone(Oid KeyHash) - { - OplogEntry Entry = {.OpKeyHash = KeyHash}; - Entry.MakeTombstone(); - - m_Oplog.Append(Entry); - } - - void Flush() - { - m_Oplog.Flush(); - m_OpBlobs.Flush(); - } - - uint32_t GetMaxLsn() const { return m_MaxLsn.load(); } - - LoggerRef Log() { return m_OwnerOplog->Log(); } - -private: - ProjectStore::Oplog* m_OwnerOplog; - std::filesystem::path m_OplogStoragePath; - mutable RwLock m_RwLock; - TCasLogFile<OplogEntry> m_Oplog; - BasicFile m_OpBlobs; - std::atomic<uint64_t> m_NextOpsOffset{0}; - uint64_t m_OpsAlign = 32; - std::atomic<uint32_t> m_MaxLsn{0}; -}; - -////////////////////////////////////////////////////////////////////////// - -ProjectStore::Oplog::Oplog(std::string_view Id, - Project* Project, - CidStore& Store, - std::filesystem::path BasePath, - const std::filesystem::path& MarkerPath) -: m_OuterProject(Project) -, m_CidStore(Store) -, m_BasePath(BasePath) -, m_MarkerPath(MarkerPath) -, m_OplogId(Id) -{ - using namespace std::literals; - - m_Storage = new OplogStorage(this, m_BasePath); - const bool StoreExists = m_Storage->Exists(); - m_Storage->Open(/* IsCreate */ !StoreExists); - - m_TempPath = m_BasePath / "temp"sv; - - CleanDirectory(m_TempPath); -} - -ProjectStore::Oplog::~Oplog() -{ - if (m_Storage) - { - Flush(); - } -} - -void -ProjectStore::Oplog::Flush() -{ - ZEN_ASSERT(m_Storage); - m_Storage->Flush(); -} - -void -ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx) -{ - std::vector<Oid> BadEntryKeys; - - using namespace std::literals; - - IterateOplogWithKey([&](int Lsn, const Oid& Key, CbObjectView Op) { - ZEN_UNUSED(Lsn); - - std::vector<IoHash> Cids; - Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); }); - - { - XXH3_128Stream KeyHasher; - Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); - XXH3_128 KeyHash128 = KeyHasher.GetHash(); - Oid KeyHash; - memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash); - - ZEN_ASSERT_FORMAT(KeyHash == Key, "oplog data does not match information from index (op:{} != index:{})", KeyHash, Key); - } - - for (const IoHash& Cid : Cids) - { - if (!m_CidStore.ContainsChunk(Cid)) - { - // oplog entry references a CAS chunk which is not - // present - BadEntryKeys.push_back(Key); - return; - } - if (Ctx.IsBadCid(Cid)) - { - // oplog entry references a CAS chunk which has been - // flagged as bad - BadEntryKeys.push_back(Key); - return; - } - } - }); - - if (!BadEntryKeys.empty()) - { - if (Ctx.RunRecovery()) - { - ZEN_WARN("scrubbing found {} bad ops in oplog @ '{}', these will be removed from the index", BadEntryKeys.size(), m_BasePath); - - // Actually perform some clean-up - RwLock::ExclusiveLockScope _(m_OplogLock); - - for (const auto& Key : BadEntryKeys) - { - m_LatestOpMap.erase(Key); - m_Storage->AppendTombstone(Key); - } - } - else - { - ZEN_WARN("scrubbing found {} bad ops in oplog @ '{}' but no cleanup will be performed", BadEntryKeys.size(), m_BasePath); - } - } -} - -void -ProjectStore::Oplog::GatherReferences(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("Store::Oplog::GatherReferences"); - if (GcCtx.SkipCid()) - { - return; - } - - std::vector<IoHash> Cids; - Cids.reserve(1024); - IterateOplog([&](CbObjectView Op) { - Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); }); - if (Cids.size() >= 1024) - { - GcCtx.AddRetainedCids(Cids); - Cids.clear(); - } - }); - GcCtx.AddRetainedCids(Cids); -} - -uint64_t -ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath) -{ - using namespace std::literals; - - uint64_t Size = OplogStorage::OpBlobsSize(BasePath); - std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::exists(StateFilePath)) - { - Size += std::filesystem::file_size(StateFilePath); - } - - return Size; -} - -uint64_t -ProjectStore::Oplog::TotalSize() const -{ - return TotalSize(m_BasePath); -} - -std::filesystem::path -ProjectStore::Oplog::PrepareForDelete(bool MoveFolder) -{ - RwLock::ExclusiveLockScope _(m_OplogLock); - m_ChunkMap.clear(); - m_MetaMap.clear(); - m_FileMap.clear(); - m_OpAddressMap.clear(); - m_LatestOpMap.clear(); - m_Storage = {}; - if (!MoveFolder) - { - return {}; - } - std::filesystem::path MovedDir; - if (PrepareDirectoryDelete(m_BasePath, MovedDir)) - { - return MovedDir; - } - return {}; -} - -bool -ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath) -{ - using namespace std::literals; - - std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - return std::filesystem::is_regular_file(StateFilePath); -} - -void -ProjectStore::Oplog::Read() -{ - using namespace std::literals; - - std::filesystem::path StateFilePath = m_BasePath / "oplog.zcb"sv; - if (std::filesystem::is_regular_file(StateFilePath)) - { - ZEN_INFO("reading config for oplog '{}' in project '{}' from {}", m_OplogId, m_OuterProject->Identifier, StateFilePath); - - BasicFile Blob; - Blob.Open(StateFilePath, BasicFile::Mode::kRead); - - IoBuffer Obj = Blob.ReadAll(); - CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); - - if (ValidationError != CbValidateError::None) - { - ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), StateFilePath); - return; - } - - CbObject Cfg = LoadCompactBinaryObject(Obj); - - m_MarkerPath = Cfg["gcpath"sv].AsString(); - } - else - { - ZEN_INFO("config for oplog '{}' in project '{}' not found at {}. Assuming legacy store", - m_OplogId, - m_OuterProject->Identifier, - StateFilePath); - } - ReplayLog(); -} - -void -ProjectStore::Oplog::Write() -{ - using namespace std::literals; - - BinaryWriter Mem; - - CbObjectWriter Cfg; - - Cfg << "gcpath"sv << PathToUtf8(m_MarkerPath); - - Cfg.Save(Mem); - - std::filesystem::path StateFilePath = m_BasePath / "oplog.zcb"sv; - - ZEN_INFO("persisting config for oplog '{}' in project '{}' to {}", m_OplogId, m_OuterProject->Identifier, StateFilePath); - - BasicFile Blob; - Blob.Open(StateFilePath, BasicFile::Mode::kTruncate); - Blob.Write(Mem.Data(), Mem.Size(), 0); - Blob.Flush(); -} - -void -ProjectStore::Oplog::Update(const std::filesystem::path& MarkerPath) -{ - if (m_MarkerPath == MarkerPath) - { - return; - } - Write(); -} - -void -ProjectStore::Oplog::ReplayLog() -{ - ZEN_LOG_SCOPE("ReplayLog '{}'", m_OplogId); - - RwLock::ExclusiveLockScope OplogLock(m_OplogLock); - if (!m_Storage) - { - return; - } - m_Storage->ReplayLog([&](CbObjectView Op, const OplogEntry& OpEntry) { RegisterOplogEntry(OplogLock, GetMapping(Op), OpEntry); }); -} - -IoBuffer -ProjectStore::Oplog::FindChunk(Oid ChunkId) -{ - RwLock::SharedLockScope OplogLock(m_OplogLock); - if (!m_Storage) - { - return IoBuffer{}; - } - - if (auto ChunkIt = m_ChunkMap.find(ChunkId); ChunkIt != m_ChunkMap.end()) - { - IoHash ChunkHash = ChunkIt->second; - OplogLock.ReleaseNow(); - - IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkHash); - Chunk.SetContentType(ZenContentType::kCompressedBinary); - - return Chunk; - } - - if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) - { - std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - - OplogLock.ReleaseNow(); - - IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath); - FileChunk.SetContentType(ZenContentType::kBinary); - - return FileChunk; - } - - if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end()) - { - IoHash ChunkHash = MetaIt->second; - OplogLock.ReleaseNow(); - - IoBuffer Chunk = m_CidStore.FindChunkByCid(ChunkHash); - Chunk.SetContentType(ZenContentType::kCompressedBinary); - - return Chunk; - } - - return {}; -} - -std::vector<ProjectStore::Oplog::ChunkInfo> -ProjectStore::Oplog::GetAllChunksInfo() -{ - // First just capture all the chunk ids - - std::vector<ChunkInfo> InfoArray; - - { - RwLock::SharedLockScope _(m_OplogLock); - - if (m_Storage) - { - const size_t NumEntries = m_FileMap.size() + m_ChunkMap.size(); - - InfoArray.reserve(NumEntries); - - for (const auto& Kv : m_FileMap) - { - InfoArray.push_back({.ChunkId = Kv.first}); - } - - for (const auto& Kv : m_ChunkMap) - { - InfoArray.push_back({.ChunkId = Kv.first}); - } - } - } - - for (ChunkInfo& Info : InfoArray) - { - if (IoBuffer Chunk = FindChunk(Info.ChunkId)) - { - Info.ChunkSize = Chunk.GetSize(); - } - } - - return InfoArray; -} - -void -ProjectStore::Oplog::IterateChunkMap(std::function<void(const Oid&, const IoHash&)>&& Fn) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return; - } - - for (const auto& Kv : m_ChunkMap) - { - Fn(Kv.first, Kv.second); - } -} - -void -ProjectStore::Oplog::IterateFileMap( - std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return; - } - - for (const auto& Kv : m_FileMap) - { - Fn(Kv.first, Kv.second.ServerPath, Kv.second.ClientPath); - } -} - -void -ProjectStore::Oplog::IterateOplog(std::function<void(CbObjectView)>&& Handler) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return; - } - - std::vector<OplogEntryAddress> Entries; - Entries.reserve(m_LatestOpMap.size()); - - for (const auto& Kv : m_LatestOpMap) - { - const auto AddressEntry = m_OpAddressMap.find(Kv.second); - ZEN_ASSERT(AddressEntry != m_OpAddressMap.end()); - - Entries.push_back(AddressEntry->second); - } - - std::sort(Entries.begin(), Entries.end(), [](const OplogEntryAddress& Lhs, const OplogEntryAddress& Rhs) { - return Lhs.Offset < Rhs.Offset; - }); - - m_Storage->ReplayLogEntries(Entries, [&](CbObjectView Op) { Handler(Op); }); -} - -void -ProjectStore::Oplog::IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Handler) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return; - } - - std::vector<OplogEntryAddress> SortedEntries; - std::vector<Oid> SortedKeys; - std::vector<int> SortedLSNs; - - { - const auto TargetEntryCount = m_LatestOpMap.size(); - - std::vector<size_t> EntryIndexes; - std::vector<OplogEntryAddress> Entries; - std::vector<Oid> Keys; - std::vector<int> LSNs; - - Entries.reserve(TargetEntryCount); - EntryIndexes.reserve(TargetEntryCount); - Keys.reserve(TargetEntryCount); - LSNs.reserve(TargetEntryCount); - - for (const auto& Kv : m_LatestOpMap) - { - const auto AddressEntry = m_OpAddressMap.find(Kv.second); - ZEN_ASSERT(AddressEntry != m_OpAddressMap.end()); - - Entries.push_back(AddressEntry->second); - Keys.push_back(Kv.first); - LSNs.push_back(Kv.second); - EntryIndexes.push_back(EntryIndexes.size()); - } - - std::sort(EntryIndexes.begin(), EntryIndexes.end(), [&Entries](const size_t& Lhs, const size_t& Rhs) { - const OplogEntryAddress& LhsEntry = Entries[Lhs]; - const OplogEntryAddress& RhsEntry = Entries[Rhs]; - return LhsEntry.Offset < RhsEntry.Offset; - }); - - SortedEntries.reserve(EntryIndexes.size()); - SortedKeys.reserve(EntryIndexes.size()); - SortedLSNs.reserve(EntryIndexes.size()); - - for (size_t Index : EntryIndexes) - { - SortedEntries.push_back(Entries[Index]); - SortedKeys.push_back(Keys[Index]); - SortedLSNs.push_back(LSNs[Index]); - } - } - - size_t EntryIndex = 0; - m_Storage->ReplayLogEntries(SortedEntries, [&](CbObjectView Op) { - Handler(SortedLSNs[EntryIndex], SortedKeys[EntryIndex], Op); - EntryIndex++; - }); -} - -int -ProjectStore::Oplog::GetOpIndexByKey(const Oid& Key) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return {}; - } - if (const auto LatestOp = m_LatestOpMap.find(Key); LatestOp != m_LatestOpMap.end()) - { - return LatestOp->second; - } - return -1; -} - -int -ProjectStore::Oplog::GetMaxOpIndex() const -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return -1; - } - return gsl::narrow<int>(m_Storage->GetMaxLsn()); -} - -std::optional<CbObject> -ProjectStore::Oplog::GetOpByKey(const Oid& Key) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return {}; - } - - if (const auto LatestOp = m_LatestOpMap.find(Key); LatestOp != m_LatestOpMap.end()) - { - const auto AddressEntry = m_OpAddressMap.find(LatestOp->second); - ZEN_ASSERT(AddressEntry != m_OpAddressMap.end()); - - return m_Storage->GetOp(AddressEntry->second); - } - - return {}; -} - -std::optional<CbObject> -ProjectStore::Oplog::GetOpByIndex(int Index) -{ - RwLock::SharedLockScope _(m_OplogLock); - if (!m_Storage) - { - return {}; - } - - if (const auto AddressEntryIt = m_OpAddressMap.find(Index); AddressEntryIt != m_OpAddressMap.end()) - { - return m_Storage->GetOp(AddressEntryIt->second); - } - - return {}; -} - -void -ProjectStore::Oplog::AddChunkMappings(const std::unordered_map<Oid, IoHash, Oid::Hasher>& ChunkMappings) -{ - RwLock::ExclusiveLockScope OplogLock(m_OplogLock); - for (const auto& It : ChunkMappings) - { - AddChunkMapping(OplogLock, It.first, It.second); - } -} - -void -ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&, - Oid FileId, - IoHash Hash, - std::string_view ServerPath, - std::string_view ClientPath) -{ - FileMapEntry Entry; - - if (Hash != IoHash::Zero) - { - m_ChunkMap.insert_or_assign(FileId, Hash); - } - else - { - Entry.ServerPath = ServerPath; - } - - Entry.ClientPath = ClientPath; - - m_FileMap[FileId] = std::move(Entry); -} - -void -ProjectStore::Oplog::AddChunkMapping(const RwLock::ExclusiveLockScope&, Oid ChunkId, IoHash Hash) -{ - m_ChunkMap.insert_or_assign(ChunkId, Hash); -} - -void -ProjectStore::Oplog::AddMetaMapping(const RwLock::ExclusiveLockScope&, Oid ChunkId, IoHash Hash) -{ - m_MetaMap.insert_or_assign(ChunkId, Hash); -} - -ProjectStore::Oplog::OplogEntryMapping -ProjectStore::Oplog::GetMapping(CbObjectView Core) -{ - using namespace std::literals; - - OplogEntryMapping Result; - - // Update chunk id maps - for (CbFieldView Field : Core) - { - std::string_view FieldName = Field.GetName(); - if (FieldName == "package"sv) - { - CbObjectView PackageObj = Field.AsObjectView(); - Oid Id = PackageObj["id"sv].AsObjectId(); - IoHash Hash = PackageObj["data"sv].AsBinaryAttachment(); - Result.Chunks.emplace_back(OplogEntryMapping::Mapping{Id, Hash}); - ZEN_DEBUG("package data {} -> {}", Id, Hash); - continue; - } - if (FieldName == "bulkdata"sv) - { - CbArrayView BulkDataArray = Field.AsArrayView(); - for (CbFieldView& Entry : BulkDataArray) - { - CbObjectView BulkObj = Entry.AsObjectView(); - Oid Id = BulkObj["id"sv].AsObjectId(); - IoHash Hash = BulkObj["data"sv].AsBinaryAttachment(); - Result.Chunks.emplace_back(OplogEntryMapping::Mapping{Id, Hash}); - ZEN_DEBUG("bulkdata {} -> {}", Id, Hash); - } - continue; - } - if (FieldName == "packagedata"sv) - { - CbArrayView PackageDataArray = Field.AsArrayView(); - for (CbFieldView& Entry : PackageDataArray) - { - CbObjectView PackageDataObj = Entry.AsObjectView(); - Oid Id = PackageDataObj["id"sv].AsObjectId(); - IoHash Hash = PackageDataObj["data"sv].AsBinaryAttachment(); - Result.Chunks.emplace_back(OplogEntryMapping::Mapping{Id, Hash}); - ZEN_DEBUG("package {} -> {}", Id, Hash); - } - continue; - } - if (FieldName == "files"sv) - { - CbArrayView FilesArray = Field.AsArrayView(); - Result.Files.reserve(FilesArray.Num()); - for (CbFieldView& Entry : FilesArray) - { - CbObjectView FileObj = Entry.AsObjectView(); - - std::string_view ServerPath = FileObj["serverpath"sv].AsString(); - std::string_view ClientPath = FileObj["clientpath"sv].AsString(); - Oid Id = FileObj["id"sv].AsObjectId(); - IoHash Hash = FileObj["data"sv].AsBinaryAttachment(); - if (ServerPath.empty() && Hash == IoHash::Zero) - { - ZEN_WARN("invalid file for entry '{}', missing both 'serverpath' and 'data' fields", Id); - continue; - } - if (ClientPath.empty()) - { - ZEN_WARN("invalid file for entry '{}', missing 'clientpath' field", Id); - continue; - } - - Result.Files.emplace_back(OplogEntryMapping::FileMapping{Id, Hash, std::string(ServerPath), std::string(ClientPath)}); - ZEN_DEBUG("file {} -> {}, ServerPath: {}, ClientPath: {}", Id, Hash, ServerPath, ClientPath); - } - continue; - } - if (FieldName == "meta"sv) - { - CbArrayView MetaArray = Field.AsArrayView(); - Result.Meta.reserve(MetaArray.Num()); - for (CbFieldView& Entry : MetaArray) - { - CbObjectView MetaObj = Entry.AsObjectView(); - Oid Id = MetaObj["id"sv].AsObjectId(); - IoHash Hash = MetaObj["data"sv].AsBinaryAttachment(); - Result.Meta.emplace_back(OplogEntryMapping::Mapping{Id, Hash}); - auto NameString = MetaObj["name"sv].AsString(); - ZEN_DEBUG("meta data ({}) {} -> {}", NameString, Id, Hash); - } - continue; - } - } - - return Result; -} - -uint32_t -ProjectStore::Oplog::RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock, - const OplogEntryMapping& OpMapping, - const OplogEntry& OpEntry) -{ - // For now we're assuming the update is all in-memory so we can hold an exclusive lock without causing - // too many problems. Longer term we'll probably want to ensure we can do concurrent updates however - - using namespace std::literals; - - // Update chunk id maps - for (const OplogEntryMapping::Mapping& Chunk : OpMapping.Chunks) - { - AddChunkMapping(OplogLock, Chunk.Id, Chunk.Hash); - } - - for (const OplogEntryMapping::FileMapping& File : OpMapping.Files) - { - AddFileMapping(OplogLock, File.Id, File.Hash, File.ServerPath, File.ClientPath); - } - - for (const OplogEntryMapping::Mapping& Meta : OpMapping.Meta) - { - AddMetaMapping(OplogLock, Meta.Id, Meta.Hash); - } - - m_OpAddressMap.emplace(OpEntry.OpLsn, OplogEntryAddress{.Offset = OpEntry.OpCoreOffset, .Size = OpEntry.OpCoreSize}); - m_LatestOpMap[OpEntry.OpKeyHash] = OpEntry.OpLsn; - - return OpEntry.OpLsn; -} - -uint32_t -ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) -{ - ZEN_TRACE_CPU("Store::Oplog::AppendNewOplogEntry"); - - const CbObject& Core = OpPackage.GetObject(); - const uint32_t EntryId = AppendNewOplogEntry(Core); - if (EntryId == 0xffffffffu) - { - // The oplog has been deleted so just drop this - return EntryId; - } - - // Persist attachments after oplog entry so GC won't find attachments without references - - uint64_t AttachmentBytes = 0; - uint64_t NewAttachmentBytes = 0; - - auto Attachments = OpPackage.GetAttachments(); - - for (const auto& Attach : Attachments) - { - ZEN_ASSERT(Attach.IsCompressedBinary()); - - CompressedBuffer AttachmentData = Attach.AsCompressedBinary(); - const uint64_t AttachmentSize = AttachmentData.DecodeRawSize(); - CidStore::InsertResult InsertResult = m_CidStore.AddChunk(AttachmentData.GetCompressed().Flatten().AsIoBuffer(), Attach.GetHash()); - - if (InsertResult.New) - { - NewAttachmentBytes += AttachmentSize; - } - AttachmentBytes += AttachmentSize; - } - - ZEN_DEBUG("oplog entry #{} attachments: {} new, {} total", EntryId, NiceBytes(NewAttachmentBytes), NiceBytes(AttachmentBytes)); - - return EntryId; -} - -uint32_t -ProjectStore::Oplog::AppendNewOplogEntry(CbObject Core) -{ - ZEN_TRACE_CPU("Store::Oplog::AppendNewOplogEntry"); - - using namespace std::literals; - - OplogEntryMapping Mapping = GetMapping(Core); - - SharedBuffer Buffer = Core.GetBuffer(); - const uint64_t WriteSize = Buffer.GetSize(); - const auto OpCoreHash = uint32_t(XXH3_64bits(Buffer.GetData(), WriteSize) & 0xffffFFFF); - - ZEN_ASSERT(WriteSize != 0); - - XXH3_128Stream KeyHasher; - Core["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); }); - XXH3_128 KeyHash128 = KeyHasher.GetHash(); - Oid KeyHash; - memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash); - - RefPtr<OplogStorage> Storage; - { - RwLock::SharedLockScope _(m_OplogLock); - Storage = m_Storage; - } - if (!m_Storage) - { - return 0xffffffffu; - } - const OplogEntry OpEntry = m_Storage->AppendOp(Buffer, OpCoreHash, KeyHash); - - RwLock::ExclusiveLockScope OplogLock(m_OplogLock); - const uint32_t EntryId = RegisterOplogEntry(OplogLock, Mapping, OpEntry); - - return EntryId; -} - -////////////////////////////////////////////////////////////////////////// - -ProjectStore::Project::Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath) -: m_ProjectStore(PrjStore) -, m_CidStore(Store) -, m_OplogStoragePath(BasePath) -, m_LastAccessTimes({std::make_pair(std::string(), GcClock::TickCount())}) -{ -} - -ProjectStore::Project::~Project() -{ - // Only write access times if we have not been explicitly deleted - if (!m_OplogStoragePath.empty()) - { - WriteAccessTimes(); - } -} - -bool -ProjectStore::Project::Exists(const std::filesystem::path& BasePath) -{ - return std::filesystem::exists(BasePath / "Project.zcb"); -} - -void -ProjectStore::Project::Read() -{ - ZEN_TRACE_CPU("Store::Project::Read"); - - using namespace std::literals; - - std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"sv; - - ZEN_INFO("reading config for project '{}' from {}", Identifier, ProjectStateFilePath); - - BasicFile Blob; - Blob.Open(ProjectStateFilePath, BasicFile::Mode::kRead); - - IoBuffer Obj = Blob.ReadAll(); - CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); - - if (ValidationError == CbValidateError::None) - { - CbObject Cfg = LoadCompactBinaryObject(Obj); - - Identifier = Cfg["id"sv].AsString(); - RootDir = Cfg["root"sv].AsString(); - ProjectRootDir = Cfg["project"sv].AsString(); - EngineRootDir = Cfg["engine"sv].AsString(); - ProjectFilePath = Cfg["projectfile"sv].AsString(); - } - else - { - ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectStateFilePath); - } - - ReadAccessTimes(); -} - -void -ProjectStore::Project::Write() -{ - ZEN_TRACE_CPU("Store::Project::Write"); - - using namespace std::literals; - - BinaryWriter Mem; - - CbObjectWriter Cfg; - Cfg << "id"sv << Identifier; - Cfg << "root"sv << PathToUtf8(RootDir); - Cfg << "project"sv << ProjectRootDir; - Cfg << "engine"sv << EngineRootDir; - Cfg << "projectfile"sv << ProjectFilePath; - - Cfg.Save(Mem); - - CreateDirectories(m_OplogStoragePath); - - std::filesystem::path ProjectStateFilePath = m_OplogStoragePath / "Project.zcb"sv; - - ZEN_INFO("persisting config for project '{}' to {}", Identifier, ProjectStateFilePath); - - BasicFile Blob; - Blob.Open(ProjectStateFilePath, BasicFile::Mode::kTruncate); - Blob.Write(Mem.Data(), Mem.Size(), 0); - Blob.Flush(); -} - -void -ProjectStore::Project::ReadAccessTimes() -{ - using namespace std::literals; - - RwLock::SharedLockScope _(m_ProjectLock); - - std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - if (!std::filesystem::exists(ProjectAccessTimesFilePath)) - { - return; - } - - ZEN_INFO("reading access times for project '{}' from {}", Identifier, ProjectAccessTimesFilePath); - - BasicFile Blob; - Blob.Open(ProjectAccessTimesFilePath, BasicFile::Mode::kRead); - - IoBuffer Obj = Blob.ReadAll(); - CbValidateError ValidationError = ValidateCompactBinary(MemoryView(Obj.Data(), Obj.Size()), CbValidateMode::All); - - if (ValidationError == CbValidateError::None) - { - CbObject Reader = LoadCompactBinaryObject(Obj); - - uint64_t Count = Reader["count"sv].AsUInt64(0); - if (Count > 0) - { - std::vector<uint64_t> Ticks; - Ticks.reserve(Count); - CbArrayView TicksArray = Reader["ticks"sv].AsArrayView(); - for (CbFieldView& TickView : TicksArray) - { - Ticks.emplace_back(TickView.AsUInt64()); - } - CbArrayView IdArray = Reader["ids"sv].AsArrayView(); - uint64_t Index = 0; - for (CbFieldView& IdView : IdArray) - { - std::string_view Id = IdView.AsString(); - m_LastAccessTimes.insert_or_assign(std::string(Id), Ticks[Index++]); - } - } - - ////// Legacy format read - { - CbArrayView LastAccessTimes = Reader["lastaccess"sv].AsArrayView(); - for (CbFieldView& Entry : LastAccessTimes) - { - CbObjectView AccessTime = Entry.AsObjectView(); - std::string_view Id = AccessTime["id"sv].AsString(); - GcClock::Tick AccessTick = AccessTime["tick"sv].AsUInt64(); - m_LastAccessTimes.insert_or_assign(std::string(Id), AccessTick); - } - } - } - else - { - ZEN_ERROR("validation error {} hit for '{}'", int(ValidationError), ProjectAccessTimesFilePath); - } -} - -void -ProjectStore::Project::WriteAccessTimes() -{ - using namespace std::literals; - - CbObjectWriter Writer; - - Writer.AddInteger("count", gsl::narrow<uint64_t>(m_LastAccessTimes.size())); - Writer.BeginArray("ids"); - - { - RwLock::SharedLockScope _(m_ProjectLock); - for (const auto& It : m_LastAccessTimes) - { - Writer << It.first; - } - Writer.EndArray(); - Writer.BeginArray("ticks"); - for (const auto& It : m_LastAccessTimes) - { - Writer << gsl::narrow<uint64_t>(It.second); - } - Writer.EndArray(); - } - - CbObject Data = Writer.Save(); - - try - { - CreateDirectories(m_OplogStoragePath); - - std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - - ZEN_INFO("persisting access times for project '{}' to {}", Identifier, ProjectAccessTimesFilePath); - - WriteFile(ProjectAccessTimesFilePath, Data.GetBuffer().AsIoBuffer()); - } - catch (std::exception& Err) - { - ZEN_WARN("writing access times FAILED, reason: '{}'", Err.what()); - } -} - -LoggerRef -ProjectStore::Project::Log() -{ - return m_ProjectStore->Log(); -} - -std::filesystem::path -ProjectStore::Project::BasePathForOplog(std::string_view OplogId) -{ - return m_OplogStoragePath / OplogId; -} - -ProjectStore::Oplog* -ProjectStore::Project::NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath) -{ - RwLock::ExclusiveLockScope _(m_ProjectLock); - - std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); - - try - { - Oplog* Log = m_Oplogs - .try_emplace(std::string{OplogId}, - std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath, MarkerPath)) - .first->second.get(); - - Log->Write(); - return Log; - } - catch (std::exception&) - { - // In case of failure we need to ensure there's no half constructed entry around - // - // (This is probably already ensured by the try_emplace implementation?) - - m_Oplogs.erase(std::string{OplogId}); - - return nullptr; - } -} - -ProjectStore::Oplog* -ProjectStore::Project::OpenOplog(std::string_view OplogId) -{ - ZEN_TRACE_CPU("Store::OpenOplog"); - { - RwLock::SharedLockScope _(m_ProjectLock); - - auto OplogIt = m_Oplogs.find(std::string(OplogId)); - - if (OplogIt != m_Oplogs.end()) - { - return OplogIt->second.get(); - } - } - - RwLock::ExclusiveLockScope _(m_ProjectLock); - - std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); - - if (Oplog::ExistsAt(OplogBasePath)) - { - // Do open of existing oplog - - try - { - Oplog* Log = - m_Oplogs - .try_emplace(std::string{OplogId}, - std::make_unique<ProjectStore::Oplog>(OplogId, this, m_CidStore, OplogBasePath, std::filesystem::path{})) - .first->second.get(); - Log->Read(); - - return Log; - } - catch (std::exception& ex) - { - ZEN_WARN("failed to open oplog '{}' @ '{}': {}", OplogId, OplogBasePath, ex.what()); - - m_Oplogs.erase(std::string{OplogId}); - } - } - - return nullptr; -} - -std::filesystem::path -ProjectStore::Project::RemoveOplog(std::string_view OplogId) -{ - RwLock::ExclusiveLockScope _(m_ProjectLock); - - std::filesystem::path DeletePath; - if (auto OplogIt = m_Oplogs.find(std::string(OplogId)); OplogIt == m_Oplogs.end()) - { - std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); - - if (Oplog::ExistsAt(OplogBasePath)) - { - std::filesystem::path MovedDir; - if (PrepareDirectoryDelete(DeletePath, MovedDir)) - { - DeletePath = MovedDir; - } - } - } - else - { - std::unique_ptr<Oplog>& Oplog = OplogIt->second; - DeletePath = Oplog->PrepareForDelete(true); - m_DeletedOplogs.emplace_back(std::move(Oplog)); - m_Oplogs.erase(OplogIt); - } - m_LastAccessTimes.erase(std::string(OplogId)); - return DeletePath; -} - -void -ProjectStore::Project::DeleteOplog(std::string_view OplogId) -{ - std::filesystem::path DeletePath = RemoveOplog(OplogId); - - // Erase content on disk - if (!DeletePath.empty()) - { - OplogStorage::Delete(DeletePath); - } -} - -std::vector<std::string> -ProjectStore::Project::ScanForOplogs() const -{ - DirectoryContent DirContent; - GetDirectoryContent(m_OplogStoragePath, DirectoryContent::IncludeDirsFlag, DirContent); - std::vector<std::string> Oplogs; - Oplogs.reserve(DirContent.Directories.size()); - for (const std::filesystem::path& DirPath : DirContent.Directories) - { - Oplogs.push_back(DirPath.filename().string()); - } - return Oplogs; -} - -void -ProjectStore::Project::IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const -{ - RwLock::SharedLockScope Lock(m_ProjectLock); - - for (auto& Kv : m_Oplogs) - { - Fn(Lock, *Kv.second); - } -} - -void -ProjectStore::Project::IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn) -{ - RwLock::SharedLockScope Lock(m_ProjectLock); - - for (auto& Kv : m_Oplogs) - { - Fn(Lock, *Kv.second); - } -} - -void -ProjectStore::Project::Flush() -{ - // We only need to flush oplogs that we have already loaded - IterateOplogs([&](const RwLock::SharedLockScope&, Oplog& Ops) { Ops.Flush(); }); - WriteAccessTimes(); -} - -void -ProjectStore::Project::ScrubStorage(ScrubContext& Ctx) -{ - // Scrubbing needs to check all existing oplogs - std::vector<std::string> OpLogs = ScanForOplogs(); - for (const std::string& OpLogId : OpLogs) - { - OpenOplog(OpLogId); - } - IterateOplogs([&](const RwLock::SharedLockScope& ProjectLock, Oplog& Ops) { - if (!IsExpired(ProjectLock, GcClock::TimePoint::min(), Ops)) - { - Ops.ScrubStorage(Ctx); - } - }); -} - -void -ProjectStore::Project::GatherReferences(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("Store::Project::GatherReferences"); - - Stopwatch Timer; - const auto Guard = MakeGuard([&] { - ZEN_DEBUG("gathered references from project store project {} in {}", Identifier, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - // GatherReferences needs to check all existing oplogs - std::vector<std::string> OpLogs = ScanForOplogs(); - for (const std::string& OpLogId : OpLogs) - { - OpenOplog(OpLogId); - } - - { - // Make sure any oplog at least have a last access time so they eventually will be GC:d if not touched - RwLock::ExclusiveLockScope _(m_ProjectLock); - for (const std::string& OpId : OpLogs) - { - if (auto It = m_LastAccessTimes.find(OpId); It == m_LastAccessTimes.end()) - { - m_LastAccessTimes[OpId] = GcClock::TickCount(); - } - } - } - - IterateOplogs([&](const RwLock::SharedLockScope& ProjectLock, Oplog& Ops) { - if (!IsExpired(ProjectLock, GcCtx.ProjectStoreExpireTime(), Ops)) - { - Ops.GatherReferences(GcCtx); - } - }); -} - -uint64_t -ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath) -{ - using namespace std::literals; - - uint64_t Size = 0; - std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv; - if (std::filesystem::exists(AccessTimesFilePath)) - { - Size += std::filesystem::file_size(AccessTimesFilePath); - } - std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectFilePath)) - { - Size += std::filesystem::file_size(ProjectFilePath); - } - - return Size; -} - -uint64_t -ProjectStore::Project::TotalSize() const -{ - uint64_t Result = TotalSize(m_OplogStoragePath); - { - std::vector<std::string> OpLogs = ScanForOplogs(); - for (const std::string& OpLogId : OpLogs) - { - std::filesystem::path OplogBasePath = m_OplogStoragePath / OpLogId; - Result += Oplog::TotalSize(OplogBasePath); - } - } - return Result; -} - -bool -ProjectStore::Project::PrepareForDelete(std::filesystem::path& OutDeletePath) -{ - RwLock::ExclusiveLockScope _(m_ProjectLock); - - for (auto& It : m_Oplogs) - { - // We don't care about the moved folder - It.second->PrepareForDelete(false); - m_DeletedOplogs.emplace_back(std::move(It.second)); - } - - m_Oplogs.clear(); - - bool Success = PrepareDirectoryDelete(m_OplogStoragePath, OutDeletePath); - if (!Success) - { - return false; - } - m_OplogStoragePath.clear(); - return true; -} - -bool -ProjectStore::Project::IsExpired(const RwLock::SharedLockScope&, - const std::string& EntryName, - const std::filesystem::path& MarkerPath, - const GcClock::TimePoint ExpireTime) -{ - if (!MarkerPath.empty()) - { - std::error_code Ec; - if (std::filesystem::exists(MarkerPath, Ec)) - { - if (Ec) - { - ZEN_WARN("Failed to check expiry via marker file '{}', assuming {} is not expired", - EntryName.empty() ? "project" : EntryName, - MarkerPath.string()); - return false; - } - return false; - } - } - - const GcClock::Tick ExpireTicks = ExpireTime.time_since_epoch().count(); - - if (auto It = m_LastAccessTimes.find(EntryName); It != m_LastAccessTimes.end()) - { - if (It->second <= ExpireTicks) - { - return true; - } - } - return false; -} - -bool -ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock, const GcClock::TimePoint ExpireTime) -{ - return IsExpired(ProjectLock, std::string(), ProjectFilePath, ExpireTime); -} - -bool -ProjectStore::Project::IsExpired(const RwLock::SharedLockScope& ProjectLock, - const GcClock::TimePoint ExpireTime, - const ProjectStore::Oplog& Oplog) -{ - return IsExpired(ProjectLock, Oplog.OplogId(), Oplog.MarkerPath(), ExpireTime); -} - -bool -ProjectStore::Project::IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog) -{ - RwLock::SharedLockScope Lock(m_ProjectLock); - return IsExpired(Lock, Oplog.OplogId(), Oplog.MarkerPath(), ExpireTime); -} - -void -ProjectStore::Project::TouchProject() const -{ - RwLock::ExclusiveLockScope _(m_ProjectLock); - m_LastAccessTimes.insert_or_assign(std::string(), GcClock::TickCount()); -}; - -void -ProjectStore::Project::TouchOplog(std::string_view Oplog) const -{ - ZEN_ASSERT(!Oplog.empty()); - RwLock::ExclusiveLockScope _(m_ProjectLock); - m_LastAccessTimes.insert_or_assign(std::string(Oplog), GcClock::TickCount()); -}; - -GcClock::TimePoint -ProjectStore::Project::LastOplogAccessTime(std::string_view Oplog) const -{ - RwLock::SharedLockScope Lock(m_ProjectLock); - if (auto It = m_LastAccessTimes.find(std::string(Oplog)); It != m_LastAccessTimes.end()) - { - return GcClock::TimePointFromTick(It->second); - } - return GcClock::TimePoint::min(); -} - -////////////////////////////////////////////////////////////////////////// - -ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue) -: m_Log(logging::Get("project")) -, m_Gc(Gc) -, m_CidStore(Store) -, m_JobQueue(JobQueue) -, m_ProjectBasePath(BasePath) -, m_DiskWriteBlocker(Gc.GetDiskWriteBlocker()) -{ - ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath); - // m_Log.set_level(spdlog::level::debug); - m_Gc.AddGcContributor(this); - m_Gc.AddGcStorage(this); - m_Gc.AddGcReferencer(*this); -} - -ProjectStore::~ProjectStore() -{ - ZEN_INFO("closing project store at '{}'", m_ProjectBasePath); - m_Gc.RemoveGcReferencer(*this); - m_Gc.RemoveGcStorage(this); - m_Gc.RemoveGcContributor(this); -} - -std::filesystem::path -ProjectStore::BasePathForProject(std::string_view ProjectId) -{ - return m_ProjectBasePath / ProjectId; -} - -void -ProjectStore::DiscoverProjects() -{ - if (!std::filesystem::exists(m_ProjectBasePath)) - { - return; - } - - DirectoryContent DirContent; - GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent); - - for (const std::filesystem::path& DirPath : DirContent.Directories) - { - std::string DirName = PathToUtf8(DirPath.filename()); - if (DirName.starts_with("[dropped]")) - { - continue; - } - OpenProject(DirName); - } -} - -void -ProjectStore::IterateProjects(std::function<void(Project& Prj)>&& Fn) -{ - RwLock::SharedLockScope _(m_ProjectsLock); - - for (auto& Kv : m_Projects) - { - Fn(*Kv.second.Get()); - } -} - -void -ProjectStore::Flush() -{ - ZEN_INFO("flushing project store at '{}'", m_ProjectBasePath); - std::vector<Ref<Project>> Projects; - { - RwLock::SharedLockScope _(m_ProjectsLock); - Projects.reserve(m_Projects.size()); - - for (auto& Kv : m_Projects) - { - Projects.push_back(Kv.second); - } - } - for (const Ref<Project>& Project : Projects) - { - Project->Flush(); - } -} - -void -ProjectStore::ScrubStorage(ScrubContext& Ctx) -{ - ZEN_INFO("scrubbing '{}'", m_ProjectBasePath); - - DiscoverProjects(); - - std::vector<Ref<Project>> Projects; - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - Projects.reserve(m_Projects.size()); - - for (auto& Kv : m_Projects) - { - if (Kv.second->IsExpired(Lock, GcClock::TimePoint::min())) - { - continue; - } - Projects.push_back(Kv.second); - } - } - for (const Ref<Project>& Project : Projects) - { - Project->ScrubStorage(Ctx); - } -} - -void -ProjectStore::GatherReferences(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("Store::GatherReferences"); - - size_t ProjectCount = 0; - size_t ExpiredProjectCount = 0; - Stopwatch Timer; - const auto Guard = MakeGuard([&] { - ZEN_DEBUG("gathered references from '{}' in {}, found {} active projects and {} expired projects", - m_ProjectBasePath.string(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - ProjectCount, - ExpiredProjectCount); - }); - - DiscoverProjects(); - - std::vector<Ref<Project>> Projects; - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - Projects.reserve(m_Projects.size()); - - for (auto& Kv : m_Projects) - { - if (Kv.second->IsExpired(Lock, GcCtx.ProjectStoreExpireTime())) - { - ExpiredProjectCount++; - continue; - } - Projects.push_back(Kv.second); - } - } - ProjectCount = Projects.size(); - for (const Ref<Project>& Project : Projects) - { - Project->GatherReferences(GcCtx); - } -} - -void -ProjectStore::CollectGarbage(GcContext& GcCtx) -{ - ZEN_TRACE_CPU("Store::CollectGarbage"); - - size_t ProjectCount = 0; - size_t ExpiredProjectCount = 0; - - Stopwatch Timer; - const auto Guard = MakeGuard([&] { - ZEN_DEBUG("garbage collect from '{}' DONE after {}, found {} active projects and {} expired projects", - m_ProjectBasePath.string(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - ProjectCount, - ExpiredProjectCount); - }); - std::vector<Ref<Project>> ExpiredProjects; - std::vector<Ref<Project>> Projects; - - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - for (auto& Kv : m_Projects) - { - if (Kv.second->IsExpired(Lock, GcCtx.ProjectStoreExpireTime())) - { - ExpiredProjects.push_back(Kv.second); - ExpiredProjectCount++; - continue; - } - Projects.push_back(Kv.second); - ProjectCount++; - } - } - - if (!GcCtx.IsDeletionMode()) - { - ZEN_DEBUG("garbage collect DISABLED, for '{}' ", m_ProjectBasePath.string()); - return; - } - - for (const Ref<Project>& Project : Projects) - { - std::vector<std::string> ExpiredOplogs; - { - RwLock::ExclusiveLockScope _(m_ProjectsLock); - Project->IterateOplogs([&GcCtx, &Project, &ExpiredOplogs](const RwLock::SharedLockScope& Lock, ProjectStore::Oplog& Oplog) { - if (Project->IsExpired(Lock, GcCtx.ProjectStoreExpireTime(), Oplog)) - { - ExpiredOplogs.push_back(Oplog.OplogId()); - } - }); - } - for (const std::string& OplogId : ExpiredOplogs) - { - ZEN_DEBUG("ProjectStore::CollectGarbage garbage collected oplog '{}' in project '{}'. Removing storage on disk", - OplogId, - Project->Identifier); - Project->DeleteOplog(OplogId); - } - Project->Flush(); - } - - if (ExpiredProjects.empty()) - { - ZEN_DEBUG("garbage collect for '{}', no expired projects found", m_ProjectBasePath.string()); - return; - } - - for (const Ref<Project>& Project : ExpiredProjects) - { - std::filesystem::path PathToRemove; - std::string ProjectId; - { - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - if (!Project->IsExpired(Lock, GcCtx.ProjectStoreExpireTime())) - { - ZEN_DEBUG("ProjectStore::CollectGarbage skipped garbage collect of project '{}'. Project no longer expired.", - ProjectId); - continue; - } - } - RwLock::ExclusiveLockScope _(m_ProjectsLock); - bool Success = Project->PrepareForDelete(PathToRemove); - if (!Success) - { - ZEN_DEBUG("ProjectStore::CollectGarbage skipped garbage collect of project '{}'. Project folder is locked.", ProjectId); - continue; - } - m_Projects.erase(Project->Identifier); - ProjectId = Project->Identifier; - } - - ZEN_DEBUG("ProjectStore::CollectGarbage garbage collected project '{}'. Removing storage on disk", ProjectId); - if (PathToRemove.empty()) - { - continue; - } - - DeleteDirectories(PathToRemove); - } -} - -GcStorageSize -ProjectStore::StorageSize() const -{ - ZEN_TRACE_CPU("Store::StorageSize"); - - using namespace std::literals; - - GcStorageSize Result; - { - if (std::filesystem::exists(m_ProjectBasePath)) - { - DirectoryContent ProjectsFolderContent; - GetDirectoryContent(m_ProjectBasePath, DirectoryContent::IncludeDirsFlag, ProjectsFolderContent); - - for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) - { - std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectStateFilePath)) - { - Result.DiskSize += Project::TotalSize(ProjectBasePath); - DirectoryContent DirContent; - GetDirectoryContent(ProjectBasePath, DirectoryContent::IncludeDirsFlag, DirContent); - for (const std::filesystem::path& OplogBasePath : DirContent.Directories) - { - Result.DiskSize += Oplog::TotalSize(OplogBasePath); - } - } - } - } - } - return Result; -} - -Ref<ProjectStore::Project> -ProjectStore::OpenProject(std::string_view ProjectId) -{ - ZEN_TRACE_CPU("Store::OpenProject"); - - { - RwLock::SharedLockScope _(m_ProjectsLock); - - auto ProjIt = m_Projects.find(std::string{ProjectId}); - - if (ProjIt != m_Projects.end()) - { - return ProjIt->second; - } - } - - RwLock::ExclusiveLockScope _(m_ProjectsLock); - - std::filesystem::path BasePath = BasePathForProject(ProjectId); - - if (Project::Exists(BasePath)) - { - try - { - ZEN_INFO("opening project {} @ {}", ProjectId, BasePath); - - Ref<Project>& Prj = - m_Projects - .try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) - .first->second; - Prj->Identifier = ProjectId; - Prj->Read(); - return Prj; - } - catch (std::exception& e) - { - ZEN_WARN("failed to open {} @ {} ({})", ProjectId, BasePath, e.what()); - m_Projects.erase(std::string{ProjectId}); - } - } - - return {}; -} - -Ref<ProjectStore::Project> -ProjectStore::NewProject(const std::filesystem::path& BasePath, - std::string_view ProjectId, - std::string_view RootDir, - std::string_view EngineRootDir, - std::string_view ProjectRootDir, - std::string_view ProjectFilePath) -{ - ZEN_TRACE_CPU("Store::NewProject"); - - RwLock::ExclusiveLockScope _(m_ProjectsLock); - - Ref<Project>& Prj = - m_Projects.try_emplace(std::string{ProjectId}, Ref<ProjectStore::Project>(new ProjectStore::Project(this, m_CidStore, BasePath))) - .first->second; - Prj->Identifier = ProjectId; - Prj->RootDir = RootDir; - Prj->EngineRootDir = EngineRootDir; - Prj->ProjectRootDir = ProjectRootDir; - Prj->ProjectFilePath = ProjectFilePath; - Prj->Write(); - - return Prj; -} - -bool -ProjectStore::UpdateProject(std::string_view ProjectId, - std::string_view RootDir, - std::string_view EngineRootDir, - std::string_view ProjectRootDir, - std::string_view ProjectFilePath) -{ - ZEN_TRACE_CPU("Store::UpdateProject"); - - ZEN_INFO("updating project {}", ProjectId); - - RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock); - - auto ProjIt = m_Projects.find(std::string{ProjectId}); - - if (ProjIt == m_Projects.end()) - { - return false; - } - Ref<ProjectStore::Project> Prj = ProjIt->second; - - Prj->RootDir = RootDir; - Prj->EngineRootDir = EngineRootDir; - Prj->ProjectRootDir = ProjectRootDir; - Prj->ProjectFilePath = ProjectFilePath; - Prj->Write(); - - return true; -} - -bool -ProjectStore::RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath) -{ - RwLock::ExclusiveLockScope ProjectsLock(m_ProjectsLock); - - auto ProjIt = m_Projects.find(std::string{ProjectId}); - - if (ProjIt == m_Projects.end()) - { - return true; - } - - bool Success = ProjIt->second->PrepareForDelete(OutDeletePath); - - if (!Success) - { - return false; - } - m_Projects.erase(ProjIt); - return true; -} - -bool -ProjectStore::DeleteProject(std::string_view ProjectId) -{ - ZEN_TRACE_CPU("Store::DeleteProject"); - - ZEN_INFO("deleting project {}", ProjectId); - - std::filesystem::path DeletePath; - if (!RemoveProject(ProjectId, DeletePath)) - { - return false; - } - - if (!DeletePath.empty()) - { - DeleteDirectories(DeletePath); - } - - return true; -} - -bool -ProjectStore::Exists(std::string_view ProjectId) -{ - return Project::Exists(BasePathForProject(ProjectId)); -} - -CbArray -ProjectStore::GetProjectsList() -{ - ZEN_TRACE_CPU("Store::GetProjectsList"); - - using namespace std::literals; - - DiscoverProjects(); - - CbWriter Response; - Response.BeginArray(); - - IterateProjects([&Response](ProjectStore::Project& Prj) { - Response.BeginObject(); - Response << "Id"sv << Prj.Identifier; - Response << "RootDir"sv << Prj.RootDir.string(); - Response << "ProjectRootDir"sv << Prj.ProjectRootDir; - Response << "EngineRootDir"sv << Prj.EngineRootDir; - Response << "ProjectFilePath"sv << Prj.ProjectFilePath; - Response.EndObject(); - }); - Response.EndArray(); - return Response.Save().AsArray(); -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::string_view OplogId, bool FilterClient, CbObject& OutPayload) -{ - ZEN_TRACE_CPU("Store::GetProjectFiles"); - - using namespace std::literals; - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Project files request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - CbObjectWriter Response; - Response.BeginArray("files"sv); - - FoundLog->IterateFileMap([&](const Oid& Id, const std::string_view& ServerPath, const std::string_view& ClientPath) { - Response.BeginObject(); - Response << "id"sv << Id; - Response << "clientpath"sv << ClientPath; - if (!FilterClient && !ServerPath.empty()) - { - Response << "serverpath"sv << ServerPath; - } - Response.EndObject(); - }); - - Response.EndArray(); - OutPayload = Response.Save(); - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetProjectChunkInfos(const std::string_view ProjectId, const std::string_view OplogId, CbObject& OutPayload) -{ - ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos"); - - using namespace std::literals; - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - std::vector<std::pair<Oid, IoHash>> ChunkInfos; - FoundLog->IterateChunkMap([&ChunkInfos](const Oid& Id, const IoHash& Hash) { ChunkInfos.push_back({Id, Hash}); }); - - CbObjectWriter Response; - Response.BeginArray("chunkinfos"sv); - - for (const auto& ChunkInfo : ChunkInfos) - { - if (IoBuffer Chunk = FoundLog->FindChunk(ChunkInfo.first)) - { - Response.BeginObject(); - Response << "id"sv << ChunkInfo.first; - Response << "rawhash"sv << ChunkInfo.second; - Response << "rawsize"sv << Chunk.GetSize(); - Response.EndObject(); - } - } - Response.EndArray(); - - OutPayload = Response.Save(); - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunkInfo(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view ChunkId, - CbObject& OutPayload) -{ - using namespace std::literals; - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk info request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) - { - return {HttpResponseCode::BadRequest, - fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)}; - } - - const Oid Obj = Oid::FromHexString(ChunkId); - - IoBuffer Chunk = FoundLog->FindChunk(Obj); - if (!Chunk) - { - return {HttpResponseCode::NotFound, {}}; - } - - uint64_t ChunkSize = Chunk.GetSize(); - if (Chunk.GetContentType() == HttpContentType::kCompressedBinary) - { - IoHash RawHash; - uint64_t RawSize; - bool IsCompressed = CompressedBuffer::ValidateCompressedHeader(Chunk, RawHash, RawSize); - if (!IsCompressed) - { - return {HttpResponseCode::InternalServerError, - fmt::format("Chunk info request for malformed chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)}; - } - ChunkSize = RawSize; - } - - CbObjectWriter Response; - Response << "size"sv << ChunkSize; - OutPayload = Response.Save(); - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - IoBuffer& OutChunk) -{ - if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)}; - } - - const Oid Obj = Oid::FromHexString(ChunkId); - - return GetChunkRange(ProjectId, OplogId, Obj, Offset, Size, AcceptType, OutChunk); -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - Oid ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - IoBuffer& OutChunk) -{ - bool IsOffset = Offset != 0 || Size != ~(0ull); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - IoBuffer Chunk = FoundLog->FindChunk(ChunkId); - if (!Chunk) - { - return {HttpResponseCode::NotFound, {}}; - } - - OutChunk = Chunk; - HttpContentType ContentType = Chunk.GetContentType(); - - if (Chunk.GetContentType() == HttpContentType::kCompressedBinary) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), RawHash, RawSize); - ZEN_ASSERT(!Compressed.IsNull()); - - if (IsOffset) - { - if ((Offset + Size) > RawSize) - { - Size = RawSize - Offset; - } - - if (AcceptType == HttpContentType::kBinary) - { - OutChunk = Compressed.Decompress(Offset, Size).AsIoBuffer(); - OutChunk.SetContentType(HttpContentType::kBinary); - } - else - { - // Value will be a range of compressed blocks that covers the requested range - // The client will have to compensate for any offsets that do not land on an even block size multiple - OutChunk = Compressed.CopyRange(Offset, Size).GetCompressed().Flatten().AsIoBuffer(); - OutChunk.SetContentType(HttpContentType::kCompressedBinary); - } - } - else - { - if (AcceptType == HttpContentType::kBinary) - { - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(HttpContentType::kBinary); - } - else - { - OutChunk = Compressed.GetCompressed().Flatten().AsIoBuffer(); - OutChunk.SetContentType(HttpContentType::kCompressedBinary); - } - } - } - else if (IsOffset) - { - if ((Offset + Size) > Chunk.GetSize()) - { - Size = Chunk.GetSize() - Offset; - } - OutChunk = IoBuffer(std::move(Chunk), Offset, Size); - OutChunk.SetContentType(ContentType); - } - - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::GetChunk(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view Cid, - ZenContentType AcceptType, - IoBuffer& OutChunk) -{ - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - if (Cid.length() != IoHash::StringLength) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, Cid)}; - } - - const IoHash Hash = IoHash::FromHexString(Cid); - OutChunk = m_CidStore.FindChunkByCid(Hash); - - if (!OutChunk) - { - return {HttpResponseCode::NotFound, fmt::format("chunk - '{}' MISSING", Cid)}; - } - - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(ZenContentType::kBinary); - } - else - { - OutChunk.SetContentType(ZenContentType::kCompressedBinary); - } - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::PutChunk(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view Cid, - ZenContentType ContentType, - IoBuffer&& Chunk) -{ - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* FoundLog = Project->OpenOplog(OplogId); - if (!FoundLog) - { - return {HttpResponseCode::NotFound, fmt::format("Chunk put request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - if (Cid.length() != IoHash::StringLength) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk put request for invalid chunk hash '{}'", Cid)}; - } - - const IoHash Hash = IoHash::FromHexString(Cid); - - if (ContentType != HttpContentType::kCompressedBinary) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid content type for chunk '{}'", Cid)}; - } - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize); - if (RawHash != Hash) - { - return {HttpResponseCode::BadRequest, fmt::format("Chunk request for invalid payload format for chunk '{}'", Cid)}; - } - - CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk, Hash); - return {Result.New ? HttpResponseCode::Created : HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::WriteOplog(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload, CbObject& OutResponse) -{ - ZEN_TRACE_CPU("Store::WriteOplog"); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId); - if (!Oplog) - { - return {HttpResponseCode::NotFound, fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - CbObject ContainerObject = LoadCompactBinaryObject(Payload); - if (!ContainerObject) - { - return {HttpResponseCode::BadRequest, "Invalid payload format"}; - } - - CidStore& ChunkStore = m_CidStore; - RwLock AttachmentsLock; - tsl::robin_set<IoHash, IoHash::Hasher> Attachments; - - auto HasAttachment = [&ChunkStore](const IoHash& RawHash) { return ChunkStore.ContainsChunk(RawHash); }; - auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - if (BlockHash != IoHash::Zero) - { - Attachments.insert(BlockHash); - } - else - { - Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); - } - }; - auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { - RwLock::ExclusiveLockScope _(AttachmentsLock); - Attachments.insert(RawHash); - }; - - RemoteProjectStore::Result RemoteResult = - SaveOplogContainer(*Oplog, ContainerObject, HasAttachment, OnNeedBlock, OnNeedAttachment, nullptr); - - if (RemoteResult.ErrorCode) - { - return ConvertResult(RemoteResult); - } - - CbObjectWriter Cbo; - Cbo.BeginArray("need"); - { - for (const IoHash& Hash : Attachments) - { - ZEN_DEBUG("Need attachment {}", Hash); - Cbo << Hash; - } - } - Cbo.EndArray(); // "need" - - OutResponse = Cbo.Save(); - return {HttpResponseCode::OK, {}}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::ReadOplog(const std::string_view ProjectId, - const std::string_view OplogId, - const HttpServerRequest::QueryParams& Params, - CbObject& OutResponse) -{ - ZEN_TRACE_CPU("Store::ReadOplog"); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId); - if (!Oplog) - { - return {HttpResponseCode::NotFound, fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - size_t MaxBlockSize = 128u * 1024u * 1024u; - if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false) - { - if (auto Value = ParseInt<size_t>(Param)) - { - MaxBlockSize = Value.value(); - } - } - size_t MaxChunkEmbedSize = 1024u * 1024u; - if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false) - { - if (auto Value = ParseInt<size_t>(Param)) - { - MaxChunkEmbedSize = Value.value(); - } - } - - CidStore& ChunkStore = m_CidStore; - - RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( - ChunkStore, - *Project.Get(), - *Oplog, - MaxBlockSize, - MaxChunkEmbedSize, - false, - [](CompressedBuffer&&, const IoHash) {}, - [](const IoHash&) {}, - [](const std::unordered_set<IoHash, IoHash::Hasher>) {}, - nullptr); - - OutResponse = std::move(ContainerResult.ContainerObject); - return ConvertResult(ContainerResult); -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::WriteBlock(const std::string_view ProjectId, const std::string_view OplogId, IoBuffer&& Payload) -{ - ZEN_TRACE_CPU("Store::WriteBlock"); - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown project '{}'", ProjectId)}; - } - Project->TouchProject(); - - ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId); - if (!Oplog) - { - return {HttpResponseCode::NotFound, fmt::format("Write block request for unknown oplog '{}/{}'", ProjectId, OplogId)}; - } - Project->TouchOplog(OplogId); - - if (!IterateBlock(std::move(Payload), [this](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { - IoBuffer Compressed = Chunk.GetCompressed().Flatten().AsIoBuffer(); - m_CidStore.AddChunk(Compressed, AttachmentRawHash); - ZEN_DEBUG("Saved attachment {} from block, size {}", AttachmentRawHash, Compressed.GetSize()); - })) - { - return {HttpResponseCode::BadRequest, "Invalid chunk in block"}; - } - - return {HttpResponseCode::OK, {}}; -} - -bool -ProjectStore::Rpc(HttpServerRequest& HttpReq, - const std::string_view ProjectId, - const std::string_view OplogId, - IoBuffer&& Payload, - AuthMgr& AuthManager) -{ - ZEN_TRACE_CPU("Store::Rpc"); - - using namespace std::literals; - HttpContentType PayloadContentType = HttpReq.RequestContentType(); - CbPackage Package; - CbObject Cb; - switch (PayloadContentType) - { - case HttpContentType::kJSON: - case HttpContentType::kUnknownContentType: - case HttpContentType::kText: - { - std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); - Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); - if (!Cb) - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected JSON format"); - return false; - } - } - break; - case HttpContentType::kCbObject: - Cb = LoadCompactBinaryObject(Payload); - if (!Cb) - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected compact binary format"); - return false; - } - break; - case HttpContentType::kCbPackage: - Package = ParsePackageMessage(Payload); - Cb = Package.GetObject(); - if (!Cb) - { - HttpReq.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Content format not supported, expected package message format"); - return false; - } - break; - default: - HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); - return false; - } - - Ref<ProjectStore::Project> Project = OpenProject(ProjectId); - if (!Project) - { - HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Rpc oplog request for unknown project '{}'", ProjectId)); - return true; - } - Project->TouchProject(); - - ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId); - if (!Oplog) - { - HttpReq.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); - return true; - } - Project->TouchOplog(OplogId); - - std::string_view Method = Cb["method"sv].AsString(); - - if (Method == "import"sv) - { - if (!AreDiskWritesAllowed()) - { - HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - return true; - } - std::pair<HttpResponseCode, std::string> Result = Import(*Project.Get(), *Oplog, Cb["params"sv].AsObjectView(), AuthManager); - if (Result.second.empty()) - { - HttpReq.WriteResponse(Result.first); - return Result.first != HttpResponseCode::BadRequest; - } - HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); - return true; - } - else if (Method == "export"sv) - { - std::pair<HttpResponseCode, std::string> Result = Export(Project, *Oplog, Cb["params"sv].AsObjectView(), AuthManager); - HttpReq.WriteResponse(Result.first, HttpContentType::kText, Result.second); - return true; - } - else if (Method == "getchunks"sv) - { - ZEN_TRACE_CPU("Store::Rpc::getchunks"); - CbPackage ResponsePackage; - { - CbArrayView ChunksArray = Cb["chunks"sv].AsArrayView(); - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("chunks"sv); - for (CbFieldView FieldView : ChunksArray) - { - IoHash RawHash = FieldView.AsHash(); - IoBuffer ChunkBuffer = m_CidStore.FindChunkByCid(RawHash); - if (ChunkBuffer) - { - ResponseWriter.AddHash(RawHash); - ResponsePackage.AddAttachment( - CbAttachment(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkBuffer)), RawHash)); - } - } - ResponseWriter.EndArray(); - ResponsePackage.SetObject(ResponseWriter.Save()); - } - CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage, FormatFlags::kDefault); - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); - return true; - } - else if (Method == "putchunks"sv) - { - ZEN_TRACE_CPU("Store::Rpc::putchunks"); - if (!AreDiskWritesAllowed()) - { - HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - return true; - } - std::span<const CbAttachment> Attachments = Package.GetAttachments(); - for (const CbAttachment& Attachment : Attachments) - { - IoHash RawHash = Attachment.GetHash(); - CompressedBuffer Compressed = Attachment.AsCompressedBinary(); - m_CidStore.AddChunk(Compressed.GetCompressed().Flatten().AsIoBuffer(), RawHash, CidStore::InsertMode::kCopyOnly); - } - HttpReq.WriteResponse(HttpResponseCode::OK); - return true; - } - else if (Method == "snapshot"sv) - { - ZEN_TRACE_CPU("Store::Rpc::snapshot"); - if (!AreDiskWritesAllowed()) - { - HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); - return true; - } - - // Snapshot all referenced files. This brings the content of all - // files into the CID store - - int OpCount = 0; - uint64_t InlinedBytes = 0; - uint64_t InlinedFiles = 0; - uint64_t TotalBytes = 0; - uint64_t TotalFiles = 0; - - std::vector<CbObject> NewOps; - std::unordered_map<Oid, IoHash, Oid::Hasher> NewChunkMappings; - - Oplog->IterateOplog([&](CbObjectView Op) { - bool OpRewritten = false; - bool AllOk = true; - - CbWriter Cbo; - Cbo.BeginArray("files"sv); - - for (CbFieldView& Field : Op["files"sv]) - { - bool CopyField = true; - - if (CbObjectView View = Field.AsObjectView()) - { - const IoHash DataHash = View["data"sv].AsHash(); - - if (DataHash == IoHash::Zero) - { - std::string_view ServerPath = View["serverpath"sv].AsString(); - std::filesystem::path FilePath = Project->RootDir / ServerPath; - BasicFile DataFile; - std::error_code Ec; - DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); - - if (Ec) - { - // Error... - - ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); - - AllOk = false; - } - else - { - // Read file contents into memory, compress and store in CidStore - - Oid ChunkId = View["id"sv].AsObjectId(); - IoBuffer FileIoBuffer = DataFile.ReadAll(); - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); - const IoHash RawHash = Compressed.DecodeRawHash(); - const uint64_t RawSize = Compressed.DecodeRawSize(); - IoBuffer CompressedBuffer = Compressed.GetCompressed().Flatten().AsIoBuffer(); - CidStore::InsertResult Result = m_CidStore.AddChunk(CompressedBuffer, RawHash); - - TotalBytes += RawSize; - ++TotalFiles; - - if (Result.New) - { - InlinedBytes += RawSize; - ++InlinedFiles; - } - - // Rewrite file array entry with new data reference - CbObjectWriter Writer; - RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { - if (Field.GetName() == "data"sv) - { - // omit this field as we will write it explicitly ourselves - return true; - } - return false; - }); - Writer.AddBinaryAttachment("data"sv, RawHash); - - CbObject RewrittenOp = Writer.Save(); - Cbo.AddObject(std::move(RewrittenOp)); - CopyField = false; - - NewChunkMappings.insert_or_assign(ChunkId, RawHash); - } - } - } - - if (CopyField) - { - Cbo.AddField(Field); - } - else - { - OpRewritten = true; - } - } - - if (OpRewritten && AllOk) - { - Cbo.EndArray(); - CbArray FilesArray = Cbo.Save().AsArray(); - - CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { - if (Field.GetName() == "files"sv) - { - NewWriter.AddArray("files"sv, FilesArray); - - return true; - } - - return false; - }); - - NewOps.push_back(std::move(RewrittenOp)); - } - - OpCount++; - }); - - // Make sure we have references to our attachments - Oplog->AddChunkMappings(NewChunkMappings); - - CbObjectWriter ResponseObj; - - // Persist rewritten oplog entries - - if (!NewOps.empty()) - { - ResponseObj.BeginArray("rewritten_ops"); - - for (CbObject& NewOp : NewOps) - { - uint32_t NewLsn = Oplog->AppendNewOplogEntry(std::move(NewOp)); - - ZEN_DEBUG("appended rewritten op at LSN: {}", NewLsn); - - ResponseObj.AddInteger(NewLsn); - } - - ResponseObj.EndArray(); - } - - ResponseObj << "inlined_bytes" << InlinedBytes << "inlined_files" << InlinedFiles; - ResponseObj << "total_bytes" << TotalBytes << "total_files" << TotalFiles; - - ZEN_INFO("rewrote {} oplog entries (out of {})", NewOps.size(), OpCount); - - HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); - return true; - } - HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("Unknown rpc method '{}'", Method)); - return true; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::Export(Ref<ProjectStore::Project> Project, ProjectStore::Oplog& Oplog, CbObjectView&& Params, AuthMgr& AuthManager) -{ - ZEN_TRACE_CPU("Store::Export"); - - using namespace std::literals; - - size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(128u * 1024u * 1024u); - size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(1024u * 1024u); - bool Force = Params["force"sv].AsBool(false); - bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); - - CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Params, AuthManager, MaxBlockSize, MaxChunkEmbedSize, Oplog.TempPath()); - - if (RemoteStoreResult.Store == nullptr) - { - return {HttpResponseCode::BadRequest, RemoteStoreResult.Description}; - } - std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - - ZEN_INFO("Saving oplog '{}/{}' to {}, maxblocksize {}, maxchunkembedsize {}", - Project->Identifier, - Oplog.OplogId(), - StoreInfo.Description, - NiceBytes(MaxBlockSize), - NiceBytes(MaxChunkEmbedSize)); - - JobId JobId = - m_JobQueue.QueueJob(fmt::format("Export oplog '{}/{}' to {}", Project->Identifier, Oplog.OplogId(), StoreInfo.Description), - [this, - ActualRemoteStore = std::move(RemoteStore), - Project, - OplogPtr = &Oplog, - MaxBlockSize, - MaxChunkEmbedSize, - EmbedLooseFile, - CreateBlocks = StoreInfo.CreateBlocks, - UseTempBlockFiles = StoreInfo.UseTempBlockFiles, - Force](JobContext& Context) { - RemoteProjectStore::Result Result = SaveOplog(m_CidStore, - *ActualRemoteStore, - *Project.Get(), - *OplogPtr, - MaxBlockSize, - MaxChunkEmbedSize, - EmbedLooseFile, - CreateBlocks, - UseTempBlockFiles, - Force, - &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) - { - throw std::runtime_error( - fmt::format("Export failed. Status '{}'. Reason: '{}'", ToString(Response.first), Response.second)); - } - }); - - return {HttpResponseCode::Accepted, fmt::format("{}", JobId.Id)}; -} - -std::pair<HttpResponseCode, std::string> -ProjectStore::Import(ProjectStore::Project& Project, ProjectStore::Oplog& Oplog, CbObjectView&& Params, AuthMgr& AuthManager) -{ - ZEN_TRACE_CPU("Store::Import"); - - using namespace std::literals; - - size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(128u * 1024u * 1024u); - size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(1024u * 1024u); - bool Force = Params["force"sv].AsBool(false); - - CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Params, AuthManager, MaxBlockSize, MaxChunkEmbedSize, Oplog.TempPath()); - - if (RemoteStoreResult.Store == nullptr) - { - return {HttpResponseCode::BadRequest, RemoteStoreResult.Description}; - } - std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); - - ZEN_INFO("Loading oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description); - JobId JobId = m_JobQueue.QueueJob( - fmt::format("Import oplog '{}/{}' from {}", Project.Identifier, Oplog.OplogId(), StoreInfo.Description), - [this, ActualRemoteStore = std::move(RemoteStore), OplogPtr = &Oplog, MaxBlockSize, MaxChunkEmbedSize, Force](JobContext& Context) { - RemoteProjectStore::Result Result = LoadOplog(m_CidStore, *ActualRemoteStore, *OplogPtr, Force, &Context); - auto Response = ConvertResult(Result); - ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); - if (!IsHttpSuccessCode(Response.first)) - { - throw std::runtime_error( - fmt::format("Import failed. Status '{}'. Reason: '{}'", ToString(Response.first), Response.second)); - } - }); - - return {HttpResponseCode::Accepted, fmt::format("{}", JobId.Id)}; -} - -bool -ProjectStore::AreDiskWritesAllowed() const -{ - return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); -} - -std::string -ProjectStore::GetGcName(GcCtx&) -{ - return fmt::format("projectstore:'{}'", m_ProjectBasePath.string()); -} - -class ProjectStoreGcStoreCompactor : public GcStoreCompactor -{ -public: - ProjectStoreGcStoreCompactor(const std::filesystem::path& BasePath, - std::vector<std::filesystem::path>&& OplogPathsToRemove, - std::vector<std::filesystem::path>&& ProjectPathsToRemove) - : m_BasePath(BasePath) - , m_OplogPathsToRemove(std::move(OplogPathsToRemove)) - , m_ProjectPathsToRemove(std::move(ProjectPathsToRemove)) - { - } - - virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>&) - { - ZEN_TRACE_CPU("Store::CompactStore"); - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [COMPACT] '{}': RemovedDisk: {} in {}", - m_BasePath, - NiceBytes(Stats.RemovedDisk), - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - if (Ctx.Settings.IsDeleteMode) - { - for (const std::filesystem::path& OplogPath : m_OplogPathsToRemove) - { - uint64_t OplogSize = ProjectStore::Oplog::TotalSize(OplogPath); - if (DeleteDirectories(OplogPath)) - { - ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed oplog folder '{}', removed {}", - m_BasePath, - OplogPath, - NiceBytes(OplogSize)); - Stats.RemovedDisk += OplogSize; - } - else - { - ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove oplog folder '{}'", m_BasePath, OplogPath); - } - } - - for (const std::filesystem::path& ProjectPath : m_ProjectPathsToRemove) - { - uint64_t ProjectSize = ProjectStore::Project::TotalSize(ProjectPath); - if (DeleteDirectories(ProjectPath)) - { - ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': removed project folder '{}', removed {}", - m_BasePath, - ProjectPath, - NiceBytes(ProjectSize)); - Stats.RemovedDisk += ProjectSize; - } - else - { - ZEN_WARN("GCV2: projectstore [COMPACT] '{}': Failed to remove project folder '{}'", m_BasePath, ProjectPath); - } - } - } - else - { - ZEN_DEBUG("GCV2: projectstore [COMPACT] '{}': Skipped deleting of {} oplogs and {} projects", - m_BasePath, - m_OplogPathsToRemove.size(), - m_ProjectPathsToRemove.size()); - } - - m_ProjectPathsToRemove.clear(); - m_OplogPathsToRemove.clear(); - } - -private: - std::filesystem::path m_BasePath; - std::vector<std::filesystem::path> m_OplogPathsToRemove; - std::vector<std::filesystem::path> m_ProjectPathsToRemove; -}; - -GcStoreCompactor* -ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) -{ - ZEN_TRACE_CPU("Store::RemoveExpiredData"); - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {} in {}", - m_ProjectBasePath, - Stats.CheckedCount, - Stats.FoundCount, - Stats.DeletedCount, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - std::vector<std::filesystem::path> OplogPathsToRemove; - std::vector<std::filesystem::path> ProjectPathsToRemove; - - std::vector<Ref<Project>> ExpiredProjects; - std::vector<Ref<Project>> Projects; - - DiscoverProjects(); - - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - for (auto& Kv : m_Projects) - { - Stats.CheckedCount++; - if (Kv.second->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime)) - { - ExpiredProjects.push_back(Kv.second); - continue; - } - Projects.push_back(Kv.second); - } - } - - for (const Ref<Project>& Project : Projects) - { - std::vector<std::string> OpLogs = Project->ScanForOplogs(); - for (const std::string& OpLogId : OpLogs) - { - Project->OpenOplog(OpLogId); - if (Ctx.IsCancelledFlag) - { - return nullptr; - } - } - } - - size_t ExpiredOplogCount = 0; - for (const Ref<Project>& Project : Projects) - { - if (Ctx.IsCancelledFlag) - { - break; - } - - std::vector<std::string> ExpiredOplogs; - { - Project->IterateOplogs( - [&Ctx, &Stats, &Project, &ExpiredOplogs](const RwLock::SharedLockScope& Lock, ProjectStore::Oplog& Oplog) { - Stats.CheckedCount++; - if (Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime, Oplog)) - { - ExpiredOplogs.push_back(Oplog.OplogId()); - } - }); - } - std::filesystem::path ProjectPath = BasePathForProject(Project->Identifier); - ExpiredOplogCount += ExpiredOplogs.size(); - if (Ctx.Settings.IsDeleteMode) - { - for (const std::string& OplogId : ExpiredOplogs) - { - std::filesystem::path RemovePath = Project->RemoveOplog(OplogId); - if (!RemovePath.empty()) - { - OplogPathsToRemove.push_back(RemovePath); - } - } - Stats.DeletedCount += ExpiredOplogs.size(); - Project->Flush(); - } - } - - if (ExpiredProjects.empty() && ExpiredOplogCount == 0) - { - ZEN_DEBUG("GCV2: projectstore [REMOVE EXPIRED] '{}': no expired projects found", m_ProjectBasePath); - return nullptr; - } - - if (Ctx.Settings.IsDeleteMode) - { - for (const Ref<Project>& Project : ExpiredProjects) - { - std::string ProjectId = Project->Identifier; - { - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - if (!Project->IsExpired(Lock, Ctx.Settings.ProjectStoreExpireTime)) - { - ZEN_DEBUG( - "GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project no longer " - "expired.", - m_ProjectBasePath, - ProjectId); - continue; - } - } - std::filesystem::path RemovePath; - bool Success = RemoveProject(ProjectId, RemovePath); - if (!Success) - { - ZEN_DEBUG( - "GCV2: projectstore [REMOVE EXPIRED] '{}': skipped garbage collect of project '{}'. Project folder is locked.", - m_ProjectBasePath, - ProjectId); - continue; - } - if (!RemovePath.empty()) - { - ProjectPathsToRemove.push_back(RemovePath); - } - } - } - Stats.DeletedCount += ExpiredProjects.size(); - } - - size_t ExpiredProjectCount = ExpiredProjects.size(); - Stats.FoundCount += ExpiredOplogCount + ExpiredProjectCount; - if (!OplogPathsToRemove.empty() || !ProjectPathsToRemove.empty()) - { - return new ProjectStoreGcStoreCompactor(m_ProjectBasePath, std::move(OplogPathsToRemove), std::move(ProjectPathsToRemove)); - } - return nullptr; -} - -class ProjectStoreReferenceChecker : public GcReferenceChecker -{ -public: - ProjectStoreReferenceChecker(ProjectStore::Oplog& Owner, bool PreCache) : m_Oplog(Owner), m_PreCache(PreCache) {} - - virtual ~ProjectStoreReferenceChecker() {} - - virtual void PreCache(GcCtx& Ctx) override - { - if (m_PreCache) - { - ZEN_TRACE_CPU("Store::PreCache"); - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [PRECACHE] '{}': precached {} references in {} from {}/{}", - m_Oplog.m_BasePath, - m_References.size(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - m_Oplog.m_OuterProject->Identifier, - m_Oplog.OplogId()); - }); - - RwLock::SharedLockScope __(m_Oplog.m_OplogLock); - if (Ctx.IsCancelledFlag) - { - return; - } - m_Oplog.IterateOplog([&](CbObjectView Op) { - Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); }); - }); - m_PreCachedLsn = m_Oplog.GetMaxOpIndex(); - } - } - - virtual void LockState(GcCtx& Ctx) override - { - ZEN_TRACE_CPU("Store::LockState"); - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [LOCKSTATE] '{}': found {} references in {} from {}/{}", - m_Oplog.m_BasePath, - m_References.size(), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()), - m_Oplog.m_OuterProject->Identifier, - m_Oplog.OplogId()); - }); - - m_OplogLock = std::make_unique<RwLock::SharedLockScope>(m_Oplog.m_OplogLock); - if (m_Oplog.GetMaxOpIndex() != m_PreCachedLsn) - { - // TODO: Maybe we could just check the added oplog entries - we might get a few extra references from obsolete entries - // but I don't think that would be critical - m_References.resize(0); - m_Oplog.IterateOplog([&](CbObjectView Op) { - Op.IterateAttachments([&](CbFieldView Visitor) { m_References.emplace_back(Visitor.AsAttachment()); }); - }); - } - } - - virtual void RemoveUsedReferencesFromSet(GcCtx& Ctx, HashSet& IoCids) override - { - ZEN_TRACE_CPU("Store::RemoveUsedReferencesFromSet"); - - ZEN_ASSERT(m_OplogLock); - - size_t InitialCount = IoCids.size(); - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}", - m_Oplog.m_BasePath, - InitialCount - IoCids.size(), - InitialCount, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - for (const IoHash& ReferenceHash : m_References) - { - if (IoCids.erase(ReferenceHash) == 1) - { - if (IoCids.empty()) - { - return; - } - } - } - } - ProjectStore::Oplog& m_Oplog; - bool m_PreCache; - std::unique_ptr<RwLock::SharedLockScope> m_OplogLock; - std::vector<IoHash> m_References; - int m_PreCachedLsn = -1; -}; - -std::vector<GcReferenceChecker*> -ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) -{ - ZEN_TRACE_CPU("Store::CreateReferenceCheckers"); - - size_t ProjectCount = 0; - size_t OplogCount = 0; - - Stopwatch Timer; - const auto _ = MakeGuard([&] { - if (!Ctx.Settings.Verbose) - { - return; - } - ZEN_INFO("GCV2: projectstore [CREATE CHECKERS] '{}': opened {} projects and {} oplogs in {}", - m_ProjectBasePath, - ProjectCount, - OplogCount, - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); - }); - - DiscoverProjects(); - - std::vector<Ref<ProjectStore::Project>> Projects; - { - RwLock::SharedLockScope Lock(m_ProjectsLock); - Projects.reserve(m_Projects.size()); - - for (auto& Kv : m_Projects) - { - Projects.push_back(Kv.second); - } - } - ProjectCount += Projects.size(); - std::vector<GcReferenceChecker*> Checkers; - try - { - for (const Ref<ProjectStore::Project>& Project : Projects) - { - std::vector<std::string> OpLogs = Project->ScanForOplogs(); - Checkers.reserve(OpLogs.size()); - for (const std::string& OpLogId : OpLogs) - { - ProjectStore::Oplog* Oplog = Project->OpenOplog(OpLogId); - GcClock::TimePoint Now = GcClock::Now(); - bool TryPreCache = Project->LastOplogAccessTime(OpLogId) < (Now - std::chrono::minutes(5)); - Checkers.emplace_back(new ProjectStoreReferenceChecker(*Oplog, TryPreCache)); - } - OplogCount += OpLogs.size(); - } - } - catch (std::exception&) - { - while (!Checkers.empty()) - { - delete Checkers.back(); - Checkers.pop_back(); - } - throw; - } - - return Checkers; -} - -////////////////////////////////////////////////////////////////////////// - -#if ZEN_WITH_TESTS - -namespace testutils { - using namespace std::literals; - - std::string OidAsString(const Oid& Id) - { - StringBuilder<25> OidStringBuilder; - Id.ToString(OidStringBuilder); - return OidStringBuilder.ToString(); - } - - CbPackage CreateOplogPackage(const Oid& Id, const std::span<const std::pair<Oid, CompressedBuffer>>& Attachments) - { - CbPackage Package; - CbObjectWriter Object; - Object << "key"sv << OidAsString(Id); - if (!Attachments.empty()) - { - Object.BeginArray("bulkdata"); - for (const auto& Attachment : Attachments) - { - CbAttachment Attach(Attachment.second, Attachment.second.DecodeRawHash()); - Object.BeginObject(); - Object << "id"sv << Attachment.first; - Object << "type"sv - << "Standard"sv; - Object << "data"sv << Attach; - Object.EndObject(); - - Package.AddAttachment(Attach); - } - Object.EndArray(); - } - Package.SetObject(Object.Save()); - return Package; - }; - - std::vector<std::pair<Oid, CompressedBuffer>> CreateAttachments(const std::span<const size_t>& Sizes) - { - std::vector<std::pair<Oid, CompressedBuffer>> Result; - Result.reserve(Sizes.size()); - for (size_t Size : Sizes) - { - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(CreateRandomBlob(Size))); - Result.emplace_back(std::pair<Oid, CompressedBuffer>(Oid::NewOid(), Compressed)); - } - return Result; - } - - uint64_t GetCompressedOffset(const CompressedBuffer& Buffer, uint64_t RawOffset) - { - if (RawOffset > 0) - { - uint64_t BlockSize = 0; - OodleCompressor Compressor; - OodleCompressionLevel CompressionLevel; - if (!Buffer.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) - { - return 0; - } - return BlockSize > 0 ? RawOffset % BlockSize : 0; - } - return 0; - } - -} // namespace testutils - -TEST_CASE("project.store.create") -{ - using namespace std::literals; - - ScopedTemporaryDirectory TempDir; - - auto JobQueue = MakeJobQueue(1, ""sv); - GcManager Gc; - CidStore CidStore(Gc); - CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); - - std::string_view ProjectName("proj1"sv); - std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue); - std::filesystem::path RootDir = TempDir.Path() / "root"; - std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; - std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; - std::filesystem::path ProjectFilePath = TempDir.Path() / "game" / "game.uproject"; - - Ref<ProjectStore::Project> Project(ProjectStore.NewProject(BasePath / ProjectName, - ProjectName, - RootDir.string(), - EngineRootDir.string(), - ProjectRootDir.string(), - ProjectFilePath.string())); - CHECK(ProjectStore.DeleteProject(ProjectName)); - CHECK(!Project->Exists(BasePath)); -} - -TEST_CASE("project.store.lifetimes") -{ - using namespace std::literals; - - ScopedTemporaryDirectory TempDir; - - auto JobQueue = MakeJobQueue(1, ""sv); - GcManager Gc; - CidStore CidStore(Gc); - CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); - - std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue); - std::filesystem::path RootDir = TempDir.Path() / "root"; - std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; - std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; - std::filesystem::path ProjectFilePath = TempDir.Path() / "game" / "game.uproject"; - - Ref<ProjectStore::Project> Project(ProjectStore.NewProject(BasePath / "proj1"sv, - "proj1"sv, - RootDir.string(), - EngineRootDir.string(), - ProjectRootDir.string(), - ProjectFilePath.string())); - ProjectStore::Oplog* Oplog = Project->NewOplog("oplog1", {}); - CHECK(Oplog != nullptr); - - std::filesystem::path DeletePath; - CHECK(Project->PrepareForDelete(DeletePath)); - CHECK(!DeletePath.empty()); - CHECK(Project->OpenOplog("oplog1") == nullptr); - // Oplog is now invalid, but pointer can still be accessed since we store old oplog pointers - CHECK(Oplog->OplogCount() == 0); - // Project is still valid since we have a Ref to it - CHECK(Project->Identifier == "proj1"sv); -} - -TEST_CASE("project.store.gc") -{ - using namespace std::literals; - using namespace testutils; - - ScopedTemporaryDirectory TempDir; - - auto JobQueue = MakeJobQueue(1, ""sv); - GcManager Gc; - CidStore CidStore(Gc); - CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); - - std::filesystem::path BasePath = TempDir.Path() / "projectstore"; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue); - std::filesystem::path RootDir = TempDir.Path() / "root"; - std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; - - std::filesystem::path Project1RootDir = TempDir.Path() / "game1"; - std::filesystem::path Project1FilePath = TempDir.Path() / "game1" / "game.uproject"; - { - CreateDirectories(Project1FilePath.parent_path()); - BasicFile ProjectFile; - ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); - } - std::filesystem::path Project1OplogPath = TempDir.Path() / "game1" / "saves" / "cooked" / ".projectstore"; - { - CreateDirectories(Project1OplogPath.parent_path()); - BasicFile OplogFile; - OplogFile.Open(Project1OplogPath, BasicFile::Mode::kTruncate); - } - - std::filesystem::path Project2RootDir = TempDir.Path() / "game2"; - std::filesystem::path Project2FilePath = TempDir.Path() / "game2" / "game.uproject"; - { - CreateDirectories(Project2FilePath.parent_path()); - BasicFile ProjectFile; - ProjectFile.Open(Project2FilePath, BasicFile::Mode::kTruncate); - } - std::filesystem::path Project2Oplog1Path = TempDir.Path() / "game1" / "saves" / "cooked" / ".projectstore"; - { - CreateDirectories(Project2Oplog1Path.parent_path()); - BasicFile OplogFile; - OplogFile.Open(Project2Oplog1Path, BasicFile::Mode::kTruncate); - } - std::filesystem::path Project2Oplog2Path = TempDir.Path() / "game2" / "saves" / "cooked" / ".projectstore"; - { - CreateDirectories(Project2Oplog2Path.parent_path()); - BasicFile OplogFile; - OplogFile.Open(Project2Oplog2Path, BasicFile::Mode::kTruncate); - } - - { - Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, - "proj1"sv, - RootDir.string(), - EngineRootDir.string(), - Project1RootDir.string(), - Project1FilePath.string())); - ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1", Project1OplogPath); - CHECK(Oplog != nullptr); - - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{77}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{7123, 583, 690, 99}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{55, 122}))); - } - - { - Ref<ProjectStore::Project> Project2(ProjectStore.NewProject(BasePath / "proj2"sv, - "proj2"sv, - RootDir.string(), - EngineRootDir.string(), - Project2RootDir.string(), - Project2FilePath.string())); - { - ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog2", Project2Oplog1Path); - CHECK(Oplog != nullptr); - - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{177}))); - Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9123, 383, 590, 96}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{535, 221}))); - } - { - ProjectStore::Oplog* Oplog = Project2->NewOplog("oplog3", Project2Oplog2Path); - CHECK(Oplog != nullptr); - - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), {})); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{137}))); - Oplog->AppendNewOplogEntry( - CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{9723, 683, 594, 98}))); - Oplog->AppendNewOplogEntry(CreateOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list<size_t>{531, 271}))); - } - } - - SUBCASE("v1") - { - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 21); - ProjectStore.CollectGarbage(GcCtx); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 21); - ProjectStore.CollectGarbage(GcCtx); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project1FilePath); - - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 21); - ProjectStore.CollectGarbage(GcCtx); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 14); - ProjectStore.CollectGarbage(GcCtx); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project2Oplog1Path); - { - GcContext GcCtx(GcClock::Now() - std::chrono::hours(24), GcClock::Now() - std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 14); - ProjectStore.CollectGarbage(GcCtx); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 7); - ProjectStore.CollectGarbage(GcCtx); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project2FilePath); - { - GcContext GcCtx(GcClock::Now() + std::chrono::hours(24), GcClock::Now() + std::chrono::hours(24)); - ProjectStore.GatherReferences(GcCtx); - size_t RefCount = 0; - GcCtx.IterateCids([&RefCount](const IoHash&) { RefCount++; }); - CHECK(RefCount == 0); - ProjectStore.CollectGarbage(GcCtx); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(!ProjectStore.OpenProject("proj2"sv)); - } - } - - SUBCASE("v2") - { - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project1FilePath); - - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(5u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), - .CollectSmallObjects = true, - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(4u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(21u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(7u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project2Oplog1Path); - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), - .CollectSmallObjects = true, - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(3u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), - .CollectSmallObjects = true, - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(3u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(0u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(ProjectStore.OpenProject("proj2"sv)); - } - - std::filesystem::remove(Project2FilePath); - { - GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), - .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), - .CollectSmallObjects = true, - .IsDeleteMode = true}; - GcResult Result = Gc.CollectGarbage(Settings); - CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.CheckedCount); - CHECK_EQ(1u, Result.ReferencerStatSum.RemoveExpiredDataStats.DeletedCount); - CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.CheckedCount); - CHECK_EQ(14u, Result.ReferenceStoreStatSum.RemoveUnreferencedDataStats.DeletedCount); - CHECK(!ProjectStore.OpenProject("proj1"sv)); - CHECK(!ProjectStore.OpenProject("proj2"sv)); - } - } -} - -TEST_CASE("project.store.partial.read") -{ - using namespace std::literals; - using namespace testutils; - - ScopedTemporaryDirectory TempDir; - - auto JobQueue = MakeJobQueue(1, ""sv); - GcManager Gc; - CidStore CidStore(Gc); - CidStoreConfiguration CidConfig = {.RootDirectory = TempDir.Path() / "cas"sv, .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; - CidStore.Initialize(CidConfig); - - std::filesystem::path BasePath = TempDir.Path() / "projectstore"sv; - ProjectStore ProjectStore(CidStore, BasePath, Gc, *JobQueue); - std::filesystem::path RootDir = TempDir.Path() / "root"sv; - std::filesystem::path EngineRootDir = TempDir.Path() / "engine"sv; - - std::filesystem::path Project1RootDir = TempDir.Path() / "game1"sv; - std::filesystem::path Project1FilePath = TempDir.Path() / "game1"sv / "game.uproject"sv; - { - CreateDirectories(Project1FilePath.parent_path()); - BasicFile ProjectFile; - ProjectFile.Open(Project1FilePath, BasicFile::Mode::kTruncate); - } - - std::vector<Oid> OpIds; - OpIds.insert(OpIds.end(), {Oid::NewOid(), Oid::NewOid(), Oid::NewOid(), Oid::NewOid()}); - std::unordered_map<Oid, std::vector<std::pair<Oid, CompressedBuffer>>, Oid::Hasher> Attachments; - { - Ref<ProjectStore::Project> Project1(ProjectStore.NewProject(BasePath / "proj1"sv, - "proj1"sv, - RootDir.string(), - EngineRootDir.string(), - Project1RootDir.string(), - Project1FilePath.string())); - ProjectStore::Oplog* Oplog = Project1->NewOplog("oplog1"sv, {}); - CHECK(Oplog != nullptr); - Attachments[OpIds[0]] = {}; - Attachments[OpIds[1]] = CreateAttachments(std::initializer_list<size_t>{77}); - Attachments[OpIds[2]] = CreateAttachments(std::initializer_list<size_t>{7123, 9583, 690, 99}); - Attachments[OpIds[3]] = CreateAttachments(std::initializer_list<size_t>{55, 122}); - for (auto It : Attachments) - { - Oplog->AppendNewOplogEntry(CreateOplogPackage(It.first, It.second)); - } - } - { - IoBuffer Chunk; - CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk) - .first == HttpResponseCode::OK); - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Attachment = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize); - CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize()); - } - - IoBuffer ChunkResult; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 0, - ~0ull, - HttpContentType::kCompressedBinary, - ChunkResult) - .first == HttpResponseCode::OK); - CHECK(ChunkResult); - CHECK(CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult)).DecodeRawSize() == - Attachments[OpIds[2]][1].second.DecodeRawSize()); - - IoBuffer PartialChunkResult; - CHECK(ProjectStore - .GetChunkRange("proj1"sv, - "oplog1"sv, - OidAsString(Attachments[OpIds[2]][1].first), - 5, - 1773, - HttpContentType::kCompressedBinary, - PartialChunkResult) - .first == HttpResponseCode::OK); - CHECK(PartialChunkResult); - IoHash PartialRawHash; - uint64_t PartialRawSize; - CompressedBuffer PartialCompressedResult = - CompressedBuffer::FromCompressed(SharedBuffer(PartialChunkResult), PartialRawHash, PartialRawSize); - CHECK(PartialRawSize >= 1773); - - uint64_t RawOffsetInPartialCompressed = GetCompressedOffset(PartialCompressedResult, 5); - SharedBuffer PartialDecompressed = PartialCompressedResult.Decompress(RawOffsetInPartialCompressed); - SharedBuffer FullDecompressed = Attachments[OpIds[2]][1].second.Decompress(); - const uint8_t* FullDataPtr = &(reinterpret_cast<const uint8_t*>(FullDecompressed.GetView().GetData())[5]); - const uint8_t* PartialDataPtr = reinterpret_cast<const uint8_t*>(PartialDecompressed.GetView().GetData()); - CHECK(FullDataPtr[0] == PartialDataPtr[0]); -} - -TEST_CASE("project.store.block") -{ - using namespace std::literals; - using namespace testutils; - - std::vector<std::size_t> AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 3466, 1093, 4269, 2257, 3685, 3489, - 7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251, - 491, 5464, 4607, 8135, 3767, 4045, 4415, 5007, 8876, 6761, 3359, 8526, 4097, 4855, 8225}); - - std::vector<std::pair<Oid, CompressedBuffer>> AttachmentsWithId = CreateAttachments(AttachmentSizes); - std::vector<SharedBuffer> Chunks; - Chunks.reserve(AttachmentSizes.size()); - for (const auto& It : AttachmentsWithId) - { - Chunks.push_back(It.second.GetCompressed().Flatten()); - } - CompressedBuffer Block = GenerateBlock(std::move(Chunks)); - IoBuffer BlockBuffer = Block.GetCompressed().Flatten().AsIoBuffer(); - CHECK(IterateBlock(std::move(BlockBuffer), [](CompressedBuffer&&, const IoHash&) {})); -} - -#endif - -void -prj_forcelink() -{ -} - -} // namespace zen diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h deleted file mode 100644 index 5ebcd420c..000000000 --- a/src/zenserver/projectstore/projectstore.h +++ /dev/null @@ -1,389 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/compactbinary.h> -#include <zencore/uid.h> -#include <zencore/xxhash.h> -#include <zenhttp/httpserver.h> -#include <zenstore/gc.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#include <map> -#include <unordered_map> - -namespace zen { - -class CbPackage; -class CidStore; -class AuthMgr; -class ScrubContext; -class JobQueue; - -enum class HttpResponseCode; - -struct OplogEntry -{ - uint32_t OpLsn; - uint32_t OpCoreOffset; // note: Multiple of alignment! - uint32_t OpCoreSize; - uint32_t OpCoreHash; // Used as checksum - Oid OpKeyHash; - uint32_t Reserved; - - inline bool IsTombstone() const { return OpCoreOffset == 0 && OpCoreSize == 0 && OpLsn == 0; } - inline void MakeTombstone() { OpLsn = OpCoreOffset = OpCoreSize = OpCoreHash = Reserved = 0; } -}; - -struct OplogEntryAddress -{ - uint64_t Offset; - uint64_t Size; -}; - -static_assert(IsPow2(sizeof(OplogEntry))); - -/** Project Store - - A project store consists of a number of Projects. - - Each project contains a number of oplogs (short for "operation log"). UE uses - one oplog per target platform to store the output of the cook process. - - An oplog consists of a sequence of "op" entries. Each entry is a structured object - containing references to attachments. Attachments are typically the serialized - package data split into separate chunks for bulk data, exports and header - information. - */ -class ProjectStore : public RefCounted, public GcStorage, public GcContributor, public GcReferencer -{ - struct OplogStorage; - -public: - ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcManager& Gc, JobQueue& JobQueue); - ~ProjectStore(); - - struct Project; - - struct Oplog - { - Oplog(std::string_view Id, - Project* Project, - CidStore& Store, - std::filesystem::path BasePath, - const std::filesystem::path& MarkerPath); - ~Oplog(); - - [[nodiscard]] static bool ExistsAt(const std::filesystem::path& BasePath); - - void Read(); - void Write(); - void Update(const std::filesystem::path& MarkerPath); - - struct ChunkInfo - { - Oid ChunkId; - uint64_t ChunkSize; - }; - - std::vector<ChunkInfo> GetAllChunksInfo(); - void IterateChunkMap(std::function<void(const Oid&, const IoHash& Hash)>&& Fn); - void IterateFileMap(std::function<void(const Oid&, const std::string_view& ServerPath, const std::string_view& ClientPath)>&& Fn); - void IterateOplog(std::function<void(CbObjectView)>&& Fn); - void IterateOplogWithKey(std::function<void(int, const Oid&, CbObjectView)>&& Fn); - std::optional<CbObject> GetOpByKey(const Oid& Key); - std::optional<CbObject> GetOpByIndex(int Index); - int GetOpIndexByKey(const Oid& Key); - int GetMaxOpIndex() const; - - IoBuffer FindChunk(Oid ChunkId); - - inline static const uint32_t kInvalidOp = ~0u; - - /** Persist a new oplog entry - * - * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected - */ - uint32_t AppendNewOplogEntry(CbPackage Op); - - uint32_t AppendNewOplogEntry(CbObject Core); - - enum UpdateType - { - kUpdateNewEntry, - kUpdateReplay - }; - - const std::string& OplogId() const { return m_OplogId; } - - const std::filesystem::path& TempPath() const { return m_TempPath; } - const std::filesystem::path& MarkerPath() const { return m_MarkerPath; } - - LoggerRef Log() { return m_OuterProject->Log(); } - void Flush(); - void ScrubStorage(ScrubContext& Ctx); - void GatherReferences(GcContext& GcCtx); - static uint64_t TotalSize(const std::filesystem::path& BasePath); - uint64_t TotalSize() const; - - std::size_t OplogCount() const - { - RwLock::SharedLockScope _(m_OplogLock); - return m_LatestOpMap.size(); - } - - std::filesystem::path PrepareForDelete(bool MoveFolder); - - void AddChunkMappings(const std::unordered_map<Oid, IoHash, Oid::Hasher>& ChunkMappings); - - private: - struct FileMapEntry - { - std::string ServerPath; - std::string ClientPath; - }; - - template<class V> - using OidMap = tsl::robin_map<Oid, V, Oid::Hasher>; - - Project* m_OuterProject = nullptr; - CidStore& m_CidStore; - std::filesystem::path m_BasePath; - std::filesystem::path m_MarkerPath; - std::filesystem::path m_TempPath; - - mutable RwLock m_OplogLock; - OidMap<IoHash> m_ChunkMap; // output data chunk id -> CAS address - OidMap<IoHash> m_MetaMap; // meta chunk id -> CAS address - OidMap<FileMapEntry> m_FileMap; // file id -> file map entry - int32_t m_ManifestVersion; // File system manifest version - tsl::robin_map<int, OplogEntryAddress> m_OpAddressMap; // Index LSN -> op data in ops blob file - OidMap<int> m_LatestOpMap; // op key -> latest op LSN for key - - RefPtr<OplogStorage> m_Storage; - std::string m_OplogId; - - /** Scan oplog and register each entry, thus updating the in-memory tracking tables - */ - void ReplayLog(); - - struct OplogEntryMapping - { - struct Mapping - { - Oid Id; - IoHash Hash; - }; - - struct FileMapping - { - Oid Id; - IoHash Hash; // This is either zero or a cid - std::string ServerPath; // If Hash is valid then this should be empty - std::string ClientPath; - }; - - std::vector<Mapping> Chunks; - std::vector<Mapping> Meta; - std::vector<FileMapping> Files; - }; - - OplogEntryMapping GetMapping(CbObjectView Core); - - /** Update tracking metadata for a new oplog entry - * - * This is used during replay (and gets called as part of new op append) - * - * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected - */ - uint32_t RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock, const OplogEntryMapping& OpMapping, const OplogEntry& OpEntry); - - void AddFileMapping(const RwLock::ExclusiveLockScope& OplogLock, - Oid FileId, - IoHash Hash, - std::string_view ServerPath, - std::string_view ClientPath); - void AddChunkMapping(const RwLock::ExclusiveLockScope& OplogLock, Oid ChunkId, IoHash Hash); - void AddMetaMapping(const RwLock::ExclusiveLockScope& OplogLock, Oid ChunkId, IoHash Hash); - - friend class ProjectStoreReferenceChecker; - }; - - struct Project : public RefCounted - { - std::string Identifier; - std::filesystem::path RootDir; - std::string EngineRootDir; - std::string ProjectRootDir; - std::string ProjectFilePath; - - Oplog* NewOplog(std::string_view OplogId, const std::filesystem::path& MarkerPath); - Oplog* OpenOplog(std::string_view OplogId); - void DeleteOplog(std::string_view OplogId); - std::filesystem::path RemoveOplog(std::string_view OplogId); - void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, const Oplog&)>&& Fn) const; - void IterateOplogs(std::function<void(const RwLock::SharedLockScope&, Oplog&)>&& Fn); - std::vector<std::string> ScanForOplogs() const; - bool IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime); - bool IsExpired(const RwLock::SharedLockScope&, const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog); - bool IsExpired(const GcClock::TimePoint ExpireTime, const ProjectStore::Oplog& Oplog); - void TouchProject() const; - void TouchOplog(std::string_view Oplog) const; - GcClock::TimePoint LastOplogAccessTime(std::string_view Oplog) const; - - Project(ProjectStore* PrjStore, CidStore& Store, std::filesystem::path BasePath); - virtual ~Project(); - - void Read(); - void Write(); - [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); - LoggerRef Log(); - void GatherReferences(GcContext& GcCtx); - static uint64_t TotalSize(const std::filesystem::path& BasePath); - uint64_t TotalSize() const; - bool PrepareForDelete(std::filesystem::path& OutDeletePath); - - private: - ProjectStore* m_ProjectStore; - CidStore& m_CidStore; - mutable RwLock m_ProjectLock; - std::map<std::string, std::unique_ptr<Oplog>> m_Oplogs; - std::vector<std::unique_ptr<Oplog>> m_DeletedOplogs; - std::filesystem::path m_OplogStoragePath; - mutable tsl::robin_map<std::string, GcClock::Tick> m_LastAccessTimes; - - std::filesystem::path BasePathForOplog(std::string_view OplogId); - bool IsExpired(const RwLock::SharedLockScope&, - const std::string& EntryName, - const std::filesystem::path& MarkerPath, - const GcClock::TimePoint ExpireTime); - void WriteAccessTimes(); - void ReadAccessTimes(); - }; - - // Oplog* OpenProjectOplog(std::string_view ProjectId, std::string_view OplogId); - - Ref<Project> OpenProject(std::string_view ProjectId); - Ref<Project> NewProject(const std::filesystem::path& BasePath, - std::string_view ProjectId, - std::string_view RootDir, - std::string_view EngineRootDir, - std::string_view ProjectRootDir, - std::string_view ProjectFilePath); - bool UpdateProject(std::string_view ProjectId, - std::string_view RootDir, - std::string_view EngineRootDir, - std::string_view ProjectRootDir, - std::string_view ProjectFilePath); - bool RemoveProject(std::string_view ProjectId, std::filesystem::path& OutDeletePath); - bool DeleteProject(std::string_view ProjectId); - bool Exists(std::string_view ProjectId); - void Flush(); - void DiscoverProjects(); - void IterateProjects(std::function<void(Project& Prj)>&& Fn); - - LoggerRef Log() { return m_Log; } - const std::filesystem::path& BasePath() const { return m_ProjectBasePath; } - - virtual void GatherReferences(GcContext& GcCtx) override; - virtual void ScrubStorage(ScrubContext& Ctx) override; - virtual void CollectGarbage(GcContext& GcCtx) override; - virtual GcStorageSize StorageSize() const override; - - virtual std::string GetGcName(GcCtx& Ctx) override; - virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; - virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; - - CbArray GetProjectsList(); - std::pair<HttpResponseCode, std::string> GetProjectFiles(const std::string_view ProjectId, - const std::string_view OplogId, - bool FilterClient, - CbObject& OutPayload); - std::pair<HttpResponseCode, std::string> GetProjectChunkInfos(const std::string_view ProjectId, - const std::string_view OplogId, - CbObject& OutPayload); - std::pair<HttpResponseCode, std::string> GetChunkInfo(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view ChunkId, - CbObject& OutPayload); - std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - const Oid ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - IoBuffer& OutChunk); - std::pair<HttpResponseCode, std::string> GetChunkRange(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view ChunkId, - uint64_t Offset, - uint64_t Size, - ZenContentType AcceptType, - IoBuffer& OutChunk); - std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view Cid, - ZenContentType AcceptType, - IoBuffer& OutChunk); - - std::pair<HttpResponseCode, std::string> PutChunk(const std::string_view ProjectId, - const std::string_view OplogId, - const std::string_view Cid, - ZenContentType ContentType, - IoBuffer&& Chunk); - - std::pair<HttpResponseCode, std::string> WriteOplog(const std::string_view ProjectId, - const std::string_view OplogId, - IoBuffer&& Payload, - CbObject& OutResponse); - - std::pair<HttpResponseCode, std::string> ReadOplog(const std::string_view ProjectId, - const std::string_view OplogId, - const HttpServerRequest::QueryParams& Params, - CbObject& OutResponse); - - std::pair<HttpResponseCode, std::string> WriteBlock(const std::string_view ProjectId, - const std::string_view OplogId, - IoBuffer&& Payload); - - bool Rpc(HttpServerRequest& HttpReq, - const std::string_view ProjectId, - const std::string_view OplogId, - IoBuffer&& Payload, - AuthMgr& AuthManager); - - std::pair<HttpResponseCode, std::string> Export(Ref<ProjectStore::Project> Project, - ProjectStore::Oplog& Oplog, - CbObjectView&& Params, - AuthMgr& AuthManager); - - std::pair<HttpResponseCode, std::string> Import(ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - CbObjectView&& Params, - AuthMgr& AuthManager); - - bool AreDiskWritesAllowed() const; - -private: - LoggerRef m_Log; - GcManager& m_Gc; - CidStore& m_CidStore; - JobQueue& m_JobQueue; - std::filesystem::path m_ProjectBasePath; - mutable RwLock m_ProjectsLock; - std::map<std::string, Ref<Project>> m_Projects; - const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; - - std::filesystem::path BasePathForProject(std::string_view ProjectId); - - friend class ProjectStoreGcStoreCompactor; -}; - -void prj_forcelink(); - -} // namespace zen diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp deleted file mode 100644 index 826c8ff51..000000000 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ /dev/null @@ -1,1699 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "remoteprojectstore.h" - -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinaryutil.h> -#include <zencore/compress.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zencore/scopeguard.h> -#include <zencore/stream.h> -#include <zencore/timer.h> -#include <zencore/workthreadpool.h> -#include <zenstore/cidstore.h> -#include <zenutil/workerpools.h> - -#include <unordered_map> - -namespace zen { - -/* - OplogContainer - Binary("ops") // Compressed CompactBinary object to hide attachment references, also makes the oplog smaller - { - CbArray("ops") - { - CbObject Op - (CbFieldType::BinaryAttachment Attachments[]) - (OpData) - } - } - CbArray("blocks") - CbObject - CbFieldType::BinaryAttachment "rawhash" // Optional, only if we are creating blocks (Jupiter/File) - CbArray("chunks") - CbFieldType::Hash // Chunk hashes - CbArray("chunks") // Optional, only if we are not creating blocks (Zen) - CbFieldType::BinaryAttachment // Chunk attachment hashes - - CompressedBinary ChunkBlock - { - VarUInt ChunkCount - VarUInt ChunkSizes[ChunkCount] - uint8_t[chunksize])[ChunkCount] - } -*/ - -////////////////////////////// AsyncRemoteResult - -struct AsyncRemoteResult -{ - void SetError(int32_t ErrorCode, const std::string& ErrorReason, const std::string ErrorText) - { - int32_t Expected = 0; - if (m_ErrorCode.compare_exchange_weak(Expected, ErrorCode ? ErrorCode : -1)) - { - m_ErrorReason = ErrorReason; - m_ErrorText = ErrorText; - } - } - bool IsError() const { return m_ErrorCode.load() != 0; } - int GetError() const { return m_ErrorCode.load(); }; - const std::string& GetErrorReason() const { return m_ErrorReason; }; - const std::string& GetErrorText() const { return m_ErrorText; }; - RemoteProjectStore::Result ConvertResult(double ElapsedSeconds = 0.0) const - { - return RemoteProjectStore::Result{m_ErrorCode, ElapsedSeconds, m_ErrorReason, m_ErrorText}; - } - -private: - std::atomic<int32_t> m_ErrorCode = 0; - std::string m_ErrorReason; - std::string m_ErrorText; -}; - -void -ReportProgress(JobContext* OptionalContext, std::string_view CurrentOp, ptrdiff_t Total, ptrdiff_t Remaining) -{ - if (OptionalContext) - { - ZEN_ASSERT(Total > 0); - OptionalContext->ReportProgress(CurrentOp, gsl::narrow<uint32_t>((100 * (Total - Remaining)) / Total)); - } - ZEN_INFO("{}", CurrentOp); -} - -void -ReportMessage(JobContext* OptionalContext, std::string_view Message) -{ - if (OptionalContext) - { - OptionalContext->ReportMessage(Message); - } - ZEN_INFO("{}", Message); -} - -bool -IsCancelled(JobContext* OptionalContext) -{ - if (!OptionalContext) - { - return false; - } - return OptionalContext->IsCancelled(); -} - -bool -IterateBlock(IoBuffer&& CompressedBlock, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor) -{ - IoBuffer BlockPayload = CompressedBuffer::FromCompressedNoValidate(std::move(CompressedBlock)).Decompress().AsIoBuffer(); - - MemoryView BlockView = BlockPayload.GetView(); - const uint8_t* ReadPtr = reinterpret_cast<const uint8_t*>(BlockView.GetData()); - uint32_t NumberSize; - uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize); - ReadPtr += NumberSize; - std::vector<uint64_t> ChunkSizes; - ChunkSizes.reserve(ChunkCount); - while (ChunkCount--) - { - ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize)); - ReadPtr += NumberSize; - } - ptrdiff_t TempBufferLength = std::distance(reinterpret_cast<const uint8_t*>(BlockView.GetData()), ReadPtr); - ZEN_ASSERT(TempBufferLength > 0); - for (uint64_t ChunkSize : ChunkSizes) - { - IoBuffer Chunk(IoBuffer::Wrap, ReadPtr, ChunkSize); - IoHash AttachmentRawHash; - uint64_t AttachmentRawSize; - CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize); - - if (!CompressedChunk) - { - ZEN_ERROR("Invalid chunk in block"); - return false; - } - Visitor(std::move(CompressedChunk), AttachmentRawHash); - ReadPtr += ChunkSize; - ZEN_ASSERT(ReadPtr <= BlockView.GetDataEnd()); - } - return true; -}; - -CompressedBuffer -GenerateBlock(std::vector<SharedBuffer>&& Chunks) -{ - size_t ChunkCount = Chunks.size(); - SharedBuffer SizeBuffer; - { - IoBuffer TempBuffer(ChunkCount * 9); - MutableMemoryView View = TempBuffer.GetMutableView(); - uint8_t* BufferStartPtr = reinterpret_cast<uint8_t*>(View.GetData()); - uint8_t* BufferEndPtr = BufferStartPtr; - BufferEndPtr += WriteVarUInt(gsl::narrow<uint64_t>(ChunkCount), BufferEndPtr); - auto It = Chunks.begin(); - while (It != Chunks.end()) - { - BufferEndPtr += WriteVarUInt(gsl::narrow<uint64_t>(It->GetSize()), BufferEndPtr); - It++; - } - ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd()); - ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr); - SizeBuffer = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow<size_t>(TempBufferLength))); - } - CompositeBuffer AllBuffers(std::move(SizeBuffer), CompositeBuffer(std::move(Chunks))); - - CompressedBuffer CompressedBlock = - CompressedBuffer::Compress(std::move(AllBuffers), OodleCompressor::Mermaid, OodleCompressionLevel::None); - - return CompressedBlock; -} - -struct Block -{ - IoHash BlockHash; - std::vector<IoHash> ChunksInBlock; -}; - -void -CreateBlock(WorkerThreadPool& WorkerPool, - Latch& OpSectionsLatch, - std::vector<SharedBuffer>&& ChunksInBlock, - RwLock& SectionsLock, - std::vector<Block>& Blocks, - size_t BlockIndex, - const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock, - AsyncRemoteResult& RemoteResult) -{ - ZEN_INFO("Generating block with {} attachments", ChunksInBlock.size()); - - OpSectionsLatch.AddCount(1); - WorkerPool.ScheduleWork( - [&Blocks, &SectionsLock, &OpSectionsLatch, BlockIndex, Chunks = std::move(ChunksInBlock), &AsyncOnBlock, &RemoteResult]() mutable { - auto _ = MakeGuard([&OpSectionsLatch] { OpSectionsLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - if (!Chunks.empty()) - { - CompressedBuffer CompressedBlock = GenerateBlock(std::move(Chunks)); // Move to callback and return IoHash - IoHash BlockHash = CompressedBlock.DecodeRawHash(); - AsyncOnBlock(std::move(CompressedBlock), BlockHash); - { - // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index - RwLock::SharedLockScope __(SectionsLock); - Blocks[BlockIndex].BlockHash = BlockHash; - } - } - }); -} - -size_t -AddBlock(RwLock& BlocksLock, std::vector<Block>& Blocks) -{ - size_t BlockIndex; - { - RwLock::ExclusiveLockScope _(BlocksLock); - BlockIndex = Blocks.size(); - Blocks.resize(BlockIndex + 1); - } - return BlockIndex; -} - -CbObject -BuildContainer(CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - bool BuildBlocks, - const std::vector<Block>& KnownBlocks, - WorkerThreadPool& WorkerPool, - const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock, - const std::function<void(const IoHash&)>& OnLargeAttachment, - const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks, - tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>* OutLooseAttachments, - JobContext* OptionalContext, - AsyncRemoteResult& RemoteResult) -{ - using namespace std::literals; - - std::unordered_set<IoHash, IoHash::Hasher> LargeChunkHashes; - CbObjectWriter SectionOpsWriter; - SectionOpsWriter.BeginArray("ops"sv); - - size_t OpCount = 0; - - CbObject OplogContainerObject; - { - RwLock BlocksLock; - std::vector<Block> Blocks; - CompressedBuffer OpsBuffer; - - std::unordered_set<IoHash, IoHash::Hasher> BlockAttachmentHashes; - - size_t BlockSize = 0; - std::vector<SharedBuffer> ChunksInBlock; - std::unordered_map<IoHash, int, IoHash::Hasher> Attachments; - - auto RewriteOp = [&](int LSN, CbObjectView Op, const std::function<void(CbObjectView)>& CB) { - bool OpRewritten = false; - CbArrayView Files = Op["files"sv].AsArrayView(); - if (Files.Num() == 0) - { - CB(Op); - return; - } - - CbWriter Cbo; - Cbo.BeginArray("files"sv); - - for (CbFieldView& Field : Files) - { - bool CopyField = true; - - if (CbObjectView View = Field.AsObjectView()) - { - IoHash DataHash = View["data"sv].AsHash(); - - if (DataHash == IoHash::Zero) - { - { - // Read file contents into memory and compress - - std::string_view ServerPath = View["serverpath"sv].AsString(); - std::filesystem::path FilePath = Project.RootDir / ServerPath; - BasicFile DataFile; - DataFile.Open(FilePath, BasicFile::Mode::kRead); - - IoBuffer FileIoBuffer = DataFile.ReadAll(); - DataFile.Close(); - - CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); - - DataHash = Compressed.DecodeRawHash(); - uint64_t PayloadSize = Compressed.GetCompressed().GetSize(); - if (PayloadSize > MaxChunkEmbedSize) - { - // Write it out as a temporary file - IoBuffer AttachmentBuffer; - std::filesystem::path AttachmentPath = Oplog.TempPath() / DataHash.ToHexString(); - if (std::filesystem::is_regular_file(AttachmentPath)) - { - AttachmentBuffer = IoBufferBuilder::MakeFromFile(AttachmentPath); - if (AttachmentBuffer.GetSize() != PayloadSize) - { - AttachmentBuffer = IoBuffer{}; - } - } - if (!AttachmentBuffer) - { - BasicFile BlockFile; - uint32_t RetriesLeft = 3; - BlockFile.Open(AttachmentPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temp attachment '{}', reason: '{}', retries left: {}.", - AttachmentPath, - Ec.message(), - RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); - uint64_t Offset = 0; - for (const SharedBuffer& Buffer : Compressed.GetCompressed().GetSegments()) - { - BlockFile.Write(Buffer.GetView(), Offset); - Offset += Buffer.GetSize(); - } - void* FileHandle = BlockFile.Detach(); - AttachmentBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); - - AttachmentBuffer.SetDeleteOnClose(true); - ZEN_DEBUG("Saved temp attachment {}, {}", DataHash, NiceBytes(PayloadSize)); - } - OutLooseAttachments->insert_or_assign(DataHash, AttachmentBuffer); - } - else - { - // If it is small we just hang on to the compressed buffer - OutLooseAttachments->insert_or_assign(DataHash, Compressed.GetCompressed().Flatten().AsIoBuffer()); - } - } - - // Rewrite file array entry with new data reference - CbObjectWriter Writer; - RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { - if (Field.GetName() == "data"sv) - { - // omit this field as we will write it explicitly ourselves - return true; - } - return false; - }); - Writer.AddBinaryAttachment("data"sv, DataHash); - - CbObject RewrittenOp = Writer.Save(); - Cbo.AddObject(std::move(RewrittenOp)); - CopyField = false; - - Attachments.insert_or_assign(DataHash, LSN); - } - } - - if (CopyField) - { - Cbo.AddField(Field); - } - else - { - OpRewritten = true; - } - } - - if (!OpRewritten) - { - CB(Op); - return; - } - - Cbo.EndArray(); - CbArray FilesArray = Cbo.Save().AsArray(); - - CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { - if (Field.GetName() == "files"sv) - { - NewWriter.AddArray("files"sv, FilesArray); - - return true; - } - - return false; - }); - CB(RewrittenOp); - }; - - ReportMessage(OptionalContext, "Building exported oplog and fetching attachments"); - - tsl::robin_map<int, std::string> OpLSNToKey; - - Oplog.IterateOplogWithKey([&](int LSN, const Oid&, CbObjectView Op) { - if (RemoteResult.IsError()) - { - return; - } - std::string_view Key = Op["key"sv].AsString(); - OpLSNToKey.insert({LSN, std::string(Key)}); - Op.IterateAttachments([&](CbFieldView FieldView) { Attachments.insert({FieldView.AsAttachment(), LSN}); }); - if (OutLooseAttachments != nullptr) - { - RewriteOp(LSN, Op, [&SectionOpsWriter](CbObjectView Op) { SectionOpsWriter << Op; }); - } - else - { - SectionOpsWriter << Op; - } - OpCount++; - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - }); - - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - return {}; - } - - if (!Attachments.empty() && !KnownBlocks.empty()) - { - ReportMessage(OptionalContext, fmt::format("Checking {} known blocks for reuse", KnownBlocks.size())); - - size_t ReusedBlockCount = 0; - for (const Block& KnownBlock : KnownBlocks) - { - size_t BlockAttachmentCount = KnownBlock.ChunksInBlock.size(); - if (BlockAttachmentCount == 0) - { - continue; - } - size_t FoundAttachmentCount = 0; - for (const IoHash& KnownHash : KnownBlock.ChunksInBlock) - { - if (Attachments.contains(KnownHash)) - { - FoundAttachmentCount++; - } - } - - size_t ReusePercent = (FoundAttachmentCount * 100) / BlockAttachmentCount; - // TODO: Configure reuse-level - if (ReusePercent > 80) - { - ZEN_DEBUG("Reusing block {}. {} attachments found, usage level: {}%", - KnownBlock.BlockHash, - FoundAttachmentCount, - ReusePercent); - for (const IoHash& KnownHash : KnownBlock.ChunksInBlock) - { - Attachments.erase(KnownHash); - } - - BlocksLock.WithExclusiveLock([&]() { Blocks.push_back(KnownBlock); }); - ReusedBlockCount++; - } - else if (FoundAttachmentCount > 0) - { - ZEN_DEBUG("Skipping block {}. {} attachments found, usage level: {}%", - KnownBlock.BlockHash, - FoundAttachmentCount, - ReusePercent); - } - } - ReportMessage(OptionalContext, fmt::format("Reusing {} out of {} known blocks", ReusedBlockCount, KnownBlocks.size())); - } - - ReportMessage(OptionalContext, fmt::format("Sorting {} attachments from {} ops", Attachments.size(), OpLSNToKey.size())); - - // Sort attachments so we get predictable blocks for the same oplog upload - std::vector<IoHash> SortedAttachments; - SortedAttachments.reserve(Attachments.size()); - for (const auto& It : Attachments) - { - SortedAttachments.push_back(It.first); - } - std::sort(SortedAttachments.begin(), SortedAttachments.end(), [&Attachments, &OpLSNToKey](const IoHash& Lhs, const IoHash& Rhs) { - auto LhsLNSIt = Attachments.find(Lhs); - ZEN_ASSERT_SLOW(LhsLNSIt != Attachments.end()); - auto RhsLNSIt = Attachments.find(Rhs); - ZEN_ASSERT_SLOW(RhsLNSIt != Attachments.end()); - if (LhsLNSIt->second == RhsLNSIt->second) - { - return Lhs < Rhs; - } - auto LhsKeyIt = OpLSNToKey.find(LhsLNSIt->second); - ZEN_ASSERT_SLOW(LhsKeyIt != OpLSNToKey.end()); - auto RhsKeyIt = OpLSNToKey.find(RhsLNSIt->second); - ZEN_ASSERT_SLOW(RhsKeyIt != OpLSNToKey.end()); - return LhsKeyIt->second < RhsKeyIt->second; - }); - - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - return {}; - } - ReportMessage(OptionalContext, - fmt::format("Assembling {} attachments from {} ops into blocks and loose attachments", - SortedAttachments.size(), - OpLSNToKey.size())); - - auto GetPayload = [&](const IoHash& AttachmentHash) { - if (OutLooseAttachments != nullptr) - { - auto PayloadIt = OutLooseAttachments->find(AttachmentHash); - if (PayloadIt != OutLooseAttachments->end()) - { - return PayloadIt->second; - } - } - return ChunkStore.FindChunkByCid(AttachmentHash); - }; - - int LastLSNOp = -1; - size_t GeneratedBlockCount = 0; - size_t LargeAttachmentCount = 0; - - Latch BlockCreateLatch(1); - for (const IoHash& AttachmentHash : SortedAttachments) - { - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - BlockCreateLatch.CountDown(); - while (!BlockCreateLatch.Wait(1000)) - { - ZEN_INFO("Aborting, {} blocks remaining...", BlockCreateLatch.Remaining()); - } - return {}; - } - - auto It = Attachments.find(AttachmentHash); - ZEN_ASSERT(It != Attachments.end()); - IoBuffer Payload = GetPayload(AttachmentHash); - if (!Payload) - { - std::optional<CbObject> Op = Oplog.GetOpByIndex(It->second); - ZEN_ASSERT(Op.has_value()); - ExtendableStringBuilder<1024> Sb; - Sb.Append("Failed to find attachment '"); - Sb.Append(AttachmentHash.ToHexString()); - Sb.Append("' for op: \n"); - Op.value().ToJson(Sb); - - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), Sb.ToString(), {}); - ZEN_ERROR("Failed to build container ({}). Reason: '{}'", RemoteResult.GetError(), RemoteResult.GetErrorReason()); - - BlockCreateLatch.CountDown(); - while (!BlockCreateLatch.Wait(1000)) - { - ZEN_INFO("Aborting, {} blocks remaining...", BlockCreateLatch.Remaining()); - } - - return {}; - } - - uint64_t PayloadSize = Payload.GetSize(); - if (PayloadSize > MaxChunkEmbedSize) - { - if (LargeChunkHashes.insert(AttachmentHash).second) - { - OnLargeAttachment(AttachmentHash); - LargeAttachmentCount++; - } - continue; - } - - if (!BlockAttachmentHashes.insert(AttachmentHash).second) - { - continue; - } - - const int CurrentOpLSN = It->second; - - BlockSize += PayloadSize; - if (BuildBlocks) - { - ChunksInBlock.emplace_back(SharedBuffer(std::move(Payload))); - } - else - { - Payload = {}; - } - - if (BlockSize >= MaxBlockSize && (CurrentOpLSN != LastLSNOp)) - { - size_t BlockIndex = AddBlock(BlocksLock, Blocks); - if (BuildBlocks) - { - CreateBlock(WorkerPool, - BlockCreateLatch, - std::move(ChunksInBlock), - BlocksLock, - Blocks, - BlockIndex, - AsyncOnBlock, - RemoteResult); - } - else - { - ZEN_INFO("Bulk group {} attachments", BlockAttachmentHashes.size()); - OnBlockChunks(BlockAttachmentHashes); - } - { - // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index - RwLock::SharedLockScope _(BlocksLock); - Blocks[BlockIndex].ChunksInBlock.insert(Blocks[BlockIndex].ChunksInBlock.end(), - BlockAttachmentHashes.begin(), - BlockAttachmentHashes.end()); - } - BlockAttachmentHashes.clear(); - ChunksInBlock.clear(); - BlockSize = 0; - GeneratedBlockCount++; - } - LastLSNOp = CurrentOpLSN; - } - if (BlockSize > 0) - { - size_t BlockIndex = AddBlock(BlocksLock, Blocks); - if (BuildBlocks) - { - CreateBlock(WorkerPool, - BlockCreateLatch, - std::move(ChunksInBlock), - BlocksLock, - Blocks, - BlockIndex, - AsyncOnBlock, - RemoteResult); - } - else - { - ZEN_INFO("Bulk group {} attachments", BlockAttachmentHashes.size()); - OnBlockChunks(BlockAttachmentHashes); - } - { - // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index - RwLock::SharedLockScope _(BlocksLock); - Blocks[BlockIndex].ChunksInBlock.insert(Blocks[BlockIndex].ChunksInBlock.end(), - BlockAttachmentHashes.begin(), - BlockAttachmentHashes.end()); - } - BlockAttachmentHashes.clear(); - ChunksInBlock.clear(); - BlockSize = 0; - GeneratedBlockCount++; - } - SectionOpsWriter.EndArray(); // "ops" - - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - BlockCreateLatch.CountDown(); - while (!BlockCreateLatch.Wait(1000)) - { - ZEN_INFO("Aborting, {} blocks remaining...", BlockCreateLatch.Remaining()); - } - return {}; - } - ReportMessage(OptionalContext, - fmt::format("Assembled {} attachments from {} ops into {} blocks and {} loose attachments", - SortedAttachments.size(), - OpLSNToKey.size(), - GeneratedBlockCount, - LargeAttachmentCount)); - - CompressedBuffer CompressedOpsSection = CompressedBuffer::Compress(SectionOpsWriter.Save().GetBuffer()); - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - BlockCreateLatch.CountDown(); - while (!BlockCreateLatch.Wait(1000)) - { - ptrdiff_t Remaining = BlockCreateLatch.Remaining(); - ReportProgress(OptionalContext, fmt::format("Aborting, {} blocks remaining...", Remaining), GeneratedBlockCount, Remaining); - } - if (GeneratedBlockCount > 0) - { - ReportProgress(OptionalContext, fmt::format("Aborting, {} blocks remaining...", 0), GeneratedBlockCount, 0); - } - return {}; - } - ReportMessage(OptionalContext, - fmt::format("Added oplog section {}, {}", - CompressedOpsSection.DecodeRawHash(), - NiceBytes(CompressedOpsSection.GetCompressedSize()))); - - BlockCreateLatch.CountDown(); - while (!BlockCreateLatch.Wait(1000)) - { - ptrdiff_t Remaining = BlockCreateLatch.Remaining(); - if (IsCancelled(OptionalContext)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - while (!BlockCreateLatch.Wait(1000)) - { - Remaining = BlockCreateLatch.Remaining(); - ReportProgress(OptionalContext, - fmt::format("Aborting, {} blocks remaining...", Remaining), - GeneratedBlockCount, - Remaining); - } - ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0); - return {}; - } - ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", Remaining), GeneratedBlockCount, Remaining); - } - if (GeneratedBlockCount > 0) - { - ReportProgress(OptionalContext, fmt::format("Creating blocks, {} remaining...", 0), GeneratedBlockCount, 0); - } - - if (!RemoteResult.IsError()) - { - CbObjectWriter OplogContinerWriter; - RwLock::SharedLockScope _(BlocksLock); - OplogContinerWriter.AddBinary("ops"sv, CompressedOpsSection.GetCompressed().Flatten().AsIoBuffer()); - - OplogContinerWriter.BeginArray("blocks"sv); - { - for (const Block& B : Blocks) - { - ZEN_ASSERT(!B.ChunksInBlock.empty()); - if (BuildBlocks) - { - ZEN_ASSERT(B.BlockHash != IoHash::Zero); - - OplogContinerWriter.BeginObject(); - { - OplogContinerWriter.AddBinaryAttachment("rawhash"sv, B.BlockHash); - OplogContinerWriter.BeginArray("chunks"sv); - { - for (const IoHash& RawHash : B.ChunksInBlock) - { - OplogContinerWriter.AddHash(RawHash); - } - } - OplogContinerWriter.EndArray(); // "chunks" - } - OplogContinerWriter.EndObject(); - continue; - } - - ZEN_ASSERT(B.BlockHash == IoHash::Zero); - OplogContinerWriter.BeginObject(); - { - OplogContinerWriter.BeginArray("chunks"sv); - { - for (const IoHash& RawHash : B.ChunksInBlock) - { - OplogContinerWriter.AddBinaryAttachment(RawHash); - } - } - OplogContinerWriter.EndArray(); - } - OplogContinerWriter.EndObject(); - } - } - OplogContinerWriter.EndArray(); // "blocks"sv - - OplogContinerWriter.BeginArray("chunks"sv); - { - for (const IoHash& AttachmentHash : LargeChunkHashes) - { - OplogContinerWriter.AddBinaryAttachment(AttachmentHash); - } - } - OplogContinerWriter.EndArray(); // "chunks" - - OplogContainerObject = OplogContinerWriter.Save(); - } - } - return OplogContainerObject; -} - -RemoteProjectStore::LoadContainerResult -BuildContainer(CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - bool BuildBlocks, - const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock, - const std::function<void(const IoHash&)>& OnLargeAttachment, - const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks, - tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>* OutOptionalTempAttachments) -{ - WorkerThreadPool& WorkerPool = GetSmallWorkerPool(); - - AsyncRemoteResult RemoteResult; - CbObject ContainerObject = BuildContainer(ChunkStore, - Project, - Oplog, - MaxBlockSize, - MaxChunkEmbedSize, - BuildBlocks, - {}, - WorkerPool, - AsyncOnBlock, - OnLargeAttachment, - OnBlockChunks, - OutOptionalTempAttachments, - nullptr, - RemoteResult); - return RemoteProjectStore::LoadContainerResult{RemoteResult.ConvertResult(), ContainerObject}; -} -void -UploadAttachments(WorkerThreadPool& WorkerPool, - CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - const std::unordered_set<IoHash, IoHash::Hasher>& LargeAttachments, - const std::vector<std::vector<IoHash>>& BlockChunks, - const std::unordered_map<IoHash, IoBuffer, IoHash::Hasher>& CreatedBlocks, - const tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>& TempAttachments, - const std::unordered_set<IoHash, IoHash::Hasher>& Needs, - bool ForceAll, - AsyncRemoteResult& RemoteResult, - JobContext* OptionalContext) -{ - using namespace std::literals; - - if (Needs.empty() && !ForceAll) - { - return; - } - - ReportMessage(OptionalContext, "Filtering needed attachments..."); - - std::unordered_set<IoHash, IoHash::Hasher> AttachmentsToUpload; - - size_t BlockAttachmentCountToUpload = 0; - size_t LargeAttachmentCountToUpload = 0; - std::atomic<ptrdiff_t> BulkAttachmentCountToUpload = 0; - AttachmentsToUpload.reserve(ForceAll ? CreatedBlocks.size() + LargeAttachments.size() : Needs.size()); - - for (const auto& CreatedBlock : CreatedBlocks) - { - if (ForceAll || Needs.contains(CreatedBlock.first)) - { - AttachmentsToUpload.insert(CreatedBlock.first); - BlockAttachmentCountToUpload++; - } - } - for (const IoHash& LargeAttachment : LargeAttachments) - { - if (ForceAll || Needs.contains(LargeAttachment)) - { - AttachmentsToUpload.insert(LargeAttachment); - LargeAttachmentCountToUpload++; - } - } - for (const std::vector<IoHash>& BlockHashes : BlockChunks) - { - if (ForceAll) - { - AttachmentsToUpload.insert(BlockHashes.begin(), BlockHashes.end()); - BulkAttachmentCountToUpload += BlockHashes.size(); - continue; - } - for (const IoHash& Hash : BlockHashes) - { - if (Needs.contains(Hash)) - { - AttachmentsToUpload.insert(Hash); - BulkAttachmentCountToUpload++; - } - } - } - - for (const IoHash& Needed : Needs) - { - if (!AttachmentsToUpload.contains(Needed)) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), - "Invalid attachment", - fmt::format("Upload requested of unknown attachment '{}'", Needed)); - ZEN_ERROR("Failed to upload attachment '{}'. ({}). Reason: '{}'", - Needed, - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - } - - if (AttachmentsToUpload.empty()) - { - ReportMessage(OptionalContext, "No attachments needed"); - return; - } - - if (IsCancelled(OptionalContext)) - { - if (!RemoteResult.IsError()) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - return; - } - - ReportMessage(OptionalContext, - fmt::format("Saving {} attachments ({} blocks, {} attachments, {} bulk attachments)", - AttachmentsToUpload.size(), - BlockAttachmentCountToUpload, - LargeAttachmentCountToUpload, - BulkAttachmentCountToUpload.load())); - - ptrdiff_t AttachmentsToSave(0); - Latch SaveAttachmentsLatch(1); - - for (const IoHash& RawHash : LargeAttachments) - { - if (RemoteResult.IsError()) - { - break; - } - if (!AttachmentsToUpload.contains(RawHash)) - { - continue; - } - - IoBuffer Payload; - if (auto BlockIt = CreatedBlocks.find(RawHash); BlockIt != CreatedBlocks.end()) - { - Payload = BlockIt->second; - } - else if (auto LooseTmpFileIt = TempAttachments.find(RawHash); LooseTmpFileIt != TempAttachments.end()) - { - Payload = LooseTmpFileIt->second; - } - - SaveAttachmentsLatch.AddCount(1); - AttachmentsToSave++; - WorkerPool.ScheduleWork( - [&ChunkStore, &RemoteStore, &SaveAttachmentsLatch, &RemoteResult, RawHash, &CreatedBlocks, TempPayload = std::move(Payload)]() { - auto _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - IoBuffer Payload = TempPayload ? TempPayload : ChunkStore.FindChunkByCid(RawHash); - if (!Payload) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::NotFound), - fmt::format("Failed to find attachment {}", RawHash), - {}); - ZEN_WARN("Failed to save attachment '{}' ({}). Reason: '{}'", - RawHash, - RemoteResult.GetErrorReason(), - RemoteResult.GetError()); - return; - } - - RemoteProjectStore::SaveAttachmentResult Result = - RemoteStore.SaveAttachment(CompositeBuffer(SharedBuffer(Payload)), RawHash); - if (Result.ErrorCode) - { - RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); - ZEN_WARN("Failed to save attachment '{}', {} ({}). Reason: '{}'", - RawHash, - NiceBytes(Payload.GetSize()), - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - ZEN_DEBUG("Saved attachment {}, {} in {}", - RawHash, - NiceBytes(Payload.GetSize()), - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000))); - return; - }); - } - - if (IsCancelled(OptionalContext)) - { - if (!RemoteResult.IsError()) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - return; - } - - for (auto& It : CreatedBlocks) - { - if (RemoteResult.IsError()) - { - break; - } - const IoHash& RawHash = It.first; - if (!AttachmentsToUpload.contains(RawHash)) - { - continue; - } - IoBuffer Payload = It.second; - ZEN_ASSERT(Payload); - SaveAttachmentsLatch.AddCount(1); - AttachmentsToSave++; - WorkerPool.ScheduleWork([&ChunkStore, &RemoteStore, &SaveAttachmentsLatch, &RemoteResult, Payload = std::move(Payload), RawHash]() { - auto _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - - RemoteProjectStore::SaveAttachmentResult Result = RemoteStore.SaveAttachment(CompositeBuffer(SharedBuffer(Payload)), RawHash); - if (Result.ErrorCode) - { - RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); - ZEN_WARN("Failed to save attachment '{}', {} ({}). Reason: '{}'", - RawHash, - NiceBytes(Payload.GetSize()), - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - - ZEN_DEBUG("Saved attachment {}, {} in {}", - RawHash, - NiceBytes(Payload.GetSize()), - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000))); - return; - }); - } - - if (IsCancelled(OptionalContext)) - { - if (!RemoteResult.IsError()) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - return; - } - - for (const std::vector<IoHash>& Chunks : BlockChunks) - { - if (RemoteResult.IsError()) - { - break; - } - - std::vector<IoHash> NeededChunks; - NeededChunks.reserve(Chunks.size()); - for (const IoHash& Chunk : Chunks) - { - if (AttachmentsToUpload.contains(Chunk)) - { - NeededChunks.push_back(Chunk); - } - } - if (NeededChunks.empty()) - { - continue; - } - - SaveAttachmentsLatch.AddCount(1); - AttachmentsToSave++; - WorkerPool.ScheduleWork([&RemoteStore, - &ChunkStore, - &SaveAttachmentsLatch, - &RemoteResult, - &Chunks, - NeededChunks = std::move(NeededChunks), - &BulkAttachmentCountToUpload]() { - auto _ = MakeGuard([&SaveAttachmentsLatch] { SaveAttachmentsLatch.CountDown(); }); - std::vector<SharedBuffer> ChunkBuffers; - ChunkBuffers.reserve(NeededChunks.size()); - for (const IoHash& Chunk : NeededChunks) - { - IoBuffer ChunkPayload = ChunkStore.FindChunkByCid(Chunk); - if (!ChunkPayload) - { - RemoteResult.SetError(static_cast<int32_t>(HttpResponseCode::NotFound), - fmt::format("Missing chunk {}"sv, Chunk), - fmt::format("Unable to fetch attachment {} required by the oplog"sv, Chunk)); - ChunkBuffers.clear(); - break; - } - ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload))); - } - RemoteProjectStore::SaveAttachmentsResult Result = RemoteStore.SaveAttachments(ChunkBuffers); - if (Result.ErrorCode) - { - RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); - ZEN_WARN("Failed to save attachments with {} chunks ({}). Reason: '{}'", - Chunks.size(), - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - ZEN_DEBUG("Saved {} bulk attachments in {}", - Chunks.size(), - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000))); - BulkAttachmentCountToUpload.fetch_sub(Chunks.size()); - }); - } - - SaveAttachmentsLatch.CountDown(); - while (!SaveAttachmentsLatch.Wait(1000)) - { - ptrdiff_t Remaining = SaveAttachmentsLatch.Remaining(); - if (IsCancelled(OptionalContext)) - { - if (!RemoteResult.IsError()) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - } - ReportProgress( - OptionalContext, - fmt::format("Saving attachments, {} remaining...", BlockChunks.empty() ? Remaining : BulkAttachmentCountToUpload.load()), - AttachmentsToSave, - Remaining); - } - if (AttachmentsToSave > 0) - { - ReportProgress(OptionalContext, fmt::format("Saving attachments, {} remaining...", 0), AttachmentsToSave, 0); - } -} - -RemoteProjectStore::Result -SaveOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - bool EmbedLooseFiles, - bool BuildBlocks, - bool UseTempBlocks, - bool ForceUpload, - JobContext* OptionalContext) -{ - using namespace std::literals; - - Stopwatch Timer; - - WorkerThreadPool& WorkerPool = GetSmallWorkerPool(); - - std::filesystem::path AttachmentTempPath; - if (UseTempBlocks) - { - AttachmentTempPath = Oplog.TempPath(); - AttachmentTempPath.append(".pending"); - CreateDirectories(AttachmentTempPath); - } - - AsyncRemoteResult RemoteResult; - RwLock AttachmentsLock; - std::unordered_set<IoHash, IoHash::Hasher> LargeAttachments; - std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> CreatedBlocks; - - auto MakeTempBlock = [AttachmentTempPath, &RemoteResult, &AttachmentsLock, &CreatedBlocks](CompressedBuffer&& CompressedBlock, - const IoHash& BlockHash) { - std::filesystem::path BlockPath = AttachmentTempPath; - BlockPath.append(BlockHash.ToHexString()); - if (!std::filesystem::exists(BlockPath)) - { - IoBuffer BlockBuffer; - try - { - BasicFile BlockFile; - uint32_t RetriesLeft = 3; - BlockFile.Open(BlockPath, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temporary oplog block '{}', reason: '{}', retries left: {}.", - BlockPath, - Ec.message(), - RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); - - uint64_t Offset = 0; - for (const SharedBuffer& Buffer : CompressedBlock.GetCompressed().GetSegments()) - { - BlockFile.Write(Buffer.GetView(), Offset); - Offset += Buffer.GetSize(); - } - void* FileHandle = BlockFile.Detach(); - BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); - } - catch (std::exception& Ex) - { - RemoteResult.SetError(gsl::narrow<int32_t>(HttpResponseCode::InternalServerError), - Ex.what(), - "Unable to create temp block file"); - return; - } - - BlockBuffer.SetDeleteOnClose(true); - { - RwLock::ExclusiveLockScope __(AttachmentsLock); - CreatedBlocks.insert({BlockHash, std::move(BlockBuffer)}); - } - ZEN_DEBUG("Saved temp block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize())); - } - }; - - auto UploadBlock = [&RemoteStore, &RemoteResult](CompressedBuffer&& CompressedBlock, const IoHash& BlockHash) { - RemoteProjectStore::SaveAttachmentResult Result = RemoteStore.SaveAttachment(CompressedBlock.GetCompressed(), BlockHash); - if (Result.ErrorCode) - { - RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); - ZEN_WARN("Failed to save attachment ({}). Reason: '{}'", RemoteResult.GetErrorReason(), RemoteResult.GetError()); - return; - } - ZEN_DEBUG("Saved block {}, {}", BlockHash, NiceBytes(CompressedBlock.GetCompressedSize())); - }; - - std::vector<std::vector<IoHash>> BlockChunks; - auto OnBlockChunks = [&BlockChunks](const std::unordered_set<IoHash, IoHash::Hasher>& Chunks) { - BlockChunks.push_back({Chunks.begin(), Chunks.end()}); - ZEN_DEBUG("Found {} block chunks", Chunks.size()); - }; - - auto OnLargeAttachment = [&AttachmentsLock, &LargeAttachments](const IoHash& AttachmentHash) { - { - RwLock::ExclusiveLockScope _(AttachmentsLock); - LargeAttachments.insert(AttachmentHash); - } - ZEN_DEBUG("Found attachment {}", AttachmentHash); - }; - - std::function<void(CompressedBuffer&&, const IoHash&)> OnBlock; - if (UseTempBlocks) - { - OnBlock = MakeTempBlock; - } - else - { - OnBlock = UploadBlock; - } - - std::vector<Block> KnownBlocks; - - if (BuildBlocks) - { - ReportMessage(OptionalContext, "Loading oplog base container"); - RemoteProjectStore::LoadContainerResult BaseContainerResult = RemoteStore.LoadBaseContainer(); - if (BaseContainerResult.ErrorCode != static_cast<int>(HttpResponseCode::NoContent)) - { - if (BaseContainerResult.ErrorCode) - { - ZEN_WARN("Failed to load oplog base container, reason: '{}', error code: {}", - BaseContainerResult.Reason, - BaseContainerResult.ErrorCode); - } - else - { - CbArrayView BlocksArray = BaseContainerResult.ContainerObject["blocks"sv].AsArrayView(); - KnownBlocks.reserve(BlocksArray.Num()); - for (CbFieldView BlockField : BlocksArray) - { - CbObjectView BlockView = BlockField.AsObjectView(); - IoHash BlockHash = BlockView["rawhash"sv].AsBinaryAttachment(); - - std::vector<IoHash> ChunksInBlock; - CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); - if (BlockHash == IoHash::Zero) - { - continue; - } - - ChunksInBlock.reserve(ChunksArray.Num()); - for (CbFieldView ChunkField : ChunksArray) - { - ChunksInBlock.push_back(ChunkField.AsHash()); - } - KnownBlocks.push_back({.BlockHash = BlockHash, .ChunksInBlock = std::move(ChunksInBlock)}); - }; - } - ReportMessage(OptionalContext, fmt::format("Loading oplog base container in {:.3} s", BaseContainerResult.ElapsedSeconds)); - } - } - - tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher> TempAttachments; - CbObject OplogContainerObject = BuildContainer(ChunkStore, - Project, - Oplog, - MaxBlockSize, - MaxChunkEmbedSize, - BuildBlocks, - KnownBlocks, - WorkerPool, - OnBlock, - OnLargeAttachment, - OnBlockChunks, - EmbedLooseFiles ? &TempAttachments : nullptr, - OptionalContext, - /* out */ RemoteResult); - if (!RemoteResult.IsError()) - { - if (IsCancelled(OptionalContext)) - { - RemoteProjectStore::Result Result = {.ErrorCode = 0, - .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500, - .Text = "Operation cancelled"}; - return Result; - } - - uint64_t ChunkCount = OplogContainerObject["chunks"sv].AsArrayView().Num(); - uint64_t BlockCount = OplogContainerObject["blocks"sv].AsArrayView().Num(); - ReportMessage(OptionalContext, fmt::format("Saving oplog container with {} attachments and {} blocks...", ChunkCount, BlockCount)); - - RemoteProjectStore::SaveResult ContainerSaveResult = RemoteStore.SaveContainer(OplogContainerObject.GetBuffer().AsIoBuffer()); - if (ContainerSaveResult.ErrorCode) - { - RemoteResult.SetError(ContainerSaveResult.ErrorCode, ContainerSaveResult.Reason, "Failed to save oplog container"); - ZEN_WARN("Failed to save oplog container ({}). Reason: '{}'", RemoteResult.GetErrorReason(), RemoteResult.GetError()); - } - else - { - ZEN_DEBUG("Saved container in {}", NiceTimeSpanMs(static_cast<uint64_t>(ContainerSaveResult.ElapsedSeconds * 1000))); - } - - UploadAttachments(WorkerPool, - ChunkStore, - RemoteStore, - LargeAttachments, - BlockChunks, - CreatedBlocks, - TempAttachments, - ContainerSaveResult.Needs, - ForceUpload, - RemoteResult, - OptionalContext); - - while (!RemoteResult.IsError()) - { - if (IsCancelled(OptionalContext)) - { - RemoteProjectStore::Result Result = {.ErrorCode = 0, - .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500, - .Text = "Operation cancelled"}; - return Result; - } - - ReportMessage(OptionalContext, "Finalizing oplog container..."); - RemoteProjectStore::FinalizeResult ContainerFinalizeResult = RemoteStore.FinalizeContainer(ContainerSaveResult.RawHash); - if (ContainerFinalizeResult.ErrorCode) - { - RemoteResult.SetError(ContainerFinalizeResult.ErrorCode, ContainerFinalizeResult.Reason, ContainerFinalizeResult.Text); - ZEN_WARN("Failed to finalize oplog container {} ({}). Reason: '{}'", - ContainerSaveResult.RawHash, - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - } - ZEN_DEBUG("Finalized container in {}", NiceTimeSpanMs(static_cast<uint64_t>(ContainerFinalizeResult.ElapsedSeconds * 1000))); - if (ContainerFinalizeResult.Needs.empty()) - { - break; - } - - if (IsCancelled(OptionalContext)) - { - RemoteProjectStore::Result Result = {.ErrorCode = 0, - .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500, - .Text = "Operation cancelled"}; - return Result; - } - - ReportMessage(OptionalContext, - fmt::format("Finalize reported {} missing attachments...", ContainerFinalizeResult.Needs.size())); - - UploadAttachments(WorkerPool, - ChunkStore, - RemoteStore, - LargeAttachments, - BlockChunks, - CreatedBlocks, - TempAttachments, - ContainerFinalizeResult.Needs, - false, - RemoteResult, - OptionalContext); - } - - TempAttachments.clear(); - CreatedBlocks.clear(); - } - RemoteProjectStore::Result Result = RemoteResult.ConvertResult(); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - ZEN_INFO("Saved oplog {} in {}", - RemoteResult.GetError() == 0 ? "SUCCESS" : "FAILURE", - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000))); - return Result; -}; - -RemoteProjectStore::Result -SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function<bool(const IoHash& RawHash)>& HasAttachment, - const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock, - const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, - JobContext* OptionalContext) -{ - using namespace std::literals; - - Stopwatch Timer; - - size_t NeedAttachmentCount = 0; - CbArrayView LargeChunksArray = ContainerObject["chunks"sv].AsArrayView(); - for (CbFieldView LargeChunksField : LargeChunksArray) - { - IoHash AttachmentHash = LargeChunksField.AsBinaryAttachment(); - if (HasAttachment(AttachmentHash)) - { - continue; - } - OnNeedAttachment(AttachmentHash); - }; - ReportMessage(OptionalContext, fmt::format("Requesting {} of {} large attachements", NeedAttachmentCount, LargeChunksArray.Num())); - - size_t NeedBlockCount = 0; - CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); - for (CbFieldView BlockField : BlocksArray) - { - CbObjectView BlockView = BlockField.AsObjectView(); - IoHash BlockHash = BlockView["rawhash"sv].AsBinaryAttachment(); - - CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); - if (BlockHash == IoHash::Zero) - { - std::vector<IoHash> NeededChunks; - NeededChunks.reserve(ChunksArray.GetSize()); - for (CbFieldView ChunkField : ChunksArray) - { - IoHash ChunkHash = ChunkField.AsBinaryAttachment(); - if (HasAttachment(ChunkHash)) - { - continue; - } - NeededChunks.emplace_back(ChunkHash); - } - - if (!NeededChunks.empty()) - { - OnNeedBlock(IoHash::Zero, std::move(NeededChunks)); - } - continue; - } - - for (CbFieldView ChunkField : ChunksArray) - { - IoHash ChunkHash = ChunkField.AsHash(); - if (HasAttachment(ChunkHash)) - { - continue; - } - - OnNeedBlock(BlockHash, {}); - break; - } - }; - ReportMessage(OptionalContext, fmt::format("Requesting {} of {} attachment blocks", NeedBlockCount, BlocksArray.Num())); - - MemoryView OpsSection = ContainerObject["ops"sv].AsBinaryView(); - IoBuffer OpsBuffer(IoBuffer::Wrap, OpsSection.GetData(), OpsSection.GetSize()); - IoBuffer SectionPayload = CompressedBuffer::FromCompressedNoValidate(std::move(OpsBuffer)).Decompress().AsIoBuffer(); - - CbObject SectionObject = LoadCompactBinaryObject(SectionPayload); - if (!SectionObject) - { - ZEN_WARN("Failed to save oplog container. Reason: '{}'", "Section has unexpected data type"); - return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest), - Timer.GetElapsedTimeMs() / 1000.500, - "Section has unexpected data type", - "Failed to save oplog container"}; - } - - CbArrayView OpsArray = SectionObject["ops"sv].AsArrayView(); - ReportMessage(OptionalContext, fmt::format("Writing {} ops to oplog", OpsArray.Num())); - for (CbFieldView OpEntry : OpsArray) - { - CbObjectView Core = OpEntry.AsObjectView(); - BinaryWriter Writer; - Core.CopyTo(Writer); - MemoryView OpView = Writer.GetView(); - IoBuffer OpBuffer(IoBuffer::Wrap, OpView.GetData(), OpView.GetSize()); - CbObject Op(SharedBuffer(OpBuffer), CbFieldType::HasFieldType); - const uint32_t OpLsn = Oplog.AppendNewOplogEntry(Op); - if (OpLsn == ProjectStore::Oplog::kInvalidOp) - { - return RemoteProjectStore::Result{gsl::narrow<int>(HttpResponseCode::BadRequest), - Timer.GetElapsedTimeMs() / 1000.500, - "Failed saving op", - "Failed to save oplog container"}; - } - ZEN_DEBUG("oplog entry #{}", OpLsn); - } - return RemoteProjectStore::Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500}; -} - -RemoteProjectStore::Result -LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - bool ForceDownload, - JobContext* OptionalContext) -{ - using namespace std::literals; - - Stopwatch Timer; - - WorkerThreadPool& WorkerPool = GetSmallWorkerPool(); - - std::unordered_set<IoHash, IoHash::Hasher> Attachments; - std::vector<std::vector<IoHash>> ChunksInBlocks; - - RemoteProjectStore::LoadContainerResult LoadContainerResult = RemoteStore.LoadContainer(); - if (LoadContainerResult.ErrorCode) - { - ZEN_WARN("Failed to load oplog container, reason: '{}', error code: {}", LoadContainerResult.Reason, LoadContainerResult.ErrorCode); - return RemoteProjectStore::Result{.ErrorCode = LoadContainerResult.ErrorCode, - .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500, - .Reason = LoadContainerResult.Reason, - .Text = LoadContainerResult.Text}; - } - ReportMessage(OptionalContext, - fmt::format("Loaded container in {}", NiceTimeSpanMs(static_cast<uint64_t>(LoadContainerResult.ElapsedSeconds * 1000)))); - - AsyncRemoteResult RemoteResult; - Latch AttachmentsWorkLatch(1); - std::atomic_size_t AttachmentCount = 0; - - auto HasAttachment = [&ChunkStore, ForceDownload](const IoHash& RawHash) { - return !ForceDownload && ChunkStore.ContainsChunk(RawHash); - }; - auto OnNeedBlock = [&RemoteStore, &ChunkStore, &WorkerPool, &ChunksInBlocks, &AttachmentsWorkLatch, &AttachmentCount, &RemoteResult]( - const IoHash& BlockHash, - std::vector<IoHash>&& Chunks) { - if (BlockHash == IoHash::Zero) - { - AttachmentsWorkLatch.AddCount(1); - AttachmentCount.fetch_add(1); - WorkerPool.ScheduleWork([&RemoteStore, &ChunkStore, &AttachmentsWorkLatch, &RemoteResult, Chunks = std::move(Chunks)]() { - auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - - RemoteProjectStore::LoadAttachmentsResult Result = RemoteStore.LoadAttachments(Chunks); - if (Result.ErrorCode) - { - RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); - ZEN_WARN("Failed to load attachments with {} chunks ({}). Reason: '{}'", - Chunks.size(), - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - ZEN_DEBUG("Loaded {} bulk attachments in {}", - Chunks.size(), - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000))); - for (const auto& It : Result.Chunks) - { - ChunkStore.AddChunk(It.second.GetCompressed().Flatten().AsIoBuffer(), It.first, CidStore::InsertMode::kCopyOnly); - } - }); - return; - } - AttachmentsWorkLatch.AddCount(1); - AttachmentCount.fetch_add(1); - WorkerPool.ScheduleWork([&AttachmentsWorkLatch, &ChunkStore, &RemoteStore, BlockHash, &RemoteResult]() { - auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash); - if (BlockResult.ErrorCode) - { - RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); - ZEN_WARN("Failed to load oplog container, missing attachment {} ({}). Reason: '{}'", - BlockHash, - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - ZEN_DEBUG("Loaded block attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(BlockResult.ElapsedSeconds * 1000))); - - if (!IterateBlock(std::move(BlockResult.Bytes), [&ChunkStore](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { - ChunkStore.AddChunk(Chunk.GetCompressed().Flatten().AsIoBuffer(), AttachmentRawHash); - })) - { - RemoteResult.SetError(gsl::narrow<int32_t>(HttpResponseCode::InternalServerError), - fmt::format("Invalid format for block {}", BlockHash), - {}); - ZEN_WARN("Failed to load oplog container, attachment {} has invalid format ({}). Reason: '{}'", - BlockHash, - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - return; - } - }); - }; - - auto OnNeedAttachment = [&RemoteStore, &ChunkStore, &WorkerPool, &AttachmentsWorkLatch, &RemoteResult, &Attachments, &AttachmentCount]( - const IoHash& RawHash) { - if (!Attachments.insert(RawHash).second) - { - return; - } - - AttachmentsWorkLatch.AddCount(1); - AttachmentCount.fetch_add(1); - WorkerPool.ScheduleWork([&RemoteStore, &ChunkStore, &RemoteResult, &AttachmentsWorkLatch, RawHash]() { - auto _ = MakeGuard([&AttachmentsWorkLatch] { AttachmentsWorkLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash); - if (AttachmentResult.ErrorCode) - { - RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text); - ZEN_WARN("Failed to download attachment {}, reason: '{}', error code: {}", - RawHash, - AttachmentResult.Reason, - AttachmentResult.ErrorCode); - return; - } - ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast<uint64_t>(AttachmentResult.ElapsedSeconds * 1000))); - ChunkStore.AddChunk(AttachmentResult.Bytes, RawHash); - }); - }; - - RemoteProjectStore::Result Result = - SaveOplogContainer(Oplog, LoadContainerResult.ContainerObject, HasAttachment, OnNeedBlock, OnNeedAttachment, OptionalContext); - if (!Attachments.empty()) - { - ReportMessage(OptionalContext, fmt::format("Found {} attachments to download", Attachments.size())); - } - - AttachmentsWorkLatch.CountDown(); - while (!AttachmentsWorkLatch.Wait(1000)) - { - ptrdiff_t Remaining = AttachmentsWorkLatch.Remaining(); - if (IsCancelled(OptionalContext)) - { - if (!RemoteResult.IsError()) - { - RemoteResult.SetError(gsl::narrow<int>(HttpResponseCode::OK), "Operation cancelled", ""); - } - } - ReportProgress(OptionalContext, fmt::format("Loading attachments, {} remaining...", Remaining), AttachmentCount.load(), Remaining); - } - if (AttachmentCount.load() > 0) - { - ReportProgress(OptionalContext, fmt::format("Loading attachments, {} remaining...", 0), AttachmentCount.load(), 0); - } - if (Result.ErrorCode == 0) - { - Result = RemoteResult.ConvertResult(); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - - ReportMessage(OptionalContext, - fmt::format("Loaded oplog {} in {}", - RemoteResult.GetError() == 0 ? "SUCCESS" : "FAILURE", - NiceTimeSpanMs(static_cast<uint64_t>(Result.ElapsedSeconds * 1000.0)))); - - return Result; -} - -////////////////////////////////////////////////////////////////////////// -// These are here to avoid vtable leakage - -RemoteProjectStore::RemoteProjectStore() -{ -} - -RemoteProjectStore::~RemoteProjectStore() -{ -} - -} // namespace zen diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h deleted file mode 100644 index be086084c..000000000 --- a/src/zenserver/projectstore/remoteprojectstore.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/jobqueue.h> -#include "projectstore.h" - -#include <unordered_set> - -namespace zen { - -class CidStore; -class WorkerThreadPool; - -class RemoteProjectStore -{ -public: - struct Result - { - int32_t ErrorCode{}; - double ElapsedSeconds{}; - std::string Reason; - std::string Text; - }; - - struct SaveResult : public Result - { - std::unordered_set<IoHash, IoHash::Hasher> Needs; - IoHash RawHash; - }; - - struct FinalizeResult : public Result - { - std::unordered_set<IoHash, IoHash::Hasher> Needs; - }; - - struct SaveAttachmentResult : public Result - { - }; - - struct SaveAttachmentsResult : public Result - { - }; - - struct LoadAttachmentResult : public Result - { - IoBuffer Bytes; - }; - - struct LoadContainerResult : public Result - { - CbObject ContainerObject; - }; - - struct LoadAttachmentsResult : public Result - { - std::vector<std::pair<IoHash, CompressedBuffer>> Chunks; - }; - - struct RemoteStoreInfo - { - bool CreateBlocks; - bool UseTempBlockFiles; - std::string Description; - }; - - RemoteProjectStore(); - virtual ~RemoteProjectStore(); - - virtual RemoteStoreInfo GetInfo() const = 0; - - virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0; - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) = 0; - virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Payloads) = 0; - - virtual LoadContainerResult LoadContainer() = 0; - virtual LoadContainerResult LoadBaseContainer() = 0; - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) = 0; -}; - -struct RemoteStoreOptions -{ - size_t MaxBlockSize = 128u * 1024u * 1024u; - size_t MaxChunkEmbedSize = 1024u * 1024u; -}; - -RemoteProjectStore::LoadContainerResult BuildContainer( - CidStore& ChunkStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - bool BuildBlocks, - const std::function<void(CompressedBuffer&&, const IoHash&)>& AsyncOnBlock, - const std::function<void(const IoHash&)>& OnLargeAttachment, - const std::function<void(const std::unordered_set<IoHash, IoHash::Hasher>)>& OnBlockChunks, - tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher>* - OutOptionalTempAttachments); // Set OutOptionalTempAttachments to nullptr to avoid embedding loose "additional files" - -class JobContext; - -RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function<bool(const IoHash& RawHash)>& HasAttachment, - const std::function<void(const IoHash& BlockHash, std::vector<IoHash>&& Chunks)>& OnNeedBlock, - const std::function<void(const IoHash& RawHash)>& OnNeedAttachment, - JobContext* OptionalContext); - -RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Project& Project, - ProjectStore::Oplog& Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize, - bool EmbedLooseFiles, - bool BuildBlocks, - bool UseTempBlocks, - bool ForceUpload, - JobContext* OptionalContext); - -RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - bool ForceDownload, - JobContext* OptionalContext); - -CompressedBuffer GenerateBlock(std::vector<SharedBuffer>&& Chunks); -bool IterateBlock(IoBuffer&& CompressedBlock, std::function<void(CompressedBuffer&& Chunk, const IoHash& AttachmentHash)> Visitor); - -} // namespace zen diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp deleted file mode 100644 index 7823010b5..000000000 --- a/src/zenserver/projectstore/zenremoteprojectstore.cpp +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zenremoteprojectstore.h" - -#include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compositebuffer.h> -#include <zencore/fmtutils.h> -#include <zencore/scopeguard.h> -#include <zencore/stream.h> -#include <zencore/timer.h> -#include <zenutil/packageformat.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <cpr/cpr.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -using namespace std::literals; - -class ZenRemoteStore : public RemoteProjectStore -{ -public: - ZenRemoteStore(std::string_view HostAddress, - std::string_view Project, - std::string_view Oplog, - size_t MaxBlockSize, - size_t MaxChunkEmbedSize) - : m_HostAddress(HostAddress) - , m_ProjectStoreUrl(fmt::format("{}/prj"sv, m_HostAddress)) - , m_Project(Project) - , m_Oplog(Oplog) - , m_MaxBlockSize(MaxBlockSize) - , m_MaxChunkEmbedSize(MaxChunkEmbedSize) - { - } - - virtual RemoteStoreInfo GetInfo() const override - { - return {.CreateBlocks = false, .UseTempBlockFiles = false, .Description = fmt::format("[zen] {}"sv, m_HostAddress)}; - } - - virtual SaveResult SaveContainer(const IoBuffer& Payload) override - { - Stopwatch Timer; - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - - std::string SaveRequest = fmt::format("{}/{}/oplog/{}/save"sv, m_ProjectStoreUrl, m_Project, m_Oplog); - Session->SetUrl({SaveRequest}); - Session->SetHeader({{"Content-Type", std::string(MapContentTypeToString(HttpContentType::kCbObject))}}); - MemoryView Data(Payload.GetView()); - Session->SetBody({reinterpret_cast<const char*>(Data.GetData()), Data.GetSize()}); - cpr::Response Response = Session->Post(); - SaveResult Result = SaveResult{ConvertResult(Response)}; - - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed saving oplog container to {}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - IoBuffer ResponsePayload(IoBuffer::Wrap, Response.text.data(), Response.text.size()); - CbObject ResponseObject = LoadCompactBinaryObject(ResponsePayload); - if (!ResponseObject) - { - Result.Reason = fmt::format("The response for {}/{}/{} is not formatted as a compact binary object"sv, - m_ProjectStoreUrl, - m_Project, - m_Oplog); - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - CbArrayView NeedsArray = ResponseObject["need"sv].AsArrayView(); - for (CbFieldView FieldView : NeedsArray) - { - IoHash ChunkHash = FieldView.AsHash(); - Result.Needs.insert(ChunkHash); - } - - Result.RawHash = IoHash::HashBuffer(Payload); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash) override - { - Stopwatch Timer; - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - - std::string SaveRequest = fmt::format("{}/{}/oplog/{}/{}"sv, m_ProjectStoreUrl, m_Project, m_Oplog, RawHash); - Session->SetUrl({SaveRequest}); - Session->SetHeader({{"Content-Type", std::string(MapContentTypeToString(HttpContentType::kCompressedBinary))}}); - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min<size_t>(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; - Session->SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(SizeLeft), ReadCallback)); - cpr::Response Response = Session->Post(); - SaveAttachmentResult Result = SaveAttachmentResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - RawHash, - Result.Reason); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual SaveAttachmentsResult SaveAttachments(const std::vector<SharedBuffer>& Chunks) override - { - Stopwatch Timer; - - CbPackage RequestPackage; - { - CbObjectWriter RequestWriter; - RequestWriter.AddString("method"sv, "putchunks"sv); - RequestWriter.BeginArray("chunks"sv); - { - for (const SharedBuffer& Chunk : Chunks) - { - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Chunk, RawHash, RawSize); - RequestWriter.AddHash(RawHash); - RequestPackage.AddAttachment(CbAttachment(Compressed, RawHash)); - } - } - RequestWriter.EndArray(); // "chunks" - RequestPackage.SetObject(RequestWriter.Save()); - } - CompositeBuffer Payload = FormatPackageMessageBuffer(RequestPackage, FormatFlags::kDefault); - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - std::string SaveRequest = fmt::format("{}/{}/oplog/{}/rpc"sv, m_ProjectStoreUrl, m_Project, m_Oplog); - Session->SetUrl({SaveRequest}); - Session->SetHeader({{"Content-Type", std::string(MapContentTypeToString(HttpContentType::kCbPackage))}}); - - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min<size_t>(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; - Session->SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(SizeLeft), ReadCallback)); - cpr::Response Response = Session->Post(); - SaveAttachmentsResult Result = SaveAttachmentsResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed saving {} oplog attachments to {}/{}/{}. Reason: '{}'", - Chunks.size(), - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual LoadAttachmentsResult LoadAttachments(const std::vector<IoHash>& RawHashes) override - { - Stopwatch Timer; - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - std::string SaveRequest = fmt::format("{}/{}/oplog/{}/rpc"sv, m_ProjectStoreUrl, m_Project, m_Oplog); - - CbObject Request; - { - CbObjectWriter RequestWriter; - RequestWriter.AddString("method"sv, "getchunks"sv); - RequestWriter.BeginArray("chunks"sv); - { - for (const IoHash& RawHash : RawHashes) - { - RequestWriter.AddHash(RawHash); - } - } - RequestWriter.EndArray(); // "chunks" - Request = RequestWriter.Save(); - } - IoBuffer Payload = Request.GetBuffer().AsIoBuffer(); - Session->SetBody(cpr::Body{(const char*)Payload.GetData(), Payload.GetSize()}); - Session->SetUrl(SaveRequest); - Session->SetHeader({{"Content-Type", std::string(MapContentTypeToString(HttpContentType::kCbObject))}, - {"Accept", std::string(MapContentTypeToString(HttpContentType::kCbPackage))}}); - - cpr::Response Response = Session->Post(); - LoadAttachmentsResult Result = LoadAttachmentsResult{ConvertResult(Response)}; - if (!Result.ErrorCode) - { - CbPackage Package = ParsePackageMessage(IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size())); - std::span<const CbAttachment> Attachments = Package.GetAttachments(); - Result.Chunks.reserve(Attachments.size()); - for (const CbAttachment& Attachment : Attachments) - { - Result.Chunks.emplace_back( - std::pair<IoHash, CompressedBuffer>{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()}); - } - } - else - { - Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'", - RawHashes.size(), - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - }; - - virtual FinalizeResult FinalizeContainer(const IoHash&) override - { - Stopwatch Timer; - - RwLock::ExclusiveLockScope _(SessionsLock); - Sessions.clear(); - return FinalizeResult{Result{.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500}}; - } - - virtual LoadContainerResult LoadContainer() override - { - Stopwatch Timer; - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - std::string SaveRequest = fmt::format("{}/{}/oplog/{}/load"sv, m_ProjectStoreUrl, m_Project, m_Oplog); - Session->SetUrl(SaveRequest); - Session->SetHeader({{"Accept", std::string(MapContentTypeToString(HttpContentType::kCbObject))}}); - Session->SetParameters( - {{"maxblocksize", fmt::format("{}", m_MaxBlockSize)}, {"maxchunkembedsize", fmt::format("{}", m_MaxChunkEmbedSize)}}); - cpr::Response Response = Session->Get(); - - LoadContainerResult Result = LoadContainerResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog container from {}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - } - else - { - Result.ContainerObject = LoadCompactBinaryObject(IoBuffer(IoBuffer::Clone, Response.text.data(), Response.text.size())); - if (!Result.ContainerObject) - { - Result.Reason = fmt::format("The response for {}/{}/{} is not formatted as a compact binary object"sv, - m_ProjectStoreUrl, - m_Project, - m_Oplog); - Result.ErrorCode = gsl::narrow<int32_t>(HttpResponseCode::InternalServerError); - } - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - - virtual LoadContainerResult LoadBaseContainer() override - { - return LoadContainerResult{{.ErrorCode = static_cast<int>(HttpResponseCode::NoContent)}}; - } - - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override - { - Stopwatch Timer; - - std::unique_ptr<cpr::Session> Session(AllocateSession()); - auto _ = MakeGuard([this, &Session]() { ReleaseSession(std::move(Session)); }); - - std::string LoadRequest = fmt::format("{}/{}/oplog/{}/{}"sv, m_ProjectStoreUrl, m_Project, m_Oplog, RawHash); - Session->SetUrl({LoadRequest}); - Session->SetHeader({{"Accept", std::string(MapContentTypeToString(HttpContentType::kCompressedBinary))}}); - cpr::Response Response = Session->Get(); - LoadAttachmentResult Result = LoadAttachmentResult{ConvertResult(Response)}; - if (!Result.ErrorCode) - { - Result.Bytes = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()); - } - if (!Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - RawHash, - Result.Reason); - } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.500; - return Result; - } - -private: - std::unique_ptr<cpr::Session> AllocateSession() - { - RwLock::ExclusiveLockScope _(SessionsLock); - if (Sessions.empty()) - { - Sessions.emplace_back(std::make_unique<cpr::Session>()); - } - std::unique_ptr<cpr::Session> Session = std::move(Sessions.back()); - Sessions.pop_back(); - return Session; - } - - void ReleaseSession(std::unique_ptr<cpr::Session>&& Session) - { - RwLock::ExclusiveLockScope _(SessionsLock); - Sessions.emplace_back(std::move(Session)); - } - - static Result ConvertResult(const cpr::Response& Response) - { - std::string Text; - std::string Reason = Response.reason; - int32_t ErrorCode = 0; - if (Response.error.code != cpr::ErrorCode::OK) - { - ErrorCode = static_cast<int32_t>(Response.error.code); - if (!Response.error.message.empty()) - { - Reason = Response.error.message; - } - } - else if (!IsHttpSuccessCode(Response.status_code)) - { - ErrorCode = static_cast<int32_t>(Response.status_code); - - if (auto It = Response.header.find("Content-Type"); It != Response.header.end()) - { - zen::HttpContentType ContentType = zen::ParseContentType(It->second); - if (ContentType == zen::HttpContentType::kText) - { - Text = Response.text; - } - } - - Reason = fmt::format("{}"sv, Response.status_code); - } - return {.ErrorCode = ErrorCode, .ElapsedSeconds = Response.elapsed, .Reason = Reason, .Text = Text}; - } - - RwLock SessionsLock; - std::vector<std::unique_ptr<cpr::Session>> Sessions; - - const std::string m_HostAddress; - const std::string m_ProjectStoreUrl; - const std::string m_Project; - const std::string m_Oplog; - const size_t m_MaxBlockSize; - const size_t m_MaxChunkEmbedSize; -}; - -std::shared_ptr<RemoteProjectStore> -CreateZenRemoteStore(const ZenRemoteStoreOptions& Options) -{ - std::string Url = Options.Url; - if (Url.find("://"sv) == std::string::npos) - { - // Assume http URL - Url = fmt::format("http://{}"sv, Url); - } - std::shared_ptr<RemoteProjectStore> RemoteStore = - std::make_shared<ZenRemoteStore>(Url, Options.ProjectId, Options.OplogId, Options.MaxBlockSize, Options.MaxChunkEmbedSize); - return RemoteStore; -} - -} // namespace zen diff --git a/src/zenserver/projectstore/zenremoteprojectstore.h b/src/zenserver/projectstore/zenremoteprojectstore.h deleted file mode 100644 index 9f079ee74..000000000 --- a/src/zenserver/projectstore/zenremoteprojectstore.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "remoteprojectstore.h" - -namespace zen { - -struct ZenRemoteStoreOptions : RemoteStoreOptions -{ - std::string Url; - std::string ProjectId; - std::string OplogId; -}; - -std::shared_ptr<RemoteProjectStore> CreateZenRemoteStore(const ZenRemoteStoreOptions& Options); - -} // namespace zen diff --git a/src/zenserver/sentryintegration.cpp b/src/zenserver/sentryintegration.cpp deleted file mode 100644 index 11bf78a75..000000000 --- a/src/zenserver/sentryintegration.cpp +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "sentryintegration.h" - -#include <zencore/config.h> -#include <zencore/logging.h> - -#include <stdarg.h> -#include <stdio.h> - -#if ZEN_PLATFORM_LINUX -# include <pwd.h> -#endif - -#if ZEN_PLATFORM_MAC -# include <pwd.h> -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include <spdlog/spdlog.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_USE_SENTRY -# define SENTRY_BUILD_STATIC 1 -ZEN_THIRD_PARTY_INCLUDES_START -# include <sentry.h> -# include <spdlog/sinks/base_sink.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace sentry { - -struct SentryAssertImpl : zen::AssertImpl -{ - ZEN_FORCENOINLINE ZEN_DEBUG_SECTION SentryAssertImpl(); - virtual ZEN_FORCENOINLINE ZEN_DEBUG_SECTION ~SentryAssertImpl(); - virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, - int LineNumber, - const char* FunctionName, - const char* Msg) override; - AssertImpl* PrevAssertImpl; -}; - -class sentry_sink final : public spdlog::sinks::base_sink<spdlog::details::null_mutex> -{ -public: - sentry_sink(); - ~sentry_sink(); - -protected: - void sink_it_(const spdlog::details::log_msg& msg) override; - void flush_() override; -}; - -////////////////////////////////////////////////////////////////////////// - -static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_INFO, - SENTRY_LEVEL_WARNING, - SENTRY_LEVEL_ERROR, - SENTRY_LEVEL_FATAL, - SENTRY_LEVEL_DEBUG}; - -sentry_sink::sentry_sink() -{ -} -sentry_sink::~sentry_sink() -{ -} - -void -sentry_sink::sink_it_(const spdlog::details::log_msg& msg) -{ - if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) - { - return; - } - try - { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw - char TmpBuffer[256]; - size_t MaxCopy = zen::Min<size_t>(msg.payload.size(), size_t(255)); - memcpy(TmpBuffer, msg.payload.data(), MaxCopy); - TmpBuffer[MaxCopy] = '\0'; - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ TmpBuffer); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} -void -sentry_sink::flush_() -{ -} - -SentryAssertImpl::SentryAssertImpl() : PrevAssertImpl(CurrentAssertImpl) -{ - CurrentAssertImpl = this; -} - -SentryAssertImpl::~SentryAssertImpl() -{ - CurrentAssertImpl = PrevAssertImpl; -} - -void -SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionName, const char* Msg) -{ - try - { - std::string Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg); - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the Msg raw - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ Msg); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} - -} // namespace sentry - -namespace zen { - -# if ZEN_USE_SENTRY -static void -SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) -{ - char LogMessageBuffer[160]; - std::string LogMessage; - const char* MessagePtr = LogMessageBuffer; - - int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); - - if (n >= int(sizeof LogMessageBuffer)) - { - LogMessage.resize(n + 1); - - n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args); - - MessagePtr = LogMessage.c_str(); - } - - switch (Level) - { - case SENTRY_LEVEL_DEBUG: - ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_INFO: - ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_WARNING: - ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_ERROR: - ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_FATAL: - ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); - break; - } -} -# endif - -SentryIntegration::SentryIntegration() -{ -} - -SentryIntegration::~SentryIntegration() -{ - if (m_IsInitialized && m_SentryErrorCode == 0) - { - logging::SetErrorLog(""); - m_SentryAssert.reset(); - sentry_close(); - } -} - -void -SentryIntegration::Initialize(std::string SentryDatabasePath, std::string SentryAttachmentPath, bool AllowPII) -{ - m_AllowPII = AllowPII; - - if (SentryDatabasePath.starts_with("\\\\?\\")) - { - SentryDatabasePath = SentryDatabasePath.substr(4); - } - sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, "https://[email protected]/5919284"); - sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); - sentry_options_set_logger(SentryOptions, SentryLogFunction, this); - if (SentryAttachmentPath.starts_with("\\\\?\\")) - { - SentryAttachmentPath = SentryAttachmentPath.substr(4); - } - sentry_options_add_attachment(SentryOptions, SentryAttachmentPath.c_str()); - sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - - // sentry_options_set_debug(SentryOptions, 1); - - m_SentryErrorCode = sentry_init(SentryOptions); - - if (m_SentryErrorCode == 0) - { - if (m_AllowPII) - { -# if ZEN_PLATFORM_WINDOWS - CHAR Buffer[511 + 1]; - DWORD BufferLength = sizeof(Buffer) / sizeof(CHAR); - BOOL OK = GetUserNameA(Buffer, &BufferLength); - if (OK && BufferLength) - { - m_SentryUserName = std::string(Buffer, BufferLength - 1); - } - BufferLength = sizeof(Buffer) / sizeof(CHAR); - OK = GetComputerNameA(Buffer, &BufferLength); - if (OK && BufferLength) - { - m_SentryHostName = std::string(Buffer, BufferLength); - } - else - { - m_SentryHostName = "unknown"; - } -# endif // ZEN_PLATFORM_WINDOWS - -# if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC) - uid_t uid = geteuid(); - struct passwd* pw = getpwuid(uid); - if (pw) - { - m_SentryUserName = std::string(pw->pw_name); - } - else - { - m_SentryUserName = "unknown"; - } - char HostNameBuffer[1023 + 1]; - int err = gethostname(HostNameBuffer, sizeof(HostNameBuffer)); - if (err == 0) - { - m_SentryHostName = std::string(HostNameBuffer); - } - else - { - m_SentryHostName = "unknown"; - } -# endif - m_SentryId = fmt::format("{}@{}", m_SentryUserName, m_SentryHostName); - sentry_value_t SentryUserObject = sentry_value_new_object(); - sentry_value_set_by_key(SentryUserObject, "id", sentry_value_new_string(m_SentryId.c_str())); - sentry_value_set_by_key(SentryUserObject, "username", sentry_value_new_string(m_SentryUserName.c_str())); - sentry_value_set_by_key(SentryUserObject, "ip_address", sentry_value_new_string("{{auto}}")); - sentry_set_user(SentryUserObject); - } - - m_SentryLogger = spdlog::create<sentry::sentry_sink>("sentry"); - logging::SetErrorLog("sentry"); - - m_SentryAssert = std::make_unique<sentry::SentryAssertImpl>(); - } - - m_IsInitialized = true; -} - -void -SentryIntegration::LogStartupInformation() -{ -# if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC) - uid_t uid = geteuid(); - struct passwd* pw = getpwuid(uid); - if (pw) - { - m_SentryUserName = std::string(pw->pw_name); - } - ZEN_INFO("Username: '{}'", m_SentryUserName); - - char HostNameBuffer[1023 + 1]; - int err = gethostname(HostNameBuffer, sizeof(HostNameBuffer)); - if (err == 0) - { - ZEN_INFO("Hostname: '{}'", HostNameBuffer); - } -# endif - if (m_IsInitialized) - { - if (m_SentryErrorCode == 0) - { - if (m_AllowPII) - { - ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); - } - else - { - ZEN_INFO("sentry initialized with anonymous reports"); - } - } - else - { - ZEN_WARN("sentry_init returned failure! (error code: {})", m_SentryErrorCode); - } - } -} - -void -SentryIntegration::ClearCaches() -{ - sentry_clear_modulecache(); -} - -} // namespace zen -#endif diff --git a/src/zenserver/sentryintegration.h b/src/zenserver/sentryintegration.h deleted file mode 100644 index dd8b87ab7..000000000 --- a/src/zenserver/sentryintegration.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/intmath.h> -#include <zencore/zencore.h> - -#if !defined(ZEN_USE_SENTRY) -# if ZEN_PLATFORM_MAC && ZEN_ARCH_ARM64 -// vcpkg's sentry-native port does not support Arm on Mac. -# define ZEN_USE_SENTRY 0 -# else -# define ZEN_USE_SENTRY 1 -# endif -#endif - -#if ZEN_USE_SENTRY - -# include <memory> - -ZEN_THIRD_PARTY_INCLUDES_START -# include <spdlog/logger.h> -ZEN_THIRD_PARTY_INCLUDES_END - -namespace sentry { - -struct SentryAssertImpl; - -} // namespace sentry - -namespace zen { - -class SentryIntegration -{ -public: - SentryIntegration(); - ~SentryIntegration(); - - void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII); - void LogStartupInformation(); - static void ClearCaches(); - -private: - int m_SentryErrorCode = 0; - bool m_IsInitialized = false; - bool m_AllowPII = false; - std::unique_ptr<sentry::SentryAssertImpl> m_SentryAssert; - std::string m_SentryUserName; - std::string m_SentryHostName; - std::string m_SentryId; - std::shared_ptr<spdlog::logger> m_SentryLogger; -}; - -} // namespace zen -#endif diff --git a/src/zenserver/stats/statsreporter.cpp b/src/zenserver/stats/statsreporter.cpp index 5d5ef4bfa..a1926eba4 100644 --- a/src/zenserver/stats/statsreporter.cpp +++ b/src/zenserver/stats/statsreporter.cpp @@ -3,6 +3,7 @@ #include "statsreporter.h" #include <zencore/logging.h> +#include <zencore/trace.h> #include <zennet/statsdclient.h> namespace zen { @@ -18,6 +19,7 @@ StatsReporter::~StatsReporter() void StatsReporter::Initialize(const ZenStatsConfig& Config) { + ZEN_TRACE_CPU("StatsReporter::Initialize"); RwLock::ExclusiveLockScope _(m_Lock); if (Config.Enabled) diff --git a/src/zenserver/upstream/jupiter.cpp b/src/zenserver/upstream/jupiter.cpp deleted file mode 100644 index a67c497ad..000000000 --- a/src/zenserver/upstream/jupiter.cpp +++ /dev/null @@ -1,1259 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "jupiter.h" - -#include "diag/logging.h" - -#include <zencore/compactbinary.h> -#include <zencore/compositebuffer.h> -#include <zencore/fmtutils.h> -#include <zencore/iobuffer.h> -#include <zencore/iohash.h> -#include <zencore/scopeguard.h> -#include <zencore/thread.h> -#include <zencore/trace.h> -#include <zenhttp/formatters.h> -#include <zenutil/basicfile.h> - -ZEN_THIRD_PARTY_INCLUDES_START -#include <cpr/cpr.h> -#include <fmt/format.h> -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_PLATFORM_WINDOWS -# pragma comment(lib, "Crypt32.lib") -# pragma comment(lib, "Wldap32.lib") -#endif - -#include <json11.hpp> - -using namespace std::literals; - -namespace zen { - -namespace detail { - struct CloudCacheSessionState - { - CloudCacheSessionState(CloudCacheClient& Client) : m_Client(Client) {} - - const CloudCacheAccessToken& GetAccessToken(bool RefreshToken) - { - if (RefreshToken) - { - m_AccessToken = m_Client.AcquireAccessToken(); - } - - return m_AccessToken; - } - - cpr::Session& GetSession() { return m_Session; } - - void Reset(std::chrono::milliseconds ConnectTimeout, std::chrono::milliseconds Timeout, bool AssumeHttp2) - { - m_Session.SetBody({}); - m_Session.SetHeader({}); - m_Session.SetConnectTimeout(ConnectTimeout); - m_Session.SetTimeout(Timeout); - if (AssumeHttp2) - { - m_Session.SetHttpVersion(cpr::HttpVersion{cpr::HttpVersionCode::VERSION_2_0_PRIOR_KNOWLEDGE}); - } - } - - private: - friend class zen::CloudCacheClient; - - CloudCacheClient& m_Client; - CloudCacheAccessToken m_AccessToken; - cpr::Session m_Session; - }; - - CloudCacheResult ConvertResponse(const cpr::Response& Response) - { - if (Response.error) - { - return {.ElapsedSeconds = Response.elapsed, - .ErrorCode = static_cast<int32_t>(Response.error.code), - .Reason = Response.error.message, - .Success = false}; - } - if (!IsHttpSuccessCode(Response.status_code)) - { - return {.ElapsedSeconds = Response.elapsed, - .ErrorCode = static_cast<int32_t>(Response.status_code), - .Reason = Response.reason.empty() ? Response.text : Response.reason, - .Success = false}; - } - return {.Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .ErrorCode = 0, - .Reason = Response.reason, - .Success = true}; - } - - cpr::Response GetWithStreaming(cpr::Session& Session, std::filesystem::path TempFolderPath, std::string_view Name, IoBuffer& OutBuffer) - { - if (TempFolderPath.empty()) - { - return Session.Get(); - } - - std::string PayloadString; - std::shared_ptr<BasicFile> PayloadFile; - - auto _ = MakeGuard([&]() { - if (PayloadFile) - { - PayloadFile.reset(); - std::filesystem::path TempPath = TempFolderPath / Name; - std::error_code Ec; - std::filesystem::remove(TempPath, Ec); - } - }); - - uint64_t Offset = 0; - Session.SetWriteCallback(cpr::WriteCallback{[&](std::string data, intptr_t) { - if (!PayloadFile && (PayloadString.length() + data.length()) > (1024 * 1024)) - { - std::filesystem::path TempPath = TempFolderPath / Name; - PayloadFile = std::make_shared<BasicFile>(); - PayloadFile->Open(TempPath, BasicFile::Mode::kTruncateDelete); - PayloadFile->Write(PayloadString.data(), PayloadString.size(), Offset); - Offset += PayloadString.size(); - PayloadString.clear(); - } - if (PayloadFile) - { - PayloadFile->Write(data.data(), data.size(), Offset); - Offset += data.size(); - } - else - { - PayloadString.append(data); - } - return true; - }}); - - cpr::Response Response = Session.Get(); - - if (!Response.error && IsHttpSuccessCode(Response.status_code)) - { - if (PayloadFile) - { - uint64_t PayloadSize = PayloadFile->FileSize(); - void* FileHandle = PayloadFile->Detach(); - PayloadFile.reset(); - OutBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, PayloadSize, /*IsWholeFile*/ true); - OutBuffer.SetDeleteOnClose(true); - } - else - { - OutBuffer = IoBufferBuilder::MakeCloneFromMemory(PayloadString.data(), PayloadString.size()); - } - return Response; - } - - Response.text.swap(PayloadString); - return Response; - } - -} // namespace detail - -CloudCacheSession::CloudCacheSession(CloudCacheClient* CacheClient) : m_Log(CacheClient->Logger()), m_CacheClient(CacheClient) -{ - m_SessionState = m_CacheClient->AllocSessionState(); -} - -CloudCacheSession::~CloudCacheSession() -{ - m_CacheClient->FreeSessionState(m_SessionState); -} - -CloudCacheResult -CloudCacheSession::Authenticate() -{ - const bool RefreshToken = true; - const CloudCacheAccessToken& AccessToken = GetAccessToken(RefreshToken); - - return {.Success = AccessToken.IsValid()}; -} - -CloudCacheResult -CloudCacheSession::GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType) -{ - const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream"; - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << Namespace << "/" << BucketId << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", ContentType}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Get(); - ZEN_DEBUG("GET {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (Result.Success) - { - Result.Response = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()); - } - else - { - ZEN_WARN( - "CloudCacheSession::GetRef failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - ContentType, - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::GetBlob(std::string_view Namespace, const IoHash& Key) -{ - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/octet-stream"}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Get(); - ZEN_DEBUG("GET {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (Result.Success) - { - Result.Response = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()); - } - else - { - ZEN_WARN( - "CloudCacheSession::GetBlob failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/octet-stream", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath) -{ - ZEN_TRACE_CPU("JupiterClient::GetCompressedBlob"); - - ExtendableStringBuilder<256> Uri; - std::string KeyString = Key.ToHexString(); - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << Namespace << "/" << KeyString; - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-comp"}}); - Session.SetOption(cpr::Body{}); - - IoBuffer Payload; - cpr::Response Response = detail::GetWithStreaming(Session, TempFolderPath, KeyString, Payload); - ZEN_DEBUG("GET {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (Result.Success) - { - Result.Response = std::move(Payload); - } - else - { - ZEN_WARN( - "CloudCacheSession::GetCompressedBlob failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-comp", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::GetInlineBlob(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - IoHash& OutPayloadHash, - std::filesystem::path TempFolderPath) -{ - ZEN_TRACE_CPU("JupiterClient::GetInlineBlob"); - - ExtendableStringBuilder<256> Uri; - std::string KeyString = Key.ToHexString(); - Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << Namespace << "/" << BucketId << "/" << KeyString; - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-jupiter-inline"}}); - Session.SetOption(cpr::Body{}); - - IoBuffer Payload; - cpr::Response Response = detail::GetWithStreaming(Session, TempFolderPath, KeyString, Payload); - ZEN_DEBUG("GET {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (Result.Success) - { - Result.Response = std::move(Payload); - } - else - { - ZEN_WARN( - "CloudCacheSession::GetInlineBlob failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-jupiter-inline", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - if (auto It = Response.header.find("X-Jupiter-InlinePayloadHash"); It != Response.header.end()) - { - const std::string& PayloadHashHeader = It->second; - if (PayloadHashHeader.length() == IoHash::StringLength) - { - OutPayloadHash = IoHash::FromHexString(PayloadHashHeader); - } - } - - return Result; -} - -CloudCacheResult -CloudCacheSession::GetObject(std::string_view Namespace, const IoHash& Key) -{ - ZEN_TRACE_CPU("JupiterClient::GetObject"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Get(); - ZEN_DEBUG("GET {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (Result.Success) - { - Result.Response = IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()); - } - else - { - ZEN_WARN( - "CloudCacheSession::GetObject failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-cb", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - return Result; -} - -PutRefResult -CloudCacheSession::PutRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType) -{ - ZEN_TRACE_CPU("JupiterClient::PutRef"); - - IoHash Hash = IoHash::HashBuffer(Ref.Data(), Ref.Size()); - - const std::string ContentType = RefType == ZenContentType::kCbObject ? "application/x-ue-cb" : "application/octet-stream"; - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << Namespace << "/" << BucketId << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption( - cpr::Header{{"Authorization", AccessToken.Value}, {"X-Jupiter-IoHash", Hash.ToHexString()}, {"Content-Type", ContentType}}); - Session.SetBody(cpr::Body{(const char*)Ref.Data(), Ref.Size()}); - - cpr::Response Response = Session.Put(); - ZEN_DEBUG("PUT {}", Response); - - PutRefResult Result = {detail::ConvertResponse(Response)}; - if (Result.Success) - { - std::string JsonError; - json11::Json Json = json11::Json::parse(Response.text, JsonError); - if (JsonError.empty()) - { - json11::Json::array Needs = Json["needs"].array_items(); - for (const auto& Need : Needs) - { - Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value())); - } - } - Result.RawHash = Hash; - } - else - { - ZEN_WARN( - "CloudCacheSession::PutRef failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-X-Jupiter-IoHash: '{}', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - Hash.ToHexString(), - ContentType, - NiceBytes(Ref.Size()), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - return Result; -} - -FinalizeRefResult -CloudCacheSession::FinalizeRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& RefHash) -{ - ZEN_TRACE_CPU("JupiterClient::FinalizeRef"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << Namespace << "/" << BucketId << "/" << Key.ToHexString() << "/finalize/" - << RefHash.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, - {"X-Jupiter-IoHash", RefHash.ToHexString()}, - {"Content-Type", "application/x-ue-cb"}}); - Session.SetBody(cpr::Body{}); - - cpr::Response Response = Session.Post(); - ZEN_DEBUG("POST {}", Response); - - FinalizeRefResult Result = {detail::ConvertResponse(Response)}; - - if (Result.Success) - { - std::string JsonError; - json11::Json Json = json11::Json::parse(Response.text, JsonError); - if (JsonError.empty()) - { - json11::Json::array Needs = Json["needs"].array_items(); - for (const auto& Need : Needs) - { - Result.Needs.emplace_back(IoHash::FromHexString(Need.string_value())); - } - } - } - else - { - ZEN_WARN( - "CloudCacheSession::FinalizeRef failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-X-Jupiter-IoHash: '{}', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - RefHash.ToHexString(), - "application/x-ue-cb", - NiceBytes(0), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - return Result; -} - -CloudCacheResult -CloudCacheSession::PutBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob) -{ - ZEN_TRACE_CPU("JupiterClient::PutBlob"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/blobs/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/octet-stream"}}); - Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()}); - - cpr::Response Response = Session.Put(); - ZEN_DEBUG("PUT {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::PutBlob failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/octet-stream", - NiceBytes(Blob.Size()), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob) -{ - ZEN_TRACE_CPU("JupiterClient::PutCompressedBlob"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-comp"}}); - - uint64_t Offset = 0; - if (Blob.IsWholeFile()) - { - auto ReadCallback = [&Blob, &Offset](char* buffer, size_t& size, intptr_t) { - size = Min<size_t>(size, Blob.GetSize() - Offset); - IoBuffer PayloadRange = IoBuffer(Blob, Offset, size); - MutableMemoryView Data(buffer, size); - Data.CopyFrom(PayloadRange.GetView()); - Offset += size; - return true; - }; - Session.SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(Blob.GetSize()), ReadCallback)); - } - else - { - Session.SetBody(cpr::Body{(const char*)Blob.Data(), Blob.Size()}); - } - - cpr::Response Response = Session.Put(); - ZEN_DEBUG("PUT {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::PutCompressedBlob failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-comp", - NiceBytes(Blob.Size()), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::PutCompressedBlob(std::string_view Namespace, const IoHash& Key, const CompositeBuffer& Payload) -{ - ZEN_TRACE_CPU("JupiterClient::PutCompressedBlob"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/compressed-blobs/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-comp"}}); - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min<size_t>(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; - Session.SetReadCallback(cpr::ReadCallback(gsl::narrow<cpr::cpr_off_t>(SizeLeft), ReadCallback)); - - cpr::Response Response = Session.Put(); - ZEN_DEBUG("PUT {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::PutCompressedBlob failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-comp", - NiceBytes(Payload.GetSize()), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::PutObject(std::string_view Namespace, const IoHash& Key, IoBuffer Object) -{ - ZEN_TRACE_CPU("JupiterClient::PutObject"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Content-Type", "application/x-ue-cb"}}); - Session.SetBody(cpr::Body{(const char*)Object.Data(), Object.Size()}); - - cpr::Response Response = Session.Put(); - ZEN_DEBUG("PUT {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::PutObject failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-ContentType: '{}', " - "ContentSize: {}, " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-cb", - NiceBytes(Object.GetSize()), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheResult -CloudCacheSession::RefExists(std::string_view Namespace, std::string_view BucketId, const IoHash& Key) -{ - ZEN_TRACE_CPU("JupiterClient::RefExists"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/refs/" << Namespace << "/" << BucketId << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Head(); - ZEN_DEBUG("HEAD {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::RefExists failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -GetObjectReferencesResult -CloudCacheSession::GetObjectReferences(std::string_view Namespace, const IoHash& Key) -{ - ZEN_TRACE_CPU("JupiterClient::GetObjectReferences"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/objects/" << Namespace << "/" << Key.ToHexString() << "/references"; - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Get(); - ZEN_DEBUG("GET {}", Response); - - GetObjectReferencesResult Result = {detail::ConvertResponse(Response)}; - - if (Result.Success) - { - IoBuffer Buffer = IoBuffer(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size()); - const CbObject ReferencesResponse = LoadCompactBinaryObject(Buffer); - for (auto& Item : ReferencesResponse["references"sv]) - { - Result.References.insert(Item.AsHash()); - } - } - else - { - ZEN_WARN( - "CloudCacheSession::GetObjectReferences failed PUT. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-cb", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - return Result; -} - -CloudCacheResult -CloudCacheSession::BlobExists(std::string_view Namespace, const IoHash& Key) -{ - return CacheTypeExists(Namespace, "blobs"sv, Key); -} - -CloudCacheResult -CloudCacheSession::CompressedBlobExists(std::string_view Namespace, const IoHash& Key) -{ - return CacheTypeExists(Namespace, "compressed-blobs"sv, Key); -} - -CloudCacheResult -CloudCacheSession::ObjectExists(std::string_view Namespace, const IoHash& Key) -{ - return CacheTypeExists(Namespace, "objects"sv, Key); -} - -CloudCacheExistsResult -CloudCacheSession::BlobExists(std::string_view Namespace, const std::set<IoHash>& Keys) -{ - return CacheTypeExists(Namespace, "blobs"sv, Keys); -} - -CloudCacheExistsResult -CloudCacheSession::CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys) -{ - return CacheTypeExists(Namespace, "compressed-blobs"sv, Keys); -} - -CloudCacheExistsResult -CloudCacheSession::ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys) -{ - return CacheTypeExists(Namespace, "objects"sv, Keys); -} - -std::vector<IoHash> -CloudCacheSession::Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes) -{ - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl(); - Uri << "/api/v1/s/" << Namespace; - - ZEN_UNUSED(BucketId, ChunkHashes); - - return {}; -} - -cpr::Session& -CloudCacheSession::GetSession() -{ - return m_SessionState->GetSession(); -} - -CloudCacheAccessToken -CloudCacheSession::GetAccessToken(bool RefreshToken) -{ - return m_SessionState->GetAccessToken(RefreshToken); -} - -CloudCacheResult -CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key) -{ - ZEN_TRACE_CPU("JupiterClient::CacheTypeExists"); - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << Namespace << "/" << Key.ToHexString(); - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption(cpr::Header{{"Authorization", AccessToken.Value}}); - Session.SetOption(cpr::Body{}); - - cpr::Response Response = Session.Head(); - ZEN_DEBUG("HEAD {}", Response); - - CloudCacheResult Result = detail::ConvertResponse(Response); - if (!Result.Success) - { - ZEN_WARN( - "CloudCacheSession::CacheTypeExists failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-cb", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - return Result; -} - -CloudCacheExistsResult -CloudCacheSession::CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys) -{ - ZEN_TRACE_CPU("JupiterClient::CacheTypeExists"); - - ExtendableStringBuilder<256> Body; - Body << "["; - for (const auto& Key : Keys) - { - Body << (Body.Size() != 1 ? ",\"" : "\"") << Key.ToHexString() << "\""; - } - Body << "]"; - - ExtendableStringBuilder<256> Uri; - Uri << m_CacheClient->ServiceUrl() << "/api/v1/" << TypeId << "/" << Namespace << "/exist"; - - cpr::Session& Session = GetSession(); - const CloudCacheAccessToken& AccessToken = GetAccessToken(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetOption( - cpr::Header{{"Authorization", AccessToken.Value}, {"Accept", "application/x-ue-cb"}, {"Content-Type", "application/json"}}); - Session.SetOption(cpr::Body(Body.ToString())); - - cpr::Response Response = Session.Post(); - ZEN_DEBUG("POST {}", Response); - CloudCacheExistsResult Result = {detail::ConvertResponse(Response)}; - - if (Result.Success) - { - IoBuffer Buffer = IoBuffer(zen::IoBuffer::Wrap, Response.text.data(), Response.text.size()); - const CbObject ExistsResponse = LoadCompactBinaryObject(Buffer); - for (auto& Item : ExistsResponse["needs"sv]) - { - Result.Needs.insert(Item.AsHash()); - } - } - else - { - ZEN_WARN( - "CloudCacheSession::CacheTypeExists failed GET. " - "Elapsed: {} s, " - "Uri: '{}', " - "Header-Authorization: '{} <redacted>', " - "Header-Accept: '{}', " - "Response.status_code: {}, " - "Response.reason: '{}', " - "Response.error.code: {}, " - "Response.error.message: '{}', " - "Response.raw_header: '{}'" - "Response.text: '{}'", - Response.elapsed, - Uri, - AccessToken.Value.substr(0, 6), - "application/x-ue-cb", - Response.status_code, - Response.reason, - gsl::narrow<int>(Response.error.code), - Response.error.message, - Response.raw_header, - Response.text); - } - - return Result; -} - -/** - * An access token provider that holds a token that will never change. - */ -class StaticTokenProvider final : public CloudCacheTokenProvider -{ -public: - StaticTokenProvider(CloudCacheAccessToken Token) : m_Token(std::move(Token)) {} - - virtual ~StaticTokenProvider() = default; - - virtual CloudCacheAccessToken AcquireAccessToken() final override { return m_Token; } - -private: - CloudCacheAccessToken m_Token; -}; - -std::unique_ptr<CloudCacheTokenProvider> -CloudCacheTokenProvider::CreateFromStaticToken(CloudCacheAccessToken Token) -{ - return std::make_unique<StaticTokenProvider>(std::move(Token)); -} - -class OAuthClientCredentialsTokenProvider final : public CloudCacheTokenProvider -{ -public: - OAuthClientCredentialsTokenProvider(const CloudCacheTokenProvider::OAuthClientCredentialsParams& Params) - { - m_Url = std::string(Params.Url); - m_ClientId = std::string(Params.ClientId); - m_ClientSecret = std::string(Params.ClientSecret); - } - - virtual ~OAuthClientCredentialsTokenProvider() = default; - - virtual CloudCacheAccessToken AcquireAccessToken() final override - { - using namespace std::chrono; - - std::string Body = - fmt::format("client_id={}&scope=cache_access&grant_type=client_credentials&client_secret={}", m_ClientId, m_ClientSecret); - - cpr::Response Response = - cpr::Post(cpr::Url{m_Url}, cpr::Header{{"Content-Type", "application/x-www-form-urlencoded"}}, cpr::Body{std::move(Body)}); - - if (Response.error || Response.status_code != 200) - { - return {}; - } - - std::string JsonError; - json11::Json Json = json11::Json::parse(Response.text, JsonError); - - if (JsonError.empty() == false) - { - return {}; - } - - std::string Token = Json["access_token"].string_value(); - int64_t ExpiresInSeconds = static_cast<int64_t>(Json["expires_in"].int_value()); - CloudCacheAccessToken::TimePoint ExpireTime = CloudCacheAccessToken::Clock::now() + seconds(ExpiresInSeconds); - - return {.Value = fmt::format("Bearer {}", Token), .ExpireTime = ExpireTime}; - } - -private: - std::string m_Url; - std::string m_ClientId; - std::string m_ClientSecret; -}; - -std::unique_ptr<CloudCacheTokenProvider> -CloudCacheTokenProvider::CreateFromOAuthClientCredentials(const OAuthClientCredentialsParams& Params) -{ - return std::make_unique<OAuthClientCredentialsTokenProvider>(Params); -} - -class CallbackTokenProvider final : public CloudCacheTokenProvider -{ -public: - CallbackTokenProvider(std::function<CloudCacheAccessToken()>&& Callback) : m_Callback(std::move(Callback)) {} - - virtual ~CallbackTokenProvider() = default; - - virtual CloudCacheAccessToken AcquireAccessToken() final override { return m_Callback(); } - -private: - std::function<CloudCacheAccessToken()> m_Callback; -}; - -std::unique_ptr<CloudCacheTokenProvider> -CloudCacheTokenProvider::CreateFromCallback(std::function<CloudCacheAccessToken()>&& Callback) -{ - return std::make_unique<CallbackTokenProvider>(std::move(Callback)); -} - -CloudCacheClient::CloudCacheClient(const CloudCacheClientOptions& Options, std::unique_ptr<CloudCacheTokenProvider> TokenProvider) -: m_Log(zen::logging::Get("jupiter")) -, m_ServiceUrl(Options.ServiceUrl) -, m_DefaultDdcNamespace(Options.DdcNamespace) -, m_DefaultBlobStoreNamespace(Options.BlobStoreNamespace) -, m_ComputeCluster(Options.ComputeCluster) -, m_ConnectTimeout(Options.ConnectTimeout) -, m_Timeout(Options.Timeout) -, m_TokenProvider(std::move(TokenProvider)) -, m_AssumeHttp2(Options.AssumeHttp2) -{ - ZEN_ASSERT(m_TokenProvider.get() != nullptr); -} - -CloudCacheClient::~CloudCacheClient() -{ - RwLock::ExclusiveLockScope _(m_SessionStateLock); - - for (auto State : m_SessionStateCache) - { - delete State; - } -} - -CloudCacheAccessToken -CloudCacheClient::AcquireAccessToken() -{ - ZEN_TRACE_CPU("JupiterClient::AcquireAccessToken"); - - return m_TokenProvider->AcquireAccessToken(); -} - -detail::CloudCacheSessionState* -CloudCacheClient::AllocSessionState() -{ - detail::CloudCacheSessionState* State = nullptr; - - bool IsTokenValid = false; - - { - RwLock::ExclusiveLockScope _(m_SessionStateLock); - - if (m_SessionStateCache.empty() == false) - { - State = m_SessionStateCache.front(); - IsTokenValid = State->m_AccessToken.IsValid(); - - m_SessionStateCache.pop_front(); - } - } - - if (State == nullptr) - { - State = new detail::CloudCacheSessionState(*this); - } - - State->Reset(m_ConnectTimeout, m_Timeout, m_AssumeHttp2); - - if (IsTokenValid == false) - { - State->m_AccessToken = m_TokenProvider->AcquireAccessToken(); - } - - return State; -} - -void -CloudCacheClient::FreeSessionState(detail::CloudCacheSessionState* State) -{ - RwLock::ExclusiveLockScope _(m_SessionStateLock); - m_SessionStateCache.push_front(State); -} - -} // namespace zen diff --git a/src/zenserver/upstream/jupiter.h b/src/zenserver/upstream/jupiter.h deleted file mode 100644 index b5aa95ed5..000000000 --- a/src/zenserver/upstream/jupiter.h +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zenbase/refcount.h> -#include <zencore/iohash.h> -#include <zencore/logging.h> -#include <zencore/thread.h> -#include <zenhttp/httpserver.h> - -#include <atomic> -#include <chrono> -#include <list> -#include <memory> -#include <set> -#include <vector> - -struct ZenCacheValue; - -namespace cpr { -class Session; -} - -namespace zen { -namespace detail { - struct CloudCacheSessionState; -} - -class CbObjectView; -class CloudCacheClient; -class IoBuffer; -struct IoHash; - -/** - * Cached access token, for use with `Authorization:` header - */ -struct CloudCacheAccessToken -{ - using Clock = std::chrono::system_clock; - using TimePoint = Clock::time_point; - - static constexpr int64_t ExpireMarginInSeconds = 30; - - std::string Value; - TimePoint ExpireTime; - - bool IsValid() const - { - return Value.empty() == false && - ExpireMarginInSeconds < std::chrono::duration_cast<std::chrono::seconds>(ExpireTime - Clock::now()).count(); - } -}; - -struct CloudCacheResult -{ - IoBuffer Response; - int64_t Bytes{}; - double ElapsedSeconds{}; - int32_t ErrorCode{}; - std::string Reason; - bool Success = false; -}; - -struct PutRefResult : CloudCacheResult -{ - std::vector<IoHash> Needs; - IoHash RawHash; -}; - -struct FinalizeRefResult : CloudCacheResult -{ - std::vector<IoHash> Needs; -}; - -struct CloudCacheExistsResult : CloudCacheResult -{ - std::set<IoHash> Needs; -}; - -struct GetObjectReferencesResult : CloudCacheResult -{ - std::set<IoHash> References; -}; - -/** - * Context for performing Jupiter operations - * - * Maintains an HTTP connection so that subsequent operations don't need to go - * through the whole connection setup process - * - */ -class CloudCacheSession -{ -public: - CloudCacheSession(CloudCacheClient* CacheClient); - ~CloudCacheSession(); - - CloudCacheResult Authenticate(); - CloudCacheResult GetRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType RefType); - CloudCacheResult GetBlob(std::string_view Namespace, const IoHash& Key); - CloudCacheResult GetCompressedBlob(std::string_view Namespace, const IoHash& Key, std::filesystem::path TempFolderPath = {}); - CloudCacheResult GetObject(std::string_view Namespace, const IoHash& Key); - CloudCacheResult GetInlineBlob(std::string_view Namespace, - std::string_view BucketId, - const IoHash& Key, - IoHash& OutPayloadHash, - std::filesystem::path TempFolderPath = {}); - - PutRefResult PutRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, IoBuffer Ref, ZenContentType RefType); - CloudCacheResult PutBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob); - CloudCacheResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, IoBuffer Blob); - CloudCacheResult PutCompressedBlob(std::string_view Namespace, const IoHash& Key, const CompositeBuffer& Blob); - CloudCacheResult PutObject(std::string_view Namespace, const IoHash& Key, IoBuffer Object); - - FinalizeRefResult FinalizeRef(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& RefHah); - - CloudCacheResult RefExists(std::string_view Namespace, std::string_view BucketId, const IoHash& Key); - - GetObjectReferencesResult GetObjectReferences(std::string_view Namespace, const IoHash& Key); - - CloudCacheResult BlobExists(std::string_view Namespace, const IoHash& Key); - CloudCacheResult CompressedBlobExists(std::string_view Namespace, const IoHash& Key); - CloudCacheResult ObjectExists(std::string_view Namespace, const IoHash& Key); - - CloudCacheExistsResult BlobExists(std::string_view Namespace, const std::set<IoHash>& Keys); - CloudCacheExistsResult CompressedBlobExists(std::string_view Namespace, const std::set<IoHash>& Keys); - CloudCacheExistsResult ObjectExists(std::string_view Namespace, const std::set<IoHash>& Keys); - - std::vector<IoHash> Filter(std::string_view Namespace, std::string_view BucketId, const std::vector<IoHash>& ChunkHashes); - - CloudCacheClient& Client() { return *m_CacheClient; }; - -private: - inline LoggerRef Log() { return m_Log; } - cpr::Session& GetSession(); - CloudCacheAccessToken GetAccessToken(bool RefreshToken = false); - - CloudCacheResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const IoHash& Key); - - CloudCacheExistsResult CacheTypeExists(std::string_view Namespace, std::string_view TypeId, const std::set<IoHash>& Keys); - - LoggerRef m_Log; - RefPtr<CloudCacheClient> m_CacheClient; - detail::CloudCacheSessionState* m_SessionState; -}; - -/** - * Access token provider interface - */ -class CloudCacheTokenProvider -{ -public: - virtual ~CloudCacheTokenProvider() = default; - - virtual CloudCacheAccessToken AcquireAccessToken() = 0; - - static std::unique_ptr<CloudCacheTokenProvider> CreateFromStaticToken(CloudCacheAccessToken Token); - - struct OAuthClientCredentialsParams - { - std::string_view Url; - std::string_view ClientId; - std::string_view ClientSecret; - }; - - static std::unique_ptr<CloudCacheTokenProvider> CreateFromOAuthClientCredentials(const OAuthClientCredentialsParams& Params); - - static std::unique_ptr<CloudCacheTokenProvider> CreateFromCallback(std::function<CloudCacheAccessToken()>&& Callback); -}; - -struct CloudCacheClientOptions -{ - std::string_view Name; - std::string_view ServiceUrl; - std::string_view DdcNamespace; - std::string_view BlobStoreNamespace; - std::string_view ComputeCluster; - std::chrono::milliseconds ConnectTimeout{5000}; - std::chrono::milliseconds Timeout{}; - bool AssumeHttp2 = false; -}; - -/** - * Jupiter upstream cache client - */ -class CloudCacheClient : public RefCounted -{ -public: - CloudCacheClient(const CloudCacheClientOptions& Options, std::unique_ptr<CloudCacheTokenProvider> TokenProvider); - ~CloudCacheClient(); - - CloudCacheAccessToken AcquireAccessToken(); - std::string_view DefaultDdcNamespace() const { return m_DefaultDdcNamespace; } - std::string_view DefaultBlobStoreNamespace() const { return m_DefaultBlobStoreNamespace; } - std::string_view ComputeCluster() const { return m_ComputeCluster; } - std::string_view ServiceUrl() const { return m_ServiceUrl; } - - LoggerRef Logger() { return m_Log; } - -private: - LoggerRef m_Log; - std::string m_ServiceUrl; - std::string m_DefaultDdcNamespace; - std::string m_DefaultBlobStoreNamespace; - std::string m_ComputeCluster; - std::chrono::milliseconds m_ConnectTimeout{}; - std::chrono::milliseconds m_Timeout{}; - std::unique_ptr<CloudCacheTokenProvider> m_TokenProvider; - bool m_AssumeHttp2; - - RwLock m_SessionStateLock; - std::list<detail::CloudCacheSessionState*> m_SessionStateCache; - - detail::CloudCacheSessionState* AllocSessionState(); - void FreeSessionState(detail::CloudCacheSessionState*); - - friend class CloudCacheSession; -}; - -} // namespace zen diff --git a/src/zenserver/upstream/upstream.h b/src/zenserver/upstream/upstream.h index a57301206..4d45687fc 100644 --- a/src/zenserver/upstream/upstream.h +++ b/src/zenserver/upstream/upstream.h @@ -2,7 +2,6 @@ #pragma once -#include <upstream/jupiter.h> #include <upstream/upstreamcache.h> #include <upstream/upstreamservice.h> #include <upstream/zen.h> diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp index dac29c273..8558e2a10 100644 --- a/src/zenserver/upstream/upstreamcache.cpp +++ b/src/zenserver/upstream/upstreamcache.cpp @@ -1,7 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "upstreamcache.h" -#include "jupiter.h" #include "zen.h" #include <zencore/blockingqueue.h> @@ -15,11 +14,15 @@ #include <zencore/timer.h> #include <zencore/trace.h> -#include <zenhttp/auth/authmgr.h> -#include <zenstore/cidstore.h> -#include <zenutil/packageformat.h> +#include <zenhttp/httpclientauth.h> +#include <zenhttp/packageformat.h> #include <zenstore/cache/structuredcachestore.h> +#include <zenstore/cidstore.h> + +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenremotestore/jupiter/jupitersession.h> + #include "cache/httpstructuredcache.h" #include "diag/logging.h" @@ -85,7 +88,7 @@ namespace detail { class JupiterUpstreamEndpoint final : public UpstreamEndpoint { public: - JupiterUpstreamEndpoint(const CloudCacheClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) + JupiterUpstreamEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) : m_AuthMgr(Mgr) , m_Log(zen::logging::Get("upstream")) { @@ -93,30 +96,27 @@ namespace detail { m_Info.Name = Options.Name; m_Info.Url = Options.ServiceUrl; - std::unique_ptr<CloudCacheTokenProvider> TokenProvider; + std::function<HttpClientAccessToken()> TokenProvider; if (AuthConfig.OAuthUrl.empty() == false) { - TokenProvider = CloudCacheTokenProvider::CreateFromOAuthClientCredentials( + TokenProvider = httpclientauth::CreateFromOAuthClientCredentials( {.Url = AuthConfig.OAuthUrl, .ClientId = AuthConfig.OAuthClientId, .ClientSecret = AuthConfig.OAuthClientSecret}); } - else if (AuthConfig.OpenIdProvider.empty() == false) + else if (!AuthConfig.OpenIdProvider.empty()) { - TokenProvider = - CloudCacheTokenProvider::CreateFromCallback([this, ProviderName = std::string(AuthConfig.OpenIdProvider)]() { - AuthMgr::OpenIdAccessToken Token = m_AuthMgr.GetOpenIdAccessToken(ProviderName); - return CloudCacheAccessToken{.Value = Token.AccessToken, .ExpireTime = Token.ExpireTime}; - }); + TokenProvider = httpclientauth::CreateFromOpenIdProvider(m_AuthMgr, AuthConfig.OpenIdProvider); + } + else if (!AuthConfig.AccessToken.empty()) + { + TokenProvider = httpclientauth::CreateFromStaticToken(AuthConfig.AccessToken); } else { - CloudCacheAccessToken AccessToken{.Value = std::string(AuthConfig.AccessToken), - .ExpireTime = CloudCacheAccessToken::TimePoint::max()}; - - TokenProvider = CloudCacheTokenProvider::CreateFromStaticToken(AccessToken); + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(m_AuthMgr); } - m_Client = new CloudCacheClient(Options, std::move(TokenProvider)); + m_Client = new JupiterClient(Options, std::move(TokenProvider)); } virtual ~JupiterUpstreamEndpoint() {} @@ -134,8 +134,8 @@ namespace detail { return {.State = UpstreamEndpointState::kOk}; } - CloudCacheSession Session(m_Client); - const CloudCacheResult Result = Session.Authenticate(); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + const JupiterResult Result = Session.Authenticate(); if (Result.Success) { @@ -152,7 +152,7 @@ namespace detail { return m_Status.EndpointStatus(); } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -160,20 +160,11 @@ namespace detail { } } - std::string_view GetActualDdcNamespace(CloudCacheSession& Session, std::string_view Namespace) + std::string_view GetActualBlobStoreNamespace(std::string_view Namespace) { if (Namespace == ZenCacheStore::DefaultNamespace) { - return Session.Client().DefaultDdcNamespace(); - } - return Namespace; - } - - std::string_view GetActualBlobStoreNamespace(CloudCacheSession& Session, std::string_view Namespace) - { - if (Namespace == ZenCacheStore::DefaultNamespace) - { - return Session.Client().DefaultBlobStoreNamespace(); + return m_Client->DefaultBlobStoreNamespace(); } return Namespace; } @@ -190,10 +181,10 @@ namespace detail { try { - CloudCacheSession Session(m_Client); - CloudCacheResult Result; + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + JupiterResult Result; - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); if (Type == ZenContentType::kCompressedBinary) { @@ -209,8 +200,9 @@ namespace detail { int NumAttachments = 0; CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - CloudCacheResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); - Result.Bytes += AttachmentResult.Bytes; + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; Result.ErrorCode = AttachmentResult.ErrorCode; @@ -248,8 +240,9 @@ namespace detail { CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { - CloudCacheResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); - Result.Bytes += AttachmentResult.Bytes; + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; Result.ErrorCode = AttachmentResult.ErrorCode; @@ -283,7 +276,9 @@ namespace detail { if (Result.ErrorCode == 0) { - return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + return {.Status = {.Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, .Value = Result.Response, .Source = &m_Info}; } @@ -292,7 +287,7 @@ namespace detail { return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; } } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -306,7 +301,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords"); - CloudCacheSession Session(m_Client); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheKeyRequest* Request : Requests) @@ -318,9 +313,8 @@ namespace detail { double ElapsedSeconds = 0.0; if (!Result.Error) { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); - CloudCacheResult RefResult = - Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + JupiterResult RefResult = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); AppendResult(RefResult, Result); ElapsedSeconds = RefResult.ElapsedSeconds; @@ -333,7 +327,7 @@ namespace detail { { Record = LoadCompactBinaryObject(RefResult.Response); Record.IterateAttachments([&](CbFieldView AttachmentHash) { - CloudCacheResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + JupiterResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); AppendResult(BlobResult, Result); m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); @@ -371,15 +365,17 @@ namespace detail { try { - CloudCacheSession Session(m_Client); - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); - const CloudCacheResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); if (Result.ErrorCode == 0) { - return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + return {.Status = {.Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, .Value = Result.Response, .Source = &m_Info}; } @@ -388,7 +384,7 @@ namespace detail { return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; } } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -402,7 +398,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks"); - CloudCacheSession Session(m_Client); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheChunkRequest* RequestPtr : CacheChunkRequests) @@ -416,8 +412,8 @@ namespace detail { bool IsCompressed = false; if (!Result.Error) { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); - const CloudCacheResult BlobResult = + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult BlobResult = Request.ChunkId == IoHash::Zero ? Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, Request.ChunkId) : Session.GetCompressedBlob(BlobStoreNamespace, Request.ChunkId); @@ -457,7 +453,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues"); - CloudCacheSession Session(m_Client); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheValueRequest* RequestPtr : CacheValueRequests) @@ -471,9 +467,9 @@ namespace detail { bool IsCompressed = false; if (!Result.Error) { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); - IoHash PayloadHash; - const CloudCacheResult BlobResult = + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + IoHash PayloadHash; + const JupiterResult BlobResult = Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, PayloadHash); ElapsedSeconds = BlobResult.ElapsedSeconds; Payload = BlobResult.Response; @@ -537,14 +533,14 @@ namespace detail { try { - CloudCacheSession Session(m_Client); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); if (CacheRecord.Type == ZenContentType::kBinary) { - CloudCacheResult Result; + JupiterResult Result; for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, CacheRecord.Namespace); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(CacheRecord.Namespace); Result = Session.PutRef(BlobStoreNamespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, @@ -555,7 +551,7 @@ namespace detail { m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); return {.Reason = std::move(Result.Reason), - .Bytes = Result.Bytes, + .Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}; } @@ -615,7 +611,7 @@ namespace detail { }); } } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -626,10 +622,10 @@ namespace detail { virtual UpstreamEndpointStats& Stats() override { return m_Stats; } private: - static void AppendResult(const CloudCacheResult& Result, GetUpstreamCacheResult& Out) + static void AppendResult(const JupiterResult& Result, GetUpstreamCacheResult& Out) { Out.Success &= Result.Success; - Out.Bytes += Result.Bytes; + Out.Bytes += gsl::narrow<int64_t>(Result.ReceivedBytes); Out.ElapsedSeconds += Result.ElapsedSeconds; if (Result.ErrorCode) @@ -639,7 +635,7 @@ namespace detail { }; PutUpstreamCacheResult PerformStructuredPut( - CloudCacheSession& Session, + JupiterSession& Session, std::string_view Namespace, const CacheKey& Key, IoBuffer ObjectBuffer, @@ -649,7 +645,7 @@ namespace detail { int64_t TotalBytes = 0ull; double TotalElapsedSeconds = 0.0; - std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Session, Namespace); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); const auto PutBlobs = [&](std::span<IoHash> ValueContentIds, std::string& OutReason) -> bool { for (const IoHash& ValueContentId : ValueContentIds) { @@ -659,7 +655,7 @@ namespace detail { return false; } - CloudCacheResult BlobResult; + JupiterResult BlobResult; for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) { BlobResult = Session.PutCompressedBlob(BlobStoreNamespace, ValueContentId, BlobBuffer); @@ -673,7 +669,7 @@ namespace detail { return false; } - TotalBytes += BlobResult.Bytes; + TotalBytes += gsl::narrow<int64_t>(BlobResult.ReceivedBytes); TotalElapsedSeconds += BlobResult.ElapsedSeconds; } @@ -694,7 +690,7 @@ namespace detail { .Success = false}; } - TotalBytes += RefResult.Bytes; + TotalBytes += gsl::narrow<int64_t>(RefResult.ReceivedBytes); TotalElapsedSeconds += RefResult.ElapsedSeconds; std::string Reason; @@ -746,7 +742,7 @@ namespace detail { } } - TotalBytes += FinalizeResult.Bytes; + TotalBytes += gsl::narrow<int64_t>(FinalizeResult.ReceivedBytes); TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = true}; @@ -754,12 +750,13 @@ namespace detail { LoggerRef Log() { return m_Log; } - AuthMgr& m_AuthMgr; - LoggerRef m_Log; - UpstreamEndpointInfo m_Info; - UpstreamStatus m_Status; - UpstreamEndpointStats m_Stats; - RefPtr<CloudCacheClient> m_Client; + AuthMgr& m_AuthMgr; + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + RefPtr<JupiterClient> m_Client; + const bool m_AllowRedirect = false; }; class ZenUpstreamEndpoint final : public UpstreamEndpoint @@ -825,7 +822,7 @@ namespace detail { return m_Status.EndpointStatus(); } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -861,7 +858,7 @@ namespace detail { return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; } } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -984,7 +981,7 @@ namespace detail { return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; } } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -1405,7 +1402,7 @@ namespace detail { .ElapsedSeconds = TotalElapsedSeconds, .Success = Result.Success}; } - catch (std::exception& Err) + catch (const std::exception& Err) { m_Status.Set(UpstreamEndpointState::kError, Err.what()); @@ -1980,7 +1977,7 @@ private: { ProcessCacheRecord(std::move(CacheRecord)); } - catch (std::exception& Err) + catch (const std::exception& Err) { ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", CacheRecord.Namespace, @@ -2052,7 +2049,7 @@ private: } } } - catch (std::exception& Err) + catch (const std::exception& Err) { ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", Err.what()); } @@ -2123,7 +2120,7 @@ UpstreamEndpoint::CreateZenEndpoint(const ZenStructuredCacheClientOptions& Optio } std::unique_ptr<UpstreamEndpoint> -UpstreamEndpoint::CreateJupiterEndpoint(const CloudCacheClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) +UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) { return std::make_unique<detail::JupiterUpstreamEndpoint>(Options, AuthConfig, Mgr); } diff --git a/src/zenserver/upstream/upstreamcache.h b/src/zenserver/upstream/upstreamcache.h index bb0193e4e..d5d61c8d9 100644 --- a/src/zenserver/upstream/upstreamcache.h +++ b/src/zenserver/upstream/upstreamcache.h @@ -8,8 +8,8 @@ #include <zencore/iohash.h> #include <zencore/stats.h> #include <zencore/zencore.h> +#include <zenstore/cache/cache.h> #include <zenstore/cache/upstreamcacheclient.h> -#include <zenutil/cache/cache.h> #include <atomic> #include <chrono> @@ -26,8 +26,8 @@ class CbPackage; class CbObjectWriter; class CidStore; class ZenCacheStore; -struct CloudCacheClientOptions; -class CloudCacheTokenProvider; +struct JupiterClientOptions; +class JupiterAccessTokenProvider; struct ZenStructuredCacheClientOptions; struct UpstreamEndpointStats @@ -128,9 +128,9 @@ public: static std::unique_ptr<UpstreamEndpoint> CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options); - static std::unique_ptr<UpstreamEndpoint> CreateJupiterEndpoint(const CloudCacheClientOptions& Options, - const UpstreamAuthConfig& AuthConfig, - AuthMgr& Mgr); + static std::unique_ptr<UpstreamEndpoint> CreateJupiterEndpoint(const JupiterClientOptions& Options, + const UpstreamAuthConfig& AuthConfig, + AuthMgr& Mgr); }; /** diff --git a/src/zenserver/upstream/upstreamservice.cpp b/src/zenserver/upstream/upstreamservice.cpp index 3d4a0f823..1dcbdb604 100644 --- a/src/zenserver/upstream/upstreamservice.cpp +++ b/src/zenserver/upstream/upstreamservice.cpp @@ -2,7 +2,6 @@ #include <upstream/upstreamservice.h> #include <upstream/upstreamcache.h> -#include <zenhttp/auth/authmgr.h> #include <zencore/compactbinarybuilder.h> #include <zencore/string.h> diff --git a/src/zenserver/upstream/zen.cpp b/src/zenserver/upstream/zen.cpp index c031a4086..25fd3a3bb 100644 --- a/src/zenserver/upstream/zen.cpp +++ b/src/zenserver/upstream/zen.cpp @@ -9,44 +9,18 @@ #include <zencore/session.h> #include <zencore/stream.h> #include <zenhttp/formatters.h> +#include <zenhttp/httpclient.h> #include <zenhttp/httpcommon.h> -#include <zenutil/packageformat.h> +#include <zenhttp/packageformat.h> #include <zenstore/cache/structuredcachestore.h> #include "diag/logging.h" -ZEN_THIRD_PARTY_INCLUDES_START -#include <cpr/cpr.h> -ZEN_THIRD_PARTY_INCLUDES_END - #include <xxhash.h> #include <gsl/gsl-lite.hpp> namespace zen { -namespace detail { - struct ZenCacheSessionState - { - ZenCacheSessionState(ZenStructuredCacheClient& Client) : OwnerClient(Client) {} - ~ZenCacheSessionState() {} - - void Reset(std::chrono::milliseconds ConnectTimeout, std::chrono::milliseconds Timeout) - { - Session.SetBody({}); - Session.SetHeader({}); - Session.SetConnectTimeout(ConnectTimeout); - Session.SetTimeout(Timeout); - } - - cpr::Session& GetSession() { return Session; } - - private: - ZenStructuredCacheClient& OwnerClient; - cpr::Session Session; - }; - -} // namespace detail - ////////////////////////////////////////////////////////////////////////// ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options) @@ -59,39 +33,6 @@ ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClien ZenStructuredCacheClient::~ZenStructuredCacheClient() { - RwLock::ExclusiveLockScope _(m_SessionStateLock); - for (auto& CacheEntry : m_SessionStateCache) - { - delete CacheEntry; - } -} - -detail::ZenCacheSessionState* -ZenStructuredCacheClient::AllocSessionState() -{ - detail::ZenCacheSessionState* State = nullptr; - - if (RwLock::ExclusiveLockScope _(m_SessionStateLock); !m_SessionStateCache.empty()) - { - State = m_SessionStateCache.front(); - m_SessionStateCache.pop_front(); - } - - if (State == nullptr) - { - State = new detail::ZenCacheSessionState(*this); - } - - State->Reset(m_ConnectTimeout, m_Timeout); - - return State; -} - -void -ZenStructuredCacheClient::FreeSessionState(detail::ZenCacheSessionState* State) -{ - RwLock::ExclusiveLockScope _(m_SessionStateLock); - m_SessionStateCache.push_front(State); } ////////////////////////////////////////////////////////////////////////// @@ -102,59 +43,54 @@ ZenStructuredCacheSession::ZenStructuredCacheSession(Ref<ZenStructuredCacheClien : m_Log(OuterClient->Log()) , m_Client(std::move(OuterClient)) { - m_SessionState = m_Client->AllocSessionState(); } ZenStructuredCacheSession::~ZenStructuredCacheSession() { - m_Client->FreeSessionState(m_SessionState); } ZenCacheResult ZenStructuredCacheSession::CheckHealth() { - ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/health/check"; + HttpClient Http{m_Client->ServiceUrl()}; - cpr::Session& Session = m_SessionState->GetSession(); - Session.SetOption(cpr::Url{Uri.c_str()}); - cpr::Response Response = Session.Get(); + HttpClient::Response Response = Http.Get("/health/check"sv); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - return {.Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Response.status_code == 200}; + return {.Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Response.StatusCode == HttpResponseCode::OK}; } ZenCacheResult ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type) { + HttpClient Http{m_Client->ServiceUrl()}; + ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/"; + Uri << "/z$/"; if (Namespace != ZenCacheStore::DefaultNamespace) { Uri << Namespace << "/"; } Uri << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->GetSession(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Accept", std::string{MapContentTypeToString(Type)}}}); - cpr::Response Response = Session.Get(); + HttpClient::Response Response = Http.Get(Uri, {{"Accept", std::string{MapContentTypeToString(Type)}}}); ZEN_DEBUG("GET {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200; - const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - return {.Response = Buffer, .Bytes = Response.downloaded_bytes, .ElapsedSeconds = Response.elapsed, .Success = Success}; + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } ZenCacheResult @@ -163,35 +99,28 @@ ZenStructuredCacheSession::GetCacheChunk(std::string_view Namespace, const IoHash& Key, const IoHash& ValueContentId) { + HttpClient Http{m_Client->ServiceUrl()}; + ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/"; + Uri << "/z$/"; if (Namespace != ZenCacheStore::DefaultNamespace) { Uri << Namespace << "/"; } Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); - cpr::Session& Session = m_SessionState->GetSession(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Accept", "application/x-ue-comp"}}); - - cpr::Response Response = Session.Get(); + HttpClient::Response Response = Http.Get(Uri, {{"Accept", "application/x-ue-comp"}}); ZEN_DEBUG("GET {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200; - const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; - return {.Response = Buffer, - .Bytes = Response.downloaded_bytes, - .ElapsedSeconds = Response.elapsed, - .Reason = Response.reason, - .Success = Success}; + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } ZenCacheResult @@ -201,33 +130,29 @@ ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, IoBuffer Value, ZenContentType Type) { + HttpClient Http{m_Client->ServiceUrl()}; + ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/"; + Uri << "/z$/"; if (Namespace != ZenCacheStore::DefaultNamespace) { Uri << Namespace << "/"; } Uri << BucketId << "/" << Key.ToHexString(); - cpr::Session& Session = m_SessionState->GetSession(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Content-Type", - Type == ZenContentType::kCbPackage ? "application/x-ue-cbpkg" - : Type == ZenContentType::kCbObject ? "application/x-ue-cb" - : "application/octet-stream"}}); - Session.SetBody(cpr::Body{static_cast<const char*>(Value.Data()), Value.Size()}); + Value.SetContentType(Type); - cpr::Response Response = Session.Put(); + HttpClient::Response Response = Http.Put(Uri, Value); ZEN_DEBUG("PUT {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Reason = Response.reason, .Success = Success}; + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } ZenCacheResult @@ -237,94 +162,89 @@ ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, const IoHash& ValueContentId, IoBuffer Payload) { + HttpClient Http{m_Client->ServiceUrl()}; + ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/"; + Uri << "/z$/"; if (Namespace != ZenCacheStore::DefaultNamespace) { Uri << Namespace << "/"; } Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); - cpr::Session& Session = m_SessionState->GetSession(); - - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-comp"}}); - Session.SetBody(cpr::Body{static_cast<const char*>(Payload.Data()), Payload.Size()}); + Payload.SetContentType(HttpContentType::kCompressedBinary); - cpr::Response Response = Session.Put(); + HttpClient::Response Response = Http.Put(Uri, Payload); ZEN_DEBUG("PUT {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200 || Response.status_code == 201; - return {.Bytes = Response.uploaded_bytes, .ElapsedSeconds = Response.elapsed, .Reason = Response.reason, .Success = Success}; + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } ZenCacheResult ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) { + HttpClient Http{m_Client->ServiceUrl()}; + ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/$rpc"; + Uri << "/z$/$rpc"; - BinaryWriter Body; - Request.CopyTo(Body); + // TODO: this seems redundant, we should be able to send the data more directly, without the BinaryWriter - cpr::Session& Session = m_SessionState->GetSession(); + BinaryWriter BodyWriter; + Request.CopyTo(BodyWriter); - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cb"}, {"Accept", "application/x-ue-cbpkg"}}); - Session.SetBody(cpr::Body{reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()}); + IoBuffer Body{IoBuffer::Wrap, BodyWriter.GetData(), BodyWriter.GetSize()}; + Body.SetContentType(HttpContentType::kCbObject); - cpr::Response Response = Session.Post(); + HttpClient::Response Response = Http.Post(Uri, Body, {{"Accept", "application/x-ue-cbpkg"}}); ZEN_DEBUG("POST {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200; - const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; return {.Response = std::move(Buffer), - .Bytes = Response.uploaded_bytes, - .ElapsedSeconds = Response.elapsed, - .Reason = Response.reason, + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } ZenCacheResult ZenStructuredCacheSession::InvokeRpc(const CbPackage& Request) { - ExtendableStringBuilder<256> Uri; - Uri << m_Client->ServiceUrl() << "/z$/$rpc"; + HttpClient Http{m_Client->ServiceUrl()}; - SharedBuffer Message = FormatPackageMessageBuffer(Request).Flatten(); - - cpr::Session& Session = m_SessionState->GetSession(); + ExtendableStringBuilder<256> Uri; + Uri << "/z$/$rpc"; - Session.SetOption(cpr::Url{Uri.c_str()}); - Session.SetHeader(cpr::Header{{"Content-Type", "application/x-ue-cbpkg"}, {"Accept", "application/x-ue-cbpkg"}}); - Session.SetBody(cpr::Body{reinterpret_cast<const char*>(Message.GetData()), Message.GetSize()}); + IoBuffer Message = FormatPackageMessageBuffer(Request).Flatten().AsIoBuffer(); + Message.SetContentType(HttpContentType::kCbPackage); - cpr::Response Response = Session.Post(); + HttpClient::Response Response = Http.Post(Uri, Message, {{"Accept", "application/x-ue-cbpkg"}}); ZEN_DEBUG("POST {}", Response); - if (Response.error) + if (auto& Error = Response.Error; Error) { - return {.ErrorCode = static_cast<int32_t>(Response.error.code), .Reason = std::move(Response.error.message)}; + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; } - const bool Success = Response.status_code == 200; - const IoBuffer Buffer = Success ? IoBufferBuilder::MakeCloneFromMemory(Response.text.data(), Response.text.size()) : IoBuffer(); + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; return {.Response = std::move(Buffer), - .Bytes = Response.uploaded_bytes, - .ElapsedSeconds = Response.elapsed, - .Reason = Response.reason, + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; } diff --git a/src/zenserver/upstream/zen.h b/src/zenserver/upstream/zen.h index c1e4fbd0f..6321b46b1 100644 --- a/src/zenserver/upstream/zen.h +++ b/src/zenserver/upstream/zen.h @@ -5,18 +5,11 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/logging.h> -#include <zencore/memory.h> -#include <zencore/thread.h> +#include <zencore/memoryview.h> #include <zencore/uid.h> #include <zencore/zencore.h> -ZEN_THIRD_PARTY_INCLUDES_START -#include <tsl/robin_map.h> -#include <asio.hpp> -ZEN_THIRD_PARTY_INCLUDES_END - #include <chrono> -#include <list> struct ZenCacheValue; @@ -29,10 +22,6 @@ class ZenStructuredCacheClient; ////////////////////////////////////////////////////////////////////////// -namespace detail { - struct ZenCacheSessionState; -} - struct ZenCacheResult { IoBuffer Response; @@ -85,7 +74,6 @@ private: LoggerRef m_Log; Ref<ZenStructuredCacheClient> m_Client; - detail::ZenCacheSessionState* m_SessionState; }; /** Zen Structured Cache client @@ -109,12 +97,6 @@ private: std::chrono::milliseconds m_ConnectTimeout; std::chrono::milliseconds m_Timeout; - RwLock m_SessionStateLock; - std::list<detail::ZenCacheSessionState*> m_SessionStateCache; - - detail::ZenCacheSessionState* AllocSessionState(); - void FreeSessionState(detail::ZenCacheSessionState*); - friend class ZenStructuredCacheSession; }; diff --git a/src/zenserver/vfs/vfsimpl.cpp b/src/zenserver/vfs/vfsimpl.cpp deleted file mode 100644 index f528b2620..000000000 --- a/src/zenserver/vfs/vfsimpl.cpp +++ /dev/null @@ -1,458 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "vfsimpl.h" -#include "vfsservice.h" - -#include "projectstore/projectstore.h" - -#include <zencore/fmtutils.h> -#include <zencore/logging.h> -#include <zenstore/cache/structuredcachestore.h> -#include <zenvfs/projfs.h> -#include <zenvfs/vfs.h> - -#include <memory> -#include <unordered_map> - -#if ZEN_WITH_VFS - -namespace zen { - -using namespace std::literals; - -////////////////////////////////////////////////////////////////////////// - -VfsOplogDataSource::VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore) -: m_ProjectId(ProjectId) -, m_OplogId(OplogId) -, m_ProjectStore(std::move(InProjectStore)) -{ -} - -void -VfsOplogDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount); -} - -void -VfsOplogDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - IoBuffer ChunkBuffer; - auto Result = - m_ProjectStore->GetChunkRange(m_ProjectId, m_OplogId, ChunkId, 0, ~0ull, ZenContentType::kCompressedBinary, /* out */ ChunkBuffer); - - if (Result.first == HttpResponseCode::OK) - { - const uint8_t* SourceBuffer = reinterpret_cast<const uint8_t*>(ChunkBuffer.GetData()); - uint64_t AvailableBufferBytes = ChunkBuffer.GetSize(); - - ZEN_ASSERT(AvailableBufferBytes >= ByteOffset); - AvailableBufferBytes -= ByteOffset; - SourceBuffer += ByteOffset; - - ZEN_ASSERT(AvailableBufferBytes >= ByteCount); - memcpy(Buffer, SourceBuffer, ByteCount); - } -} - -void -VfsOplogDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) -{ - // This should never be called - ZEN_UNUSED(NodePath, DirNode); -} - -////////////////////////////////////////////////////////////////////////// - -VfsCacheDataSource::VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore) -: m_NamespaceId(NamespaceId) -, m_BucketId(BucketId) -, m_CacheStore(std::move(InCacheStore)) -{ -} - -VfsCacheDataSource::~VfsCacheDataSource() -{ -} - -void -VfsCacheDataSource::ReadNamedData(std::string_view Name, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - if (auto DotIndex = Name.find_first_of('.'); DotIndex != std::string_view::npos) - { - Name = Name.substr(0, DotIndex); - } - - IoHash HashKey = IoHash::FromHexString(Name); - - CacheRequestContext CacheContext{}; - - ZenCacheValue Value; - if (m_CacheStore->Get(CacheContext, m_NamespaceId, m_BucketId, HashKey, /* out */ Value)) - { - // TODO bounds check! - auto DataPtr = reinterpret_cast<const uint8_t*>(Value.Value.GetData()) + ByteOffset; - - memcpy(Buffer, DataPtr, ByteCount); - - return; - } -} - -void -VfsCacheDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount); -} - -void -VfsCacheDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) -{ - ZEN_UNUSED(NodePath, DirNode); -} - -////////////////////////////////////////////////////////////////////////// - -VfsService::Impl::Impl() -{ -} - -VfsService::Impl::~Impl() -{ - Unmount(); -} - -void -VfsService::Impl::Mount(std::string_view MountPoint) -{ - ZEN_INFO("VFS mount requested at '{}'", MountPoint); - -# if ZEN_PLATFORM_WINDOWS - if (!IsProjFsAvailable()) - { - throw std::runtime_error("Projected File System component not available"); - } -# endif - - if (!m_MountpointPath.empty()) - { - throw std::runtime_error("VFS already mounted"); - } - - m_MountpointPath = MountPoint; - - RefreshVfs(); -} - -void -VfsService::Impl::Unmount() -{ - if (m_MountpointPath.empty()) - { - return; - } - - ZEN_INFO("unmounting VFS from '{}'", m_MountpointPath); - - m_MountpointPath.clear(); - - RefreshVfs(); -} - -void -VfsService::Impl::AddService(Ref<ProjectStore>&& Ps) -{ - m_ProjectStore = std::move(Ps); - - RefreshVfs(); -} - -void -VfsService::Impl::AddService(Ref<ZenCacheStore>&& Z$) -{ - m_ZenCacheStore = std::move(Z$); - - RefreshVfs(); -} - -void -VfsService::Impl::RefreshVfs() -{ - if (m_VfsHost && m_MountpointPath.empty()) - { - m_VfsHost->RequestStop(); - m_VfsThread.join(); - m_VfsHost.reset(); - m_VfsThreadRunning.Reset(); - m_VfsDataSource = nullptr; - - return; - } - - if (!m_VfsHost && !m_MountpointPath.empty()) - { - m_VfsThread = std::thread(&VfsService::Impl::VfsThread, this); - m_VfsThreadRunning.Wait(); - - // At this stage, m_VfsHost should be initialized - - ZEN_ASSERT(m_VfsHost); - } - - if (m_ProjectStore && m_VfsHost) - { - if (!m_VfsDataSource) - { - m_VfsDataSource = new VfsServiceDataSource(this); - } - - m_VfsHost->AddMount("projects"sv, m_VfsDataSource); - } - - if (m_ZenCacheStore && m_VfsHost) - { - if (!m_VfsDataSource) - { - m_VfsDataSource = new VfsServiceDataSource(this); - } - - m_VfsHost->AddMount("ddc_cache"sv, m_VfsDataSource); - } -} - -void -VfsService::Impl::VfsThread() -{ - SetCurrentThreadName("VFS"); - - ZEN_INFO("VFS service thread now RUNNING"); - - try - { - m_VfsHost = std::make_unique<VfsHost>(m_MountpointPath); - m_VfsHost->Initialize(); - - m_VfsThreadRunning.Set(); - m_VfsHost->Run(); - } - catch (std::exception& Ex) - { - ZEN_WARN("exception caught in VFS thread: {}", Ex.what()); - - m_VfsThreadException = std::current_exception(); - } - - if (m_VfsHost) - { - m_VfsHost->Cleanup(); - } - - ZEN_INFO("VFS service thread now EXITING"); -} - -////////////////////////////////////////////////////////////////////////// - -Ref<VfsOplogDataSource> -VfsServiceDataSource::GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId) -{ - ExtendableStringBuilder<256> Key; - Key << ProjectId << "." << OplogId; - std::string StdKey{Key}; - - RwLock::ExclusiveLockScope _(m_Lock); - - if (auto It = m_OplogSourceMap.find(StdKey); It == m_OplogSourceMap.end()) - { - Ref<VfsOplogDataSource> NewSource{new VfsOplogDataSource(ProjectId, OplogId, m_VfsImpl->m_ProjectStore)}; - m_OplogSourceMap[StdKey] = NewSource; - return NewSource; - } - else - { - return It->second; - } -} - -Ref<VfsCacheDataSource> -VfsServiceDataSource::GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId) -{ - ExtendableStringBuilder<256> Key; - Key << NamespaceId << "." << BucketId; - std::string StdKey{Key}; - - RwLock::ExclusiveLockScope _(m_Lock); - - if (auto It = m_CacheSourceMap.find(StdKey); It == m_CacheSourceMap.end()) - { - Ref<VfsCacheDataSource> NewSource{new VfsCacheDataSource(NamespaceId, BucketId, m_VfsImpl->m_ZenCacheStore)}; - m_CacheSourceMap[StdKey] = NewSource; - return NewSource; - } - else - { - return It->second; - } -} - -void -VfsServiceDataSource::ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - ZEN_UNUSED(Path, Buffer, ByteOffset, ByteCount); -} - -void -VfsServiceDataSource::ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) -{ - ZEN_UNUSED(ChunkId, Buffer, ByteOffset, ByteCount); -} - -void -VfsServiceDataSource::PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) -{ - if (NodePath == "projects"sv) - { - // Project enumeration - - m_VfsImpl->m_ProjectStore->DiscoverProjects(); - - m_VfsImpl->m_ProjectStore->IterateProjects( - [&](ProjectStore::Project& Project) { DirNode.AddVirtualNode(Project.Identifier, m_VfsImpl->m_VfsDataSource); }); - } - else if (NodePath.starts_with("projects\\"sv)) - { - std::string_view ProjectId{NodePath}; - ProjectId = ProjectId.substr(9); // Skip "projects\" - - if (std::string_view::size_type SlashOffset = ProjectId.find_first_of('\\'); SlashOffset == std::string_view::npos) - { - Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId); - - if (!Project) - { - // No such project found? - - return; - } - - // Oplog enumeration - - std::vector<std::string> Oplogs = Project->ScanForOplogs(); - - for (auto& Oplog : Oplogs) - { - DirNode.AddVirtualNode(Oplog, m_VfsImpl->m_VfsDataSource); - } - } - else - { - std::string_view OplogId = ProjectId.substr(SlashOffset + 1); - ProjectId = ProjectId.substr(0, SlashOffset); - - Ref<ProjectStore::Project> Project = m_VfsImpl->m_ProjectStore->OpenProject(ProjectId); - - if (!Project) - { - // No such project found? - - return; - } - - // Oplog contents enumeration - - if (ProjectStore::Oplog* Oplog = Project->OpenOplog(OplogId)) - { - Ref<VfsOplogDataSource> DataSource = GetOplogDataSource(ProjectId, OplogId); - - // Get metadata for all chunks - std::vector<ProjectStore::Oplog::ChunkInfo> ChunkInfos = Oplog->GetAllChunksInfo(); - - std::unordered_map<zen::Oid, uint64_t> ChunkSizes; - - for (const auto& Ci : ChunkInfos) - { - ChunkSizes[Ci.ChunkId] = Ci.ChunkSize; - } - - auto EmitFilesForDataArray = [&](zen::CbArrayView DataArray) { - for (auto DataIter : DataArray) - { - if (zen::CbObjectView Data = DataIter.AsObjectView()) - { - std::string_view FileName = Data["filename"sv].AsString(); - zen::Oid ChunkId = Data["id"sv].AsObjectId(); - - if (auto FindIt = ChunkSizes.find(ChunkId); FindIt != ChunkSizes.end()) - { - DirNode.AddFileNode(FileName, FindIt->second /* file size */, ChunkId, DataSource); - } - else - { - ZEN_WARN("no chunk metadata found for chunk {} (file: '{}')", ChunkId, FileName); - } - } - } - }; - - Oplog->IterateOplog([&](CbObjectView Op) { - EmitFilesForDataArray(Op["packagedata"sv].AsArrayView()); - EmitFilesForDataArray(Op["bulkdata"sv].AsArrayView()); - }); - - DirNode.AddFileNode("stats.json", 42, Oid::Zero); - } - } - } - else if (NodePath == "ddc_cache"sv) - { - // Namespace enumeration - - std::vector<std::string> Namespaces = m_VfsImpl->m_ZenCacheStore->GetNamespaces(); - - for (auto& Namespace : Namespaces) - { - DirNode.AddVirtualNode(Namespace, m_VfsImpl->m_VfsDataSource); - } - } - else if (NodePath.starts_with("ddc_cache\\"sv)) - { - std::string_view NamespaceId{NodePath}; - NamespaceId = NamespaceId.substr(10); // Skip "ddc_cache\" - - auto& Cache = m_VfsImpl->m_ZenCacheStore; - - if (std::string_view::size_type SlashOffset = NamespaceId.find_first_of('\\'); SlashOffset == std::string_view::npos) - { - // Bucket enumeration - - if (auto NsInfo = Cache->GetNamespaceInfo(NamespaceId)) - { - for (auto& BucketName : NsInfo->BucketNames) - { - DirNode.AddVirtualNode(BucketName, m_VfsImpl->m_VfsDataSource); - } - } - } - else - { - // Bucket contents enumeration - - std::string_view BucketId = NamespaceId.substr(SlashOffset + 1); - NamespaceId = NamespaceId.substr(0, SlashOffset); - - Ref<VfsCacheDataSource> DataSource = GetCacheDataSource(NamespaceId, BucketId); - - auto Enumerator = [&](const IoHash& Key, const CacheValueDetails::ValueDetails& Details) { - ExtendableStringBuilder<64> KeyString; - Key.ToHexString(KeyString); - KeyString.Append(".udd"); - DirNode.AddFileNode(KeyString, Details.Size, Oid::Zero, DataSource); - }; - - Cache->EnumerateBucketContents(NamespaceId, BucketId, Enumerator); - } - } -} - -} // namespace zen -#endif diff --git a/src/zenserver/vfs/vfsimpl.h b/src/zenserver/vfs/vfsimpl.h deleted file mode 100644 index c33df100b..000000000 --- a/src/zenserver/vfs/vfsimpl.h +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "vfsservice.h" - -#include "projectstore/projectstore.h" - -#include <zencore/logging.h> -#include <zenvfs/vfs.h> - -#if ZEN_WITH_VFS - -# include <memory> -# include <unordered_map> - -namespace zen { - -struct VfsOplogDataSource : public VfsTreeDataSource -{ - VfsOplogDataSource(std::string_view ProjectId, std::string_view OplogId, Ref<ProjectStore> InProjectStore); - - virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; - -private: - std::string m_ProjectId; - std::string m_OplogId; - Ref<ProjectStore> m_ProjectStore; -}; - -struct VfsCacheDataSource : public VfsTreeDataSource -{ - VfsCacheDataSource(std::string_view NamespaceId, std::string_view BucketId, Ref<ZenCacheStore> InCacheStore); - ~VfsCacheDataSource(); - - virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; - -private: - std::string m_NamespaceId; - std::string m_BucketId; - Ref<ZenCacheStore> m_CacheStore; -}; - -struct VfsServiceDataSource : public VfsTreeDataSource -{ - VfsServiceDataSource(VfsService::Impl* VfsImpl) : m_VfsImpl(VfsImpl) {} - - virtual void ReadNamedData(std::string_view Path, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void ReadChunkData(const Oid& ChunkId, void* Buffer, uint64_t ByteOffset, uint64_t ByteCount) override; - virtual void PopulateDirectory(std::string NodePath, VfsTreeNode& DirNode) override; - -private: - VfsService::Impl* m_VfsImpl = nullptr; - - RwLock m_Lock; - std::unordered_map<std::string, Ref<VfsOplogDataSource>> m_OplogSourceMap; - std::unordered_map<std::string, Ref<VfsCacheDataSource>> m_CacheSourceMap; - - Ref<VfsOplogDataSource> GetOplogDataSource(std::string_view ProjectId, std::string_view OplogId); - Ref<VfsCacheDataSource> GetCacheDataSource(std::string_view NamespaceId, std::string_view BucketId); -}; - -////////////////////////////////////////////////////////////////////////// - -struct VfsService::Impl -{ - Impl(); - ~Impl(); - - void Mount(std::string_view MountPoint); - void Unmount(); - void AddService(Ref<ProjectStore>&&); - void AddService(Ref<ZenCacheStore>&&); - - inline std::string GetMountpointPath() { return m_MountpointPath; } - inline bool IsVfsRunning() const { return !!m_VfsHost.get(); } - -private: - Ref<ProjectStore> m_ProjectStore; - Ref<ZenCacheStore> m_ZenCacheStore; - Ref<VfsServiceDataSource> m_VfsDataSource; - std::string m_MountpointPath; - - std::unique_ptr<VfsHost> m_VfsHost; - std::thread m_VfsThread; - Event m_VfsThreadRunning; - std::exception_ptr m_VfsThreadException; - - void RefreshVfs(); - void VfsThread(); - - friend struct VfsServiceDataSource; -}; - -} // namespace zen - -#endif diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp index 04ba29ed2..863ec348a 100644 --- a/src/zenserver/vfs/vfsservice.cpp +++ b/src/zenserver/vfs/vfsservice.cpp @@ -1,7 +1,8 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "vfsservice.h" -#include "vfsimpl.h" + +#include <zenstore/vfsimpl.h> #include <zencore/compactbinarybuilder.h> @@ -61,10 +62,8 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl) { - m_Impl = new Impl; - m_Router.RegisterRoute( "info", [&](HttpRouterRequest& Request) { @@ -105,7 +104,7 @@ VfsService::VfsService() { m_Impl->Mount(Mountpath); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); } @@ -123,7 +122,7 @@ VfsService::VfsService() { m_Impl->Unmount(); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); } @@ -136,70 +135,25 @@ VfsService::VfsService() } }, HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); } VfsService::~VfsService() { - delete m_Impl; -} - -void -VfsService::Mount(std::string_view MountPoint) -{ - m_Impl->Mount(MountPoint); -} - -void -VfsService::Unmount() -{ - m_Impl->Unmount(); -} - -void -VfsService::AddService(Ref<ProjectStore>&& Ps) -{ - m_Impl->AddService(std::move(Ps)); -} - -void -VfsService::AddService(Ref<ZenCacheStore>&& Z$) -{ - m_Impl->AddService(std::move(Z$)); + m_StatusService.UnregisterHandler("vfs", *this); } #else -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService) { + ZEN_UNUSED(ServiceImpl); } VfsService::~VfsService() { } -void -VfsService::Mount(std::string_view MountPoint) -{ - ZEN_UNUSED(MountPoint); -} - -void -VfsService::Unmount() -{ -} - -void -VfsService::AddService(Ref<ProjectStore>&& Ps) -{ - ZEN_UNUSED(Ps); -} - -void -VfsService::AddService(Ref<ZenCacheStore>&& Z$) -{ - ZEN_UNUSED(Z$); -} - #endif const char* @@ -209,6 +163,14 @@ VfsService::BaseUri() const } void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) { m_Router.HandleRequest(HttpServiceRequest); diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h index dcdc71e81..4e06da878 100644 --- a/src/zenserver/vfs/vfsservice.h +++ b/src/zenserver/vfs/vfsservice.h @@ -4,7 +4,7 @@ #include <zenbase/refcount.h> #include <zenhttp/httpserver.h> -#include <zenvfs/vfs.h> +#include <zenhttp/httpstatus.h> #include <memory> @@ -12,6 +12,7 @@ namespace zen { class ProjectStore; class ZenCacheStore; +struct VfsServiceImpl; /** Virtual File System service @@ -24,27 +25,22 @@ class ZenCacheStore; */ -class VfsService : public HttpService +class VfsService : public HttpService, public IHttpStatusProvider { public: - VfsService(); + explicit VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl); ~VfsService(); - void Mount(std::string_view MountPoint); - void Unmount(); - - void AddService(Ref<ProjectStore>&&); - void AddService(Ref<ZenCacheStore>&&); - protected: virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: - struct Impl; - Impl* m_Impl = nullptr; + VfsServiceImpl* m_Impl = nullptr; - HttpRequestRouter m_Router; + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; friend struct VfsServiceDataSource; }; diff --git a/src/zenserver/windows/service.cpp b/src/zenserver/windows/service.cpp deleted file mode 100644 index cb87df1f6..000000000 --- a/src/zenserver/windows/service.cpp +++ /dev/null @@ -1,648 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "service.h" - -#include <zencore/zencore.h> - -#if ZEN_PLATFORM_WINDOWS - -# include <zencore/except.h> -# include <zencore/thread.h> - -# include <stdio.h> -# include <tchar.h> -# include <zencore/windows.h> - -# define SVCNAME L"Zen Store" - -SERVICE_STATUS gSvcStatus; -SERVICE_STATUS_HANDLE gSvcStatusHandle; -HANDLE ghSvcStopEvent = NULL; - -void SvcInstall(void); - -void ReportSvcStatus(DWORD, DWORD, DWORD); -void SvcReportEvent(LPTSTR); - -WindowsService::WindowsService() -{ -} - -WindowsService::~WindowsService() -{ -} - -// -// Purpose: -// Installs a service in the SCM database -// -// Parameters: -// None -// -// Return value: -// None -// -VOID -WindowsService::Install() -{ - SC_HANDLE schSCManager; - SC_HANDLE schService; - TCHAR szPath[MAX_PATH]; - - if (!GetModuleFileName(NULL, szPath, MAX_PATH)) - { - printf("Cannot install service (%d)\n", GetLastError()); - return; - } - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Create the service - - schService = CreateService(schSCManager, // SCM database - SVCNAME, // name of service - SVCNAME, // service name to display - SERVICE_ALL_ACCESS, // desired access - SERVICE_WIN32_OWN_PROCESS, // service type - SERVICE_DEMAND_START, // start type - SERVICE_ERROR_NORMAL, // error control type - szPath, // path to service's binary - NULL, // no load ordering group - NULL, // no tag identifier - NULL, // no dependencies - NULL, // LocalSystem account - NULL); // no password - - if (schService == NULL) - { - printf("CreateService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - else - printf("Service installed successfully\n"); - - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} - -void -WindowsService::Delete() -{ - SC_HANDLE schSCManager; - SC_HANDLE schService; - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Get a handle to the service. - - schService = OpenService(schSCManager, // SCM database - SVCNAME, // name of service - DELETE); // need delete access - - if (schService == NULL) - { - printf("OpenService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - - // Delete the service. - - if (!DeleteService(schService)) - { - printf("DeleteService failed (%d)\n", GetLastError()); - } - else - printf("Service deleted successfully\n"); - - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} - -WindowsService* gSvc; - -void WINAPI -CallMain(DWORD, LPSTR*) -{ - gSvc->SvcMain(); -} - -int -WindowsService::ServiceMain() -{ - zen::SetCurrentThreadName("svc-main"); - - gSvc = this; - - SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; - - // This call returns when the service has stopped. - // The process should simply terminate when the call returns. - - if (!StartServiceCtrlDispatcher(DispatchTable)) - { - const DWORD dwError = zen::GetLastError(); - - if (dwError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) - { - // Not actually running as a service - gSvc = nullptr; - - zen::SetIsInteractiveSession(true); - - return Run(); - } - else - { - zen::ThrowSystemError(dwError, "StartServiceCtrlDispatcher failed"); - } - } - - zen::SetIsInteractiveSession(false); - - return zen::ApplicationExitCode(); -} - -int -WindowsService::SvcMain() -{ - // Register the handler function for the service - - gSvcStatusHandle = RegisterServiceCtrlHandler(SVCNAME, SvcCtrlHandler); - - if (!gSvcStatusHandle) - { - SvcReportEvent((LPTSTR)TEXT("RegisterServiceCtrlHandler")); - - return 1; - } - - // These SERVICE_STATUS members remain as set here - - gSvcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - gSvcStatus.dwServiceSpecificExitCode = 0; - - // Report initial status to the SCM - - ReportSvcStatus(SERVICE_START_PENDING, NO_ERROR, 3000); - - // Create an event. The control handler function, SvcCtrlHandler, - // signals this event when it receives the stop control code. - - ghSvcStopEvent = CreateEvent(NULL, // default security attributes - TRUE, // manual reset event - FALSE, // not signaled - NULL); // no name - - if (ghSvcStopEvent == NULL) - { - ReportSvcStatus(SERVICE_STOPPED, GetLastError(), 0); - - return 1; - } - - // Report running status when initialization is complete. - - ReportSvcStatus(SERVICE_RUNNING, NO_ERROR, 0); - - int ReturnCode = Run(); - - ReportSvcStatus(SERVICE_STOPPED, NO_ERROR, 0); - - return ReturnCode; -} - -// -// Purpose: -// Retrieves and displays the current service configuration. -// -// Parameters: -// None -// -// Return value: -// None -// -void -DoQuerySvc() -{ - SC_HANDLE schSCManager{}; - SC_HANDLE schService{}; - LPQUERY_SERVICE_CONFIG lpsc{}; - LPSERVICE_DESCRIPTION lpsd{}; - DWORD dwBytesNeeded{}, cbBufSize{}, dwError{}; - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Get a handle to the service. - - schService = OpenService(schSCManager, // SCM database - SVCNAME, // name of service - SERVICE_QUERY_CONFIG); // need query config access - - if (schService == NULL) - { - printf("OpenService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - - // Get the configuration information. - - if (!QueryServiceConfig(schService, NULL, 0, &dwBytesNeeded)) - { - dwError = GetLastError(); - if (ERROR_INSUFFICIENT_BUFFER == dwError) - { - cbBufSize = dwBytesNeeded; - lpsc = (LPQUERY_SERVICE_CONFIG)LocalAlloc(LMEM_FIXED, cbBufSize); - } - else - { - printf("QueryServiceConfig failed (%d)", dwError); - goto cleanup; - } - } - - if (!QueryServiceConfig(schService, lpsc, cbBufSize, &dwBytesNeeded)) - { - printf("QueryServiceConfig failed (%d)", GetLastError()); - goto cleanup; - } - - if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &dwBytesNeeded)) - { - dwError = GetLastError(); - if (ERROR_INSUFFICIENT_BUFFER == dwError) - { - cbBufSize = dwBytesNeeded; - lpsd = (LPSERVICE_DESCRIPTION)LocalAlloc(LMEM_FIXED, cbBufSize); - } - else - { - printf("QueryServiceConfig2 failed (%d)", dwError); - goto cleanup; - } - } - - if (!QueryServiceConfig2(schService, SERVICE_CONFIG_DESCRIPTION, (LPBYTE)lpsd, cbBufSize, &dwBytesNeeded)) - { - printf("QueryServiceConfig2 failed (%d)", GetLastError()); - goto cleanup; - } - - // Print the configuration information. - - _tprintf(TEXT("%s configuration: \n"), SVCNAME); - _tprintf(TEXT(" Type: 0x%x\n"), lpsc->dwServiceType); - _tprintf(TEXT(" Start Type: 0x%x\n"), lpsc->dwStartType); - _tprintf(TEXT(" Error Control: 0x%x\n"), lpsc->dwErrorControl); - _tprintf(TEXT(" Binary path: %s\n"), lpsc->lpBinaryPathName); - _tprintf(TEXT(" Account: %s\n"), lpsc->lpServiceStartName); - - if (lpsd->lpDescription != NULL && lstrcmp(lpsd->lpDescription, TEXT("")) != 0) - _tprintf(TEXT(" Description: %s\n"), lpsd->lpDescription); - if (lpsc->lpLoadOrderGroup != NULL && lstrcmp(lpsc->lpLoadOrderGroup, TEXT("")) != 0) - _tprintf(TEXT(" Load order group: %s\n"), lpsc->lpLoadOrderGroup); - if (lpsc->dwTagId != 0) - _tprintf(TEXT(" Tag ID: %d\n"), lpsc->dwTagId); - if (lpsc->lpDependencies != NULL && lstrcmp(lpsc->lpDependencies, TEXT("")) != 0) - _tprintf(TEXT(" Dependencies: %s\n"), lpsc->lpDependencies); - - LocalFree(lpsc); - LocalFree(lpsd); - -cleanup: - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} - -// -// Purpose: -// Disables the service. -// -// Parameters: -// None -// -// Return value: -// None -// -void -DoDisableSvc() -{ - SC_HANDLE schSCManager; - SC_HANDLE schService; - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Get a handle to the service. - - schService = OpenService(schSCManager, // SCM database - SVCNAME, // name of service - SERVICE_CHANGE_CONFIG); // need change config access - - if (schService == NULL) - { - printf("OpenService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - - // Change the service start type. - - if (!ChangeServiceConfig(schService, // handle of service - SERVICE_NO_CHANGE, // service type: no change - SERVICE_DISABLED, // service start type - SERVICE_NO_CHANGE, // error control: no change - NULL, // binary path: no change - NULL, // load order group: no change - NULL, // tag ID: no change - NULL, // dependencies: no change - NULL, // account name: no change - NULL, // password: no change - NULL)) // display name: no change - { - printf("ChangeServiceConfig failed (%d)\n", GetLastError()); - } - else - printf("Service disabled successfully.\n"); - - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} - -// -// Purpose: -// Enables the service. -// -// Parameters: -// None -// -// Return value: -// None -// -VOID __stdcall DoEnableSvc() -{ - SC_HANDLE schSCManager; - SC_HANDLE schService; - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Get a handle to the service. - - schService = OpenService(schSCManager, // SCM database - SVCNAME, // name of service - SERVICE_CHANGE_CONFIG); // need change config access - - if (schService == NULL) - { - printf("OpenService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - - // Change the service start type. - - if (!ChangeServiceConfig(schService, // handle of service - SERVICE_NO_CHANGE, // service type: no change - SERVICE_DEMAND_START, // service start type - SERVICE_NO_CHANGE, // error control: no change - NULL, // binary path: no change - NULL, // load order group: no change - NULL, // tag ID: no change - NULL, // dependencies: no change - NULL, // account name: no change - NULL, // password: no change - NULL)) // display name: no change - { - printf("ChangeServiceConfig failed (%d)\n", GetLastError()); - } - else - printf("Service enabled successfully.\n"); - - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} -// -// Purpose: -// Updates the service description to "This is a test description". -// -// Parameters: -// None -// -// Return value: -// None -// -void -DoUpdateSvcDesc() -{ - SC_HANDLE schSCManager; - SC_HANDLE schService; - SERVICE_DESCRIPTION sd; - TCHAR szDesc[] = TEXT("This is a test description"); - - // Get a handle to the SCM database. - - schSCManager = OpenSCManager(NULL, // local computer - NULL, // ServicesActive database - SC_MANAGER_ALL_ACCESS); // full access rights - - if (NULL == schSCManager) - { - printf("OpenSCManager failed (%d)\n", GetLastError()); - return; - } - - // Get a handle to the service. - - schService = OpenService(schSCManager, // SCM database - SVCNAME, // name of service - SERVICE_CHANGE_CONFIG); // need change config access - - if (schService == NULL) - { - printf("OpenService failed (%d)\n", GetLastError()); - CloseServiceHandle(schSCManager); - return; - } - - // Change the service description. - - sd.lpDescription = szDesc; - - if (!ChangeServiceConfig2(schService, // handle to service - SERVICE_CONFIG_DESCRIPTION, // change: description - &sd)) // new description - { - printf("ChangeServiceConfig2 failed\n"); - } - else - printf("Service description updated successfully.\n"); - - CloseServiceHandle(schService); - CloseServiceHandle(schSCManager); -} - -// -// Purpose: -// Sets the current service status and reports it to the SCM. -// -// Parameters: -// dwCurrentState - The current state (see SERVICE_STATUS) -// dwWin32ExitCode - The system error code -// dwWaitHint - Estimated time for pending operation, -// in milliseconds -// -// Return value: -// None -// -VOID -ReportSvcStatus(DWORD dwCurrentState, DWORD dwWin32ExitCode, DWORD dwWaitHint) -{ - static DWORD dwCheckPoint = 1; - - // Fill in the SERVICE_STATUS structure. - - gSvcStatus.dwCurrentState = dwCurrentState; - gSvcStatus.dwWin32ExitCode = dwWin32ExitCode; - gSvcStatus.dwWaitHint = dwWaitHint; - - if (dwCurrentState == SERVICE_START_PENDING) - gSvcStatus.dwControlsAccepted = 0; - else - gSvcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP; - - if ((dwCurrentState == SERVICE_RUNNING) || (dwCurrentState == SERVICE_STOPPED)) - gSvcStatus.dwCheckPoint = 0; - else - gSvcStatus.dwCheckPoint = dwCheckPoint++; - - // Report the status of the service to the SCM. - SetServiceStatus(gSvcStatusHandle, &gSvcStatus); -} - -void -WindowsService::SvcCtrlHandler(DWORD dwCtrl) -{ - // Handle the requested control code. - // - // Called by SCM whenever a control code is sent to the service - // using the ControlService function. - - switch (dwCtrl) - { - case SERVICE_CONTROL_STOP: - ReportSvcStatus(SERVICE_STOP_PENDING, NO_ERROR, 0); - - // Signal the service to stop. - - SetEvent(ghSvcStopEvent); - zen::RequestApplicationExit(0); - - ReportSvcStatus(gSvcStatus.dwCurrentState, NO_ERROR, 0); - return; - - case SERVICE_CONTROL_INTERROGATE: - break; - - default: - break; - } -} - -// -// Purpose: -// Logs messages to the event log -// -// Parameters: -// szFunction - name of function that failed -// -// Return value: -// None -// -// Remarks: -// The service must have an entry in the Application event log. -// -VOID -SvcReportEvent(LPTSTR szFunction) -{ - ZEN_UNUSED(szFunction); - - // HANDLE hEventSource; - // LPCTSTR lpszStrings[2]; - // TCHAR Buffer[80]; - - // hEventSource = RegisterEventSource(NULL, SVCNAME); - - // if (NULL != hEventSource) - //{ - // StringCchPrintf(Buffer, 80, TEXT("%s failed with %d"), szFunction, GetLastError()); - - // lpszStrings[0] = SVCNAME; - // lpszStrings[1] = Buffer; - - // ReportEvent(hEventSource, // event log handle - // EVENTLOG_ERROR_TYPE, // event type - // 0, // event category - // SVC_ERROR, // event identifier - // NULL, // no security identifier - // 2, // size of lpszStrings array - // 0, // no binary data - // lpszStrings, // array of strings - // NULL); // no binary data - - // DeregisterEventSource(hEventSource); - //} -} - -#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zenserver/windows/service.h b/src/zenserver/windows/service.h deleted file mode 100644 index 7c9610983..000000000 --- a/src/zenserver/windows/service.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -class WindowsService -{ -public: - WindowsService(); - ~WindowsService(); - - virtual int Run() = 0; - - int ServiceMain(); - - static void Install(); - static void Delete(); - - int SvcMain(); - static void __stdcall SvcCtrlHandler(unsigned long); -}; diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp new file mode 100644 index 000000000..7ef84743e --- /dev/null +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -0,0 +1,1211 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <workspaces/httpworkspaces.h> + +#include <zencore/basicfile.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenstore/workspaces.h> +#include <zenutil/chunkrequests.h> +#include <zenutil/workerpools.h> + +#include <unordered_set> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogFs, "fs"sv); + +namespace { + + std::filesystem::path GetPathParameter(HttpServerRequest& ServerRequest, std::string_view Name) + { + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue(Name); !Value.empty()) + { + return std::filesystem::path(HttpServerRequest::Decode(Value)); + } + return {}; + } + + void WriteWorkspaceConfig(CbWriter& Writer, const Workspaces::WorkspaceConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "root_path" << Config.RootPath.string(); // utf8? + Writer << "allow_share_creation_from_http" << Config.AllowShareCreationFromHttp; + }; + + void WriteWorkspaceShareConfig(CbWriter& Writer, const Workspaces::WorkspaceShareConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "share_path" << Config.SharePath.string(); // utf8? + if (!Config.Alias.empty()) + { + Writer << "alias" << Config.Alias; + } + }; + + void WriteWorkspaceAndSharesConfig(CbWriter& Writer, Workspaces& Workspaces, const Workspaces::WorkspaceConfiguration& WorkspaceConfig) + { + WriteWorkspaceConfig(Writer, WorkspaceConfig); + if (std::optional<std::vector<Oid>> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) + { + Writer.BeginArray("shares"); + { + for (const Oid& ShareId : *ShareIds) + { + if (std::optional<Workspaces::WorkspaceShareConfiguration> WorkspaceShareConfig = + Workspaces.GetWorkspaceShareConfiguration(WorkspaceConfig.Id, ShareId); + WorkspaceShareConfig) + { + Writer.BeginObject(); + { + WriteWorkspaceShareConfig(Writer, *WorkspaceShareConfig); + } + Writer.EndObject(); + } + } + } + Writer.EndArray(); + } + } + +} // namespace + +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) +: m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_Config(Cfg) +, m_Workspaces(Workspaces) +{ + Initialize(); +} + +HttpWorkspacesService::~HttpWorkspacesService() +{ + m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); +} + +const char* +HttpWorkspacesService::BaseUri() const +{ + return "/ws/"; +} + +void +HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogFs, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("WorkspacesService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("workspaces"); + { + Cbo.BeginObject("workspace"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceReadCount << "writecount" << m_WorkspacesStats.WorkspaceWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("workspaceshare"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceShareReadCount << "writecount" << m_WorkspacesStats.WorkspaceShareWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceShareDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_WorkspacesStats.WorkspaceShareChunkHitCount << "misscount" + << m_WorkspacesStats.WorkspaceShareChunkMissCount; + } + Cbo.EndObject(); + + Cbo << "filescount" << m_WorkspacesStats.WorkspaceShareFilesReadCount; + Cbo << "entriescount" << m_WorkspacesStats.WorkspaceShareEntriesReadCount; + Cbo << "batchcount" << m_WorkspacesStats.WorkspaceShareBatchReadCount; + + Cbo << "requestcount" << m_WorkspacesStats.RequestCount; + Cbo << "badrequestcount" << m_WorkspacesStats.BadRequestCount; + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::Initialize() +{ + using namespace std::literals; + + ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); + + m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)"); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/files", + [this](HttpRouterRequest& Req) { FilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}/info", + [this](HttpRouterRequest& Req) { ChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/batch", + [this](HttpRouterRequest& Req) { BatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/entries", + [this](HttpRouterRequest& Req) { EntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}", + [this](HttpRouterRequest& Req) { ChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}/files", + [this](HttpRouterRequest& Req) { ShareAliasFilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}/info", + [this](HttpRouterRequest& Req) { ShareAliasChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/batch", + [this](HttpRouterRequest& Req) { ShareAliasBatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "share/{share_alias}/entries", + [this](HttpRouterRequest& Req) { ShareAliasEntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}", + [this](HttpRouterRequest& Req) { ShareAliasChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}", + [this](HttpRouterRequest& Req) { ShareAliasRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}", + [this](HttpRouterRequest& Req) { ShareRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}", + [this](HttpRouterRequest& Req) { WorkspaceRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "refresh", + [this](HttpRouterRequest& Req) { RefreshRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "", + [this](HttpRouterRequest& Req) { WorkspacesRequest(Req); }, + HttpVerb::kGet); + + RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); +} + +std::filesystem::path +HttpWorkspacesService::GetStatePath() const +{ + return m_Config.SystemRootDir / "workspaces"; +} + +void +HttpWorkspacesService::RefreshState() +{ + if (!m_Config.SystemRootDir.empty()) + { + m_Workspaces.RefreshState(GetStatePath()); + } +} + +bool +HttpWorkspacesService::MayChangeConfiguration(const HttpServerRequest& Req) const +{ + ZEN_UNUSED(Req); + return m_Config.AllowConfigurationChanges; +} + +void +HttpWorkspacesService::RefreshRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpWorkspacesService::WorkspacesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + std::vector<Oid> WorkspaceIds = m_Workspaces.GetWorkspaces(); + CbObjectWriter Response; + Response.BeginArray("workspaces"); + for (const Oid& WorkspaceId : WorkspaceIds) + { + if (std::optional<Workspaces::WorkspaceConfiguration> WorkspaceConfig = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + WorkspaceConfig) + { + Response.BeginObject(); + { + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *WorkspaceConfig); + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + FilesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkInfoRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + BatchRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + EntriesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + Oid ShareId = Oid::Zero; + if (Req.GetCapture(2) != Oid::Zero.ToString()) + { + ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + } + ShareRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::WorkspaceRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path WorkspacePath = GetPathParameter(ServerRequest, "root_path"sv); + if (WorkspacePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'root_path' parameter"); + } + + if (Req.GetCapture(1) == Oid::Zero.ToString()) + { + // Synthesize Id + WorkspaceId = Workspaces::PathToId(WorkspacePath); + ZEN_INFO("Generated workspace id from path '{}': {}", WorkspacePath, WorkspaceId); + } + else if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace {} is not allowed", WorkspaceId)); + } + bool AllowShareCreationFromHttp = false; + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue("allow_share_creation_from_http"); Value == "true"sv) + { + AllowShareCreationFromHttp = true; + } + + m_WorkspacesStats.WorkspaceWriteCount++; + Workspaces::WorkspaceConfiguration OldConfig = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + Workspaces::WorkspaceConfiguration NewConfig = {.Id = WorkspaceId, + .RootPath = WorkspacePath, + .AllowShareCreationFromHttp = AllowShareCreationFromHttp}; + if (OldConfig.Id == WorkspaceId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with root path '{}'", WorkspaceId, OldConfig.RootPath)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceConfiguration ConfigWithSameRoot = + Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspacePath); + ConfigWithSameRoot.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with same root path '{}'", ConfigWithSameRoot.Id, WorkspacePath)); + } + } + + bool Created = Workspaces::AddWorkspace(Log(), GetStatePath(), NewConfig); + if (Created) + { + ZEN_ASSERT(OldConfig.Id == Oid::Zero); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceReadCount++; + std::optional<Workspaces::WorkspaceConfiguration> Workspace = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + if (Workspace) + { + CbObjectWriter Response; + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *Workspace); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace {} is not allowed", WorkspaceId)); + } + + m_WorkspacesStats.WorkspaceDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +void +HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + FilesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkInfoRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + BatchRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + EntriesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + ShareRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + m_WorkspacesStats.WorkspaceShareFilesReadCount++; + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = ServerRequest.GetQueryParams().GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); + const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); + + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + Response.BeginArray("files"sv); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + if (WantsSizeField) + { + Response << "size"sv << Entry.Size; + } + if (WantsRawSizeField) + { + Response << "rawsize"sv << Entry.Size; + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Workspaces::ShareFile File = + m_Workspaces.GetWorkspaceShareChunkInfo(WorkspaceId, ShareId, ChunkId, GetSmallWorkerPool(EWorkloadType::Burst)); + if (File.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "size"sv << File.Size; + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + IoBuffer Payload = ServerRequest.ReadPayload(); + std::optional<std::vector<RequestChunkEntry>> ChunkRequests = ParseChunkBatchRequest(Payload); + if (!ChunkRequests.has_value()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); + } + m_WorkspacesStats.WorkspaceShareBatchReadCount++; + std::vector<Workspaces::ChunkRequest> Requests; + Requests.reserve(ChunkRequests.value().size()); + std::transform(ChunkRequests.value().begin(), + ChunkRequests.value().end(), + std::back_inserter(Requests), + [](const RequestChunkEntry& Entry) { + return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; + }); + std::vector<IoBuffer> Chunks = + m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool(EWorkloadType::Burst)); + if (Chunks.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + for (const IoBuffer& Buffer : Chunks) + { + if (Buffer) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + } + else + { + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + } + } + std::vector<IoBuffer> Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); + if (!Response.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); + } + return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + fmt::format("failed formatting response for batch of {} chunks", Chunks.size())); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view OpKey = ServerRequest.GetQueryParams().GetValue("opkey"sv); + if (!OpKey.empty() && OpKey != "file_manifest") + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldfilter")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + m_WorkspacesStats.WorkspaceShareEntriesReadCount++; + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + + CbObjectWriter Response; + + if (OpKey.empty()) + { + Response.BeginArray("entries"sv); + Response.BeginObject(); + } + else + { + Response.BeginObject("entry"sv); + } + { + // Synthesize a fake op + Response << "key" + << "file_manifest"; + + Response.BeginArray("files"); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + { + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + } + Response.EndObject(); + } + } + Response.EndArray(); + } + + if (OpKey.empty()) + { + Response.EndObject(); + Response.EndArray(); + } + else + { + Response.EndObject(); + } + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + if (auto OffsetParm = ServerRequest.GetQueryParams().GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid offset parameter '{}'", OffsetParm)); + } + } + + if (auto SizeParm = ServerRequest.GetQueryParams().GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt<uint64_t>(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid size parameter '{}'", SizeParm)); + } + } + + std::vector<IoBuffer> Response = m_Workspaces.GetWorkspaceShareChunks( + WorkspaceId, + ShareId, + std::vector<Workspaces::ChunkRequest>{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, + GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Response.empty() && Response[0]) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + if (Response[0].GetSize() == 0) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response[0].GetContentType(), Response); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId) +{ + Oid ShareId = InShareId; + + HttpServerRequest& ServerRequest = Req.ServerRequest(); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path SharePath = GetPathParameter(ServerRequest, "share_path"sv); + if (SharePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'share_path' parameter"); + } + + if (ShareId == Oid::Zero) + { + // Synthesize Id + ShareId = Workspaces::PathToId(SharePath); + ZEN_INFO("Generated workspace id from path '{}': {}", SharePath, ShareId); + } + + std::string Alias = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("alias"sv)); + if (!AsciiSet::HasOnly(Alias, Workspaces::ValidAliasCharactersSet)) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid 'alias' parameter"); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Workspace '{}' does not exist", WorkspaceId)); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareWriteCount++; + + const Workspaces::WorkspaceShareConfiguration OldConfig = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, ShareId); + const Workspaces::WorkspaceShareConfiguration NewConfig = {.Id = ShareId, + .SharePath = SharePath, + .Alias = std::string(Alias)}; + + if (OldConfig.Id == ShareId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceShareConfiguration ConfigWithSamePath = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, SharePath); + ConfigWithSamePath.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with same share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + } + + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("directory {} does not exist in workspace {} root '{}'", + NewConfig.SharePath, + WorkspaceId, + Workspace.RootPath)); + } + + bool Created = Workspaces::AddWorkspaceShare(Log(), Workspace.RootPath, NewConfig); + if (Created) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", ShareId)); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", ShareId)); + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + m_WorkspacesStats.WorkspaceShareReadCount++; + std::optional<Workspaces::WorkspaceShareConfiguration> Config = + m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); + if (!Config) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + WriteWorkspaceShareConfig(Response, *Config); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspaceShare(Log(), Workspace.RootPath, ShareId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +} // namespace zen diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h new file mode 100644 index 000000000..89a8e8bdc --- /dev/null +++ b/src/zenserver/workspaces/httpworkspaces.h @@ -0,0 +1,97 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> + +namespace zen { + +class Workspaces; + +struct WorkspacesServeConfig +{ + std::filesystem::path SystemRootDir; + bool AllowConfigurationChanges = false; +}; + +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); + virtual ~HttpWorkspacesService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct WorkspacesStats + { + std::atomic_uint64_t WorkspaceReadCount{}; + std::atomic_uint64_t WorkspaceWriteCount{}; + std::atomic_uint64_t WorkspaceDeleteCount{}; + std::atomic_uint64_t WorkspaceShareReadCount{}; + std::atomic_uint64_t WorkspaceShareWriteCount{}; + std::atomic_uint64_t WorkspaceShareDeleteCount{}; + std::atomic_uint64_t WorkspaceShareFilesReadCount{}; + std::atomic_uint64_t WorkspaceShareEntriesReadCount{}; + std::atomic_uint64_t WorkspaceShareBatchReadCount{}; + std::atomic_uint64_t WorkspaceShareChunkHitCount{}; + std::atomic_uint64_t WorkspaceShareChunkMissCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void Initialize(); + std::filesystem::path GetStatePath() const; + void RefreshState(); + // void WriteState(); + + bool MayChangeConfiguration(const HttpServerRequest& Req) const; + + void WorkspacesRequest(HttpRouterRequest& Req); + void RefreshRequest(HttpRouterRequest& Req); + void FilesRequest(HttpRouterRequest& Req); + void ChunkInfoRequest(HttpRouterRequest& Req); + void BatchRequest(HttpRouterRequest& Req); + void EntriesRequest(HttpRouterRequest& Req); + void ChunkRequest(HttpRouterRequest& Req); + void ShareRequest(HttpRouterRequest& Req); + void WorkspaceRequest(HttpRouterRequest& Req); + + void ShareAliasFilesRequest(HttpRouterRequest& Req); + void ShareAliasChunkInfoRequest(HttpRouterRequest& Req); + void ShareAliasBatchRequest(HttpRouterRequest& Req); + void ShareAliasEntriesRequest(HttpRouterRequest& Req); + void ShareAliasChunkRequest(HttpRouterRequest& Req); + void ShareAliasRequest(HttpRouterRequest& Req); + + void FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + const WorkspacesServeConfig m_Config; + HttpRequestRouter m_Router; + Workspaces& m_Workspaces; + WorkspacesStats m_WorkspacesStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index c42f305ee..57105045d 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -5,6 +5,7 @@ target("zenserver") add_deps("zencore", "zenhttp", "zennet", + "zenremotestore", "zenstore", "zenutil", "zenvfs") @@ -28,8 +29,6 @@ target("zenserver") add_cxxflags("/bigobj") add_links("delayimp", "projectedfslib") add_ldflags("/delayload:ProjectedFSLib.dll") - - add_links("dbghelp", "winhttp", "version") -- for Sentry else remove_files("windows/**") end @@ -41,7 +40,6 @@ target("zenserver") add_ldflags("-framework Foundation") add_ldflags("-framework Security") add_ldflags("-framework SystemConfiguration") - add_syslinks("bsm") end add_options("compute") @@ -57,14 +55,30 @@ target("zenserver") "vcpkg::sol2" ) - if has_config("zensentry") then - add_packages("vcpkg::sentry-native") - end + -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to + -- our use of setsid() at startup we pass in `--no-detach` to zenserver + -- ensure that it recieves signals when the user requests termination + on_run(function(target) + -- the following is roughly cribbed from xmake/actions/run/xmake.lua + -- it would be nicer if we had the option of amending the arguments + -- via before_run for instance, but I can't figure out a way to do that + import("core.base.option") - if is_plat("linux") then - -- As sentry_native uses symbols from breakpad_client, the latter must - -- be specified after the former with GCC-like toolchains. xmake however - -- is unaware of this and simply globs files from vcpkg's output. The - -- line below forces breakpad_client to be to the right of sentry_native - add_syslinks("breakpad_client") - end + -- get the run directory of target + local rundir = target:rundir() + + -- get the absolute target file path + local targetfile = path.absolute(target:targetfile()) + + -- get run arguments + local args = table.wrap(option.get("arguments") or target:get("runargs")) + + table.insert(args, "--detach=false") + + -- debugging? + if option.get("debug") then + debugger.run(targetfile, args, {curdir = rundir}) + else + os.execv(targetfile, args, {curdir = rundir, detach = option.get("detach")}) + end + end) diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 37b3f0531..29581b192 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -2,18 +2,19 @@ #include "zenserver.h" -#include "sentryintegration.h" - #include <zenbase/refcount.h> +#include <zencore/basicfile.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryvalidation.h> #include <zencore/config.h> +#include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iobuffer.h> #include <zencore/jobqueue.h> #include <zencore/logging.h> #include <zencore/scopeguard.h> +#include <zencore/sentryintegration.h> #include <zencore/session.h> #include <zencore/string.h> #include <zencore/thread.h> @@ -21,9 +22,12 @@ #include <zencore/trace.h> #include <zencore/workthreadpool.h> #include <zenhttp/httpserver.h> +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenstore/buildstore/buildstore.h> #include <zenstore/cidstore.h> #include <zenstore/scrubcontext.h> -#include <zenutil/basicfile.h> +#include <zenstore/vfsimpl.h> +#include <zenstore/workspaces.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> @@ -52,8 +56,18 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "config.h" #include "diag/logging.h" +#include <zencore/memory/llm.h> + namespace zen { +static const FLLMTag& +GetZenserverTag() +{ + static FLLMTag _("zenserver"); + + return _; +} + namespace utils { extern std::atomic_uint32_t SignalCounter[NSIG]; } @@ -115,7 +129,18 @@ ZenServer::OnReady() int ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry) { - m_UseSentry = ServerOptions.NoSentry == false; + ZEN_TRACE_CPU("ZenServer::Initialize"); + + ZEN_MEMSCOPE(GetZenserverTag()); + const std::string MutexName = fmt::format("zen_{}", ServerOptions.BasePort); + + if (NamedMutex::Exists(MutexName)) + { + ZEN_WARN("Mutex '{}' already exists - is another instance already running?", MutexName); + return -1; + } + + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; @@ -124,12 +149,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen if (ParentPid) { - ProcessHandle OwnerProcess; - OwnerProcess.Initialize(ParentPid); + std::error_code Ec; + ProcessHandle OwnerProcess; + OwnerProcess.Initialize(ParentPid, /* out */ Ec); if (!OwnerProcess.IsValid()) { - ZEN_WARN("Unable to initialize process handle for specified parent pid #{}", ParentPid); + ZEN_WARN("Unable to initialize process handle for specified parent pid #{}. Reason: '{}'", ParentPid, Ec.message()); // If the pid is not reachable should we just shut down immediately? the intended owner process // could have been killed or somehow crashed already @@ -144,11 +170,9 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen // Initialize/check mutex based on base port - std::string MutexName = fmt::format("zen_{}", ServerOptions.BasePort); - - if (NamedMutex::Exists(MutexName) || ((m_ServerMutex.Create(MutexName) == false))) + if (m_ServerMutex.Create(MutexName) == false) { - throw std::runtime_error(fmt::format("Failed to create mutex '{}' - is another instance already running?", MutexName).c_str()); + ThrowLastError(fmt::format("Failed to create mutex '{}'", MutexName).c_str()); } InitializeState(ServerOptions); @@ -164,16 +188,28 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http = CreateHttpServer(ServerOptions.HttpServerConfig); int EffectiveBasePort = m_Http->Initialize(ServerOptions.BasePort, ServerOptions.DataDir); + if (EffectiveBasePort == 0) + { + ZEN_WARN("Failed to initialize http service '{}' using base port {} and data dir {}", + ServerOptions.HttpServerConfig.ServerClass, + ServerOptions.BasePort, + ServerOptions.DataDir); + return -1; + } // Setup authentication manager { + ZEN_TRACE_CPU("Zenserver::InitAuth"); std::string EncryptionKey = ServerOptions.EncryptionKey; if (EncryptionKey.empty()) { EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; - ZEN_WARN("using default encryption key"); + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption key for authentication state"); + } } std::string EncryptionIV = ServerOptions.EncryptionIV; @@ -182,7 +218,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen { EncryptionIV = "0123456789abcdef"; - ZEN_WARN("using default encryption initialization vector"); + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption initialization vector for authentication state"); + } } m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth", @@ -196,18 +235,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen } m_AuthService = std::make_unique<HttpAuthService>(*m_AuthMgr); - m_Http->RegisterService(*m_AuthService); - m_Http->RegisterService(m_HealthService); - - m_Http->RegisterService(m_StatsService); m_StatsReporter.Initialize(ServerOptions.StatsConfig); if (ServerOptions.StatsConfig.Enabled) { EnqueueStatsReportingTimer(); } - m_Http->RegisterService(m_StatusService); m_StatusService.RegisterHandler("status", *this); // Initialize storage and services @@ -222,34 +256,41 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ZEN_INFO("instantiating project service"); - m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, *m_JobQueue); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{}); + m_HttpProjectService.reset( + new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr, *m_OpenProcessCache, *m_JobQueue}); - if (ServerOptions.StructuredCacheConfig.Enabled) + if (ServerOptions.WorksSpacesConfig.Enabled) { - InitializeStructuredCache(ServerOptions); + m_Workspaces.reset(new Workspaces()); + m_HttpWorkspacesService.reset( + new HttpWorkspacesService(m_StatusService, + m_StatsService, + {.SystemRootDir = ServerOptions.SystemRootDir, + .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, + *m_Workspaces)); } - else + + if (ServerOptions.BuildStoreConfig.Enabled) { - ZEN_INFO("NOT instantiating structured cache service"); + CidStoreConfiguration BuildCidConfig; + BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; + m_BuildCidStore = std::make_unique<CidStore>(m_GcManager); + m_BuildCidStore->Initialize(BuildCidConfig); + + BuildStoreConfig BuildsCfg; + BuildsCfg.RootDirectory = m_DataRoot / "builds"; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; + m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); } - m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics - -#if ZEN_WITH_TESTS - m_Http->RegisterService(m_TestingService); -#endif - - if (m_HttpProjectService) + if (ServerOptions.StructuredCacheConfig.Enabled) { - m_Http->RegisterService(*m_HttpProjectService); + InitializeStructuredCache(ServerOptions); } - - m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); - - if (m_FrontendService) + else { - m_Http->RegisterService(*m_FrontendService); + ZEN_INFO("NOT instantiating structured cache service"); } if (ServerOptions.ObjectStoreEnabled) @@ -264,16 +305,21 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique<HttpObjectStoreService>(std::move(ObjCfg)); - m_Http->RegisterService(*m_ObjStoreService); + m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg)); + } + + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore); } #if ZEN_WITH_VFS - m_VfsService = std::make_unique<VfsService>(); - m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore)); - m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore)); - m_Http->RegisterService(*m_VfsService); -#endif + m_VfsServiceImpl = std::make_unique<VfsServiceImpl>(); + m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore)); + m_VfsServiceImpl->AddService(Ref<ZenCacheStore>(m_CacheStore)); + + m_VfsService = std::make_unique<VfsService>(m_StatusService, m_VfsServiceImpl.get()); +#endif // ZEN_WITH_VFS ZEN_INFO("initializing GC, enabled '{}', interval {}, lightweight interval {}", ServerOptions.GcConfig.Enabled, @@ -285,36 +331,97 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, .Enabled = ServerOptions.GcConfig.Enabled, .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, .DiskSizeSoftLimit = ServerOptions.GcConfig.DiskSizeSoftLimit, .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, .LightweightInterval = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds), - .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1, + .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1_Deprecated, .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, - .Verbose = ServerOptions.GcConfig.Verbose}; + .Verbose = ServerOptions.GcConfig.Verbose, + .SingleThreaded = ServerOptions.GcConfig.SingleThreaded, + .AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount}; m_GcScheduler.Initialize(GcConfig); // Create and register admin interface last to make sure all is properly initialized - m_AdminService = - std::make_unique<HttpAdminService>(m_GcScheduler, - *m_JobQueue, - m_CacheStore.Get(), - m_CidStore.get(), - m_ProjectStore, - HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, - .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", - .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, - ServerOptions); + m_AdminService = std::make_unique<HttpAdminService>( + m_GcScheduler, + *m_JobQueue, + m_CacheStore.Get(), + [this]() { Flush(); }, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, + ServerOptions); + + // Register all services when all initialization for all services are done + + m_Http->RegisterService(*m_AuthService); + + m_Http->RegisterService(m_StatsService); + m_Http->RegisterService(m_StatusService); + m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics + +#if ZEN_WITH_TESTS + m_Http->RegisterService(m_TestingService); +#endif + + if (m_StructuredCacheService) + { + m_Http->RegisterService(*m_StructuredCacheService); + } + + if (m_UpstreamService) + { + m_Http->RegisterService(*m_UpstreamService); + } + + if (m_HttpProjectService) + { + m_Http->RegisterService(*m_HttpProjectService); + } + + if (m_HttpWorkspacesService) + { + m_Http->RegisterService(*m_HttpWorkspacesService); + } + + m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService); + + if (m_FrontendService) + { + m_Http->RegisterService(*m_FrontendService); + } + + if (m_ObjStoreService) + { + m_Http->RegisterService(*m_ObjStoreService); + } + + if (m_BuildStoreService) + { + m_Http->RegisterService(*m_BuildStoreService); + } +#if ZEN_WITH_VFS + m_Http->RegisterService(*m_VfsService); +#endif // ZEN_WITH_VFS + m_Http->RegisterService(*m_AdminService); + // Register health service last so if we return "OK" for health it means all services have been properly initialized + + m_Http->RegisterService(m_HealthService); + return EffectiveBasePort; } void ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { + ZEN_TRACE_CPU("ZenServer::InitializeState"); + EnqueueSigIntTimer(); // Check root manifest to deal with schema versioning @@ -358,10 +465,10 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); ValidationResult != CbValidateError::None) { - ZEN_WARN("Manifest validation failed: {}, state will be wiped", uint32_t(ValidationResult)); + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); WipeState = true; - WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, uint32_t(ValidationResult)); + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); } else { @@ -374,7 +481,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -423,7 +530,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -447,9 +554,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (UpdateManifest) { - IoBuffer ManifestBuffer = m_RootManifest.GetBuffer().AsIoBuffer(); - - WriteFile(ManifestPath, ManifestBuffer); + TemporaryFile::SafeWriteFile(ManifestPath, m_RootManifest.GetBuffer().GetView()); } if (!ServerOptions.IsTest) @@ -474,11 +579,15 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) EnqueueStateMarkerTimer(); } + + EnqueueStateExitFlagTimer(); } void ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) { + ZEN_TRACE_CPU("ZenServer::InitializeStructuredCache"); + using namespace std::literals; ZEN_INFO("instantiating structured cache service"); @@ -486,8 +595,26 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.AllowAutomaticCreationOfNamespaces = true; Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.EnableReferenceCaching = ServerOptions.StructuredCacheConfig.EnableReferenceCaching; - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, + + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) + { + const std::string& BucketName = It.first; + const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; + ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, + .PayloadAlignment = ZenBucketConfig.PayloadAlignment, + .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, + .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; + Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); + } + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; @@ -497,7 +624,8 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = 128 * 1024 * 1024; } - m_CacheStore = new ZenCacheStore(m_GcManager, *m_JobQueue, m_DataRoot / "cache", Config, m_GcManager.GetDiskWriteBlocker()); + m_CacheStore = new ZenCacheStore(m_GcManager, *m_JobQueue, m_DataRoot / "cache", Config, m_GcManager.GetDiskWriteBlocker()); + m_OpenProcessCache = std::make_unique<OpenProcessCache>(); const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; @@ -555,12 +683,12 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) { std::string_view EndpointName = UpstreamConfig.JupiterConfig.Name.empty() ? "Jupiter"sv : UpstreamConfig.JupiterConfig.Name; - auto Options = CloudCacheClientOptions{.Name = EndpointName, - .ServiceUrl = UpstreamConfig.JupiterConfig.Url, - .DdcNamespace = UpstreamConfig.JupiterConfig.DdcNamespace, - .BlobStoreNamespace = UpstreamConfig.JupiterConfig.Namespace, - .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), - .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; + auto Options = JupiterClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.JupiterConfig.Url, + .DdcNamespace = UpstreamConfig.JupiterConfig.DdcNamespace, + .BlobStoreNamespace = UpstreamConfig.JupiterConfig.Namespace, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; auto AuthConfig = UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, @@ -579,13 +707,12 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) m_StatsService, m_StatusService, *m_UpstreamCache, - m_GcManager.GetDiskWriteBlocker()); - - m_Http->RegisterService(*m_StructuredCacheService); - m_Http->RegisterService(*m_UpstreamService); + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); m_StatsReporter.AddProvider(m_CacheStore.Get()); m_StatsReporter.AddProvider(m_CidStore.get()); + m_StatsReporter.AddProvider(m_BuildCidStore.get()); } void @@ -608,6 +735,14 @@ ZenServer::Run() } ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); + +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); + } +#endif + #if ZEN_USE_SENTRY ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); if (m_UseSentry) @@ -693,7 +828,6 @@ ZenServer::Run() if (m_IsPowerCycle) { ZEN_INFO("Power cycle mode enabled -- shutting down"); - RequestExit(0); } @@ -707,16 +841,19 @@ ZenServer::Run() void ZenServer::RequestExit(int ExitCode) { - RequestApplicationExit(ExitCode); - if (m_Http) + if (RequestApplicationExit(ExitCode)) { - m_Http->RequestExit(); + if (m_Http) + { + m_Http->RequestExit(); + } } } void ZenServer::Cleanup() { + ZEN_TRACE_CPU("ZenServer::Cleanup"); ZEN_INFO(ZEN_APP_NAME " cleaning up"); try { @@ -740,27 +877,36 @@ ZenServer::Cleanup() Flush(); - ShutdownWorkerPools(); - m_AdminService.reset(); m_VfsService.reset(); + m_VfsServiceImpl.reset(); m_ObjStoreService.reset(); m_FrontendService.reset(); + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_BuildCidStore.reset(); + m_StructuredCacheService.reset(); m_UpstreamService.reset(); m_UpstreamCache.reset(); m_CacheStore = {}; + m_OpenProcessCache.reset(); + m_HttpWorkspacesService.reset(); + m_Workspaces.reset(); m_HttpProjectService.reset(); m_ProjectStore = {}; m_CidStore.reset(); m_AuthService.reset(); m_AuthMgr.reset(); m_Http = {}; + + ShutdownWorkerPools(); + m_JobQueue.reset(); } - catch (std::exception& Ex) + catch (const std::exception& Ex) { ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); } @@ -769,6 +915,7 @@ ZenServer::Cleanup() void ZenServer::EnsureIoRunner() { + ZEN_MEMSCOPE(GetZenserverTag()); if (!m_IoRunner.joinable()) { m_IoRunner = std::thread{[this] { @@ -779,8 +926,9 @@ ZenServer::EnsureIoRunner() } void -ZenServer::EnqueueTimer() +ZenServer::EnqueueProcessMonitorTimer() { + ZEN_MEMSCOPE(GetZenserverTag()); m_PidCheckTimer.expires_after(std::chrono::seconds(1)); m_PidCheckTimer.async_wait([this](const asio::error_code&) { CheckOwnerPid(); }); @@ -790,6 +938,7 @@ ZenServer::EnqueueTimer() void ZenServer::EnqueueStateMarkerTimer() { + ZEN_MEMSCOPE(GetZenserverTag()); m_StateMakerTimer.expires_after(std::chrono::seconds(5)); m_StateMakerTimer.async_wait([this](const asio::error_code&) { CheckStateMarker(); }); EnsureIoRunner(); @@ -798,14 +947,25 @@ ZenServer::EnqueueStateMarkerTimer() void ZenServer::EnqueueSigIntTimer() { + ZEN_MEMSCOPE(GetZenserverTag()); m_SigIntTimer.expires_after(std::chrono::milliseconds(500)); m_SigIntTimer.async_wait([this](const asio::error_code&) { CheckSigInt(); }); EnsureIoRunner(); } void +ZenServer::EnqueueStateExitFlagTimer() +{ + ZEN_MEMSCOPE(GetZenserverTag()); + m_StateExitFlagTimer.expires_after(std::chrono::milliseconds(500)); + m_StateExitFlagTimer.async_wait([this](const asio::error_code&) { CheckStateExitFlag(); }); + EnsureIoRunner(); +} + +void ZenServer::EnqueueStatsReportingTimer() { + ZEN_MEMSCOPE(GetZenserverTag()); m_StatsReportingTimer.expires_after(std::chrono::milliseconds(500)); m_StatsReportingTimer.async_wait([this](const asio::error_code& Ec) { if (!Ec) @@ -820,17 +980,18 @@ ZenServer::EnqueueStatsReportingTimer() void ZenServer::CheckStateMarker() { + ZEN_MEMSCOPE(GetZenserverTag()); std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); return; } } - catch (std::exception& Ex) + catch (const std::exception& Ex) { ZEN_WARN("state marker at {} could not be checked, reason: '{}'", StateMarkerPath, Ex.what()); RequestExit(1); @@ -848,14 +1009,30 @@ ZenServer::CheckSigInt() RequestExit(128 + SIGINT); return; } + if (utils::SignalCounter[SIGTERM] > 0) + { + ZEN_INFO("SIGTERM triggered for process {}, exiting", zen::GetCurrentProcessId()); + RequestExit(128 + SIGTERM); + return; + } EnqueueSigIntTimer(); } void -ZenServer::CheckOwnerPid() +ZenServer::CheckStateExitFlag() { - // Pick up any new "owner" processes + if (m_ServerEntry && m_ServerEntry->IsShutdownRequested()) + { + RequestExit(0); + return; + } + EnqueueStateExitFlagTimer(); +} +bool +ZenServer::UpdateProcessMonitor() +{ + // Pick up any new "owner" processes std::set<uint32_t> AddedPids; for (auto& PidEntry : m_ServerEntry->SponsorPids) @@ -868,48 +1045,46 @@ ZenServer::CheckOwnerPid() { m_ProcessMonitor.AddPid(ThisPid); - ZEN_INFO("added process with pid #{} as a sponsor process", ThisPid); + ZEN_INFO("added process with pid {} as a sponsor process", ThisPid); } } } } + return m_ProcessMonitor.IsRunning(); +} - if (m_ProcessMonitor.IsRunning()) +void +ZenServer::CheckOwnerPid() +{ + bool IsRunning = UpdateProcessMonitor(); + + if (IsRunning) { - EnqueueTimer(); + m_FoundNoActiveSponsors = false; + EnqueueProcessMonitorTimer(); } else { - ZEN_INFO(ZEN_APP_NAME " exiting since sponsor processes are all gone"); - - RequestExit(0); + // Delay exit one iteration to avoid race conditions where one process detaches + // and another attaches + if (m_FoundNoActiveSponsors) + { + ZEN_INFO(ZEN_APP_NAME " exiting since sponsor processes are all gone"); + RequestExit(0); + } + else + { + m_FoundNoActiveSponsors = true; + EnqueueProcessMonitorTimer(); + } } } void -ZenServer::ScrubStorage() -{ - Stopwatch Timer; - ZEN_INFO("Storage validation STARTING"); - - WorkerThreadPool ThreadPool{1, "Scrub"}; - ScrubContext Ctx{ThreadPool}; - m_CidStore->ScrubStorage(Ctx); - m_ProjectStore->ScrubStorage(Ctx); - m_StructuredCacheService->ScrubStorage(Ctx); - - const uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); - - ZEN_INFO("Storage validation DONE in {}, ({} in {} chunks - {})", - NiceTimeSpanMs(ElapsedTimeMs), - NiceBytes(Ctx.ScrubbedBytes()), - Ctx.ScrubbedChunks(), - NiceByteRate(Ctx.ScrubbedBytes(), ElapsedTimeMs)); -} - -void ZenServer::Flush() { + ZEN_TRACE_CPU("ZenServer::Flush"); + if (m_CidStore) m_CidStore->Flush(); @@ -918,6 +1093,9 @@ ZenServer::Flush() if (m_ProjectStore) m_ProjectStore->Flush(); + + if (m_BuildCidStore) + m_BuildCidStore->Flush(); } void @@ -945,4 +1123,13 @@ ZenServer::ToString(ServerState Value) } } +#if ZEN_WITH_TESTS + +void +zenserver_forcelinktests() +{ +} + +#endif + } // namespace zen diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 6ff13cfff..ba76c5fff 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -24,16 +24,18 @@ ZEN_THIRD_PARTY_INCLUDES_END #include <zenhttp/httptest.h> #include <zenstore/cache/structuredcachestore.h> #include <zenstore/gc.h> +#include <zenstore/projectstore.h> #include "admin/admin.h" +#include "buildstore/httpbuildstore.h" #include "cache/httpstructuredcache.h" #include "diag/diagsvcs.h" #include "frontend/frontend.h" #include "objectstore/objectstore.h" #include "projectstore/httpprojectstore.h" -#include "projectstore/projectstore.h" #include "stats/statsreporter.h" #include "upstream/upstream.h" #include "vfs/vfsservice.h" +#include "workspaces/httpworkspaces.h" #ifndef ZEN_APP_NAME # define ZEN_APP_NAME "Unreal Zen Storage Server" @@ -55,7 +57,6 @@ public: int Initialize(const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); void InitializeState(const ZenServerOptions& ServerOptions); void InitializeStructuredCache(const ZenServerOptions& ServerOptions); - void InitializeCompute(const ZenServerOptions& ServerOptions); void Run(); void RequestExit(int ExitCode); @@ -71,15 +72,16 @@ public: void OnReady(); void EnsureIoRunner(); - void EnqueueTimer(); + void EnqueueProcessMonitorTimer(); void EnqueueStateMarkerTimer(); void EnqueueSigIntTimer(); + void EnqueueStateExitFlagTimer(); void EnqueueStatsReportingTimer(); void CheckStateMarker(); void CheckSigInt(); + void CheckStateExitFlag(); void CheckOwnerPid(); - - void ScrubStorage(); + bool UpdateProcessMonitor(); void Flush(); virtual void HandleStatusRequest(HttpServerRequest& Request) override; @@ -96,10 +98,12 @@ private: asio::io_context m_IoContext; asio::steady_timer m_PidCheckTimer{m_IoContext}; asio::steady_timer m_StateMakerTimer{m_IoContext}; + asio::steady_timer m_StateExitFlagTimer{m_IoContext}; asio::steady_timer m_SigIntTimer{m_IoContext}; asio::steady_timer m_StatsReportingTimer{m_IoContext}; ProcessMonitor m_ProcessMonitor; NamedMutex m_ServerMutex; + bool m_FoundNoActiveSponsors = false; enum ServerState { @@ -111,28 +115,36 @@ private: inline void SetNewState(ServerState NewState) { m_CurrentState = NewState; } static std::string_view ToString(ServerState Value); - StatsReporter m_StatsReporter; - Ref<HttpServer> m_Http; - std::unique_ptr<AuthMgr> m_AuthMgr; - std::unique_ptr<HttpAuthService> m_AuthService; - HttpStatusService m_StatusService; - HttpStatsService m_StatsService; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - std::unique_ptr<CidStore> m_CidStore; - Ref<ZenCacheStore> m_CacheStore; - HttpTestService m_TestService; + StatsReporter m_StatsReporter; + Ref<HttpServer> m_Http; + std::unique_ptr<AuthMgr> m_AuthMgr; + std::unique_ptr<HttpAuthService> m_AuthService; + HttpStatusService m_StatusService; + HttpStatsService m_StatsService; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr<CidStore> m_CidStore; + Ref<ZenCacheStore> m_CacheStore; + std::unique_ptr<OpenProcessCache> m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr<CidStore> m_BuildCidStore; + std::unique_ptr<BuildStore> m_BuildStore; + #if ZEN_WITH_TESTS HttpTestingService m_TestingService; #endif RefPtr<ProjectStore> m_ProjectStore; + std::unique_ptr<VfsServiceImpl> m_VfsServiceImpl; std::unique_ptr<HttpProjectService> m_HttpProjectService; + std::unique_ptr<Workspaces> m_Workspaces; + std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService; std::unique_ptr<UpstreamCache> m_UpstreamCache; std::unique_ptr<HttpUpstreamService> m_UpstreamService; std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService; HttpHealthService m_HealthService; std::unique_ptr<HttpFrontendService> m_FrontendService; std::unique_ptr<HttpObjectStoreService> m_ObjStoreService; + std::unique_ptr<HttpBuildStoreService> m_BuildStoreService; std::unique_ptr<VfsService> m_VfsService; std::unique_ptr<JobQueue> m_JobQueue; std::unique_ptr<HttpAdminService> m_AdminService; @@ -143,4 +155,6 @@ private: std::string m_StartupScrubOptions; }; +void zenserver_forcelinktests(); + } // namespace zen |