diff options
Diffstat (limited to 'src/zenserver/storage')
19 files changed, 495 insertions, 423 deletions
diff --git a/src/zenserver/storage/admin/admin.h b/src/zenserver/storage/admin/admin.h index ee3da4579..361153e42 100644 --- a/src/zenserver/storage/admin/admin.h +++ b/src/zenserver/storage/admin/admin.h @@ -13,7 +13,7 @@ class JobQueue; class ZenCacheStore; struct ZenServerConfig; -class HttpAdminService : public zen::HttpService +class HttpAdminService : public HttpService { public: struct LogPaths @@ -31,7 +31,7 @@ public: ~HttpAdminService(); virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleRequest(HttpServerRequest& Request) override; private: HttpRequestRouter m_Router; diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index de9589078..f935e2c6b 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -162,96 +162,81 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) fmt::format("Invalid blob hash '{}'", Hash)); } - std::vector<std::pair<uint64_t, uint64_t>> OffsetAndLengthPairs; + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("Blob {} not found", Hash)); + } + m_BuildStoreStats.BlobHitCount++; + if (ServerRequest.RequestVerb() == HttpVerb::kPost) { + if (ServerRequest.AcceptContentType() != HttpContentType::kCbPackage) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Accept type '{}' is not supported for blob {}, expected '{}'", + ToString(ServerRequest.AcceptContentType()), + Hash, + ToString(HttpContentType::kCbPackage))); + } + CbObject RangePayload = ServerRequest.ReadPayloadObject(); - if (RangePayload) + if (!RangePayload) { - CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView(); - OffsetAndLengthPairs.reserve(RangesArray.Num()); - for (CbFieldView FieldView : RangesArray) - { - CbObjectView RangeView = FieldView.AsObjectView(); - uint64_t RangeOffset = RangeView["offset"sv].AsUInt64(); - uint64_t RangeLength = RangeView["length"sv].AsUInt64(); - OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength)); - } - if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported) - { - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}", - OffsetAndLengthPairs.size(), - MaxRangeCountPerRequestSupported)); - } + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Missing payload for range request on blob {}", BlobHash)); } - if (OffsetAndLengthPairs.empty()) + + CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView(); + const uint64_t RangeCount = RangesArray.Num(); + if (RangeCount == 0) { m_BuildStoreStats.BadRequestCount++; return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, - "Fetching blob without ranges must be done with the GET verb"); + "POST request must include a non-empty 'ranges' array"); } - } - else - { - HttpRanges Ranges; - bool HasRange = ServerRequest.TryGetRanges(Ranges); - if (HasRange) + if (RangeCount > MaxRangeCountPerRequestSupported) { - if (Ranges.size() > 1) - { - // Only a single http range is supported, we have limited support for http multirange responses - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Multiple ranges in blob request is only supported for {} accept type", - ToString(HttpContentType::kCbPackage))); - } - const HttpRange& FirstRange = Ranges.front(); - OffsetAndLengthPairs.push_back(std::make_pair<uint64_t, uint64_t>(FirstRange.Start, FirstRange.End - FirstRange.Start + 1)); + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Range count {} exceeds maximum of {}", RangeCount, MaxRangeCountPerRequestSupported)); } - } - - m_BuildStoreStats.BlobReadCount++; - IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); - if (!Blob) - { - return ServerRequest.WriteResponse(HttpResponseCode::NotFound, - HttpContentType::kText, - fmt::format("Blob with hash '{}' could not be found", Hash)); - } - m_BuildStoreStats.BlobHitCount++; - if (OffsetAndLengthPairs.empty()) - { - return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); - } + const uint64_t BlobSize = Blob.GetSize(); + std::vector<IoBuffer> RangeBuffers; + RangeBuffers.reserve(RangeCount); - if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage) - { - const uint64_t BlobSize = Blob.GetSize(); + CbPackage ResponsePackage; + CbObjectWriter Writer; - CbPackage ResponsePackage; - std::vector<IoBuffer> RangeBuffers; - CbObjectWriter Writer; Writer.BeginArray("ranges"sv); - for (const std::pair<uint64_t, uint64_t>& Range : OffsetAndLengthPairs) + for (CbFieldView FieldView : RangesArray) { - const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0; - const uint64_t RangeSize = Min(Range.second, MaxBlobSize); + CbObjectView RangeView = FieldView.AsObjectView(); + uint64_t RangeOffset = RangeView["offset"sv].AsUInt64(); + uint64_t RangeLength = RangeView["length"sv].AsUInt64(); + + const uint64_t MaxBlobSize = RangeOffset < BlobSize ? BlobSize - RangeOffset : 0; + const uint64_t RangeSize = Min(RangeLength, MaxBlobSize); Writer.BeginObject(); { - if (Range.first + RangeSize <= BlobSize) + if (RangeOffset + RangeSize <= BlobSize) { - RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize)); - Writer.AddInteger("offset"sv, Range.first); + RangeBuffers.push_back(IoBuffer(Blob, RangeOffset, RangeSize)); + Writer.AddInteger("offset"sv, RangeOffset); Writer.AddInteger("length"sv, RangeSize); } else { - Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("offset"sv, RangeOffset); Writer.AddInteger("length"sv, 0); } } @@ -259,7 +244,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) } Writer.EndArray(); - CompositeBuffer Ranges(RangeBuffers); + CompositeBuffer Ranges(std::move(RangeBuffers)); CbAttachment PayloadAttachment(std::move(Ranges), BlobHash); Writer.AddAttachment("payload", PayloadAttachment); @@ -269,32 +254,21 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) ResponsePackage.SetObject(HeaderObject); CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); - uint64_t ResponseSize = RpcResponseBuffer.GetSize(); - ZEN_UNUSED(ResponseSize); return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); } else { - if (OffsetAndLengthPairs.size() != 1) + HttpRanges RequestedRangeHeader; + bool HasRange = ServerRequest.TryGetRanges(RequestedRangeHeader); + if (HasRange) { - // Only a single http range is supported, we have limited support for http multirange responses - m_BuildStoreStats.BadRequestCount++; - return ServerRequest.WriteResponse( - HttpResponseCode::BadRequest, - HttpContentType::kText, - fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage))); + // Standard HTTP GET with Range header: framework handles 206, Content-Range, and 416 on OOB. + return ServerRequest.WriteResponse(HttpContentType::kBinary, Blob, RequestedRangeHeader); } - - const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengthPairs.front(); - const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0; - const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize); - if (OffsetAndLength.first + RangeSize > BlobSize) + else { - return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); } - Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize); - return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); } } @@ -605,6 +579,26 @@ HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); } +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Cbo.BeginObject("capabilities"); + { + Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported; + } + Cbo.EndObject(); // capabilities + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + Request.WriteResponse(HttpResponseCode::OK, CollectStats()); +} + CbObject HttpBuildStoreService::CollectStats() { @@ -663,24 +657,10 @@ HttpBuildStoreService::CollectStats() return Cbo.Save(); } -void -HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) -{ - Request.WriteResponse(HttpResponseCode::OK, CollectStats()); -} - -void -HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +uint64_t +HttpBuildStoreService::GetActivityCounter() { - ZEN_TRACE_CPU("HttpBuildStoreService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Cbo.BeginObject("capabilities"); - { - Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported; - } - Cbo.EndObject(); // capabilities - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + return m_HttpRequests.Count(); } } // namespace zen diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h index 2a09b71cf..864d12edc 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.h +++ b/src/zenserver/storage/buildstore/httpbuildstore.h @@ -13,18 +13,19 @@ namespace zen { class BuildStore; -class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider +class HttpBuildStoreService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); virtual ~HttpBuildStoreService(); virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleRequest(HttpServerRequest& Request) override; - virtual CbObject CollectStats() override; - virtual void HandleStatsRequest(HttpServerRequest& Request) override; virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; private: struct BuildStoreStats diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp index bbdb03ba4..8ad48225b 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.cpp +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -80,7 +80,8 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach HttpStatusService& StatusService, UpstreamCache& UpstreamCache, const DiskWriteBlocker* InDiskWriteBlocker, - OpenProcessCache& InOpenProcessCache) + OpenProcessCache& InOpenProcessCache, + const ILocalRefPolicy* InLocalRefPolicy) : m_Log(logging::Get("cache")) , m_CacheStore(InCacheStore) , m_StatsService(StatsService) @@ -90,6 +91,7 @@ HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCach , m_DiskWriteBlocker(InDiskWriteBlocker) , m_OpenProcessCache(InOpenProcessCache) , m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) +, m_LocalRefPolicy(InLocalRefPolicy) { m_StatsService.RegisterHandler("z$", *this); m_StatusService.RegisterHandler("z$", *this); @@ -114,6 +116,18 @@ HttpStructuredCacheService::BaseUri() const return "/z$/"; } +bool +HttpStructuredCacheService::AcceptsLocalFileReferences() const +{ + return true; +} + +const ILocalRefPolicy* +HttpStructuredCacheService::GetLocalRefPolicy() const +{ + return m_LocalRefPolicy; +} + void HttpStructuredCacheService::Flush() { @@ -1827,113 +1841,12 @@ HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::st } } -CbObject -HttpStructuredCacheService::CollectStats() +void +HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) { - ZEN_MEMSCOPE(GetCacheHttpTag()); - CbObjectWriter Cbo; - - EmitSnapshot("requests", m_HttpRequests, Cbo); - - const uint64_t HitCount = m_CacheStats.HitCount; - const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; - const uint64_t MissCount = m_CacheStats.MissCount; - const uint64_t WriteCount = m_CacheStats.WriteCount; - const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; - struct CidStoreStats StoreStats = m_CidStore.Stats(); - const uint64_t ChunkHitCount = StoreStats.HitCount; - const uint64_t ChunkMissCount = StoreStats.MissCount; - const uint64_t ChunkWriteCount = StoreStats.WriteCount; - const uint64_t TotalCount = HitCount + MissCount; - - const uint64_t RpcRequests = m_CacheStats.RpcRequests; - const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; - const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; - const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; - const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; - const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; - const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; - - const CidStoreSize CidSize = m_CidStore.TotalSize(); - const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); - - Cbo.BeginObject("cache"); - { - Cbo << "badrequestcount" << BadRequestCount; - Cbo.BeginObject("rpc"); - Cbo << "count" << RpcRequests; - Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; - Cbo.BeginObject("records"); - Cbo << "count" << RpcRecordRequests; - Cbo << "ops" << RpcRecordBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("values"); - Cbo << "count" << RpcValueRequests; - Cbo << "ops" << RpcValueBatchRequests; - Cbo.EndObject(); - Cbo.BeginObject("chunks"); - Cbo << "count" << RpcChunkRequests; - Cbo << "ops" << RpcChunkBatchRequests; - Cbo.EndObject(); - Cbo.EndObject(); - - Cbo.BeginObject("size"); - { - Cbo << "disk" << CacheSize.DiskSize; - Cbo << "memory" << CacheSize.MemorySize; - } - Cbo.EndObject(); - - Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; - Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); - - if (m_UpstreamCache.IsActive()) - { - Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); - Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; - } - - Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; - - { - ZenCacheStore::CacheStoreStats StoreStatsData = m_CacheStore.Stats(); - Cbo.BeginObject("store"); - Cbo << "hits" << StoreStatsData.HitCount << "misses" << StoreStatsData.MissCount << "writes" << StoreStatsData.WriteCount - << "rejected_writes" << StoreStatsData.RejectedWriteCount << "rejected_reads" << StoreStatsData.RejectedReadCount; - const uint64_t StoreTotal = StoreStatsData.HitCount + StoreStatsData.MissCount; - Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(StoreStatsData.HitCount) / double(StoreTotal)) : 0.0); - EmitSnapshot("read", StoreStatsData.GetOps, Cbo); - EmitSnapshot("write", StoreStatsData.PutOps, Cbo); - Cbo.EndObject(); - } - } - Cbo.EndObject(); - - if (m_UpstreamCache.IsActive()) - { - EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); - Cbo.BeginObject("upstream"); - { - m_UpstreamCache.GetStatus(Cbo); - } - Cbo.EndObject(); - } - - Cbo.BeginObject("cid"); - { - Cbo.BeginObject("size"); - { - Cbo << "tiny" << CidSize.TinySize; - Cbo << "small" << CidSize.SmallSize; - Cbo << "large" << CidSize.LargeSize; - Cbo << "total" << CidSize.TotalSize; - } - Cbo.EndObject(); - } - Cbo.EndObject(); - - return Cbo.Save(); + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } void @@ -1944,12 +1857,6 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; - if (!ShowCidStoreStats && !ShowCacheStoreStats) - { - Request.WriteResponse(HttpResponseCode::OK, CollectStats()); - return; - } - // Full stats with optional detailed store/cid breakdowns CbObjectWriter Cbo; @@ -2156,12 +2063,38 @@ HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -void -HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) +CbObject +HttpStructuredCacheService::CollectStats() { + ZEN_TRACE_CPU("HttpStructuredCacheService::Stats"); + ZEN_MEMSCOPE(GetCacheHttpTag()); + CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); + + Cbo.BeginObject("cache"); + { + Cbo.BeginObject("size"); + { + Cbo << "disk" << CacheSize.DiskSize; + Cbo << "memory" << CacheSize.MemorySize; + } + Cbo.EndObject(); + + Cbo << "hits" << m_CacheStats.HitCount << "misses" << m_CacheStats.MissCount; + } + Cbo.EndObject(); + + return Cbo.Save(); +} + +uint64_t +HttpStructuredCacheService::GetActivityCounter() +{ + return m_HttpRequests.Count(); } bool diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h index d462415d4..f606126d6 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.h +++ b/src/zenserver/storage/cache/httpstructuredcache.h @@ -76,11 +76,14 @@ public: HttpStatusService& StatusService, UpstreamCache& UpstreamCache, const DiskWriteBlocker* InDiskWriteBlocker, - OpenProcessCache& InOpenProcessCache); + OpenProcessCache& InOpenProcessCache, + const ILocalRefPolicy* InLocalRefPolicy = nullptr); ~HttpStructuredCacheService(); - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + virtual bool AcceptsLocalFileReferences() const override; + virtual const ILocalRefPolicy* GetLocalRefPolicy() const override; void Flush(); @@ -105,9 +108,10 @@ private: void HandleCacheRequest(HttpServerRequest& Request); void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace); void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket); - virtual CbObject CollectStats() override; - virtual void HandleStatsRequest(HttpServerRequest& Request) override; virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; bool AreDiskWritesAllowed() const; @@ -124,6 +128,7 @@ private: const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; OpenProcessCache& m_OpenProcessCache; CacheRpcHandler m_RpcHandler; + const ILocalRefPolicy* m_LocalRefPolicy = nullptr; void ReplayRequestRecorder(const CacheRequestContext& Context, cache::IRpcRequestReplayer& Replayer, uint32_t ThreadCount); diff --git a/src/zenserver/storage/localrefpolicy.cpp b/src/zenserver/storage/localrefpolicy.cpp new file mode 100644 index 000000000..47ef13b28 --- /dev/null +++ b/src/zenserver/storage/localrefpolicy.cpp @@ -0,0 +1,29 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "localrefpolicy.h" + +#include <zencore/except_fmt.h> +#include <zencore/fmtutils.h> + +#include <filesystem> + +namespace zen { + +DataRootLocalRefPolicy::DataRootLocalRefPolicy(const std::filesystem::path& DataRoot) +: m_CanonicalRoot(std::filesystem::weakly_canonical(DataRoot).string()) +{ +} + +void +DataRootLocalRefPolicy::ValidatePath(const std::filesystem::path& Path) const +{ + std::filesystem::path CanonicalFile = std::filesystem::weakly_canonical(Path); + std::string FileStr = CanonicalFile.string(); + + if (FileStr.size() < m_CanonicalRoot.size() || FileStr.compare(0, m_CanonicalRoot.size(), m_CanonicalRoot) != 0) + { + throw zen::invalid_argument("local file reference '{}' is outside allowed data root", CanonicalFile); + } +} + +} // namespace zen diff --git a/src/zenserver/storage/localrefpolicy.h b/src/zenserver/storage/localrefpolicy.h new file mode 100644 index 000000000..3686d1880 --- /dev/null +++ b/src/zenserver/storage/localrefpolicy.h @@ -0,0 +1,25 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/localrefpolicy.h> + +#include <filesystem> +#include <string> + +namespace zen { + +/// Local ref policy that restricts file paths to a canonical data root directory. +/// Uses weakly_canonical + string prefix comparison to detect path traversal. +class DataRootLocalRefPolicy : public ILocalRefPolicy +{ +public: + explicit DataRootLocalRefPolicy(const std::filesystem::path& DataRoot); + + void ValidatePath(const std::filesystem::path& Path) const override; + +private: + std::string m_CanonicalRoot; +}; + +} // namespace zen diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp index 493326a32..1115c1cd6 100644 --- a/src/zenserver/storage/objectstore/objectstore.cpp +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -14,6 +14,7 @@ #include "zencore/compactbinarybuilder.h" #include "zenhttp/httpcommon.h" #include "zenhttp/httpserver.h" +#include "zenhttp/httpstats.h" #include <filesystem> #include <thread> @@ -220,17 +221,20 @@ private: StringBuilderBase& Builder; }; -HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) -: m_StatusService(StatusService) +HttpObjectStoreService::HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatsService(StatsService) +, m_StatusService(StatusService) , m_Cfg(std::move(Cfg)) { - Inititalize(); + Initialize(); + m_StatsService.RegisterHandler("obj", *this); m_StatusService.RegisterHandler("obj", *this); } HttpObjectStoreService::~HttpObjectStoreService() { m_StatusService.UnregisterHandler("obj", *this); + m_StatsService.UnregisterHandler("obj", *this); } const char* @@ -240,8 +244,10 @@ HttpObjectStoreService::BaseUri() const } void -HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) +HttpObjectStoreService::HandleRequest(HttpServerRequest& Request) { + metrics::OperationTiming::Scope $(m_HttpRequests); + if (m_Router.HandleRequest(Request) == false) { ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); @@ -258,12 +264,36 @@ HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) } void -HttpObjectStoreService::Inititalize() +HttpObjectStoreService::HandleStatsRequest(HttpServerRequest& Request) { - ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); + Request.WriteResponse(HttpResponseCode::OK, CollectStats()); +} + +CbObject +HttpObjectStoreService::CollectStats() +{ + ZEN_TRACE_CPU("HttpObjectStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + Cbo << "total_bytes_served" << m_TotalBytesServed.load(); + + return Cbo.Save(); +} + +uint64_t +HttpObjectStoreService::GetActivityCounter() +{ + return m_HttpRequests.Count(); +} + +void +HttpObjectStoreService::Initialize() +{ + ZEN_TRACE_CPU("HttpObjectStoreService::Initialize"); namespace fs = std::filesystem; - ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); + ZEN_LOG_INFO(LogObj, "Initializing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; if (!IsDir(BucketsPath)) @@ -281,27 +311,27 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "", - [this](zen::HttpRouterRequest& Request) { ListBuckets(Request); }, + [this](HttpRouterRequest& Request) { ListBuckets(Request); }, HttpVerb::kGet); m_Router.RegisterRoute( "bucket", - [this](zen::HttpRouterRequest& Request) { ListBuckets(Request); }, + [this](HttpRouterRequest& Request) { ListBuckets(Request); }, HttpVerb::kGet); m_Router.RegisterRoute( "bucket", - [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, + [this](HttpRouterRequest& Request) { CreateBucket(Request); }, HttpVerb::kPost | HttpVerb::kPut); m_Router.RegisterRoute( "bucket", - [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, + [this](HttpRouterRequest& Request) { DeleteBucket(Request); }, HttpVerb::kDelete); m_Router.RegisterRoute( "bucket/{path}", - [this](zen::HttpRouterRequest& Request) { + [this](HttpRouterRequest& Request) { const std::string_view Path = Request.GetCapture(1); const auto Sep = Path.find_last_of('.'); const bool IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0; @@ -319,7 +349,7 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "bucket/{bucket}/{path}", - [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, + [this](HttpRouterRequest& Request) { PutObject(Request); }, HttpVerb::kPost | HttpVerb::kPut); } @@ -327,7 +357,7 @@ std::filesystem::path HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) { { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), std::end(m_Cfg.Buckets), @@ -342,7 +372,7 @@ HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) } void -HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request) +HttpObjectStoreService::ListBuckets(HttpRouterRequest& Request) { namespace fs = std::filesystem; @@ -351,7 +381,7 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request) CbObjectWriter Response; Response.BeginArray("buckets"); { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); // Configured buckets for (const ObjectStoreConfig::BucketConfig& Bucket : m_Cfg.Buckets) @@ -428,13 +458,13 @@ HttpObjectStoreService::ListBuckets(zen::HttpRouterRequest& Request) } Response.EndArray(); - Response << "total_bytes_served" << TotalBytesServed.load(); + Response << "total_bytes_served" << m_TotalBytesServed.load(); return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); } void -HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) +HttpObjectStoreService::CreateBucket(HttpRouterRequest& Request) { namespace fs = std::filesystem; @@ -448,7 +478,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); @@ -462,7 +492,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) +HttpObjectStoreService::ListBucket(HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; @@ -533,7 +563,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s if (IsDir(FullPath)) { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); } CbObject Result = FileVisitor.GetResult(); @@ -552,7 +582,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s } void -HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) +HttpObjectStoreService::DeleteBucket(HttpRouterRequest& Request) { namespace fs = std::filesystem; @@ -566,7 +596,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); DeleteDirectories(BucketPath); } @@ -575,7 +605,7 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) +HttpObjectStoreService::GetObject(HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; @@ -606,16 +636,12 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } - zen::HttpRanges Ranges; - if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) - { - // Only a single range is supported - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); - } + HttpRanges Ranges; + Request.ServerRequest().TryGetRanges(Ranges); FileContents File; { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); File = ReadFile(FilePath); } @@ -635,46 +661,53 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st if (Ranges.empty()) { - const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); - + const uint64_t TotalServed = m_TotalBytesServed.fetch_add(FileBuf.GetSize()) + FileBuf.GetSize(); ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' ({}) [OK] (Served: {})", BucketName, RelativeBucketPath, - NiceBytes(FileBuf.Size()), + NiceBytes(FileBuf.GetSize()), NiceBytes(TotalServed)); - - Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); } else { - const auto Range = Ranges[0]; - const uint64_t RangeSize = 1 + (Range.End - Range.Start); - const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; - - ZEN_LOG_DEBUG(LogObj, - "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", - BucketName, - RelativeBucketPath, - Range.Start, - Range.End, - NiceBytes(RangeSize), - NiceBytes(FileBuf.Size()), - NiceBytes(TotalServed)); - - MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); - if (RangeView.GetSize() != RangeSize) + const uint64_t TotalSize = FileBuf.GetSize(); + uint64_t ServedBytes = 0; + for (const HttpRange& Range : Ranges) { - return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + const uint64_t RangeEnd = (Range.End != ~uint64_t(0)) ? Range.End : TotalSize - 1; + if (RangeEnd < TotalSize && Range.Start <= RangeEnd) + { + ServedBytes += 1 + (RangeEnd - Range.Start); + } + } + if (ServedBytes > 0) + { + const uint64_t TotalServed = m_TotalBytesServed.fetch_add(ServedBytes) + ServedBytes; + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' (Ranges: {}) ({}/{}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + Ranges.size(), + NiceBytes(ServedBytes), + NiceBytes(TotalSize), + NiceBytes(TotalServed)); + } + else + { + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' (Ranges: {}) [416] ({})", + BucketName, + RelativeBucketPath, + Ranges.size(), + NiceBytes(TotalSize)); } - - IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); - Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); } + Request.ServerRequest().WriteResponse(HttpContentType::kBinary, FileBuf, Ranges); } void -HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) +HttpObjectStoreService::PutObject(HttpRouterRequest& Request) { namespace fs = std::filesystem; @@ -699,7 +732,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) const fs::path FileDirectory = FilePath.parent_path(); { - std::lock_guard _(BucketsMutex); + std::lock_guard _(m_BucketsMutex); if (!IsDir(FileDirectory)) { diff --git a/src/zenserver/storage/objectstore/objectstore.h b/src/zenserver/storage/objectstore/objectstore.h index cc47b50c4..f51254357 100644 --- a/src/zenserver/storage/objectstore/objectstore.h +++ b/src/zenserver/storage/objectstore/objectstore.h @@ -11,6 +11,7 @@ namespace zen { class HttpRouterRequest; +class HttpStatsService; struct ObjectStoreConfig { @@ -24,31 +25,36 @@ struct ObjectStoreConfig std::vector<BucketConfig> Buckets; }; -class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider +class HttpObjectStoreService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); + HttpObjectStoreService(HttpStatsService& StatsService, HttpStatusService& StatusService, ObjectStoreConfig Cfg); virtual ~HttpObjectStoreService(); virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; private: - void Inititalize(); + void Initialize(); std::filesystem::path GetBucketDirectory(std::string_view BucketName); - void ListBuckets(zen::HttpRouterRequest& Request); - void CreateBucket(zen::HttpRouterRequest& Request); - void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); - void DeleteBucket(zen::HttpRouterRequest& Request); - void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); - void PutObject(zen::HttpRouterRequest& Request); - - HttpStatusService& m_StatusService; - ObjectStoreConfig m_Cfg; - std::mutex BucketsMutex; - HttpRequestRouter m_Router; - std::atomic_uint64_t TotalBytesServed{0}; + void ListBuckets(HttpRouterRequest& Request); + void CreateBucket(HttpRouterRequest& Request); + void ListBucket(HttpRouterRequest& Request, const std::string_view Path); + void DeleteBucket(HttpRouterRequest& Request); + void GetObject(HttpRouterRequest& Request, const std::string_view Path); + void PutObject(HttpRouterRequest& Request); + + HttpStatsService& m_StatsService; + HttpStatusService& m_StatusService; + ObjectStoreConfig m_Cfg; + std::mutex m_BucketsMutex; + HttpRequestRouter m_Router; + std::atomic_uint64_t m_TotalBytesServed{0}; + metrics::OperationTiming m_HttpRequests; }; } // namespace zen diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 03b8aa382..9844d02f0 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -16,9 +16,9 @@ #include <zenhttp/httpclientauth.h> #include <zenhttp/packageformat.h> #include <zenremotestore/builds/buildstoragecache.h> +#include <zenremotestore/builds/buildstorageresolve.h> #include <zenremotestore/builds/buildstorageutil.h> #include <zenremotestore/jupiter/jupiterhost.h> -#include <zenremotestore/operationlogoutput.h> #include <zenremotestore/projectstore/buildsremoteprojectstore.h> #include <zenremotestore/projectstore/fileremoteprojectstore.h> #include <zenremotestore/projectstore/jupiterremoteprojectstore.h> @@ -279,7 +279,7 @@ namespace { { ZEN_MEMSCOPE(GetProjectHttpTag()); - auto Log = [InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -566,11 +566,9 @@ namespace { .AllowResume = true, .RetryCount = 2}; - std::unique_ptr<OperationLogOutput> Output(CreateStandardLogOutput(Log())); - try { - ResolveResult = ResolveBuildStorage(*Output, + ResolveResult = ResolveBuildStorage(Log(), ClientSettings, Host, OverrideHost, @@ -636,11 +634,6 @@ namespace { return Result; } - static uint64_t GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory) - { - return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u; - } - } // namespace ////////////////////////////////////////////////////////////////////////// @@ -656,7 +649,8 @@ HttpProjectService::HttpProjectService(CidStore& Store, JobQueue& InJobQueue, bool InRestrictContentTypes, const std::filesystem::path& InOidcTokenExePath, - bool InAllowExternalOidcTokenExe) + bool InAllowExternalOidcTokenExe, + const ILocalRefPolicy* InLocalRefPolicy) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) @@ -668,6 +662,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, , m_RestrictContentTypes(InRestrictContentTypes) , m_OidcTokenExePath(InOidcTokenExePath) , m_AllowExternalOidcTokenExe(InAllowExternalOidcTokenExe) +, m_LocalRefPolicy(InLocalRefPolicy) { ZEN_MEMSCOPE(GetProjectHttpTag()); @@ -785,22 +780,22 @@ HttpProjectService::HttpProjectService(CidStore& Store, HttpVerb::kPost); m_Router.RegisterRoute( - "details\\$", + "details$", [this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}", + "details$/{project}", [this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}/{log}", + "details$/{project}/{log}", [this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); }, HttpVerb::kGet); m_Router.RegisterRoute( - "details\\$/{project}/{log}/{chunk}", + "details$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); @@ -820,6 +815,18 @@ HttpProjectService::BaseUri() const return "/prj/"; } +bool +HttpProjectService::AcceptsLocalFileReferences() const +{ + return true; +} + +const ILocalRefPolicy* +HttpProjectService::GetLocalRefPolicy() const +{ + return m_LocalRefPolicy; +} + void HttpProjectService::HandleRequest(HttpServerRequest& Request) { @@ -836,8 +843,17 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) } } -CbObject -HttpProjectService::CollectStats() +void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) { ZEN_TRACE_CPU("ProjectService::Stats"); @@ -848,6 +864,8 @@ HttpProjectService::CollectStats() EmitSnapshot("requests", m_HttpRequests, Cbo); + Cbo << "project_count" << (uint64_t)m_ProjectStore->ProjectCount(); + Cbo.BeginObject("store"); { Cbo.BeginObject("size"); @@ -903,22 +921,25 @@ HttpProjectService::CollectStats() } Cbo.EndObject(); - return Cbo.Save(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -void -HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) +CbObject +HttpProjectService::CollectStats() { - HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats()); + CbObjectWriter Cbo; + // CollectStats does not use the HandleStatsRequest implementation to get stats since it uses some heavy operations such as + // m_ProjectStore->StorageSize(); + EmitSnapshot("requests", m_HttpRequests, Cbo); + Cbo << "project_count" << (uint64_t)m_ProjectStore->ProjectCount(); + + return Cbo.Save(); } -void -HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +uint64_t +HttpProjectService::GetActivityCounter() { - ZEN_TRACE_CPU("HttpProjectService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + return m_HttpRequests.Count(); } void @@ -1236,7 +1257,7 @@ HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) const Oid Obj = Oid::FromHexString(ChunkId); - CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); + CbObject ResponsePayload = ProjectStore::GetChunkInfo(*Project, *FoundLog, Obj); if (ResponsePayload) { m_ProjectStats.ChunkHitCount++; @@ -1325,7 +1346,7 @@ HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) HttpContentType AcceptType = HttpReq.AcceptContentType(); ProjectStore::GetChunkRangeResult Result = - ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); + ProjectStore::GetChunkRange(*Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); switch (Result.Error) { @@ -1654,7 +1675,8 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) CbPackage Package; - if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) + const bool ValidateHashes = false; + if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver, ValidateHashes)) { CbValidateError ValidateResult; if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); @@ -2662,6 +2684,7 @@ HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) try { CbObject ContainerObject = BuildContainer( + Log(), m_CidStore, *Project, *Oplog, @@ -2749,7 +2772,11 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) case HttpContentType::kCbPackage: try { - Package = ParsePackageMessage(Payload); + ParseFlags PkgFlags = (HttpReq.IsLocalMachineRequest() && AcceptsLocalFileReferences()) ? ParseFlags::kAllowLocalReferences + : ParseFlags::kDefault; + const ILocalRefPolicy* PkgPolicy = + EnumHasAllFlags(PkgFlags, ParseFlags::kAllowLocalReferences) ? GetLocalRefPolicy() : nullptr; + Package = ParsePackageMessage(Payload, {}, PkgFlags, PkgPolicy); Cb = Package.GetObject(); } catch (const std::invalid_argument& ex) @@ -2858,6 +2885,7 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) try { LoadOplog(LoadOplogContext{ + .Log = Log(), .ChunkStore = m_CidStore, .RemoteStore = *RemoteStoreResult->Store, .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr, @@ -2983,7 +3011,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) try { - SaveOplog(m_CidStore, + SaveOplog(Log(), + m_CidStore, *ActualRemoteStore, *Project, *Oplog, diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h index 917337324..8aa345fa7 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.h +++ b/src/zenserver/storage/projectstore/httpprojectstore.h @@ -47,15 +47,19 @@ public: JobQueue& InJobQueue, bool InRestrictContentTypes, const std::filesystem::path& InOidcTokenExePath, - bool AllowExternalOidcTokenExe); + bool AllowExternalOidcTokenExe, + const ILocalRefPolicy* InLocalRefPolicy = nullptr); ~HttpProjectService(); - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + virtual bool AcceptsLocalFileReferences() const override; + virtual const ILocalRefPolicy* GetLocalRefPolicy() const override; - virtual CbObject CollectStats() override; - virtual void HandleStatsRequest(HttpServerRequest& Request) override; virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; private: struct ProjectStats @@ -116,6 +120,7 @@ private: bool m_RestrictContentTypes; std::filesystem::path m_OidcTokenExePath; bool m_AllowExternalOidcTokenExe; + const ILocalRefPolicy* m_LocalRefPolicy; Ref<TransferThreadWorkers> GetThreadWorkers(bool BoostWorkers, bool SingleThreaded); }; diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp index 0dbb45164..bb4f053e4 100644 --- a/src/zenserver/storage/storageconfig.cpp +++ b/src/zenserver/storage/storageconfig.cpp @@ -57,6 +57,12 @@ ZenStorageServerConfigurator::ValidateOptions() ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); ServerOptions.GcConfig.UseGCV2 = true; } + if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 100) + { + throw OptionParseException(fmt::format("'--buildstore-disksizelimit-percent' ('{}') is invalid, must be between 1 and 100.", + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent), + {}); + } } class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue @@ -382,6 +388,9 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions) ////// buildsstore LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + LuaOptions.AddOption("server.buildstore.disksizelimitpercent"sv, + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent, + "buildstore-disksizelimit-percent"); ////// cache LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); @@ -477,7 +486,7 @@ ZenStorageServerConfigurator::AddConfigOptions(LuaConfig::Options& LuaOptions) ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, "gc-compactblock-threshold"sv); LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); - LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); + LuaOptions.AddOption("gc.singlethreaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); LuaOptions.AddOption("gc.projectstore.attachment.store"sv, ServerOptions.GcConfig.StoreProjectAttachmentMetaData, @@ -1035,6 +1044,13 @@ ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit-percent", + "Max percentage (1-100) of total drive capacity (of --data-dir drive) before build store entries get evicted. " + "0 (default) disables this limit. When combined with --buildstore-disksizelimit, the lower value wins.", + cxxopts::value<uint32_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent)->default_value("0"), + ""); } void diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h index 18af4f096..fec8fd70b 100644 --- a/src/zenserver/storage/storageconfig.h +++ b/src/zenserver/storage/storageconfig.h @@ -135,8 +135,9 @@ struct ZenProjectStoreConfig struct ZenBuildStoreConfig { - bool Enabled = false; - uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB + uint32_t MaxDiskSpaceLimitPercent = 0; }; struct ZenWorkspacesConfig diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp index b26c57414..a516c452c 100644 --- a/src/zenserver/storage/upstream/upstreamcache.cpp +++ b/src/zenserver/storage/upstream/upstreamcache.cpp @@ -772,7 +772,7 @@ namespace detail { UpstreamEndpointInfo m_Info; UpstreamStatus m_Status; UpstreamEndpointStats m_Stats; - RefPtr<JupiterClient> m_Client; + Ref<JupiterClient> m_Client; const bool m_AllowRedirect = false; }; @@ -1446,7 +1446,7 @@ namespace detail { // Make sure we safely bump the refcount inside a scope lock RwLock::SharedLockScope _(m_ClientLock); ZEN_ASSERT(m_Client); - Ref<ZenStructuredCacheClient> ClientRef(m_Client); + Ref<ZenStructuredCacheClient> ClientRef(m_Client.Get()); _.ReleaseNow(); return ClientRef; } @@ -1485,15 +1485,15 @@ namespace detail { LoggerRef Log() { return m_Log; } - LoggerRef m_Log; - UpstreamEndpointInfo m_Info; - UpstreamStatus m_Status; - UpstreamEndpointStats m_Stats; - std::vector<ZenEndpoint> m_Endpoints; - std::chrono::milliseconds m_ConnectTimeout; - std::chrono::milliseconds m_Timeout; - RwLock m_ClientLock; - RefPtr<ZenStructuredCacheClient> m_Client; + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + std::vector<ZenEndpoint> m_Endpoints; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; + RwLock m_ClientLock; + Ref<ZenStructuredCacheClient> m_Client; }; } // namespace detail diff --git a/src/zenserver/storage/upstream/upstreamservice.h b/src/zenserver/storage/upstream/upstreamservice.h index f1da03c8c..c0063c055 100644 --- a/src/zenserver/storage/upstream/upstreamservice.h +++ b/src/zenserver/storage/upstream/upstreamservice.h @@ -9,14 +9,14 @@ namespace zen { class AuthMgr; class UpstreamCache; -class HttpUpstreamService final : public zen::HttpService +class HttpUpstreamService final : public HttpService { public: HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr); virtual ~HttpUpstreamService(); virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleRequest(HttpServerRequest& Request) override; private: UpstreamCache& m_Upstream; diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp index 785dd62f0..12e7bae73 100644 --- a/src/zenserver/storage/workspaces/httpworkspaces.cpp +++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp @@ -110,10 +110,18 @@ HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) } } -CbObject -HttpWorkspacesService::CollectStats() +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) { - ZEN_TRACE_CPU("WorkspacesService::Stats"); CbObjectWriter Cbo; EmitSnapshot("requests", m_HttpRequests, Cbo); @@ -150,22 +158,26 @@ HttpWorkspacesService::CollectStats() } Cbo.EndObject(); - return Cbo.Save(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -void -HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) +CbObject +HttpWorkspacesService::CollectStats() { - HttpReq.WriteResponse(HttpResponseCode::OK, CollectStats()); + ZEN_TRACE_CPU("HttpWorkspacesService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo << "workspaces" << m_Workspaces.GetWorkspaces().size(); + + return Cbo.Save(); } -void -HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +uint64_t +HttpWorkspacesService::GetActivityCounter() { - ZEN_TRACE_CPU("HttpWorkspacesService::Status"); - CbObjectWriter Cbo; - Cbo << "ok" << true; - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + return m_HttpRequests.Count(); } void diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h index 7c5ddeff1..4af1316f8 100644 --- a/src/zenserver/storage/workspaces/httpworkspaces.h +++ b/src/zenserver/storage/workspaces/httpworkspaces.h @@ -29,9 +29,10 @@ public: virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; - virtual CbObject CollectStats() override; - virtual void HandleStatsRequest(HttpServerRequest& Request) override; virtual void HandleStatusRequest(HttpServerRequest& Request) override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual CbObject CollectStats() override; + virtual uint64_t GetActivityCounter() override; private: struct WorkspacesStats diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index de00eb1c2..7b52f2832 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -37,8 +37,6 @@ #include <zenutil/sessionsclient.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> -#include "sessions/inprocsessionlogsink.h" -#include "sessions/sessions.h" #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> @@ -165,12 +163,7 @@ ZenStorageServer::RegisterServices() m_Http->RegisterService(*m_HttpWorkspacesService); } - if (m_HttpSessionsService) - { - m_Http->RegisterService(*m_HttpSessionsService); - } - - m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService); + m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatsService, m_StatusService); if (m_FrontendService) { @@ -223,12 +216,13 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions ZEN_INFO("instantiating project service"); + m_LocalRefPolicy = std::make_unique<DataRootLocalRefPolicy>(m_DataRoot); m_JobQueue = MakeJobQueue(8, "bgjobs"); m_OpenProcessCache = std::make_unique<OpenProcessCache>(); m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{}); m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, - m_ProjectStore, + m_ProjectStore.Get(), m_StatusService, m_StatsService, *m_AuthMgr, @@ -236,7 +230,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions *m_JobQueue, ServerOptions.RestrictContentTypes, ServerOptions.OidcTokenExecutable, - ServerOptions.AllowExternalOidcTokenExe}); + ServerOptions.AllowExternalOidcTokenExe, + m_LocalRefPolicy.get()}); if (ServerOptions.WorksSpacesConfig.Enabled) { @@ -251,16 +246,6 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions *m_Workspaces)); } - { - m_SessionsService = std::make_unique<SessionsService>(); - m_HttpSessionsService = std::make_unique<HttpSessionsService>(m_StatusService, m_StatsService, *m_SessionsService, m_IoContext); - m_HttpSessionsService->SetSelfSessionId(GetSessionId()); - - m_InProcSessionLogSink = logging::SinkPtr(new InProcSessionLogSink(*m_SessionsService)); - m_InProcSessionLogSink->SetLevel(logging::Info); - GetDefaultBroadcastSink()->AddSink(m_InProcSessionLogSink); - } - if (!ServerOptions.SessionsTargetUrl.empty()) { m_SessionsClient = std::make_unique<SessionsServiceClient>(SessionsServiceClient::Options{ @@ -281,7 +266,31 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions BuildStoreConfig BuildsCfg; BuildsCfg.RootDirectory = m_DataRoot / "builds"; BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; - m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); + + if (ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent > 0) + { + DiskSpace Space; + if (DiskSpaceInfo(m_DataRoot, Space) && Space.Total > 0) + { + uint64_t PercentLimit = Space.Total * ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent / 100; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit > 0 + ? std::min(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, PercentLimit) + : PercentLimit; + ZEN_INFO("buildstore disk limit: {}% of {} = {} (effective limit: {})", + ServerOptions.BuildStoreConfig.MaxDiskSpaceLimitPercent, + NiceBytes(Space.Total), + NiceBytes(PercentLimit), + NiceBytes(BuildsCfg.MaxDiskSpaceLimit)); + } + else + { + ZEN_WARN("buildstore-disksizelimit-percent: failed to query disk space for {}, using absolute limit {}", + m_DataRoot.string(), + NiceBytes(BuildsCfg.MaxDiskSpaceLimit)); + } + } + + m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); } if (ServerOptions.StructuredCacheConfig.Enabled) @@ -307,7 +316,7 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg)); + m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatsService, m_StatusService, std::move(ObjCfg)); } if (ServerOptions.BuildStoreConfig.Enabled) @@ -323,13 +332,13 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions ZEN_OTEL_SPAN("InitializeComputeService"); m_HttpComputeService = - std::make_unique<compute::HttpComputeService>(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions"); + std::make_unique<compute::HttpComputeService>(*m_CidStore, *m_CidStore, m_StatsService, ServerOptions.DataDir / "functions"); } #endif #if ZEN_WITH_VFS m_VfsServiceImpl = std::make_unique<VfsServiceImpl>(); - m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore)); + m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore.Get())); m_VfsServiceImpl->AddService(Ref<ZenCacheStore>(m_CacheStore)); m_VfsService = std::make_unique<VfsService>(m_StatusService, m_VfsServiceImpl.get()); @@ -713,7 +722,8 @@ ZenStorageServer::InitializeStructuredCache(const ZenStorageServerConfig& Server m_StatusService, *m_UpstreamCache, m_GcManager.GetDiskWriteBlocker(), - *m_OpenProcessCache); + *m_OpenProcessCache, + m_LocalRefPolicy.get()); m_StatsReporter.AddProvider(m_CacheStore.Get()); m_StatsReporter.AddProvider(m_CidStore.get()); @@ -838,7 +848,7 @@ ZenStorageServer::Run() OnReady(); - m_SessionsService->RegisterSession(GetSessionId(), "zenserver", GetServerMode(), Oid::Zero, {}); + StartSelfSession("zenserver"); if (m_SessionsClient) { @@ -888,11 +898,6 @@ ZenStorageServer::Cleanup() m_Http->Close(); } - if (m_InProcSessionLogSink) - { - GetDefaultBroadcastSink()->RemoveSink(m_InProcSessionLogSink); - m_InProcSessionLogSink = {}; - } if (m_SessionLogSink) { GetDefaultBroadcastSink()->RemoveSink(m_SessionLogSink); @@ -904,11 +909,6 @@ ZenStorageServer::Cleanup() m_SessionsClient.reset(); } - if (m_SessionsService) - { - m_SessionsService->RemoveSession(GetSessionId()); - } - ShutdownServices(); if (m_JobQueue) @@ -940,8 +940,6 @@ ZenStorageServer::Cleanup() m_UpstreamCache.reset(); m_CacheStore = {}; - m_HttpSessionsService.reset(); - m_SessionsService.reset(); m_HttpWorkspacesService.reset(); m_Workspaces.reset(); m_HttpProjectService.reset(); diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h index fad22ad54..9fa46ba9b 100644 --- a/src/zenserver/storage/zenstorageserver.h +++ b/src/zenserver/storage/zenstorageserver.h @@ -11,6 +11,7 @@ #include <zenstore/cache/structuredcachestore.h> #include <zenstore/gc.h> #include <zenstore/projectstore.h> +#include "localrefpolicy.h" #include "admin/admin.h" #include "buildstore/httpbuildstore.h" @@ -19,7 +20,6 @@ #include "frontend/frontend.h" #include "objectstore/objectstore.h" #include "projectstore/httpprojectstore.h" -#include "sessions/httpsessions.h" #include "stats/statsreporter.h" #include "upstream/upstream.h" #include "vfs/vfsservice.h" @@ -65,27 +65,26 @@ private: void InitializeServices(const ZenStorageServerConfig& ServerOptions); void RegisterServices(); - std::unique_ptr<JobQueue> m_JobQueue; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - std::unique_ptr<CidStore> m_CidStore; - Ref<ZenCacheStore> m_CacheStore; - std::unique_ptr<OpenProcessCache> m_OpenProcessCache; - HttpTestService m_TestService; - std::unique_ptr<CidStore> m_BuildCidStore; - std::unique_ptr<BuildStore> m_BuildStore; + std::unique_ptr<DataRootLocalRefPolicy> m_LocalRefPolicy; + std::unique_ptr<JobQueue> m_JobQueue; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr<CidStore> m_CidStore; + Ref<ZenCacheStore> m_CacheStore; + std::unique_ptr<OpenProcessCache> m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr<CidStore> m_BuildCidStore; + std::unique_ptr<BuildStore> m_BuildStore; #if ZEN_WITH_TESTS HttpTestingService m_TestingService; #endif - RefPtr<ProjectStore> m_ProjectStore; + Ref<ProjectStore> m_ProjectStore; std::unique_ptr<VfsServiceImpl> m_VfsServiceImpl; std::unique_ptr<HttpProjectService> m_HttpProjectService; std::unique_ptr<Workspaces> m_Workspaces; std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService; - std::unique_ptr<SessionsService> m_SessionsService; - std::unique_ptr<HttpSessionsService> m_HttpSessionsService; std::unique_ptr<UpstreamCache> m_UpstreamCache; std::unique_ptr<HttpUpstreamService> m_UpstreamService; std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService; @@ -98,7 +97,6 @@ private: std::unique_ptr<SessionsServiceClient> m_SessionsClient; logging::SinkPtr m_SessionLogSink; - logging::SinkPtr m_InProcSessionLogSink; asio::steady_timer m_SessionAnnounceTimer{m_IoContext}; void EnqueueSessionAnnounceTimer(); |