From 49309fa7bbd38746d33cc48acfc5cd59598dbbe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Obr=C4=99bski?= Date: Mon, 3 Feb 2025 09:38:50 +0100 Subject: Fix workspace shares reply array (#280) Workspace shares were sent in the '/ws' reply as repeating objects 'shares' instead of a 'shares' array of objects --- src/zenserver/workspaces/httpworkspaces.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 2d59c9357..905ba5ab2 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -51,9 +51,9 @@ namespace { WriteWorkspaceConfig(Writer, WorkspaceConfig); if (std::optional> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) { - for (const Oid& ShareId : *ShareIds) + Writer.BeginArray("shares"); { - Writer.BeginArray("shares"); + for (const Oid& ShareId : *ShareIds) { if (std::optional WorkspaceShareConfig = Workspaces.GetWorkspaceShareConfiguration(WorkspaceConfig.Id, ShareId); @@ -66,8 +66,8 @@ namespace { Writer.EndObject(); } } - Writer.EndArray(); } + Writer.EndArray(); } } -- cgit v1.2.3 From d39724eb644cab4ec5bbf19a703cb770b34e68c4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 08:58:52 +0100 Subject: improved builds api interface in jupiter (#281) * multipart upload/download iterface in jupiter * review fixes --- src/zenhttp/httpclient.cpp | 311 ++++++++++++--------- src/zenhttp/httpclientauth.cpp | 3 + src/zenhttp/include/zenhttp/httpclient.h | 4 +- .../include/zenutil/jupiter/jupitersession.h | 70 +++-- src/zenutil/jupiter/jupitersession.cpp | 293 +++++++++++++++++++ 5 files changed, 518 insertions(+), 163 deletions(-) (limited to 'src') diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 8052a8fd5..211a5d05c 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -282,7 +282,7 @@ AsCprBody(const IoBuffer& Obj) ////////////////////////////////////////////////////////////////////////// static HttpClient::Response -ResponseWithPayload(cpr::Response& HttpResponse, const HttpResponseCode WorkResponseCode, IoBuffer&& Payload) +ResponseWithPayload(std::string_view SessionId, cpr::Response& HttpResponse, const HttpResponseCode WorkResponseCode, IoBuffer&& Payload) { // This ends up doing a memcpy, would be good to get rid of it by streaming results // into buffer directly @@ -297,7 +297,7 @@ ResponseWithPayload(cpr::Response& HttpResponse, const HttpResponseCode WorkResp if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound) { - ZEN_WARN("HttpClient request failed: {}", HttpResponse); + ZEN_WARN("HttpClient request failed (session: {}): {}", SessionId, HttpResponse); } return HttpClient::Response{.StatusCode = WorkResponseCode, @@ -309,12 +309,12 @@ ResponseWithPayload(cpr::Response& HttpResponse, const HttpResponseCode WorkResp } static HttpClient::Response -CommonResponse(cpr::Response&& HttpResponse, IoBuffer&& Payload = {}) +CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffer&& Payload = {}) { const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) { - ZEN_WARN("HttpClient client error: {}", HttpResponse); + ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); // Client side failure code return HttpClient::Response{ @@ -339,6 +339,7 @@ CommonResponse(cpr::Response&& HttpResponse, IoBuffer&& Payload = {}) else { return ResponseWithPayload( + SessionId, HttpResponse, WorkResponseCode, Payload ? std::move(Payload) : IoBufferBuilder::MakeCloneFromMemory(HttpResponse.text.data(), HttpResponse.text.size())); @@ -448,7 +449,7 @@ ValidatePayload(cpr::Response& Response, std::unique_ptr&& Func, uint8_t RetryCount) +DoWithRetry(std::string_view SessionId, std::function&& Func, uint8_t RetryCount) { uint8_t Attempt = 0; cpr::Response Result = Func(); @@ -456,14 +457,17 @@ DoWithRetry(std::function&& Func, uint8_t RetryCount) { Sleep(100 * (Attempt + 1)); Attempt++; - ZEN_INFO("{} Attempt {}/{}", CommonResponse(std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1); + ZEN_INFO("{} Attempt {}/{}", CommonResponse(SessionId, std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1); Result = Func(); } return Result; } static cpr::Response -DoWithRetry(std::function&& Func, std::unique_ptr& PayloadFile, uint8_t RetryCount) +DoWithRetry(std::string_view SessionId, + std::function&& Func, + std::unique_ptr& PayloadFile, + uint8_t RetryCount) { uint8_t Attempt = 0; cpr::Response Result = Func(); @@ -482,7 +486,7 @@ DoWithRetry(std::function&& Func, std::unique_ptrAllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - Sess->SetBody(AsCprBody(Payload)); - Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); - return Sess.Put(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Sess->SetBody(AsCprBody(Payload)); + Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); + return Sess.Put(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -845,31 +852,36 @@ HttpClient::Put(std::string_view Url, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::Put"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, - Url, - m_ConnectionSettings, - {{"Content-Length", "0"}}, - Parameters, - m_SessionId, - GetAccessToken()); - return Sess.Put(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse(m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, + Url, + m_ConnectionSettings, + {{"Content-Length", "0"}}, + Parameters, + m_SessionId, + GetAccessToken()); + return Sess.Put(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response HttpClient::Get(std::string_view Url, const KeyValueMap& AdditionalHeader, const KeyValueMap& Parameters) { ZEN_TRACE_CPU("HttpClient::Get"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); - return Sess.Get(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); + return Sess.Get(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -877,13 +889,16 @@ HttpClient::Head(std::string_view Url, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Head"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - return Sess.Head(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + return Sess.Head(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -891,13 +906,16 @@ HttpClient::Delete(std::string_view Url, const KeyValueMap& AdditionalHeader) { ZEN_TRACE_CPU("HttpClient::Delete"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - return Sess.Delete(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + return Sess.Delete(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -905,13 +923,16 @@ HttpClient::Post(std::string_view Url, const KeyValueMap& AdditionalHeader, cons { ZEN_TRACE_CPU("HttpClient::PostNoPayload"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); - return Sess.Post(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); + return Sess.Post(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -925,16 +946,19 @@ HttpClient::Post(std::string_view Url, const IoBuffer& Payload, ZenContentType C { ZEN_TRACE_CPU("HttpClient::PostWithPayload"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - - Sess->SetBody(AsCprBody(Payload)); - Sess->UpdateHeader({HeaderContentType(ContentType)}); - return Sess.Post(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + + Sess->SetBody(AsCprBody(Payload)); + Sess->UpdateHeader({HeaderContentType(ContentType)}); + return Sess.Post(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -942,16 +966,19 @@ HttpClient::Post(std::string_view Url, CbObject Payload, const KeyValueMap& Addi { ZEN_TRACE_CPU("HttpClient::PostObjectPayload"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - - Sess->SetBody(AsCprBody(Payload)); - Sess->UpdateHeader({HeaderContentType(ZenContentType::kCbObject)}); - return Sess.Post(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + + Sess->SetBody(AsCprBody(Payload)); + Sess->UpdateHeader({HeaderContentType(ZenContentType::kCbObject)}); + return Sess.Post(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -965,24 +992,27 @@ HttpClient::Post(std::string_view Url, const CompositeBuffer& Payload, ZenConten { ZEN_TRACE_CPU("HttpClient::Post"); - return CommonResponse(DoWithRetry( - [&]() { - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - Sess->UpdateHeader({HeaderContentType(ContentType)}); - - return Sess.Post(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + uint64_t SizeLeft = Payload.GetSize(); + CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); + auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { + size = Min(size, SizeLeft); + MutableMemoryView Data(buffer, size); + Payload.CopyTo(Data, BufferIt); + SizeLeft -= size; + return true; + }; + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Sess->UpdateHeader({HeaderContentType(ContentType)}); + + return Sess.Post(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -990,29 +1020,32 @@ HttpClient::Upload(std::string_view Url, const IoBuffer& Payload, const KeyValue { ZEN_TRACE_CPU("HttpClient::Upload"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); - - uint64_t Offset = 0; - if (Payload.IsWholeFile()) - { - auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { - size = Min(size, Payload.GetSize() - Offset); - IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); - MutableMemoryView Data(buffer, size); - Data.CopyFrom(PayloadRange.GetView()); - Offset += size; - return true; - }; - return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); - } - Sess->SetBody(AsCprBody(Payload)); - return Sess.Put(); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); + + uint64_t Offset = 0; + if (Payload.IsWholeFile()) + { + auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { + size = Min(size, Payload.GetSize() - Offset); + IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); + MutableMemoryView Data(buffer, size); + Data.CopyFrom(PayloadRange.GetView()); + Offset += size; + return true; + }; + return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); + } + Sess->SetBody(AsCprBody(Payload)); + return Sess.Put(); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -1020,24 +1053,27 @@ HttpClient::Upload(std::string_view Url, const CompositeBuffer& Payload, ZenCont { ZEN_TRACE_CPU("HttpClient::Upload"); - return CommonResponse(DoWithRetry( - [&]() { - Impl::Session Sess = - m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - Sess->UpdateHeader({HeaderContentType(ContentType)}); - - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; - return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); - }, - m_ConnectionSettings.RetryCount)); + return CommonResponse( + m_SessionId, + DoWithRetry( + m_SessionId, + [&]() { + Impl::Session Sess = + m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Sess->UpdateHeader({HeaderContentType(ContentType)}); + + uint64_t SizeLeft = Payload.GetSize(); + CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); + auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { + size = Min(size, SizeLeft); + MutableMemoryView Data(buffer, size); + Payload.CopyTo(Data, BufferIt); + SizeLeft -= size; + return true; + }; + return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); + }, + m_ConnectionSettings.RetryCount)); } HttpClient::Response @@ -1048,6 +1084,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold std::string PayloadString; std::unique_ptr PayloadFile; cpr::Response Response = DoWithRetry( + m_SessionId, [&]() { auto GetHeader = [&](std::string header) -> std::pair { size_t DelimiterPos = header.find(':'); @@ -1249,7 +1286,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold PayloadFile, m_ConnectionSettings.RetryCount); - return CommonResponse(std::move(Response), PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{}); + return CommonResponse(m_SessionId, std::move(Response), PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{}); } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 04ac2ad3f..7fb3224f1 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -2,6 +2,7 @@ #include +#include #include ZEN_THIRD_PARTY_INCLUDES_START @@ -41,6 +42,7 @@ namespace zen { namespace httpclientauth { if (Response.error || Response.status_code != 200) { + ZEN_WARN("Failed fetching OAuth access token {}. Reason: '{}'", OAuthParams.Url, Response.reason); return HttpClientAccessToken{}; } @@ -49,6 +51,7 @@ namespace zen { namespace httpclientauth { if (JsonError.empty() == false) { + ZEN_WARN("Unable to parse OAuth json response from {}. Reason: '{}'", OAuthParams.Url, JsonError); return HttpClientAccessToken{}; } diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 1cf77d794..a46b9fd83 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -60,9 +60,6 @@ struct HttpClientSettings class HttpClient { public: - struct Settings - { - }; HttpClient(std::string_view BaseUri, const HttpClientSettings& Connectionsettings = {}); ~HttpClient(); @@ -180,6 +177,7 @@ public: LoggerRef Logger() { return m_Log; } std::string_view GetBaseUri() const { return m_BaseUri; } bool Authenticate(); + std::string_view GetSessionId() const { return m_SessionId; } private: const std::optional GetAccessToken(); diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 6a80332f4..075c35b40 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -102,29 +102,48 @@ public: std::vector Filter(std::string_view Namespace, std::string_view BucketId, const std::vector& ChunkHashes); - JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); - JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); - JupiterResult FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); - PutBuildPartResult PutBuildPart(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - std::string_view PartName, - const IoBuffer& Payload); - JupiterResult GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); - JupiterResult PutBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - const IoHash& Hash, - ZenContentType ContentType, - const CompositeBuffer& Payload); - JupiterResult GetBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - const IoHash& Hash, - std::filesystem::path TempFolderPath); + JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); + JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); + JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + PutBuildPartResult PutBuildPart(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + std::string_view PartName, + const IoBuffer& Payload); + JupiterResult GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); + JupiterResult PutBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + ZenContentType ContentType, + const CompositeBuffer& Payload); + JupiterResult GetBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + std::filesystem::path TempFolderPath); + + JupiterResult PutMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::vector>& OutWorkItems); + JupiterResult GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& Receiver, + std::vector>& OutWorkItems); JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, @@ -137,6 +156,11 @@ public: const Oid& PartId, const IoHash& RawHash); JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); + JupiterResult GetBlockMetadata(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + IoBuffer Payload); private: inline LoggerRef Log() { return m_Log; } diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index f706a7efc..d56927b44 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include #include @@ -354,6 +356,16 @@ JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view Typ return Result; } +JupiterResult +JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload) +{ + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); + HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/search", Namespace, BucketId), + Payload, + {HttpClient::Accept(ZenContentType::kCbObject)}); + return detail::ConvertResponse(Response, "JupiterSession::ListBuilds"sv); +} + JupiterResult JupiterSession::PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload) { @@ -436,6 +448,271 @@ JupiterSession::PutBuildBlob(std::string_view Namespace, return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv); } +JupiterResult +JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::vector>& OutWorkItems) +{ + struct MultipartUploadResponse + { + struct Part + { + uint64_t FirstByte; + uint64_t LastByte; + std::string PartId; + std::string QueryString; + }; + + std::string UploadId; + std::string BlobName; + std::vector Parts; + + static MultipartUploadResponse Parse(CbObject& Payload) + { + MultipartUploadResponse Result; + Result.UploadId = Payload["uploadId"sv].AsString(); + Result.BlobName = Payload["blobName"sv].AsString(); + CbArrayView PartsArray = Payload["parts"sv].AsArrayView(); + Result.Parts.reserve(PartsArray.Num()); + for (CbFieldView PartView : PartsArray) + { + CbObjectView PartObject = PartView.AsObjectView(); + Result.Parts.emplace_back(Part{ + .FirstByte = PartObject["firstByte"sv].AsUInt64(), + .LastByte = PartObject["lastByte"sv].AsUInt64(), + .PartId = std::string(PartObject["partId"sv].AsString()), + .QueryString = std::string(PartObject["queryString"sv].AsString()), + }); + } + return Result; + } + }; + + CbObjectWriter StartMultipartPayloadWriter; + StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize); + CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save(); + + std::string StartMultipartResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/startMultipartUpload", + Namespace, + BucketId, + BuildId, + PartId, + Hash.ToHexString()); + // ZEN_INFO("POST: {}", StartMultipartResponseRequestString); + HttpClient::Response StartMultipartResponse = + m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject)); + if (!StartMultipartResponse.IsSuccess()) + { + ZEN_WARN("{}", StartMultipartResponse.ErrorMessage("startMultipartUpload: ")); + return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv); + } + CbObject ResponseObject = LoadCompactBinaryObject(StartMultipartResponse.ResponsePayload); + + struct WorkloadData + { + MultipartUploadResponse PartDescription; + std::function Transmitter; + std::atomic PartsLeft; + }; + + std::shared_ptr Workload(std::make_shared()); + + Workload->PartDescription = MultipartUploadResponse::Parse(ResponseObject); + Workload->Transmitter = std::move(Transmitter); + Workload->PartsLeft = Workload->PartDescription.Parts.size(); + + for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++) + { + OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, PartId, Hash, ContentType, Workload, PartIndex]( + bool& OutIsComplete) -> JupiterResult { + const MultipartUploadResponse::Part& Part = Workload->PartDescription.Parts[PartIndex]; + IoBuffer PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte); + std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}", + Namespace, + BucketId, + BuildId, + PartId, + Hash.ToHexString(), + Part.QueryString); + // ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString); + HttpClient::Response MultipartUploadResponse = m_HttpClient.Put(MultipartUploadResponseRequestString, PartPayload); + if (!MultipartUploadResponse.IsSuccess()) + { + ZEN_WARN("{}", MultipartUploadResponse.ErrorMessage(MultipartUploadResponseRequestString)); + } + OutIsComplete = Workload->PartsLeft.fetch_sub(1) == 1; + if (OutIsComplete) + { + int64_t TotalUploadedBytes = MultipartUploadResponse.UploadedBytes; + int64_t TotalDownloadedBytes = MultipartUploadResponse.DownloadedBytes; + double TotalElapsedSeconds = MultipartUploadResponse.ElapsedSeconds; + HttpClient::Response MultipartEndResponse = MultipartUploadResponse; + while (MultipartEndResponse.IsSuccess()) + { + CbObjectWriter CompletePayloadWriter; + CompletePayloadWriter.AddString("blobName"sv, Workload->PartDescription.BlobName); + CompletePayloadWriter.AddString("uploadId"sv, Workload->PartDescription.UploadId); + CompletePayloadWriter.AddBool("isCompressed"sv, ContentType == ZenContentType::kCompressedBinary); + CompletePayloadWriter.BeginArray("partIds"sv); + std::unordered_map PartNameToIndex; + for (size_t UploadPartIndex = 0; UploadPartIndex < Workload->PartDescription.Parts.size(); UploadPartIndex++) + { + const MultipartUploadResponse::Part& PartDescription = Workload->PartDescription.Parts[UploadPartIndex]; + PartNameToIndex.insert({PartDescription.PartId, UploadPartIndex}); + CompletePayloadWriter.AddString(PartDescription.PartId); + } + CompletePayloadWriter.EndArray(); // "partIds" + CbObject CompletePayload = CompletePayloadWriter.Save(); + + std::string MultipartEndResponseRequestString = + fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/completeMultipart", + Namespace, + BucketId, + BuildId, + PartId, + Hash.ToHexString()); + + MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString, + CompletePayload, + HttpClient::Accept(ZenContentType::kCbObject)); + TotalUploadedBytes += MultipartEndResponse.UploadedBytes; + TotalDownloadedBytes += MultipartEndResponse.DownloadedBytes; + TotalElapsedSeconds += MultipartEndResponse.ElapsedSeconds; + if (MultipartEndResponse.IsSuccess()) + { + CbObject ResponseObject = MultipartEndResponse.AsObject(); + CbArrayView MissingPartsArrayView = ResponseObject["missingParts"sv].AsArrayView(); + if (MissingPartsArrayView.Num() == 0) + { + break; + } + else + { + for (CbFieldView PartIdView : MissingPartsArrayView) + { + std::string RetryPartId(PartIdView.AsString()); + size_t RetryPartIndex = PartNameToIndex.at(RetryPartId); + const MultipartUploadResponse::Part& RetryPart = Workload->PartDescription.Parts[RetryPartIndex]; + IoBuffer RetryPartPayload = + Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte); + std::string RetryMultipartUploadResponseRequestString = + fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}", + Namespace, + BucketId, + BuildId, + RetryPartId, + Hash.ToHexString(), + RetryPart.QueryString); + + MultipartUploadResponse = m_HttpClient.Put(RetryMultipartUploadResponseRequestString, RetryPartPayload); + TotalUploadedBytes = MultipartUploadResponse.UploadedBytes; + TotalDownloadedBytes = MultipartUploadResponse.DownloadedBytes; + TotalElapsedSeconds = MultipartUploadResponse.ElapsedSeconds; + if (!MultipartUploadResponse.IsSuccess()) + { + ZEN_WARN("{}", MultipartUploadResponse.ErrorMessage(RetryMultipartUploadResponseRequestString)); + MultipartEndResponse = MultipartUploadResponse; + } + } + } + } + else + { + ZEN_WARN("{}", MultipartEndResponse.ErrorMessage(MultipartEndResponseRequestString)); + } + } + MultipartEndResponse.UploadedBytes = TotalUploadedBytes; + MultipartEndResponse.DownloadedBytes = TotalDownloadedBytes; + MultipartEndResponse.ElapsedSeconds = TotalElapsedSeconds; + return detail::ConvertResponse(MultipartEndResponse, "JupiterSession::PutMultipartBuildBlob"sv); + } + return detail::ConvertResponse(MultipartUploadResponse, "JupiterSession::PutMultipartBuildBlob"sv); + }); + } + return detail::ConvertResponse(StartMultipartResponse, "JupiterSession::PutMultipartBuildBlob"sv); +} + +JupiterResult +JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& Receiver, + std::vector>& OutWorkItems) +{ + std::string RequestUrl = + fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()); + HttpClient::Response Response = + m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}})); + if (Response.IsSuccess()) + { + if (std::string_view ContentRange = Response.Header.Entries["Content-Range"]; !ContentRange.empty()) + { + if (std::string_view::size_type SizeDelimiterPos = ContentRange.find('/'); SizeDelimiterPos != std::string_view::npos) + { + if (std::optional TotalSizeMaybe = ParseInt(ContentRange.substr(SizeDelimiterPos + 1)); + TotalSizeMaybe.has_value()) + { + uint64_t TotalSize = TotalSizeMaybe.value(); + uint64_t PayloadSize = Response.ResponsePayload.GetSize(); + + Receiver(0, Response.ResponsePayload, TotalSize); + + if (TotalSize > PayloadSize) + { + struct WorkloadData + { + std::function Receiver; + std::atomic BytesRemaining; + }; + + std::shared_ptr Workload(std::make_shared()); + Workload->Receiver = std::move(Receiver); + Workload->BytesRemaining = TotalSize - PayloadSize; + + uint64_t Offset = PayloadSize; + while (Offset < TotalSize) + { + uint64_t PartSize = Min(ChunkSize, TotalSize - Offset); + OutWorkItems.emplace_back( + [this, Namespace, BucketId, BuildId, PartId, Hash, TotalSize, Workload, Offset, PartSize]() + -> JupiterResult { + std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", + Namespace, + BucketId, + BuildId, + PartId, + Hash.ToHexString()); + HttpClient::Response Response = m_HttpClient.Get( + RequestUrl, + HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); + if (Response.IsSuccess()) + { + uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize()); + Workload->Receiver(Offset, Response.ResponsePayload, ByteRemaning); + } + return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); + }); + Offset += PartSize; + } + } + return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); + } + } + } + Receiver(0, Response.ResponsePayload, Response.ResponsePayload.GetSize()); + } + return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); +} + JupiterResult JupiterSession::GetBuildBlob(std::string_view Namespace, std::string_view BucketId, @@ -447,6 +724,7 @@ JupiterSession::GetBuildBlob(std::string_view Namespace, HttpClient::Response Response = m_HttpClient.Download( fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()), TempFolderPath); + return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv); } @@ -502,4 +780,19 @@ JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv); } +JupiterResult +JupiterSession::GetBlockMetadata(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + IoBuffer Payload) +{ + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); + HttpClient::Response Response = + m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId, PartId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv); +} + } // namespace zen -- cgit v1.2.3 From da9179d330a37132488f6deb8d8068783b087256 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 09:02:35 +0100 Subject: moving and small refactor of chunk blocks to prepare for builds api (#282) --- .../projectstore/buildsremoteprojectstore.cpp | 77 +--- .../projectstore/fileremoteprojectstore.cpp | 6 +- .../projectstore/jupiterremoteprojectstore.cpp | 4 +- src/zenserver/projectstore/projectstore.cpp | 16 +- src/zenserver/projectstore/remoteprojectstore.cpp | 226 ++++----- src/zenserver/projectstore/remoteprojectstore.h | 31 +- .../projectstore/zenremoteprojectstore.cpp | 2 +- src/zenstore/chunkedfile.cpp | 505 -------------------- src/zenstore/chunking.cpp | 382 --------------- src/zenstore/chunking.h | 56 --- src/zenstore/include/zenstore/chunkedfile.h | 54 --- src/zenutil/chunkblock.cpp | 166 +++++++ src/zenutil/chunkedfile.cpp | 510 +++++++++++++++++++++ src/zenutil/chunking.cpp | 382 +++++++++++++++ src/zenutil/chunking.h | 56 +++ src/zenutil/include/zenutil/chunkblock.h | 32 ++ src/zenutil/include/zenutil/chunkedfile.h | 58 +++ src/zenutil/zenutil.cpp | 2 + 18 files changed, 1328 insertions(+), 1237 deletions(-) delete mode 100644 src/zenstore/chunkedfile.cpp delete mode 100644 src/zenstore/chunking.cpp delete mode 100644 src/zenstore/chunking.h delete mode 100644 src/zenstore/include/zenstore/chunkedfile.h create mode 100644 src/zenutil/chunkblock.cpp create mode 100644 src/zenutil/chunkedfile.cpp create mode 100644 src/zenutil/chunking.cpp create mode 100644 src/zenutil/chunking.h create mode 100644 src/zenutil/include/zenutil/chunkblock.h create mode 100644 src/zenutil/include/zenutil/chunkedfile.h (limited to 'src') diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index 302b81729..412769174 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -3,6 +3,7 @@ #include "buildsremoteprojectstore.h" #include +#include #include #include @@ -114,7 +115,9 @@ public: return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, + const IoHash& RawHash, + ChunkBlockDescription&& Block) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); @@ -139,44 +142,10 @@ public: if (Block.BlockHash == RawHash) { - ZEN_ASSERT(Block.ChunkLengths.size() == Block.ChunkHashes.size()); - CbObjectWriter Writer; - Writer.AddHash("rawHash"sv, RawHash); - Writer.BeginArray("rawHashes"sv); - { - for (const IoHash& ChunkHash : Block.ChunkHashes) - { - Writer.AddHash(ChunkHash); - } - } - Writer.EndArray(); - Writer.BeginArray("chunkLengths"); - { - for (uint32_t ChunkSize : Block.ChunkLengths) - { - Writer.AddInteger(ChunkSize); - } - } - Writer.EndArray(); - Writer.BeginArray("chunkOffsets"); - { - ZEN_ASSERT(Block.FirstChunkOffset != (uint32_t)-1); - uint32_t Offset = Block.FirstChunkOffset; - for (uint32_t ChunkSize : Block.ChunkLengths) - { - Writer.AddInteger(Offset); - Offset += ChunkSize; - } - } - Writer.EndArray(); + CbObjectWriter BlockMetaData; + BlockMetaData.AddString("createdBy", GetRunningExecutablePath().stem().string()); - Writer.BeginObject("metadata"sv); - { - Writer.AddString("createdBy", "zenserver"); - } - Writer.EndObject(); - - IoBuffer MetaPayload = Writer.Save().GetBuffer().AsIoBuffer(); + IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer(); MetaPayload.SetContentType(ZenContentType::kCbObject); JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload); @@ -357,8 +326,7 @@ public: Result.Reason); return Result; } - CbObject BlocksObject = LoadCompactBinaryObject(FindResult.Response); - if (!BlocksObject) + if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv, @@ -369,25 +337,20 @@ public: m_OplogBuildPartId); return Result; } - - CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); - Result.Blocks.reserve(Blocks.Num()); - for (CbFieldView BlockView : Blocks) + std::optional> Blocks = + ParseChunkBlockDescriptionList(LoadCompactBinaryObject(FindResult.Response)); + if (!Blocks) { - CbObjectView BlockObject = BlockView.AsObjectView(); - IoHash BlockHash = BlockObject["rawHash"sv].AsHash(); - if (BlockHash != IoHash::Zero) - { - CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView(); - std::vector ChunkHashes; - ChunkHashes.reserve(ChunksArray.Num()); - for (CbFieldView ChunkView : ChunksArray) - { - ChunkHashes.push_back(ChunkView.AsHash()); - } - Result.Blocks.emplace_back(Block{.BlockHash = BlockHash, .ChunkHashes = ChunkHashes}); - } + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a list of blocks"sv, + m_JupiterClient->ServiceUrl(), + m_Namespace, + m_Bucket, + m_BuildId, + m_OplogBuildPartId); + return Result; } + Result.Blocks = std::move(Blocks.value()); return Result; } diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 0fe739a12..5a21a7540 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -106,7 +106,7 @@ public: return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override { Stopwatch Timer; SaveAttachmentResult Result; @@ -192,8 +192,8 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; } - std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); - GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; + std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; Result.Blocks = std::move(KnownBlocks); return Result; } diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index e906127ff..2b6a437d1 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -92,7 +92,7 @@ public: return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override { JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); JupiterResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); @@ -193,7 +193,7 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}}; } - std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); GetKnownBlocksResult Result{ {.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}}; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 46a236af9..f6f7eba99 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -5347,7 +5347,7 @@ ProjectStore::ReadOplog(const std::string_view ProjectId, /* BuildBlocks */ false, /* IgnoreMissingAttachments */ false, /* AllowChunking*/ false, - [](CompressedBuffer&&, RemoteProjectStore::Block&&) {}, + [](CompressedBuffer&&, ChunkBlockDescription&&) {}, [](const IoHash&, TGetAttachmentBufferFunc&&) {}, [](std::vector>&&) {}, /* EmbedLooseFiles*/ false); @@ -8621,14 +8621,14 @@ TEST_CASE("project.store.block") Chunks.reserve(AttachmentSizes.size()); for (const auto& It : AttachmentsWithId) { - Chunks.push_back(std::make_pair(It.second.DecodeRawHash(), - [Buffer = It.second.GetCompressed().Flatten().AsIoBuffer()](const IoHash&) -> CompositeBuffer { - return CompositeBuffer(SharedBuffer(Buffer)); - })); + Chunks.push_back( + std::make_pair(It.second.DecodeRawHash(), [Buffer = It.second](const IoHash&) -> std::pair { + return {Buffer.DecodeRawSize(), Buffer}; + })); } - RemoteProjectStore::Block Block; - CompressedBuffer BlockBuffer = GenerateBlock(std::move(Chunks), Block); - CHECK(IterateBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {})); + ChunkBlockDescription Block; + CompressedBuffer BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block); + CHECK(IterateChunkBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {})); } TEST_CASE("project.store.iterateoplog") diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index 0589fdc5f..5b75a840e 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -12,8 +12,8 @@ #include #include #include -#include #include +#include #include #include @@ -143,7 +143,7 @@ namespace remotestore_impl { NiceBytes(Stats.m_PeakReceivedBytes)); } - size_t AddBlock(RwLock& BlocksLock, std::vector& Blocks) + size_t AddBlock(RwLock& BlocksLock, std::vector& Blocks) { size_t BlockIndex; { @@ -573,7 +573,7 @@ namespace remotestore_impl { return; } - bool StoreChunksOK = IterateBlock( + bool StoreChunksOK = IterateChunkBlock( BlockPayload, [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { @@ -738,14 +738,14 @@ namespace remotestore_impl { }); }; - void CreateBlock(WorkerThreadPool& WorkerPool, - Latch& OpSectionsLatch, - std::vector>&& ChunksInBlock, - RwLock& SectionsLock, - std::vector& Blocks, - size_t BlockIndex, - const std::function& AsyncOnBlock, - AsyncRemoteResult& RemoteResult) + void CreateBlock(WorkerThreadPool& WorkerPool, + Latch& OpSectionsLatch, + std::vector>&& ChunksInBlock, + RwLock& SectionsLock, + std::vector& Blocks, + size_t BlockIndex, + const std::function& AsyncOnBlock, + AsyncRemoteResult& RemoteResult) { OpSectionsLatch.AddCount(1); WorkerPool.ScheduleWork([&Blocks, @@ -764,10 +764,10 @@ namespace remotestore_impl { try { ZEN_ASSERT(ChunkCount > 0); - Stopwatch Timer; - RemoteProjectStore::Block Block; - CompressedBuffer CompressedBlock = GenerateBlock(std::move(Chunks), Block); - IoHash BlockHash = CompressedBlock.DecodeRawHash(); + Stopwatch Timer; + ChunkBlockDescription Block; + CompressedBuffer CompressedBlock = GenerateChunkBlock(std::move(Chunks), Block); + IoHash BlockHash = CompressedBlock.DecodeRawHash(); { // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index RwLock::SharedLockScope __(SectionsLock); @@ -800,8 +800,8 @@ namespace remotestore_impl { struct CreatedBlock { - IoBuffer Payload; - RemoteProjectStore::Block Block; + IoBuffer Payload; + ChunkBlockDescription Block; }; void UploadAttachments(WorkerThreadPool& WorkerPool, @@ -931,8 +931,8 @@ namespace remotestore_impl { } try { - IoBuffer Payload; - RemoteProjectStore::Block Block; + IoBuffer Payload; + ChunkBlockDescription Block; if (auto BlockIt = CreatedBlocks.find(RawHash); BlockIt != CreatedBlocks.end()) { Payload = BlockIt->second.Payload; @@ -1058,7 +1058,7 @@ namespace remotestore_impl { { auto It = BulkBlockAttachmentsToUpload.find(Chunk); ZEN_ASSERT(It != BulkBlockAttachmentsToUpload.end()); - CompositeBuffer ChunkPayload = It->second(It->first); + CompressedBuffer ChunkPayload = It->second(It->first).second; if (!ChunkPayload) { RemoteResult.SetError(static_cast(HttpResponseCode::NotFound), @@ -1067,8 +1067,8 @@ namespace remotestore_impl { ChunkBuffers.clear(); break; } - ChunksSize += ChunkPayload.GetSize(); - ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload).Flatten().AsIoBuffer())); + ChunksSize += ChunkPayload.GetCompressedSize(); + ChunkBuffers.emplace_back(SharedBuffer(std::move(ChunkPayload).GetCompressed().Flatten().AsIoBuffer())); } RemoteProjectStore::SaveAttachmentsResult Result = RemoteStore.SaveAttachments(ChunkBuffers); if (Result.ErrorCode) @@ -1139,54 +1139,13 @@ namespace remotestore_impl { } } // namespace remotestore_impl -bool -IterateBlock(const SharedBuffer& BlockPayload, std::function Visitor) -{ - ZEN_ASSERT(BlockPayload); - if (BlockPayload.GetSize() < 1) - { - return false; - } - - MemoryView BlockView = BlockPayload.GetView(); - const uint8_t* ReadPtr = reinterpret_cast(BlockView.GetData()); - uint32_t NumberSize; - uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize); - ReadPtr += NumberSize; - std::vector ChunkSizes; - ChunkSizes.reserve(ChunkCount); - while (ChunkCount--) - { - ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize)); - ReadPtr += NumberSize; - } - ptrdiff_t TempBufferLength = std::distance(reinterpret_cast(BlockView.GetData()), ReadPtr); - ZEN_ASSERT(TempBufferLength > 0); - for (uint64_t ChunkSize : ChunkSizes) - { - IoBuffer Chunk(IoBuffer::Wrap, ReadPtr, ChunkSize); - IoHash AttachmentRawHash; - uint64_t AttachmentRawSize; - CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize); - - if (!CompressedChunk) - { - ZEN_ERROR("Invalid chunk in block"); - return false; - } - Visitor(std::move(CompressedChunk), AttachmentRawHash); - ReadPtr += ChunkSize; - ZEN_ASSERT(ReadPtr <= BlockView.GetDataEnd()); - } - return true; -}; std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject) { using namespace std::literals; - std::vector Result; - CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); + std::vector Result; + CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); std::vector BlockHashes; BlockHashes.reserve(BlocksArray.Num()); @@ -1199,11 +1158,11 @@ GetBlockHashesFromOplog(CbObjectView ContainerObject) return BlockHashes; } -std::vector +std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes) { using namespace std::literals; - std::vector Result; + std::vector Result; CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); tsl::robin_set IncludeSet; IncludeSet.insert(IncludeBlockHashes.begin(), IncludeBlockHashes.end()); @@ -1232,47 +1191,6 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span Include return Result; } -CompressedBuffer -GenerateBlock(std::vector>&& FetchChunks, RemoteProjectStore::Block& OutBlock) -{ - const size_t ChunkCount = FetchChunks.size(); - - std::vector ChunkSegments; - ChunkSegments.resize(1); - ChunkSegments.reserve(1 + ChunkCount); - OutBlock.ChunkHashes.reserve(ChunkCount); - OutBlock.ChunkLengths.reserve(ChunkCount); - { - IoBuffer TempBuffer(ChunkCount * 9); - MutableMemoryView View = TempBuffer.GetMutableView(); - uint8_t* BufferStartPtr = reinterpret_cast(View.GetData()); - uint8_t* BufferEndPtr = BufferStartPtr; - BufferEndPtr += WriteVarUInt(gsl::narrow(ChunkCount), BufferEndPtr); - for (const auto& It : FetchChunks) - { - CompositeBuffer Chunk = It.second(It.first); - uint64_t ChunkSize = 0; - std::span Segments = Chunk.GetSegments(); - for (const SharedBuffer& Segment : Segments) - { - ChunkSize += Segment.GetSize(); - ChunkSegments.push_back(Segment); - } - BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr); - OutBlock.ChunkHashes.push_back(It.first); - OutBlock.ChunkLengths.push_back(gsl::narrow(ChunkSize)); - } - ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd()); - ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr); - ChunkSegments[0] = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow(TempBufferLength))); - } - CompressedBuffer CompressedBlock = - CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None); - OutBlock.BlockHash = CompressedBlock.DecodeRawHash(); - OutBlock.FirstChunkOffset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + ChunkSegments[0].GetSize()); - return CompressedBlock; -} - CbObject BuildContainer(CidStore& ChunkStore, ProjectStore::Project& Project, @@ -1283,9 +1201,9 @@ BuildContainer(CidStore& ChunkStore, bool BuildBlocks, bool IgnoreMissingAttachments, bool AllowChunking, - const std::vector& KnownBlocks, + const std::vector& KnownBlocks, WorkerThreadPool& WorkerPool, - const std::function& AsyncOnBlock, + const std::function& AsyncOnBlock, const std::function& OnLargeAttachment, const std::function>&&)>& OnBlockChunks, bool EmbedLooseFiles, @@ -1307,9 +1225,9 @@ BuildContainer(CidStore& ChunkStore, std::unordered_map UploadAttachments; - RwLock BlocksLock; - std::vector Blocks; - CompressedBuffer OpsBuffer; + RwLock BlocksLock; + std::vector Blocks; + CompressedBuffer OpsBuffer; std::filesystem::path AttachmentTempPath = Oplog.TempPath(); AttachmentTempPath.append(".pending"); @@ -1525,7 +1443,7 @@ BuildContainer(CidStore& ChunkStore, return {}; } - auto FindReuseBlocks = [](const std::vector& KnownBlocks, + auto FindReuseBlocks = [](const std::vector& KnownBlocks, const std::unordered_set& Attachments, JobContext* OptionalContext) -> std::vector { std::vector ReuseBlockIndexes; @@ -1538,8 +1456,8 @@ BuildContainer(CidStore& ChunkStore, for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++) { - const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex]; - size_t BlockAttachmentCount = KnownBlock.ChunkHashes.size(); + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + size_t BlockAttachmentCount = KnownBlock.ChunkHashes.size(); if (BlockAttachmentCount == 0) { continue; @@ -1586,7 +1504,7 @@ BuildContainer(CidStore& ChunkStore, std::vector ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { - const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex]; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; for (const IoHash& KnownHash : KnownBlock.ChunkHashes) { if (UploadAttachments.erase(KnownHash) == 1) @@ -1632,12 +1550,12 @@ BuildContainer(CidStore& ChunkStore, return Chunked; }; - RwLock ResolveLock; - std::unordered_set ChunkedHashes; - std::unordered_set LargeChunkHashes; - std::unordered_map ChunkedUploadAttachments; - std::unordered_map LooseUploadAttachments; - std::unordered_set MissingHashes; + RwLock ResolveLock; + std::unordered_set ChunkedHashes; + std::unordered_set LargeChunkHashes; + std::unordered_map ChunkedUploadAttachments; + std::unordered_map, IoHash::Hasher> LooseUploadAttachments; + std::unordered_set MissingHashes; remotestore_impl::ReportMessage(OptionalContext, fmt::format("Resolving {} attachments from {} ops", UploadAttachments.size(), TotalOpCount)); @@ -1730,7 +1648,7 @@ BuildContainer(CidStore& ChunkStore, } else { - size_t RawSize = RawData.GetSize(); + uint64_t RawSize = RawData.GetSize(); CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(RawData), OodleCompressor::Mermaid, OodleCompressionLevel::VeryFast); @@ -1753,8 +1671,8 @@ BuildContainer(CidStore& ChunkStore, { UploadAttachment->Size = Compressed.GetCompressedSize(); ResolveLock.WithExclusiveLock( - [RawHash, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() { - LooseUploadAttachments.insert_or_assign(RawHash, std::move(Data)); + [RawHash, RawSize, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() { + LooseUploadAttachments.insert_or_assign(RawHash, std::make_pair(RawSize, std::move(Data))); }); } } @@ -1927,7 +1845,7 @@ BuildContainer(CidStore& ChunkStore, std::vector ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { - const RemoteProjectStore::Block& KnownBlock = KnownBlocks[KnownBlockIndex]; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; for (const IoHash& KnownHash : KnownBlock.ChunkHashes) { if (ChunkedHashes.erase(KnownHash) == 1) @@ -2109,16 +2027,25 @@ BuildContainer(CidStore& ChunkStore, { if (auto It = LooseUploadAttachments.find(RawHash); It != LooseUploadAttachments.end()) { - ChunksInBlock.emplace_back(std::make_pair(RawHash, [IoBuffer = SharedBuffer(It->second)](const IoHash&) { - return CompositeBuffer(IoBuffer); - })); + ChunksInBlock.emplace_back(std::make_pair( + RawHash, + [RawSize = It->second.first, + IoBuffer = SharedBuffer(It->second.second)](const IoHash&) -> std::pair { + return std::make_pair(RawSize, CompressedBuffer::FromCompressedNoValidate(IoBuffer.AsIoBuffer())); + })); LooseUploadAttachments.erase(It); } else { - ChunksInBlock.emplace_back(std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) { - return CompositeBuffer(SharedBuffer(ChunkStore.FindChunkByCid(RawHash))); - })); + ChunksInBlock.emplace_back( + std::make_pair(RawHash, [&ChunkStore](const IoHash& RawHash) -> std::pair { + IoBuffer Chunk = ChunkStore.FindChunkByCid(RawHash); + IoHash _; + uint64_t RawSize = 0; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Chunk)), _, RawSize); + ZEN_ASSERT(Compressed); + return {RawSize, Compressed}; + })); } BlockSize += PayloadSize; @@ -2169,14 +2096,15 @@ BuildContainer(CidStore& ChunkStore, if (BlockAttachmentHashes.insert(ChunkHash).second) { const ChunkSource& Source = Chunked.ChunkSources[ChunkIndex]; - ChunksInBlock.emplace_back(std::make_pair( - ChunkHash, - [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size](const IoHash&) { - return CompressedBuffer::Compress(SharedBuffer(IoBuffer(Source, Offset, Size)), - OodleCompressor::Mermaid, - OodleCompressionLevel::None) - .GetCompressed(); - })); + ChunksInBlock.emplace_back( + std::make_pair(ChunkHash, + [Source = ChunkedFile.Source, Offset = Source.Offset, Size = Source.Size]( + const IoHash&) -> std::pair { + return {Size, + CompressedBuffer::Compress(SharedBuffer(IoBuffer(Source, Offset, Size)), + OodleCompressor::Mermaid, + OodleCompressionLevel::None)}; + })); BlockSize += CompressedBuffer::GetHeaderSizeForNoneEncoder() + Source.Size; if (BuildBlocks) { @@ -2298,7 +2226,7 @@ BuildContainer(CidStore& ChunkStore, OplogContinerWriter.AddBinary("ops"sv, CompressedOpsSection.GetCompressed().Flatten().AsIoBuffer()); OplogContinerWriter.BeginArray("blocks"sv); { - for (const RemoteProjectStore::Block& B : Blocks) + for (const ChunkBlockDescription& B : Blocks) { ZEN_ASSERT(!B.ChunkHashes.empty()); if (BuildBlocks) @@ -2392,7 +2320,7 @@ BuildContainer(CidStore& ChunkStore, bool BuildBlocks, bool IgnoreMissingAttachments, bool AllowChunking, - const std::function& AsyncOnBlock, + const std::function& AsyncOnBlock, const std::function& OnLargeAttachment, const std::function>&&)>& OnBlockChunks, bool EmbedLooseFiles) @@ -2458,8 +2386,8 @@ SaveOplog(CidStore& ChunkStore, std::unordered_map CreatedBlocks; tsl::robin_map LooseLargeFiles; - auto MakeTempBlock = [AttachmentTempPath, &RemoteResult, &AttachmentsLock, &CreatedBlocks](CompressedBuffer&& CompressedBlock, - RemoteProjectStore::Block&& Block) { + auto MakeTempBlock = [AttachmentTempPath, &RemoteResult, &AttachmentsLock, &CreatedBlocks](CompressedBuffer&& CompressedBlock, + ChunkBlockDescription&& Block) { std::filesystem::path BlockPath = AttachmentTempPath; BlockPath.append(Block.BlockHash.ToHexString()); try @@ -2478,8 +2406,8 @@ SaveOplog(CidStore& ChunkStore, } }; - auto UploadBlock = [&RemoteStore, &RemoteResult, &Info, OptionalContext](CompressedBuffer&& CompressedBlock, - RemoteProjectStore::Block&& Block) { + auto UploadBlock = [&RemoteStore, &RemoteResult, &Info, OptionalContext](CompressedBuffer&& CompressedBlock, + ChunkBlockDescription&& Block) { IoHash BlockHash = Block.BlockHash; RemoteProjectStore::SaveAttachmentResult Result = RemoteStore.SaveAttachment(CompressedBlock.GetCompressed(), BlockHash, std::move(Block)); @@ -2512,7 +2440,7 @@ SaveOplog(CidStore& ChunkStore, ZEN_DEBUG("Found attachment {}", AttachmentHash); }; - std::function OnBlock; + std::function OnBlock; if (RemoteStoreInfo.UseTempBlockFiles) { OnBlock = MakeTempBlock; @@ -2522,7 +2450,7 @@ SaveOplog(CidStore& ChunkStore, OnBlock = UploadBlock; } - std::vector KnownBlocks; + std::vector KnownBlocks; uint64_t TransferWallTimeMS = 0; diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h index e05cb9923..1ef0416b7 100644 --- a/src/zenserver/projectstore/remoteprojectstore.h +++ b/src/zenserver/projectstore/remoteprojectstore.h @@ -5,6 +5,8 @@ #include #include "projectstore.h" +#include + #include namespace zen { @@ -16,14 +18,6 @@ struct ChunkedInfo; class RemoteProjectStore { public: - struct Block - { - IoHash BlockHash; - std::vector ChunkHashes; - std::vector ChunkLengths; - uint32_t FirstChunkOffset = (uint32_t)-1; - }; - struct Result { int32_t ErrorCode{}; @@ -72,7 +66,7 @@ public: struct GetKnownBlocksResult : public Result { - std::vector Blocks; + std::vector Blocks; }; struct RemoteStoreInfo @@ -101,11 +95,11 @@ public: virtual RemoteStoreInfo GetInfo() const = 0; virtual Stats GetStats() const = 0; - virtual CreateContainerResult CreateContainer() = 0; - virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0; - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&& Block) = 0; - virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; - virtual SaveAttachmentsResult SaveAttachments(const std::vector& Payloads) = 0; + virtual CreateContainerResult CreateContainer() = 0; + virtual SaveResult SaveContainer(const IoBuffer& Payload) = 0; + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&& Block) = 0; + virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; + virtual SaveAttachmentsResult SaveAttachments(const std::vector& Payloads) = 0; virtual LoadContainerResult LoadContainer() = 0; virtual GetKnownBlocksResult GetKnownBlocks() = 0; @@ -125,7 +119,6 @@ struct RemoteStoreOptions }; typedef std::function TGetAttachmentBufferFunc; -typedef std::function FetchChunkFunc; RemoteProjectStore::LoadContainerResult BuildContainer( CidStore& ChunkStore, @@ -137,7 +130,7 @@ RemoteProjectStore::LoadContainerResult BuildContainer( bool BuildBlocks, bool IgnoreMissingAttachments, bool AllowChunking, - const std::function& AsyncOnBlock, + const std::function& AsyncOnBlock, const std::function& OnLargeAttachment, const std::function>&&)>& OnBlockChunks, bool EmbedLooseFiles); @@ -173,9 +166,7 @@ RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, bool CleanOplog, JobContext* OptionalContext); -CompressedBuffer GenerateBlock(std::vector>&& FetchChunks, RemoteProjectStore::Block& OutBlock); -bool IterateBlock(const SharedBuffer& BlockPayload, std::function Visitor); -std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); -std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); +std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); +std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); } // namespace zen diff --git a/src/zenserver/projectstore/zenremoteprojectstore.cpp b/src/zenserver/projectstore/zenremoteprojectstore.cpp index 42519b108..2ebf58a5d 100644 --- a/src/zenserver/projectstore/zenremoteprojectstore.cpp +++ b/src/zenserver/projectstore/zenremoteprojectstore.cpp @@ -93,7 +93,7 @@ public: return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, Block&&) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override { std::string SaveRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); HttpClient::Response Response = m_Client.Post(SaveRequest, Payload, ZenContentType::kCompressedBinary); diff --git a/src/zenstore/chunkedfile.cpp b/src/zenstore/chunkedfile.cpp deleted file mode 100644 index f200bc1ec..000000000 --- a/src/zenstore/chunkedfile.cpp +++ /dev/null @@ -1,505 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include -#include - -#include "chunking.h" - -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -namespace { - struct ChunkedHeader - { - static constexpr uint32_t ExpectedMagic = 0x646b6863; // chkd - static constexpr uint32_t CurrentVersion = 1; - - uint32_t Magic = ExpectedMagic; - uint32_t Version = CurrentVersion; - uint32_t ChunkSequenceLength; - uint32_t ChunkHashCount; - uint64_t ChunkSequenceOffset; - uint64_t ChunkHashesOffset; - uint64_t RawSize = 0; - IoHash RawHash; - }; -} // namespace - -IoBuffer -SerializeChunkedInfo(const ChunkedInfo& Info) -{ - size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) + - RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16); - IoBuffer HeaderData(HeaderSize); - - ChunkedHeader Header; - Header.ChunkSequenceLength = gsl::narrow(Info.ChunkSequence.size()); - Header.ChunkHashCount = gsl::narrow(Info.ChunkHashes.size()); - Header.ChunkSequenceOffset = RoundUp(sizeof(ChunkedHeader), 16); - Header.ChunkHashesOffset = RoundUp(Header.ChunkSequenceOffset + sizeof(uint32_t) * Header.ChunkSequenceLength, 16); - Header.RawSize = Info.RawSize; - Header.RawHash = Info.RawHash; - - MutableMemoryView WriteView = HeaderData.GetMutableView(); - { - MutableMemoryView HeaderWriteView = WriteView.Left(sizeof(Header)); - HeaderWriteView.CopyFrom(MemoryView(&Header, sizeof(Header))); - } - { - MutableMemoryView ChunkSequenceWriteView = WriteView.Mid(Header.ChunkSequenceOffset, sizeof(uint32_t) * Header.ChunkSequenceLength); - ChunkSequenceWriteView.CopyFrom(MemoryView(Info.ChunkSequence.data(), ChunkSequenceWriteView.GetSize())); - } - { - MutableMemoryView ChunksWriteView = WriteView.Mid(Header.ChunkHashesOffset, sizeof(IoHash) * Header.ChunkHashCount); - ChunksWriteView.CopyFrom(MemoryView(Info.ChunkHashes.data(), ChunksWriteView.GetSize())); - } - - return HeaderData; -} - -ChunkedInfo -DeserializeChunkedInfo(IoBuffer& Buffer) -{ - MemoryView View = Buffer.GetView(); - ChunkedHeader Header; - { - MutableMemoryView HeaderWriteView(&Header, sizeof(Header)); - HeaderWriteView.CopyFrom(View.Left(sizeof(Header))); - } - if (Header.Magic != ChunkedHeader::ExpectedMagic) - { - return {}; - } - if (Header.Version != ChunkedHeader::CurrentVersion) - { - return {}; - } - ChunkedInfo Info; - Info.RawSize = Header.RawSize; - Info.RawHash = Header.RawHash; - Info.ChunkSequence.resize(Header.ChunkSequenceLength); - Info.ChunkHashes.resize(Header.ChunkHashCount); - { - MutableMemoryView ChunkSequenceWriteView(Info.ChunkSequence.data(), sizeof(uint32_t) * Header.ChunkSequenceLength); - ChunkSequenceWriteView.CopyFrom(View.Mid(Header.ChunkSequenceOffset, ChunkSequenceWriteView.GetSize())); - } - { - MutableMemoryView ChunksWriteView(Info.ChunkHashes.data(), sizeof(IoHash) * Header.ChunkHashCount); - ChunksWriteView.CopyFrom(View.Mid(Header.ChunkHashesOffset, ChunksWriteView.GetSize())); - } - - return Info; -} - -void -Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function GetChunk) -{ - BasicFile Reconstructed; - Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate); - BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024); - uint64_t Offset = 0; - for (uint32_t SequenceIndex : Info.ChunkSequence) - { - IoBuffer Chunk = GetChunk(Info.ChunkHashes[SequenceIndex]); - ReconstructedWriter.Write(Chunk.GetData(), Chunk.GetSize(), Offset); - Offset += Chunk.GetSize(); - } -} - -ChunkedInfoWithSource -ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params) -{ - ChunkedInfoWithSource Result; - tsl::robin_map FoundChunks; - - ZenChunkHelper Chunker; - Chunker.SetUseThreshold(Params.UseThreshold); - Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize); - size_t End = Offset + Size; - const size_t ScanBufferSize = 1u * 1024 * 1024; // (Params.MaxSize * 9) / 3;//1 * 1024 * 1024; - BasicFileBuffer RawBuffer(RawData, ScanBufferSize); - MemoryView SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset); - ZEN_ASSERT(!SliceView.IsEmpty()); - size_t SliceSize = SliceView.GetSize(); - IoHashStream RawHashStream; - while (Offset < End) - { - size_t ScanLength = Chunker.ScanChunk(SliceView.GetData(), SliceSize); - if (ScanLength == ZenChunkHelper::kNoBoundaryFound) - { - if (Offset + SliceSize == End) - { - ScanLength = SliceSize; - } - else - { - SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset); - SliceSize = SliceView.GetSize(); - Chunker.Reset(); - continue; - } - } - uint32_t ChunkLength = gsl::narrow(ScanLength); // +HashedLength); - MemoryView ChunkView = SliceView.Left(ScanLength); - RawHashStream.Append(ChunkView); - IoHash ChunkHash = IoHash::HashBuffer(ChunkView); - SliceView.RightChopInline(ScanLength); - if (auto It = FoundChunks.find(ChunkHash); It != FoundChunks.end()) - { - Result.Info.ChunkSequence.push_back(It->second); - } - else - { - uint32_t ChunkIndex = gsl::narrow(Result.Info.ChunkHashes.size()); - FoundChunks.insert_or_assign(ChunkHash, ChunkIndex); - Result.Info.ChunkHashes.push_back(ChunkHash); - Result.ChunkSources.push_back(ChunkSource{.Offset = Offset, .Size = ChunkLength}); - Result.Info.ChunkSequence.push_back(ChunkIndex); - } - - SliceSize = SliceView.GetSize(); - Offset += ChunkLength; - } - Result.Info.RawSize = Size; - Result.Info.RawHash = RawHashStream.GetHash(); - return Result; -} - -} // namespace zen - -#if ZEN_WITH_TESTS -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include "chunking.h" - -ZEN_THIRD_PARTY_INCLUDES_START -# include -# include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { -# if 0 -TEST_CASE("chunkedfile.findparams") -{ -# if 1 - DirectoryContent SourceContent1; - GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1); - const std::vector& SourceFiles1 = SourceContent1.Files; - DirectoryContent SourceContent2; - GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2); - const std::vector& SourceFiles2 = SourceContent2.Files; -# else - std::filesystem::path SourcePath1 = - "E:\\Temp\\ChunkingTestData\\31375996\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; - std::filesystem::path SourcePath2 = - "E:\\Temp\\ChunkingTestData\\31379208\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; - const std::vector& SourceFiles1 = {SourcePath1}; - const std::vector& SourceFiles2 = {SourcePath2}; -# endif - ChunkedParams Params[] = {ChunkedParams{.UseThreshold = false, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}, - ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 35598}, - ChunkedParams{.UseThreshold = false, .MinSize = 16848, .MaxSize = 135168, .AvgSize = 39030}, - ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 36222}, - ChunkedParams{.UseThreshold = false, .MinSize = 15744, .MaxSize = 126976, .AvgSize = 36600}, - ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 35442}, - ChunkedParams{.UseThreshold = false, .MinSize = 16464, .MaxSize = 131072, .AvgSize = 37950}, - ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 38914}, - ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 35556}, - ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 35520}, - ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 35478}, - ChunkedParams{.UseThreshold = false, .MinSize = 16896, .MaxSize = 135168, .AvgSize = 39072}, - ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 38880}, - ChunkedParams{.UseThreshold = false, .MinSize = 15840, .MaxSize = 126976, .AvgSize = 36678}, - ChunkedParams{.UseThreshold = false, .MinSize = 16800, .MaxSize = 135168, .AvgSize = 38994}, - ChunkedParams{.UseThreshold = false, .MinSize = 15888, .MaxSize = 126976, .AvgSize = 36714}, - ChunkedParams{.UseThreshold = false, .MinSize = 15792, .MaxSize = 126976, .AvgSize = 36636}, - ChunkedParams{.UseThreshold = false, .MinSize = 14880, .MaxSize = 118784, .AvgSize = 37609}, - ChunkedParams{.UseThreshold = false, .MinSize = 15936, .MaxSize = 126976, .AvgSize = 36756}, - ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 38955}, - ChunkedParams{.UseThreshold = false, .MinSize = 15984, .MaxSize = 126976, .AvgSize = 36792}, - ChunkedParams{.UseThreshold = false, .MinSize = 14400, .MaxSize = 114688, .AvgSize = 36338}, - ChunkedParams{.UseThreshold = false, .MinSize = 14832, .MaxSize = 118784, .AvgSize = 37568}, - ChunkedParams{.UseThreshold = false, .MinSize = 16944, .MaxSize = 135168, .AvgSize = 39108}, - ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 36297}, - ChunkedParams{.UseThreshold = false, .MinSize = 14208, .MaxSize = 114688, .AvgSize = 36188}, - ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 36372}, - ChunkedParams{.UseThreshold = false, .MinSize = 13296, .MaxSize = 106496, .AvgSize = 36592}, - ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 38805}, - ChunkedParams{.UseThreshold = false, .MinSize = 14304, .MaxSize = 114688, .AvgSize = 36263}, - ChunkedParams{.UseThreshold = false, .MinSize = 14784, .MaxSize = 118784, .AvgSize = 37534}, - ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 38839}, - ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 39360}, - ChunkedParams{.UseThreshold = false, .MinSize = 13776, .MaxSize = 110592, .AvgSize = 37976}, - ChunkedParams{.UseThreshold = false, .MinSize = 14736, .MaxSize = 118784, .AvgSize = 37493}, - ChunkedParams{.UseThreshold = false, .MinSize = 14928, .MaxSize = 118784, .AvgSize = 37643}, - ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 39504}, - ChunkedParams{.UseThreshold = false, .MinSize = 13392, .MaxSize = 106496, .AvgSize = 36664}, - ChunkedParams{.UseThreshold = false, .MinSize = 13872, .MaxSize = 110592, .AvgSize = 38048}, - ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 39432}, - ChunkedParams{.UseThreshold = false, .MinSize = 13200, .MaxSize = 106496, .AvgSize = 36520}, - ChunkedParams{.UseThreshold = false, .MinSize = 17328, .MaxSize = 139264, .AvgSize = 36378}, - ChunkedParams{.UseThreshold = false, .MinSize = 17376, .MaxSize = 139264, .AvgSize = 36421}, - ChunkedParams{.UseThreshold = false, .MinSize = 17424, .MaxSize = 139264, .AvgSize = 36459}, - ChunkedParams{.UseThreshold = false, .MinSize = 17472, .MaxSize = 139264, .AvgSize = 36502}, - ChunkedParams{.UseThreshold = false, .MinSize = 17520, .MaxSize = 139264, .AvgSize = 36540}, - ChunkedParams{.UseThreshold = false, .MinSize = 17808, .MaxSize = 143360, .AvgSize = 37423}, - ChunkedParams{.UseThreshold = false, .MinSize = 17856, .MaxSize = 143360, .AvgSize = 37466}, - ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 25834}, - ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 21917}, - ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 29751}, - ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 33668}, - ChunkedParams{.UseThreshold = false, .MinSize = 17952, .MaxSize = 143360, .AvgSize = 37547}, - ChunkedParams{.UseThreshold = false, .MinSize = 17904, .MaxSize = 143360, .AvgSize = 37504}, - ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 22371}, - ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 37585}, - ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 26406}, - ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 26450}, - ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 30615}, - ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 30441}, - ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 22417}, - ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 22557}, - ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 30528}, - ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 27112}, - ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 34644}, - ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 34476}, - ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 35408}, - ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 38592}, - ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 30483}, - ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 26586}, - ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 26496}, - ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 31302}, - ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 34516}, - ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 22964}, - ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 35448}, - ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 38630}, - ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 23010}, - ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 31260}, - ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 34600}, - ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 27156}, - ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 30570}, - ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 38549}, - ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 22510}, - ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 38673}, - ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 34560}, - ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 22464}, - ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 26540}, - ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 38511}, - ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 23057}, - ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 27202}, - ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 31347}, - ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 35492}, - ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 31389}, - ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 27246}, - ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 23103}, - ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 35532}, - ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 23150}, - ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 27292}, - ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 31434}, - ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 35576}, - ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 27336}, - ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 23196}, - ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 31476}, - ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 35616}, - ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 27862}, - ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 32121}, - ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 23603}, - ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 36380}, - ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 27908}, - ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 23650}, - ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 32166}, - ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 36424}, - ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 23696}, - ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 32253}, - ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 32208}, - ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 23743}, - ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 36548}, - ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 28042}, - ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 23789}, - ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 32295}, - ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 36508}, - ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 27952}, - ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 27998}, - ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 36464}}; - - static const size_t ParamsCount = sizeof(Params) / sizeof(ChunkedParams); - std::vector Infos1(SourceFiles1.size()); - std::vector Infos2(SourceFiles2.size()); - - WorkerThreadPool WorkerPool(32); - - for (size_t I = 0; I < ParamsCount; I++) - { - for (int UseThreshold = 0; UseThreshold < 2; UseThreshold++) - { - Latch WorkLatch(1); - ChunkedParams Param = Params[I]; - Param.UseThreshold = UseThreshold == 1; - Stopwatch Timer; - for (size_t F = 0; F < SourceFiles1.size(); F++) - { - WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles1, &Infos1]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - BasicFile SourceData1; - SourceData1.Open(SourceFiles1[F], BasicFile::Mode::kRead); - Infos1[F] = ChunkData(SourceData1, 0, SourceData1.FileSize(), Param); - }); - } - for (size_t F = 0; F < SourceFiles2.size(); F++) - { - WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles2, &Infos2]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - BasicFile SourceData2; - SourceData2.Open(SourceFiles2[F], BasicFile::Mode::kRead); - Infos2[F] = ChunkData(SourceData2, 0, SourceData2.FileSize(), Param); - }); - } - WorkLatch.CountDown(); - WorkLatch.Wait(); - uint64_t ChunkTimeMS = Timer.GetElapsedTimeMs(); - - uint64_t Raw1Size = 0; - tsl::robin_set Chunks1; - size_t ChunkedSize1 = 0; - for (size_t F = 0; F < SourceFiles1.size(); F++) - { - const ChunkedInfoWithSource& Info = Infos1[F]; - Raw1Size += Info.Info.RawSize; - for (uint32_t Chunk1Index = 0; Chunk1Index < Info.Info.ChunkHashes.size(); ++Chunk1Index) - { - const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk1Index]; - if (Chunks1.insert(ChunkHash).second) - { - ChunkedSize1 += Info.ChunkSources[Chunk1Index].Size; - } - } - } - - uint64_t Raw2Size = 0; - tsl::robin_set Chunks2; - size_t ChunkedSize2 = 0; - size_t DiffSize = 0; - for (size_t F = 0; F < SourceFiles2.size(); F++) - { - const ChunkedInfoWithSource& Info = Infos2[F]; - Raw2Size += Info.Info.RawSize; - for (uint32_t Chunk2Index = 0; Chunk2Index < Info.Info.ChunkHashes.size(); ++Chunk2Index) - { - const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk2Index]; - if (Chunks2.insert(ChunkHash).second) - { - ChunkedSize2 += Info.ChunkSources[Chunk2Index].Size; - if (!Chunks1.contains(ChunkHash)) - { - DiffSize += Info.ChunkSources[Chunk2Index].Size; - } - } - } - } - - ZEN_INFO( - "Diff = {}, Chunks1 = {}, Chunks2 = {}, .UseThreshold = {}, .MinSize = {}, .MaxSize = {}, .AvgSize = {}, RawSize(1) = {}, " - "RawSize(2) = {}, " - "Saved(1) = {}, Saved(2) = {} in {}", - NiceBytes(DiffSize), - Chunks1.size(), - Chunks2.size(), - Param.UseThreshold, - Param.MinSize, - Param.MaxSize, - Param.AvgSize, - NiceBytes(Raw1Size), - NiceBytes(Raw2Size), - NiceBytes(Raw1Size - ChunkedSize1), - NiceBytes(Raw2Size - ChunkedSize2), - NiceTimeSpanMs(ChunkTimeMS)); - } - } - -# if 0 - for (int64_t MinSizeBase = (12u * 1024u); MinSizeBase <= (32u * 1024u); MinSizeBase += 512) - { - for (int64_t Wiggle = -132; Wiggle < 126; Wiggle += 2) - { - // size_t MinSize = 7 * 1024 - 61; // (size_t)(MinSizeBase + Wiggle); - // size_t MaxSize = 16 * (7 * 1024); // 8 * 7 * 1024;// MinSizeBase * 6; - // size_t AvgSize = MaxSize / 2; // 4 * 7 * 1024;// MinSizeBase * 3; - size_t MinSize = (size_t)(MinSizeBase + Wiggle); - //for (size_t MaxSize = (MinSize * 4) - 768; MaxSize < (MinSize * 5) + 768; MaxSize += 64) - size_t MaxSize = 8u * MinSizeBase; - { - for (size_t AvgSize = (MaxSize - MinSize) / 32 + MinSize; AvgSize < (MaxSize - MinSize) / 4 + MinSize; AvgSize += (MaxSize - MinSize) / 32) -// size_t AvgSize = (MaxSize - MinSize) / 4 + MinSize; - { - WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([&WorkLatch, MinSize, MaxSize, AvgSize, SourcePath1, SourcePath2]() - { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - ChunkedParams Params{ .UseThreshold = true, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize }; - BasicFile SourceData1; - SourceData1.Open(SourcePath1, BasicFile::Mode::kRead); - BasicFile SourceData2; - SourceData2.Open(SourcePath2, BasicFile::Mode::kRead); - ChunkedInfoWithSource Info1 = ChunkData(SourceData1, Params); - ChunkedInfoWithSource Info2 = ChunkData(SourceData2, Params); - - tsl::robin_set Chunks1; - Chunks1.reserve(Info1.Info.ChunkHashes.size()); - Chunks1.insert(Info1.Info.ChunkHashes.begin(), Info1.Info.ChunkHashes.end()); - size_t ChunkedSize1 = 0; - for (uint32_t Chunk1Index = 0; Chunk1Index < Info1.Info.ChunkHashes.size(); ++Chunk1Index) - { - ChunkedSize1 += Info1.ChunkSources[Chunk1Index].Size; - } - size_t DiffSavedSize = 0; - size_t ChunkedSize2 = 0; - for (uint32_t Chunk2Index = 0; Chunk2Index < Info2.Info.ChunkHashes.size(); ++Chunk2Index) - { - ChunkedSize2 += Info2.ChunkSources[Chunk2Index].Size; - if (Chunks1.find(Info2.Info.ChunkHashes[Chunk2Index]) == Chunks1.end()) - { - DiffSavedSize += Info2.ChunkSources[Chunk2Index].Size; - } - } - ZEN_INFO("Diff {}, Chunks1: {}, Chunks2: {}, Min: {}, Max: {}, Avg: {}, Saved(1) {}, Saved(2) {}", - NiceBytes(DiffSavedSize), - Info1.Info.ChunkHashes.size(), - Info2.Info.ChunkHashes.size(), - MinSize, - MaxSize, - AvgSize, - NiceBytes(Info1.Info.RawSize - ChunkedSize1), - NiceBytes(Info2.Info.RawSize - ChunkedSize2)); - }); - } - } - } - } -# endif // 0 - - // WorkLatch.CountDown(); - // WorkLatch.Wait(); -} -# endif // 0 - -void -chunkedfile_forcelink() -{ -} - -} // namespace zen - -#endif diff --git a/src/zenstore/chunking.cpp b/src/zenstore/chunking.cpp deleted file mode 100644 index 30edd322a..000000000 --- a/src/zenstore/chunking.cpp +++ /dev/null @@ -1,382 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "chunking.h" - -#include - -#include - -namespace zen::detail { - -static const uint32_t BuzhashTable[] = { - 0x458be752, 0xc10748cc, 0xfbbcdbb8, 0x6ded5b68, 0xb10a82b5, 0x20d75648, 0xdfc5665f, 0xa8428801, 0x7ebf5191, 0x841135c7, 0x65cc53b3, - 0x280a597c, 0x16f60255, 0xc78cbc3e, 0x294415f5, 0xb938d494, 0xec85c4e6, 0xb7d33edc, 0xe549b544, 0xfdeda5aa, 0x882bf287, 0x3116737c, - 0x05569956, 0xe8cc1f68, 0x0806ac5e, 0x22a14443, 0x15297e10, 0x50d090e7, 0x4ba60f6f, 0xefd9f1a7, 0x5c5c885c, 0x82482f93, 0x9bfd7c64, - 0x0b3e7276, 0xf2688e77, 0x8fad8abc, 0xb0509568, 0xf1ada29f, 0xa53efdfe, 0xcb2b1d00, 0xf2a9e986, 0x6463432b, 0x95094051, 0x5a223ad2, - 0x9be8401b, 0x61e579cb, 0x1a556a14, 0x5840fdc2, 0x9261ddf6, 0xcde002bb, 0x52432bb0, 0xbf17373e, 0x7b7c222f, 0x2955ed16, 0x9f10ca59, - 0xe840c4c9, 0xccabd806, 0x14543f34, 0x1462417a, 0x0d4a1f9c, 0x087ed925, 0xd7f8f24c, 0x7338c425, 0xcf86c8f5, 0xb19165cd, 0x9891c393, - 0x325384ac, 0x0308459d, 0x86141d7e, 0xc922116a, 0xe2ffa6b6, 0x53f52aed, 0x2cd86197, 0xf5b9f498, 0xbf319c8f, 0xe0411fae, 0x977eb18c, - 0xd8770976, 0x9833466a, 0xc674df7f, 0x8c297d45, 0x8ca48d26, 0xc49ed8e2, 0x7344f874, 0x556f79c7, 0x6b25eaed, 0xa03e2b42, 0xf68f66a4, - 0x8e8b09a2, 0xf2e0e62a, 0x0d3a9806, 0x9729e493, 0x8c72b0fc, 0x160b94f6, 0x450e4d3d, 0x7a320e85, 0xbef8f0e1, 0x21d73653, 0x4e3d977a, - 0x1e7b3929, 0x1cc6c719, 0xbe478d53, 0x8d752809, 0xe6d8c2c6, 0x275f0892, 0xc8acc273, 0x4cc21580, 0xecc4a617, 0xf5f7be70, 0xe795248a, - 0x375a2fe9, 0x425570b6, 0x8898dcf8, 0xdc2d97c4, 0x0106114b, 0x364dc22f, 0x1e0cad1f, 0xbe63803c, 0x5f69fac2, 0x4d5afa6f, 0x1bc0dfb5, - 0xfb273589, 0x0ea47f7b, 0x3c1c2b50, 0x21b2a932, 0x6b1223fd, 0x2fe706a8, 0xf9bd6ce2, 0xa268e64e, 0xe987f486, 0x3eacf563, 0x1ca2018c, - 0x65e18228, 0x2207360a, 0x57cf1715, 0x34c37d2b, 0x1f8f3cde, 0x93b657cf, 0x31a019fd, 0xe69eb729, 0x8bca7b9b, 0x4c9d5bed, 0x277ebeaf, - 0xe0d8f8ae, 0xd150821c, 0x31381871, 0xafc3f1b0, 0x927db328, 0xe95effac, 0x305a47bd, 0x426ba35b, 0x1233af3f, 0x686a5b83, 0x50e072e5, - 0xd9d3bb2a, 0x8befc475, 0x487f0de6, 0xc88dff89, 0xbd664d5e, 0x971b5d18, 0x63b14847, 0xd7d3c1ce, 0x7f583cf3, 0x72cbcb09, 0xc0d0a81c, - 0x7fa3429b, 0xe9158a1b, 0x225ea19a, 0xd8ca9ea3, 0xc763b282, 0xbb0c6341, 0x020b8293, 0xd4cd299d, 0x58cfa7f8, 0x91b4ee53, 0x37e4d140, - 0x95ec764c, 0x30f76b06, 0x5ee68d24, 0x679c8661, 0xa41979c2, 0xf2b61284, 0x4fac1475, 0x0adb49f9, 0x19727a23, 0x15a7e374, 0xc43a18d5, - 0x3fb1aa73, 0x342fc615, 0x924c0793, 0xbee2d7f0, 0x8a279de9, 0x4aa2d70c, 0xe24dd37f, 0xbe862c0b, 0x177c22c2, 0x5388e5ee, 0xcd8a7510, - 0xf901b4fd, 0xdbc13dbc, 0x6c0bae5b, 0x64efe8c7, 0x48b02079, 0x80331a49, 0xca3d8ae6, 0xf3546190, 0xfed7108b, 0xc49b941b, 0x32baf4a9, - 0xeb833a4a, 0x88a3f1a5, 0x3a91ce0a, 0x3cc27da1, 0x7112e684, 0x4a3096b1, 0x3794574c, 0xa3c8b6f3, 0x1d213941, 0x6e0a2e00, 0x233479f1, - 0x0f4cd82f, 0x6093edd2, 0x5d7d209e, 0x464fe319, 0xd4dcac9e, 0x0db845cb, 0xfb5e4bc3, 0xe0256ce1, 0x09fb4ed1, 0x0914be1e, 0xa5bdb2c3, - 0xc6eb57bb, 0x30320350, 0x3f397e91, 0xa67791bc, 0x86bc0e2c, 0xefa0a7e2, 0xe9ff7543, 0xe733612c, 0xd185897b, 0x329e5388, 0x91dd236b, - 0x2ecb0d93, 0xf4d82a3d, 0x35b5c03f, 0xe4e606f0, 0x05b21843, 0x37b45964, 0x5eff22f4, 0x6027f4cc, 0x77178b3c, 0xae507131, 0x7bf7cabc, - 0xf9c18d66, 0x593ade65, 0xd95ddf11, -}; - -// ROL operation (compiler turns this into a ROL when optimizing) -ZEN_FORCEINLINE static uint32_t -Rotate32(uint32_t Value, size_t RotateCount) -{ - RotateCount &= 31; - - return ((Value) << (RotateCount)) | ((Value) >> (32 - RotateCount)); -} - -} // namespace zen::detail - -namespace zen { - -void -ZenChunkHelper::Reset() -{ - InternalReset(); - - m_BytesScanned = 0; -} - -void -ZenChunkHelper::InternalReset() -{ - m_CurrentHash = 0; - m_CurrentChunkSize = 0; - m_WindowSize = 0; -} - -void -ZenChunkHelper::SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize) -{ - if (m_WindowSize) - return; // Already started - - static_assert(kChunkSizeLimitMin > kWindowSize); - - if (AvgSize) - { - // TODO: Validate AvgSize range - } - else - { - if (MinSize && MaxSize) - { - AvgSize = std::lrint(std::pow(2, (std::log2(MinSize) + std::log2(MaxSize)) / 2)); - } - else if (MinSize) - { - AvgSize = MinSize * 4; - } - else if (MaxSize) - { - AvgSize = MaxSize / 4; - } - else - { - AvgSize = kDefaultAverageChunkSize; - } - } - - if (MinSize) - { - // TODO: Validate MinSize range - } - else - { - MinSize = std::max(AvgSize / 4, kChunkSizeLimitMin); - } - - if (MaxSize) - { - // TODO: Validate MaxSize range - } - else - { - MaxSize = std::min(AvgSize * 4, kChunkSizeLimitMax); - } - - m_Discriminator = gsl::narrow(AvgSize - MinSize); - - if (m_Discriminator < MinSize) - { - m_Discriminator = gsl::narrow(MinSize); - } - - if (m_Discriminator > MaxSize) - { - m_Discriminator = gsl::narrow(MaxSize); - } - - m_Threshold = gsl::narrow((uint64_t(std::numeric_limits::max()) + 1) / m_Discriminator); - - m_ChunkSizeMin = MinSize; - m_ChunkSizeMax = MaxSize; - m_ChunkSizeAvg = AvgSize; -} - -size_t -ZenChunkHelper::ScanChunk(const void* DataBytesIn, size_t ByteCount) -{ - size_t Result = InternalScanChunk(DataBytesIn, ByteCount); - - if (Result == kNoBoundaryFound) - { - m_BytesScanned += ByteCount; - } - else - { - m_BytesScanned += Result; - } - - return Result; -} - -size_t -ZenChunkHelper::InternalScanChunk(const void* DataBytesIn, size_t ByteCount) -{ - size_t CurrentOffset = 0; - const uint8_t* CursorPtr = reinterpret_cast(DataBytesIn); - - // There's no point in updating the hash if we know we're not - // going to have a cut point, so just skip the data. This logic currently - // provides roughly a 20% speedup on my machine - - const size_t NeedHashOffset = m_ChunkSizeMin - kWindowSize; - - if (m_CurrentChunkSize < NeedHashOffset) - { - const uint32_t SkipBytes = gsl::narrow(std::min(ByteCount, NeedHashOffset - m_CurrentChunkSize)); - - ByteCount -= SkipBytes; - m_CurrentChunkSize += SkipBytes; - CurrentOffset += SkipBytes; - CursorPtr += SkipBytes; - - m_WindowSize = 0; - - if (ByteCount == 0) - { - return kNoBoundaryFound; - } - } - - // Fill window first - - if (m_WindowSize < kWindowSize) - { - const uint32_t FillBytes = uint32_t(std::min(ByteCount, kWindowSize - m_WindowSize)); - - memcpy(&m_Window[m_WindowSize], CursorPtr, FillBytes); - - CursorPtr += FillBytes; - - m_WindowSize += FillBytes; - m_CurrentChunkSize += FillBytes; - - CurrentOffset += FillBytes; - ByteCount -= FillBytes; - - if (m_WindowSize < kWindowSize) - { - return kNoBoundaryFound; - } - - // We have a full window, initialize hash - - uint32_t CurrentHash = 0; - - for (int i = 1; i < kWindowSize; ++i) - { - CurrentHash ^= detail::Rotate32(detail::BuzhashTable[m_Window[i - 1]], kWindowSize - i); - } - - m_CurrentHash = CurrentHash ^ detail::BuzhashTable[m_Window[kWindowSize - 1]]; - } - - // Scan for boundaries (i.e points where the hash matches the value determined by - // the discriminator) - - uint32_t CurrentHash = m_CurrentHash; - uint32_t CurrentChunkSize = m_CurrentChunkSize; - - size_t Index = CurrentChunkSize % kWindowSize; - - if (m_Threshold && m_UseThreshold) - { - // This is roughly 4x faster than the general modulo approach on my - // TR 3990X (~940MB/sec) and doesn't require any special parameters to - // achieve max performance - - while (ByteCount) - { - const uint8_t NewByte = *CursorPtr; - const uint8_t OldByte = m_Window[Index]; - - CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ - detail::BuzhashTable[NewByte]; - - CurrentChunkSize++; - CurrentOffset++; - - if (CurrentChunkSize >= m_ChunkSizeMin) - { - bool FoundBoundary; - - if (CurrentChunkSize >= m_ChunkSizeMax) - { - FoundBoundary = true; - } - else - { - FoundBoundary = CurrentHash <= m_Threshold; - } - - if (FoundBoundary) - { - // Boundary found! - InternalReset(); - - return CurrentOffset; - } - } - - m_Window[Index++] = *CursorPtr; - - if (Index == kWindowSize) - { - Index = 0; - } - - ++CursorPtr; - --ByteCount; - } - } - else if ((m_Discriminator & (m_Discriminator - 1)) == 0) - { - // This is quite a bit faster than the generic modulo path, but - // requires a very specific average chunk size to be used. If you - // pass in an even power-of-two divided by 0.75 as the average - // chunk size you'll hit this path - - const uint32_t Mask = m_Discriminator - 1; - - while (ByteCount) - { - const uint8_t NewByte = *CursorPtr; - const uint8_t OldByte = m_Window[Index]; - - CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ - detail::BuzhashTable[NewByte]; - - CurrentChunkSize++; - CurrentOffset++; - - if (CurrentChunkSize >= m_ChunkSizeMin) - { - bool FoundBoundary; - - if (CurrentChunkSize >= m_ChunkSizeMax) - { - FoundBoundary = true; - } - else - { - FoundBoundary = (CurrentHash & Mask) == Mask; - } - - if (FoundBoundary) - { - // Boundary found! - InternalReset(); - - return CurrentOffset; - } - } - - m_Window[Index++] = *CursorPtr; - - if (Index == kWindowSize) - { - Index = 0; - } - - ++CursorPtr; - --ByteCount; - } - } - else - { - // This is the slowest path, which caps out around 250MB/sec for large sizes - // on my TR3900X - - while (ByteCount) - { - const uint8_t NewByte = *CursorPtr; - const uint8_t OldByte = m_Window[Index]; - - CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ - detail::BuzhashTable[NewByte]; - - CurrentChunkSize++; - CurrentOffset++; - - if (CurrentChunkSize >= m_ChunkSizeMin) - { - bool FoundBoundary; - - if (CurrentChunkSize >= m_ChunkSizeMax) - { - FoundBoundary = true; - } - else - { - FoundBoundary = (CurrentHash % m_Discriminator) == (m_Discriminator - 1); - } - - if (FoundBoundary) - { - // Boundary found! - InternalReset(); - - return CurrentOffset; - } - } - - m_Window[Index++] = *CursorPtr; - - if (Index == kWindowSize) - { - Index = 0; - } - - ++CursorPtr; - --ByteCount; - } - } - - m_CurrentChunkSize = CurrentChunkSize; - m_CurrentHash = CurrentHash; - - return kNoBoundaryFound; -} - -} // namespace zen diff --git a/src/zenstore/chunking.h b/src/zenstore/chunking.h deleted file mode 100644 index 09c56454f..000000000 --- a/src/zenstore/chunking.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once -#include - -namespace zen { - -/** Content-defined chunking helper - */ -class ZenChunkHelper -{ -public: - void SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize); - size_t ScanChunk(const void* DataBytes, size_t ByteCount); - void Reset(); - - // This controls which chunking approach is used - threshold or - // modulo based. Threshold is faster and generates similarly sized - // chunks - void SetUseThreshold(bool NewState) { m_UseThreshold = NewState; } - - inline size_t ChunkSizeMin() const { return m_ChunkSizeMin; } - inline size_t ChunkSizeMax() const { return m_ChunkSizeMax; } - inline size_t ChunkSizeAvg() const { return m_ChunkSizeAvg; } - inline uint64_t BytesScanned() const { return m_BytesScanned; } - - static constexpr size_t kNoBoundaryFound = size_t(~0ull); - -private: - size_t m_ChunkSizeMin = 0; - size_t m_ChunkSizeMax = 0; - size_t m_ChunkSizeAvg = 0; - - uint32_t m_Discriminator = 0; // Computed in SetChunkSize() - uint32_t m_Threshold = 0; // Computed in SetChunkSize() - - bool m_UseThreshold = true; - - static constexpr size_t kChunkSizeLimitMax = 64 * 1024 * 1024; - static constexpr size_t kChunkSizeLimitMin = 1024; - static constexpr size_t kDefaultAverageChunkSize = 64 * 1024; - - static constexpr int kWindowSize = 48; - uint8_t m_Window[kWindowSize]; - uint32_t m_WindowSize = 0; - - uint32_t m_CurrentHash = 0; - uint32_t m_CurrentChunkSize = 0; - - uint64_t m_BytesScanned = 0; - - size_t InternalScanChunk(const void* DataBytes, size_t ByteCount); - void InternalReset(); -}; - -} // namespace zen diff --git a/src/zenstore/include/zenstore/chunkedfile.h b/src/zenstore/include/zenstore/chunkedfile.h deleted file mode 100644 index c6330bdbd..000000000 --- a/src/zenstore/include/zenstore/chunkedfile.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include - -#include -#include - -namespace zen { - -class BasicFile; - -struct ChunkedInfo -{ - uint64_t RawSize = 0; - IoHash RawHash; - std::vector ChunkSequence; - std::vector ChunkHashes; -}; - -struct ChunkSource -{ - uint64_t Offset; // 8 - uint32_t Size; // 4 -}; - -struct ChunkedInfoWithSource -{ - ChunkedInfo Info; - std::vector ChunkSources; -}; - -struct ChunkedParams -{ - bool UseThreshold = true; - size_t MinSize = (2u * 1024u) - 128u; - size_t MaxSize = (16u * 1024u); - size_t AvgSize = (3u * 1024u); -}; - -static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}; - -ChunkedInfoWithSource ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params = {}); -void Reconstruct(const ChunkedInfo& Info, - const std::filesystem::path& TargetPath, - std::function GetChunk); -IoBuffer SerializeChunkedInfo(const ChunkedInfo& Info); -ChunkedInfo DeserializeChunkedInfo(IoBuffer& Buffer); - -void chunkedfile_forcelink(); -} // namespace zen diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp new file mode 100644 index 000000000..6dae5af11 --- /dev/null +++ b/src/zenutil/chunkblock.cpp @@ -0,0 +1,166 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +#include + +namespace zen { + +using namespace std::literals; + +ChunkBlockDescription +ParseChunkBlockDescription(const CbObjectView& BlockObject) +{ + ChunkBlockDescription Result; + Result.BlockHash = BlockObject["rawHash"sv].AsHash(); + if (Result.BlockHash != IoHash::Zero) + { + CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView(); + Result.ChunkHashes.reserve(ChunksArray.Num()); + for (CbFieldView ChunkView : ChunksArray) + { + Result.ChunkHashes.push_back(ChunkView.AsHash()); + } + + CbArrayView ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView(); + std::vector ChunkLengths; + Result.ChunkRawLengths.reserve(ChunkRawLengthsArray.Num()); + for (CbFieldView ChunkView : ChunkRawLengthsArray) + { + Result.ChunkRawLengths.push_back(ChunkView.AsUInt32()); + } + } + return Result; +} + +std::vector +ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) +{ + if (!BlocksObject) + { + return {}; + } + std::vector Result; + CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); + Result.reserve(Blocks.Num()); + for (CbFieldView BlockView : Blocks) + { + CbObjectView BlockObject = BlockView.AsObjectView(); + Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + } + return Result; +} + +CbObject +BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData) +{ + ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkHashes.size()); + + CbObjectWriter Writer; + Writer.AddHash("rawHash"sv, Block.BlockHash); + Writer.BeginArray("rawHashes"sv); + { + for (const IoHash& ChunkHash : Block.ChunkHashes) + { + Writer.AddHash(ChunkHash); + } + } + Writer.EndArray(); + Writer.BeginArray("chunkRawLengths"); + { + for (uint32_t ChunkSize : Block.ChunkRawLengths) + { + Writer.AddInteger(ChunkSize); + } + } + Writer.EndArray(); + + Writer.AddObject("metadata", MetaData); + + return Writer.Save(); +} + +CompressedBuffer +GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock) +{ + const size_t ChunkCount = FetchChunks.size(); + + std::vector ChunkSegments; + ChunkSegments.resize(1); + ChunkSegments.reserve(1 + ChunkCount); + OutBlock.ChunkHashes.reserve(ChunkCount); + OutBlock.ChunkRawLengths.reserve(ChunkCount); + { + IoBuffer TempBuffer(ChunkCount * 9); + MutableMemoryView View = TempBuffer.GetMutableView(); + uint8_t* BufferStartPtr = reinterpret_cast(View.GetData()); + uint8_t* BufferEndPtr = BufferStartPtr; + BufferEndPtr += WriteVarUInt(gsl::narrow(ChunkCount), BufferEndPtr); + for (const auto& It : FetchChunks) + { + std::pair Chunk = It.second(It.first); + uint64_t ChunkSize = 0; + std::span Segments = Chunk.second.GetCompressed().GetSegments(); + for (const SharedBuffer& Segment : Segments) + { + ChunkSize += Segment.GetSize(); + ChunkSegments.push_back(Segment); + } + BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr); + OutBlock.ChunkHashes.push_back(It.first); + OutBlock.ChunkRawLengths.push_back(gsl::narrow(Chunk.first)); + } + ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd()); + ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr); + ChunkSegments[0] = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow(TempBufferLength))); + } + CompressedBuffer CompressedBlock = + CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None); + OutBlock.BlockHash = CompressedBlock.DecodeRawHash(); + return CompressedBlock; +} + +bool +IterateChunkBlock(const SharedBuffer& BlockPayload, std::function Visitor) +{ + ZEN_ASSERT(BlockPayload); + if (BlockPayload.GetSize() < 1) + { + return false; + } + + MemoryView BlockView = BlockPayload.GetView(); + const uint8_t* ReadPtr = reinterpret_cast(BlockView.GetData()); + uint32_t NumberSize; + uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize); + ReadPtr += NumberSize; + std::vector ChunkSizes; + ChunkSizes.reserve(ChunkCount); + while (ChunkCount--) + { + ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize)); + ReadPtr += NumberSize; + } + for (uint64_t ChunkSize : ChunkSizes) + { + IoBuffer Chunk(IoBuffer::Wrap, ReadPtr, ChunkSize); + IoHash AttachmentRawHash; + uint64_t AttachmentRawSize; + CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), AttachmentRawHash, AttachmentRawSize); + + if (!CompressedChunk) + { + ZEN_ERROR("Invalid chunk in block"); + return false; + } + Visitor(std::move(CompressedChunk), AttachmentRawHash); + ReadPtr += ChunkSize; + ZEN_ASSERT(ReadPtr <= BlockView.GetDataEnd()); + } + return true; +}; + +} // namespace zen diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp new file mode 100644 index 000000000..c08492eb0 --- /dev/null +++ b/src/zenutil/chunkedfile.cpp @@ -0,0 +1,510 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +#include "chunking.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +namespace { + struct ChunkedHeader + { + static constexpr uint32_t ExpectedMagic = 0x646b6863; // chkd + static constexpr uint32_t CurrentVersion = 1; + + uint32_t Magic = ExpectedMagic; + uint32_t Version = CurrentVersion; + uint32_t ChunkSequenceLength; + uint32_t ChunkHashCount; + uint64_t ChunkSequenceOffset; + uint64_t ChunkHashesOffset; + uint64_t RawSize = 0; + IoHash RawHash; + }; +} // namespace + +IoBuffer +SerializeChunkedInfo(const ChunkedInfo& Info) +{ + size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) + + RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16); + IoBuffer HeaderData(HeaderSize); + + ChunkedHeader Header; + Header.ChunkSequenceLength = gsl::narrow(Info.ChunkSequence.size()); + Header.ChunkHashCount = gsl::narrow(Info.ChunkHashes.size()); + Header.ChunkSequenceOffset = RoundUp(sizeof(ChunkedHeader), 16); + Header.ChunkHashesOffset = RoundUp(Header.ChunkSequenceOffset + sizeof(uint32_t) * Header.ChunkSequenceLength, 16); + Header.RawSize = Info.RawSize; + Header.RawHash = Info.RawHash; + + MutableMemoryView WriteView = HeaderData.GetMutableView(); + { + MutableMemoryView HeaderWriteView = WriteView.Left(sizeof(Header)); + HeaderWriteView.CopyFrom(MemoryView(&Header, sizeof(Header))); + } + { + MutableMemoryView ChunkSequenceWriteView = WriteView.Mid(Header.ChunkSequenceOffset, sizeof(uint32_t) * Header.ChunkSequenceLength); + ChunkSequenceWriteView.CopyFrom(MemoryView(Info.ChunkSequence.data(), ChunkSequenceWriteView.GetSize())); + } + { + MutableMemoryView ChunksWriteView = WriteView.Mid(Header.ChunkHashesOffset, sizeof(IoHash) * Header.ChunkHashCount); + ChunksWriteView.CopyFrom(MemoryView(Info.ChunkHashes.data(), ChunksWriteView.GetSize())); + } + + return HeaderData; +} + +ChunkedInfo +DeserializeChunkedInfo(IoBuffer& Buffer) +{ + MemoryView View = Buffer.GetView(); + ChunkedHeader Header; + { + MutableMemoryView HeaderWriteView(&Header, sizeof(Header)); + HeaderWriteView.CopyFrom(View.Left(sizeof(Header))); + } + if (Header.Magic != ChunkedHeader::ExpectedMagic) + { + return {}; + } + if (Header.Version != ChunkedHeader::CurrentVersion) + { + return {}; + } + ChunkedInfo Info; + Info.RawSize = Header.RawSize; + Info.RawHash = Header.RawHash; + Info.ChunkSequence.resize(Header.ChunkSequenceLength); + Info.ChunkHashes.resize(Header.ChunkHashCount); + { + MutableMemoryView ChunkSequenceWriteView(Info.ChunkSequence.data(), sizeof(uint32_t) * Header.ChunkSequenceLength); + ChunkSequenceWriteView.CopyFrom(View.Mid(Header.ChunkSequenceOffset, ChunkSequenceWriteView.GetSize())); + } + { + MutableMemoryView ChunksWriteView(Info.ChunkHashes.data(), sizeof(IoHash) * Header.ChunkHashCount); + ChunksWriteView.CopyFrom(View.Mid(Header.ChunkHashesOffset, ChunksWriteView.GetSize())); + } + + return Info; +} + +void +Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function GetChunk) +{ + BasicFile Reconstructed; + Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate); + BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024); + uint64_t Offset = 0; + for (uint32_t SequenceIndex : Info.ChunkSequence) + { + IoBuffer Chunk = GetChunk(Info.ChunkHashes[SequenceIndex]); + ReconstructedWriter.Write(Chunk.GetData(), Chunk.GetSize(), Offset); + Offset += Chunk.GetSize(); + } +} + +ChunkedInfoWithSource +ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params, std::atomic* BytesProcessed) +{ + ChunkedInfoWithSource Result; + tsl::robin_map FoundChunks; + + ZenChunkHelper Chunker; + Chunker.SetUseThreshold(Params.UseThreshold); + Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize); + size_t End = Offset + Size; + const size_t ScanBufferSize = 1u * 1024 * 1024; // (Params.MaxSize * 9) / 3;//1 * 1024 * 1024; + BasicFileBuffer RawBuffer(RawData, ScanBufferSize); + MemoryView SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset); + ZEN_ASSERT(!SliceView.IsEmpty()); + size_t SliceSize = SliceView.GetSize(); + IoHashStream RawHashStream; + while (Offset < End) + { + size_t ScanLength = Chunker.ScanChunk(SliceView.GetData(), SliceSize); + if (ScanLength == ZenChunkHelper::kNoBoundaryFound) + { + if (Offset + SliceSize == End) + { + ScanLength = SliceSize; + } + else + { + SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset); + SliceSize = SliceView.GetSize(); + Chunker.Reset(); + continue; + } + } + uint32_t ChunkLength = gsl::narrow(ScanLength); // +HashedLength); + MemoryView ChunkView = SliceView.Left(ScanLength); + RawHashStream.Append(ChunkView); + IoHash ChunkHash = IoHash::HashBuffer(ChunkView); + SliceView.RightChopInline(ScanLength); + if (auto It = FoundChunks.find(ChunkHash); It != FoundChunks.end()) + { + Result.Info.ChunkSequence.push_back(It->second); + } + else + { + uint32_t ChunkIndex = gsl::narrow(Result.Info.ChunkHashes.size()); + FoundChunks.insert_or_assign(ChunkHash, ChunkIndex); + Result.Info.ChunkHashes.push_back(ChunkHash); + Result.ChunkSources.push_back(ChunkSource{.Offset = Offset, .Size = ChunkLength}); + Result.Info.ChunkSequence.push_back(ChunkIndex); + } + + SliceSize = SliceView.GetSize(); + Offset += ChunkLength; + if (BytesProcessed != nullptr) + { + BytesProcessed->fetch_add(ChunkLength); + } + } + Result.Info.RawSize = Size; + Result.Info.RawHash = RawHashStream.GetHash(); + return Result; +} + +} // namespace zen + +#if ZEN_WITH_TESTS +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include "chunking.h" + +ZEN_THIRD_PARTY_INCLUDES_START +# include +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { +# if 0 +TEST_CASE("chunkedfile.findparams") +{ +# if 1 + DirectoryContent SourceContent1; + GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208", DirectoryContentFlags::IncludeFiles, SourceContent1); + const std::vector& SourceFiles1 = SourceContent1.Files; + DirectoryContent SourceContent2; + GetDirectoryContent("E:\\Temp\\ChunkingTestData\\31379208_2", DirectoryContentFlags::IncludeFiles, SourceContent2); + const std::vector& SourceFiles2 = SourceContent2.Files; +# else + std::filesystem::path SourcePath1 = + "E:\\Temp\\ChunkingTestData\\31375996\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; + std::filesystem::path SourcePath2 = + "E:\\Temp\\ChunkingTestData\\31379208\\ShaderArchive-FortniteGame_Chunk10-PCD3D_SM6-PCD3D_SM6.ushaderbytecode"; + const std::vector& SourceFiles1 = {SourcePath1}; + const std::vector& SourceFiles2 = {SourcePath2}; +# endif + ChunkedParams Params[] = {ChunkedParams{.UseThreshold = false, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}, + ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 35598}, + ChunkedParams{.UseThreshold = false, .MinSize = 16848, .MaxSize = 135168, .AvgSize = 39030}, + ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 36222}, + ChunkedParams{.UseThreshold = false, .MinSize = 15744, .MaxSize = 126976, .AvgSize = 36600}, + ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 35442}, + ChunkedParams{.UseThreshold = false, .MinSize = 16464, .MaxSize = 131072, .AvgSize = 37950}, + ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 38914}, + ChunkedParams{.UseThreshold = false, .MinSize = 15408, .MaxSize = 122880, .AvgSize = 35556}, + ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 35520}, + ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 35478}, + ChunkedParams{.UseThreshold = false, .MinSize = 16896, .MaxSize = 135168, .AvgSize = 39072}, + ChunkedParams{.UseThreshold = false, .MinSize = 15360, .MaxSize = 122880, .AvgSize = 38880}, + ChunkedParams{.UseThreshold = false, .MinSize = 15840, .MaxSize = 126976, .AvgSize = 36678}, + ChunkedParams{.UseThreshold = false, .MinSize = 16800, .MaxSize = 135168, .AvgSize = 38994}, + ChunkedParams{.UseThreshold = false, .MinSize = 15888, .MaxSize = 126976, .AvgSize = 36714}, + ChunkedParams{.UseThreshold = false, .MinSize = 15792, .MaxSize = 126976, .AvgSize = 36636}, + ChunkedParams{.UseThreshold = false, .MinSize = 14880, .MaxSize = 118784, .AvgSize = 37609}, + ChunkedParams{.UseThreshold = false, .MinSize = 15936, .MaxSize = 126976, .AvgSize = 36756}, + ChunkedParams{.UseThreshold = false, .MinSize = 15456, .MaxSize = 122880, .AvgSize = 38955}, + ChunkedParams{.UseThreshold = false, .MinSize = 15984, .MaxSize = 126976, .AvgSize = 36792}, + ChunkedParams{.UseThreshold = false, .MinSize = 14400, .MaxSize = 114688, .AvgSize = 36338}, + ChunkedParams{.UseThreshold = false, .MinSize = 14832, .MaxSize = 118784, .AvgSize = 37568}, + ChunkedParams{.UseThreshold = false, .MinSize = 16944, .MaxSize = 135168, .AvgSize = 39108}, + ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 36297}, + ChunkedParams{.UseThreshold = false, .MinSize = 14208, .MaxSize = 114688, .AvgSize = 36188}, + ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 36372}, + ChunkedParams{.UseThreshold = false, .MinSize = 13296, .MaxSize = 106496, .AvgSize = 36592}, + ChunkedParams{.UseThreshold = false, .MinSize = 15264, .MaxSize = 122880, .AvgSize = 38805}, + ChunkedParams{.UseThreshold = false, .MinSize = 14304, .MaxSize = 114688, .AvgSize = 36263}, + ChunkedParams{.UseThreshold = false, .MinSize = 14784, .MaxSize = 118784, .AvgSize = 37534}, + ChunkedParams{.UseThreshold = false, .MinSize = 15312, .MaxSize = 122880, .AvgSize = 38839}, + ChunkedParams{.UseThreshold = false, .MinSize = 14256, .MaxSize = 114688, .AvgSize = 39360}, + ChunkedParams{.UseThreshold = false, .MinSize = 13776, .MaxSize = 110592, .AvgSize = 37976}, + ChunkedParams{.UseThreshold = false, .MinSize = 14736, .MaxSize = 118784, .AvgSize = 37493}, + ChunkedParams{.UseThreshold = false, .MinSize = 14928, .MaxSize = 118784, .AvgSize = 37643}, + ChunkedParams{.UseThreshold = false, .MinSize = 14448, .MaxSize = 114688, .AvgSize = 39504}, + ChunkedParams{.UseThreshold = false, .MinSize = 13392, .MaxSize = 106496, .AvgSize = 36664}, + ChunkedParams{.UseThreshold = false, .MinSize = 13872, .MaxSize = 110592, .AvgSize = 38048}, + ChunkedParams{.UseThreshold = false, .MinSize = 14352, .MaxSize = 114688, .AvgSize = 39432}, + ChunkedParams{.UseThreshold = false, .MinSize = 13200, .MaxSize = 106496, .AvgSize = 36520}, + ChunkedParams{.UseThreshold = false, .MinSize = 17328, .MaxSize = 139264, .AvgSize = 36378}, + ChunkedParams{.UseThreshold = false, .MinSize = 17376, .MaxSize = 139264, .AvgSize = 36421}, + ChunkedParams{.UseThreshold = false, .MinSize = 17424, .MaxSize = 139264, .AvgSize = 36459}, + ChunkedParams{.UseThreshold = false, .MinSize = 17472, .MaxSize = 139264, .AvgSize = 36502}, + ChunkedParams{.UseThreshold = false, .MinSize = 17520, .MaxSize = 139264, .AvgSize = 36540}, + ChunkedParams{.UseThreshold = false, .MinSize = 17808, .MaxSize = 143360, .AvgSize = 37423}, + ChunkedParams{.UseThreshold = false, .MinSize = 17856, .MaxSize = 143360, .AvgSize = 37466}, + ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 25834}, + ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 21917}, + ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 29751}, + ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 33668}, + ChunkedParams{.UseThreshold = false, .MinSize = 17952, .MaxSize = 143360, .AvgSize = 37547}, + ChunkedParams{.UseThreshold = false, .MinSize = 17904, .MaxSize = 143360, .AvgSize = 37504}, + ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 22371}, + ChunkedParams{.UseThreshold = false, .MinSize = 18000, .MaxSize = 143360, .AvgSize = 37585}, + ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 26406}, + ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 26450}, + ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 30615}, + ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 30441}, + ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 22417}, + ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 22557}, + ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 30528}, + ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 27112}, + ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 34644}, + ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 34476}, + ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 35408}, + ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 38592}, + ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 30483}, + ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 26586}, + ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 26496}, + ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 31302}, + ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 34516}, + ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 22964}, + ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 35448}, + ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 38630}, + ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 23010}, + ChunkedParams{.UseThreshold = false, .MinSize = 18816, .MaxSize = 151552, .AvgSize = 31260}, + ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 34600}, + ChunkedParams{.UseThreshold = false, .MinSize = 18864, .MaxSize = 151552, .AvgSize = 27156}, + ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 30570}, + ChunkedParams{.UseThreshold = false, .MinSize = 18384, .MaxSize = 147456, .AvgSize = 38549}, + ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 22510}, + ChunkedParams{.UseThreshold = false, .MinSize = 18528, .MaxSize = 147456, .AvgSize = 38673}, + ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 34560}, + ChunkedParams{.UseThreshold = false, .MinSize = 18432, .MaxSize = 147456, .AvgSize = 22464}, + ChunkedParams{.UseThreshold = false, .MinSize = 18480, .MaxSize = 147456, .AvgSize = 26540}, + ChunkedParams{.UseThreshold = false, .MinSize = 18336, .MaxSize = 147456, .AvgSize = 38511}, + ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 23057}, + ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 27202}, + ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 31347}, + ChunkedParams{.UseThreshold = false, .MinSize = 18912, .MaxSize = 151552, .AvgSize = 35492}, + ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 31389}, + ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 27246}, + ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 23103}, + ChunkedParams{.UseThreshold = false, .MinSize = 18960, .MaxSize = 151552, .AvgSize = 35532}, + ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 23150}, + ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 27292}, + ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 31434}, + ChunkedParams{.UseThreshold = false, .MinSize = 19008, .MaxSize = 151552, .AvgSize = 35576}, + ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 27336}, + ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 23196}, + ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 31476}, + ChunkedParams{.UseThreshold = false, .MinSize = 19056, .MaxSize = 151552, .AvgSize = 35616}, + ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 27862}, + ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 32121}, + ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 23603}, + ChunkedParams{.UseThreshold = false, .MinSize = 19344, .MaxSize = 155648, .AvgSize = 36380}, + ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 27908}, + ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 23650}, + ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 32166}, + ChunkedParams{.UseThreshold = false, .MinSize = 19392, .MaxSize = 155648, .AvgSize = 36424}, + ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 23696}, + ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 32253}, + ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 32208}, + ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 23743}, + ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 36548}, + ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 28042}, + ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 23789}, + ChunkedParams{.UseThreshold = false, .MinSize = 19536, .MaxSize = 155648, .AvgSize = 32295}, + ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 36508}, + ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 27952}, + ChunkedParams{.UseThreshold = false, .MinSize = 19488, .MaxSize = 155648, .AvgSize = 27998}, + ChunkedParams{.UseThreshold = false, .MinSize = 19440, .MaxSize = 155648, .AvgSize = 36464}}; + + static const size_t ParamsCount = sizeof(Params) / sizeof(ChunkedParams); + std::vector Infos1(SourceFiles1.size()); + std::vector Infos2(SourceFiles2.size()); + + WorkerThreadPool WorkerPool(32); + + for (size_t I = 0; I < ParamsCount; I++) + { + for (int UseThreshold = 0; UseThreshold < 2; UseThreshold++) + { + Latch WorkLatch(1); + ChunkedParams Param = Params[I]; + Param.UseThreshold = UseThreshold == 1; + Stopwatch Timer; + for (size_t F = 0; F < SourceFiles1.size(); F++) + { + WorkLatch.AddCount(1); + WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles1, &Infos1]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + BasicFile SourceData1; + SourceData1.Open(SourceFiles1[F], BasicFile::Mode::kRead); + Infos1[F] = ChunkData(SourceData1, 0, SourceData1.FileSize(), Param); + }); + } + for (size_t F = 0; F < SourceFiles2.size(); F++) + { + WorkLatch.AddCount(1); + WorkerPool.ScheduleWork([&WorkLatch, F, Param, &SourceFiles2, &Infos2]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + BasicFile SourceData2; + SourceData2.Open(SourceFiles2[F], BasicFile::Mode::kRead); + Infos2[F] = ChunkData(SourceData2, 0, SourceData2.FileSize(), Param); + }); + } + WorkLatch.CountDown(); + WorkLatch.Wait(); + uint64_t ChunkTimeMS = Timer.GetElapsedTimeMs(); + + uint64_t Raw1Size = 0; + tsl::robin_set Chunks1; + size_t ChunkedSize1 = 0; + for (size_t F = 0; F < SourceFiles1.size(); F++) + { + const ChunkedInfoWithSource& Info = Infos1[F]; + Raw1Size += Info.Info.RawSize; + for (uint32_t Chunk1Index = 0; Chunk1Index < Info.Info.ChunkHashes.size(); ++Chunk1Index) + { + const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk1Index]; + if (Chunks1.insert(ChunkHash).second) + { + ChunkedSize1 += Info.ChunkSources[Chunk1Index].Size; + } + } + } + + uint64_t Raw2Size = 0; + tsl::robin_set Chunks2; + size_t ChunkedSize2 = 0; + size_t DiffSize = 0; + for (size_t F = 0; F < SourceFiles2.size(); F++) + { + const ChunkedInfoWithSource& Info = Infos2[F]; + Raw2Size += Info.Info.RawSize; + for (uint32_t Chunk2Index = 0; Chunk2Index < Info.Info.ChunkHashes.size(); ++Chunk2Index) + { + const IoHash ChunkHash = Info.Info.ChunkHashes[Chunk2Index]; + if (Chunks2.insert(ChunkHash).second) + { + ChunkedSize2 += Info.ChunkSources[Chunk2Index].Size; + if (!Chunks1.contains(ChunkHash)) + { + DiffSize += Info.ChunkSources[Chunk2Index].Size; + } + } + } + } + + ZEN_INFO( + "Diff = {}, Chunks1 = {}, Chunks2 = {}, .UseThreshold = {}, .MinSize = {}, .MaxSize = {}, .AvgSize = {}, RawSize(1) = {}, " + "RawSize(2) = {}, " + "Saved(1) = {}, Saved(2) = {} in {}", + NiceBytes(DiffSize), + Chunks1.size(), + Chunks2.size(), + Param.UseThreshold, + Param.MinSize, + Param.MaxSize, + Param.AvgSize, + NiceBytes(Raw1Size), + NiceBytes(Raw2Size), + NiceBytes(Raw1Size - ChunkedSize1), + NiceBytes(Raw2Size - ChunkedSize2), + NiceTimeSpanMs(ChunkTimeMS)); + } + } + +# if 0 + for (int64_t MinSizeBase = (12u * 1024u); MinSizeBase <= (32u * 1024u); MinSizeBase += 512) + { + for (int64_t Wiggle = -132; Wiggle < 126; Wiggle += 2) + { + // size_t MinSize = 7 * 1024 - 61; // (size_t)(MinSizeBase + Wiggle); + // size_t MaxSize = 16 * (7 * 1024); // 8 * 7 * 1024;// MinSizeBase * 6; + // size_t AvgSize = MaxSize / 2; // 4 * 7 * 1024;// MinSizeBase * 3; + size_t MinSize = (size_t)(MinSizeBase + Wiggle); + //for (size_t MaxSize = (MinSize * 4) - 768; MaxSize < (MinSize * 5) + 768; MaxSize += 64) + size_t MaxSize = 8u * MinSizeBase; + { + for (size_t AvgSize = (MaxSize - MinSize) / 32 + MinSize; AvgSize < (MaxSize - MinSize) / 4 + MinSize; AvgSize += (MaxSize - MinSize) / 32) +// size_t AvgSize = (MaxSize - MinSize) / 4 + MinSize; + { + WorkLatch.AddCount(1); + WorkerPool.ScheduleWork([&WorkLatch, MinSize, MaxSize, AvgSize, SourcePath1, SourcePath2]() + { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + ChunkedParams Params{ .UseThreshold = true, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize }; + BasicFile SourceData1; + SourceData1.Open(SourcePath1, BasicFile::Mode::kRead); + BasicFile SourceData2; + SourceData2.Open(SourcePath2, BasicFile::Mode::kRead); + ChunkedInfoWithSource Info1 = ChunkData(SourceData1, Params); + ChunkedInfoWithSource Info2 = ChunkData(SourceData2, Params); + + tsl::robin_set Chunks1; + Chunks1.reserve(Info1.Info.ChunkHashes.size()); + Chunks1.insert(Info1.Info.ChunkHashes.begin(), Info1.Info.ChunkHashes.end()); + size_t ChunkedSize1 = 0; + for (uint32_t Chunk1Index = 0; Chunk1Index < Info1.Info.ChunkHashes.size(); ++Chunk1Index) + { + ChunkedSize1 += Info1.ChunkSources[Chunk1Index].Size; + } + size_t DiffSavedSize = 0; + size_t ChunkedSize2 = 0; + for (uint32_t Chunk2Index = 0; Chunk2Index < Info2.Info.ChunkHashes.size(); ++Chunk2Index) + { + ChunkedSize2 += Info2.ChunkSources[Chunk2Index].Size; + if (Chunks1.find(Info2.Info.ChunkHashes[Chunk2Index]) == Chunks1.end()) + { + DiffSavedSize += Info2.ChunkSources[Chunk2Index].Size; + } + } + ZEN_INFO("Diff {}, Chunks1: {}, Chunks2: {}, Min: {}, Max: {}, Avg: {}, Saved(1) {}, Saved(2) {}", + NiceBytes(DiffSavedSize), + Info1.Info.ChunkHashes.size(), + Info2.Info.ChunkHashes.size(), + MinSize, + MaxSize, + AvgSize, + NiceBytes(Info1.Info.RawSize - ChunkedSize1), + NiceBytes(Info2.Info.RawSize - ChunkedSize2)); + }); + } + } + } + } +# endif // 0 + + // WorkLatch.CountDown(); + // WorkLatch.Wait(); +} +# endif // 0 + +void +chunkedfile_forcelink() +{ +} + +} // namespace zen + +#endif diff --git a/src/zenutil/chunking.cpp b/src/zenutil/chunking.cpp new file mode 100644 index 000000000..30edd322a --- /dev/null +++ b/src/zenutil/chunking.cpp @@ -0,0 +1,382 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "chunking.h" + +#include + +#include + +namespace zen::detail { + +static const uint32_t BuzhashTable[] = { + 0x458be752, 0xc10748cc, 0xfbbcdbb8, 0x6ded5b68, 0xb10a82b5, 0x20d75648, 0xdfc5665f, 0xa8428801, 0x7ebf5191, 0x841135c7, 0x65cc53b3, + 0x280a597c, 0x16f60255, 0xc78cbc3e, 0x294415f5, 0xb938d494, 0xec85c4e6, 0xb7d33edc, 0xe549b544, 0xfdeda5aa, 0x882bf287, 0x3116737c, + 0x05569956, 0xe8cc1f68, 0x0806ac5e, 0x22a14443, 0x15297e10, 0x50d090e7, 0x4ba60f6f, 0xefd9f1a7, 0x5c5c885c, 0x82482f93, 0x9bfd7c64, + 0x0b3e7276, 0xf2688e77, 0x8fad8abc, 0xb0509568, 0xf1ada29f, 0xa53efdfe, 0xcb2b1d00, 0xf2a9e986, 0x6463432b, 0x95094051, 0x5a223ad2, + 0x9be8401b, 0x61e579cb, 0x1a556a14, 0x5840fdc2, 0x9261ddf6, 0xcde002bb, 0x52432bb0, 0xbf17373e, 0x7b7c222f, 0x2955ed16, 0x9f10ca59, + 0xe840c4c9, 0xccabd806, 0x14543f34, 0x1462417a, 0x0d4a1f9c, 0x087ed925, 0xd7f8f24c, 0x7338c425, 0xcf86c8f5, 0xb19165cd, 0x9891c393, + 0x325384ac, 0x0308459d, 0x86141d7e, 0xc922116a, 0xe2ffa6b6, 0x53f52aed, 0x2cd86197, 0xf5b9f498, 0xbf319c8f, 0xe0411fae, 0x977eb18c, + 0xd8770976, 0x9833466a, 0xc674df7f, 0x8c297d45, 0x8ca48d26, 0xc49ed8e2, 0x7344f874, 0x556f79c7, 0x6b25eaed, 0xa03e2b42, 0xf68f66a4, + 0x8e8b09a2, 0xf2e0e62a, 0x0d3a9806, 0x9729e493, 0x8c72b0fc, 0x160b94f6, 0x450e4d3d, 0x7a320e85, 0xbef8f0e1, 0x21d73653, 0x4e3d977a, + 0x1e7b3929, 0x1cc6c719, 0xbe478d53, 0x8d752809, 0xe6d8c2c6, 0x275f0892, 0xc8acc273, 0x4cc21580, 0xecc4a617, 0xf5f7be70, 0xe795248a, + 0x375a2fe9, 0x425570b6, 0x8898dcf8, 0xdc2d97c4, 0x0106114b, 0x364dc22f, 0x1e0cad1f, 0xbe63803c, 0x5f69fac2, 0x4d5afa6f, 0x1bc0dfb5, + 0xfb273589, 0x0ea47f7b, 0x3c1c2b50, 0x21b2a932, 0x6b1223fd, 0x2fe706a8, 0xf9bd6ce2, 0xa268e64e, 0xe987f486, 0x3eacf563, 0x1ca2018c, + 0x65e18228, 0x2207360a, 0x57cf1715, 0x34c37d2b, 0x1f8f3cde, 0x93b657cf, 0x31a019fd, 0xe69eb729, 0x8bca7b9b, 0x4c9d5bed, 0x277ebeaf, + 0xe0d8f8ae, 0xd150821c, 0x31381871, 0xafc3f1b0, 0x927db328, 0xe95effac, 0x305a47bd, 0x426ba35b, 0x1233af3f, 0x686a5b83, 0x50e072e5, + 0xd9d3bb2a, 0x8befc475, 0x487f0de6, 0xc88dff89, 0xbd664d5e, 0x971b5d18, 0x63b14847, 0xd7d3c1ce, 0x7f583cf3, 0x72cbcb09, 0xc0d0a81c, + 0x7fa3429b, 0xe9158a1b, 0x225ea19a, 0xd8ca9ea3, 0xc763b282, 0xbb0c6341, 0x020b8293, 0xd4cd299d, 0x58cfa7f8, 0x91b4ee53, 0x37e4d140, + 0x95ec764c, 0x30f76b06, 0x5ee68d24, 0x679c8661, 0xa41979c2, 0xf2b61284, 0x4fac1475, 0x0adb49f9, 0x19727a23, 0x15a7e374, 0xc43a18d5, + 0x3fb1aa73, 0x342fc615, 0x924c0793, 0xbee2d7f0, 0x8a279de9, 0x4aa2d70c, 0xe24dd37f, 0xbe862c0b, 0x177c22c2, 0x5388e5ee, 0xcd8a7510, + 0xf901b4fd, 0xdbc13dbc, 0x6c0bae5b, 0x64efe8c7, 0x48b02079, 0x80331a49, 0xca3d8ae6, 0xf3546190, 0xfed7108b, 0xc49b941b, 0x32baf4a9, + 0xeb833a4a, 0x88a3f1a5, 0x3a91ce0a, 0x3cc27da1, 0x7112e684, 0x4a3096b1, 0x3794574c, 0xa3c8b6f3, 0x1d213941, 0x6e0a2e00, 0x233479f1, + 0x0f4cd82f, 0x6093edd2, 0x5d7d209e, 0x464fe319, 0xd4dcac9e, 0x0db845cb, 0xfb5e4bc3, 0xe0256ce1, 0x09fb4ed1, 0x0914be1e, 0xa5bdb2c3, + 0xc6eb57bb, 0x30320350, 0x3f397e91, 0xa67791bc, 0x86bc0e2c, 0xefa0a7e2, 0xe9ff7543, 0xe733612c, 0xd185897b, 0x329e5388, 0x91dd236b, + 0x2ecb0d93, 0xf4d82a3d, 0x35b5c03f, 0xe4e606f0, 0x05b21843, 0x37b45964, 0x5eff22f4, 0x6027f4cc, 0x77178b3c, 0xae507131, 0x7bf7cabc, + 0xf9c18d66, 0x593ade65, 0xd95ddf11, +}; + +// ROL operation (compiler turns this into a ROL when optimizing) +ZEN_FORCEINLINE static uint32_t +Rotate32(uint32_t Value, size_t RotateCount) +{ + RotateCount &= 31; + + return ((Value) << (RotateCount)) | ((Value) >> (32 - RotateCount)); +} + +} // namespace zen::detail + +namespace zen { + +void +ZenChunkHelper::Reset() +{ + InternalReset(); + + m_BytesScanned = 0; +} + +void +ZenChunkHelper::InternalReset() +{ + m_CurrentHash = 0; + m_CurrentChunkSize = 0; + m_WindowSize = 0; +} + +void +ZenChunkHelper::SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize) +{ + if (m_WindowSize) + return; // Already started + + static_assert(kChunkSizeLimitMin > kWindowSize); + + if (AvgSize) + { + // TODO: Validate AvgSize range + } + else + { + if (MinSize && MaxSize) + { + AvgSize = std::lrint(std::pow(2, (std::log2(MinSize) + std::log2(MaxSize)) / 2)); + } + else if (MinSize) + { + AvgSize = MinSize * 4; + } + else if (MaxSize) + { + AvgSize = MaxSize / 4; + } + else + { + AvgSize = kDefaultAverageChunkSize; + } + } + + if (MinSize) + { + // TODO: Validate MinSize range + } + else + { + MinSize = std::max(AvgSize / 4, kChunkSizeLimitMin); + } + + if (MaxSize) + { + // TODO: Validate MaxSize range + } + else + { + MaxSize = std::min(AvgSize * 4, kChunkSizeLimitMax); + } + + m_Discriminator = gsl::narrow(AvgSize - MinSize); + + if (m_Discriminator < MinSize) + { + m_Discriminator = gsl::narrow(MinSize); + } + + if (m_Discriminator > MaxSize) + { + m_Discriminator = gsl::narrow(MaxSize); + } + + m_Threshold = gsl::narrow((uint64_t(std::numeric_limits::max()) + 1) / m_Discriminator); + + m_ChunkSizeMin = MinSize; + m_ChunkSizeMax = MaxSize; + m_ChunkSizeAvg = AvgSize; +} + +size_t +ZenChunkHelper::ScanChunk(const void* DataBytesIn, size_t ByteCount) +{ + size_t Result = InternalScanChunk(DataBytesIn, ByteCount); + + if (Result == kNoBoundaryFound) + { + m_BytesScanned += ByteCount; + } + else + { + m_BytesScanned += Result; + } + + return Result; +} + +size_t +ZenChunkHelper::InternalScanChunk(const void* DataBytesIn, size_t ByteCount) +{ + size_t CurrentOffset = 0; + const uint8_t* CursorPtr = reinterpret_cast(DataBytesIn); + + // There's no point in updating the hash if we know we're not + // going to have a cut point, so just skip the data. This logic currently + // provides roughly a 20% speedup on my machine + + const size_t NeedHashOffset = m_ChunkSizeMin - kWindowSize; + + if (m_CurrentChunkSize < NeedHashOffset) + { + const uint32_t SkipBytes = gsl::narrow(std::min(ByteCount, NeedHashOffset - m_CurrentChunkSize)); + + ByteCount -= SkipBytes; + m_CurrentChunkSize += SkipBytes; + CurrentOffset += SkipBytes; + CursorPtr += SkipBytes; + + m_WindowSize = 0; + + if (ByteCount == 0) + { + return kNoBoundaryFound; + } + } + + // Fill window first + + if (m_WindowSize < kWindowSize) + { + const uint32_t FillBytes = uint32_t(std::min(ByteCount, kWindowSize - m_WindowSize)); + + memcpy(&m_Window[m_WindowSize], CursorPtr, FillBytes); + + CursorPtr += FillBytes; + + m_WindowSize += FillBytes; + m_CurrentChunkSize += FillBytes; + + CurrentOffset += FillBytes; + ByteCount -= FillBytes; + + if (m_WindowSize < kWindowSize) + { + return kNoBoundaryFound; + } + + // We have a full window, initialize hash + + uint32_t CurrentHash = 0; + + for (int i = 1; i < kWindowSize; ++i) + { + CurrentHash ^= detail::Rotate32(detail::BuzhashTable[m_Window[i - 1]], kWindowSize - i); + } + + m_CurrentHash = CurrentHash ^ detail::BuzhashTable[m_Window[kWindowSize - 1]]; + } + + // Scan for boundaries (i.e points where the hash matches the value determined by + // the discriminator) + + uint32_t CurrentHash = m_CurrentHash; + uint32_t CurrentChunkSize = m_CurrentChunkSize; + + size_t Index = CurrentChunkSize % kWindowSize; + + if (m_Threshold && m_UseThreshold) + { + // This is roughly 4x faster than the general modulo approach on my + // TR 3990X (~940MB/sec) and doesn't require any special parameters to + // achieve max performance + + while (ByteCount) + { + const uint8_t NewByte = *CursorPtr; + const uint8_t OldByte = m_Window[Index]; + + CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ + detail::BuzhashTable[NewByte]; + + CurrentChunkSize++; + CurrentOffset++; + + if (CurrentChunkSize >= m_ChunkSizeMin) + { + bool FoundBoundary; + + if (CurrentChunkSize >= m_ChunkSizeMax) + { + FoundBoundary = true; + } + else + { + FoundBoundary = CurrentHash <= m_Threshold; + } + + if (FoundBoundary) + { + // Boundary found! + InternalReset(); + + return CurrentOffset; + } + } + + m_Window[Index++] = *CursorPtr; + + if (Index == kWindowSize) + { + Index = 0; + } + + ++CursorPtr; + --ByteCount; + } + } + else if ((m_Discriminator & (m_Discriminator - 1)) == 0) + { + // This is quite a bit faster than the generic modulo path, but + // requires a very specific average chunk size to be used. If you + // pass in an even power-of-two divided by 0.75 as the average + // chunk size you'll hit this path + + const uint32_t Mask = m_Discriminator - 1; + + while (ByteCount) + { + const uint8_t NewByte = *CursorPtr; + const uint8_t OldByte = m_Window[Index]; + + CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ + detail::BuzhashTable[NewByte]; + + CurrentChunkSize++; + CurrentOffset++; + + if (CurrentChunkSize >= m_ChunkSizeMin) + { + bool FoundBoundary; + + if (CurrentChunkSize >= m_ChunkSizeMax) + { + FoundBoundary = true; + } + else + { + FoundBoundary = (CurrentHash & Mask) == Mask; + } + + if (FoundBoundary) + { + // Boundary found! + InternalReset(); + + return CurrentOffset; + } + } + + m_Window[Index++] = *CursorPtr; + + if (Index == kWindowSize) + { + Index = 0; + } + + ++CursorPtr; + --ByteCount; + } + } + else + { + // This is the slowest path, which caps out around 250MB/sec for large sizes + // on my TR3900X + + while (ByteCount) + { + const uint8_t NewByte = *CursorPtr; + const uint8_t OldByte = m_Window[Index]; + + CurrentHash = detail::Rotate32(CurrentHash, 1) ^ detail::Rotate32(detail::BuzhashTable[OldByte], m_WindowSize) ^ + detail::BuzhashTable[NewByte]; + + CurrentChunkSize++; + CurrentOffset++; + + if (CurrentChunkSize >= m_ChunkSizeMin) + { + bool FoundBoundary; + + if (CurrentChunkSize >= m_ChunkSizeMax) + { + FoundBoundary = true; + } + else + { + FoundBoundary = (CurrentHash % m_Discriminator) == (m_Discriminator - 1); + } + + if (FoundBoundary) + { + // Boundary found! + InternalReset(); + + return CurrentOffset; + } + } + + m_Window[Index++] = *CursorPtr; + + if (Index == kWindowSize) + { + Index = 0; + } + + ++CursorPtr; + --ByteCount; + } + } + + m_CurrentChunkSize = CurrentChunkSize; + m_CurrentHash = CurrentHash; + + return kNoBoundaryFound; +} + +} // namespace zen diff --git a/src/zenutil/chunking.h b/src/zenutil/chunking.h new file mode 100644 index 000000000..09c56454f --- /dev/null +++ b/src/zenutil/chunking.h @@ -0,0 +1,56 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once +#include + +namespace zen { + +/** Content-defined chunking helper + */ +class ZenChunkHelper +{ +public: + void SetChunkSize(size_t MinSize, size_t MaxSize, size_t AvgSize); + size_t ScanChunk(const void* DataBytes, size_t ByteCount); + void Reset(); + + // This controls which chunking approach is used - threshold or + // modulo based. Threshold is faster and generates similarly sized + // chunks + void SetUseThreshold(bool NewState) { m_UseThreshold = NewState; } + + inline size_t ChunkSizeMin() const { return m_ChunkSizeMin; } + inline size_t ChunkSizeMax() const { return m_ChunkSizeMax; } + inline size_t ChunkSizeAvg() const { return m_ChunkSizeAvg; } + inline uint64_t BytesScanned() const { return m_BytesScanned; } + + static constexpr size_t kNoBoundaryFound = size_t(~0ull); + +private: + size_t m_ChunkSizeMin = 0; + size_t m_ChunkSizeMax = 0; + size_t m_ChunkSizeAvg = 0; + + uint32_t m_Discriminator = 0; // Computed in SetChunkSize() + uint32_t m_Threshold = 0; // Computed in SetChunkSize() + + bool m_UseThreshold = true; + + static constexpr size_t kChunkSizeLimitMax = 64 * 1024 * 1024; + static constexpr size_t kChunkSizeLimitMin = 1024; + static constexpr size_t kDefaultAverageChunkSize = 64 * 1024; + + static constexpr int kWindowSize = 48; + uint8_t m_Window[kWindowSize]; + uint32_t m_WindowSize = 0; + + uint32_t m_CurrentHash = 0; + uint32_t m_CurrentChunkSize = 0; + + uint64_t m_BytesScanned = 0; + + size_t InternalScanChunk(const void* DataBytes, size_t ByteCount); + void InternalReset(); +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h new file mode 100644 index 000000000..9b7414629 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +#include +#include + +namespace zen { + +struct ChunkBlockDescription +{ + IoHash BlockHash; + std::vector ChunkHashes; + std::vector ChunkRawLengths; +}; + +std::vector ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); +ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); +CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); + +typedef std::function(const IoHash& RawHash)> FetchChunkFunc; + +CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); +bool IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor); + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h new file mode 100644 index 000000000..7110ad317 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkedfile.h @@ -0,0 +1,58 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace zen { + +class BasicFile; + +struct ChunkedInfo +{ + uint64_t RawSize = 0; + IoHash RawHash; + std::vector ChunkSequence; + std::vector ChunkHashes; +}; + +struct ChunkSource +{ + uint64_t Offset; // 8 + uint32_t Size; // 4 +}; + +struct ChunkedInfoWithSource +{ + ChunkedInfo Info; + std::vector ChunkSources; +}; + +struct ChunkedParams +{ + bool UseThreshold = true; + size_t MinSize = (2u * 1024u) - 128u; + size_t MaxSize = (16u * 1024u); + size_t AvgSize = (3u * 1024u); +}; + +static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}; + +ChunkedInfoWithSource ChunkData(BasicFile& RawData, + uint64_t Offset, + uint64_t Size, + ChunkedParams Params = {}, + std::atomic* BytesProcessed = nullptr); +void Reconstruct(const ChunkedInfo& Info, + const std::filesystem::path& TargetPath, + std::function GetChunk); +IoBuffer SerializeChunkedInfo(const ChunkedInfo& Info); +ChunkedInfo DeserializeChunkedInfo(IoBuffer& Buffer); + +void chunkedfile_forcelink(); +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index c54144549..19eb63ce9 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -6,6 +6,7 @@ # include # include +# include namespace zen { @@ -15,6 +16,7 @@ zenutil_forcelinktests() cachepolicy_forcelink(); cache::rpcrecord_forcelink(); cacherequests_forcelink(); + chunkedfile_forcelink(); } } // namespace zen -- cgit v1.2.3 From 800305f7f2afe01e9ca038a71a68e9318c52ee77 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 09:05:07 +0100 Subject: move WriteToTempFile to basicfile.h (#283) add helper constructors to BasicFile --- src/zencore/basicfile.cpp | 64 +++++++++++++- src/zencore/include/zencore/basicfile.h | 10 ++- src/zenserver/projectstore/remoteprojectstore.cpp | 100 +++++++++------------- 3 files changed, 111 insertions(+), 63 deletions(-) (limited to 'src') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index c2a21ae90..b3bffd34d 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -28,6 +28,20 @@ BasicFile::~BasicFile() { Close(); } +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode) +{ + Open(FileName, Mode); +} + +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec) +{ + Open(FileName, Mode, Ec); +} + +BasicFile::BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback) +{ + Open(FileName, Mode, std::move(RetryCallback)); +} void BasicFile::Open(const std::filesystem::path& FileName, Mode Mode) @@ -267,7 +281,7 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& RetryCallback) +{ + if (std::filesystem::is_regular_file(Path)) + { + IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path)); + if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize()) + { + ExistingTempFile.SetDeleteOnClose(true); + return ExistingTempFile; + } + } + BasicFile BlockFile; + BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback)); + uint64_t Offset = 0; + { + static const uint64_t BufferingSize = 256u * 1024u; + BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + for (const SharedBuffer& Segment : Buffer.GetSegments()) + { + size_t SegmentSize = Segment.GetSize(); + + IoBufferFileReference FileRef; + if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef)) + { + ScanFile(FileRef.FileHandle, + FileRef.FileChunkOffset, + FileRef.FileChunkSize, + BufferingSize, + [&BufferedOutput, &Offset](const void* Data, size_t Size) { + BufferedOutput.Write(Data, Size, Offset); + Offset += Size; + }); + } + else + { + BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); + Offset += SegmentSize; + } + } + } + void* FileHandle = BlockFile.Detach(); + IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); + BlockBuffer.SetDeleteOnClose(true); + return BlockBuffer; +} + ////////////////////////////////////////////////////////////////////////// /* diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 03c5605df..7edd40c9c 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -46,6 +46,10 @@ public: kPreventWrite = 0x2000'0000, // Do not open with write sharing mode (prevent other processes from writing to file while open) }; + BasicFile(const std::filesystem::path& FileName, Mode Mode); + BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec); + BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback); + void Open(const std::filesystem::path& FileName, Mode Mode); void Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec); void Open(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback); @@ -56,7 +60,7 @@ public: void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(MemoryView Data, uint64_t FileOffset); void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec); - uint64_t Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec); + uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec); void Flush(); @@ -180,6 +184,10 @@ private: uint64_t m_BufferEnd; }; +IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, + const std::filesystem::path& Path, + std::function&& RetryCallback); + ZENCORE_API void basicfile_forcelink(); } // namespace zen diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index 5b75a840e..0285cc22f 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -154,63 +154,6 @@ namespace remotestore_impl { return BlockIndex; } - IoBuffer WriteToTempFile(CompressedBuffer&& CompressedBuffer, std::filesystem::path Path) - { - if (std::filesystem::is_regular_file(Path)) - { - IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path)); - if (ExistingTempFile && ExistingTempFile.GetSize() == CompressedBuffer.GetCompressedSize()) - { - ExistingTempFile.SetDeleteOnClose(true); - return ExistingTempFile; - } - } - IoBuffer BlockBuffer; - BasicFile BlockFile; - uint32_t RetriesLeft = 3; - BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", Path, Ec.message(), RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); - uint64_t Offset = 0; - { - CompositeBuffer Compressed = std::move(CompressedBuffer).GetCompressed(); - for (const SharedBuffer& Segment : Compressed.GetSegments()) - { - size_t SegmentSize = Segment.GetSize(); - static const uint64_t BufferingSize = 256u * 1024u; - - IoBufferFileReference FileRef; - if (SegmentSize >= (BufferingSize + BufferingSize / 2) && Segment.GetFileReference(FileRef)) - { - ScanFile(FileRef.FileHandle, - FileRef.FileChunkOffset, - FileRef.FileChunkSize, - BufferingSize, - [&BlockFile, &Offset](const void* Data, size_t Size) { - BlockFile.Write(Data, Size, Offset); - Offset += Size; - }); - } - else - { - BlockFile.Write(Segment.GetData(), SegmentSize, Offset); - Offset += SegmentSize; - } - } - } - void* FileHandle = BlockFile.Detach(); - BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); - BlockBuffer.SetDeleteOnClose(true); - return BlockBuffer; - } - RemoteProjectStore::Result WriteOplogSection(ProjectStore::Oplog& Oplog, const CbObjectView& SectionObject, JobContext* OptionalContext) { using namespace std::literals; @@ -1635,9 +1578,22 @@ BuildContainer(CidStore& ChunkStore, std::filesystem::path AttachmentPath = AttachmentTempPath; AttachmentPath.append(RawHash.ToHexString()); + uint32_t RetriesLeft = 3; IoBuffer TempAttachmentBuffer = - remotestore_impl::WriteToTempFile(std::move(Compressed), AttachmentPath); + WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) { + if (RetriesLeft == 0) + { + return false; + } + ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.", + AttachmentPath, + Ec.message(), + RetriesLeft); + Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + RetriesLeft--; + return true; + }); ZEN_INFO("Saved temp attachment to '{}', {} ({})", AttachmentPath, NiceBytes(RawSize), @@ -1656,7 +1612,21 @@ BuildContainer(CidStore& ChunkStore, std::filesystem::path AttachmentPath = AttachmentTempPath; AttachmentPath.append(RawHash.ToHexString()); - IoBuffer TempAttachmentBuffer = remotestore_impl::WriteToTempFile(std::move(Compressed), AttachmentPath); + uint32_t RetriesLeft = 3; + IoBuffer TempAttachmentBuffer = + WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) { + if (RetriesLeft == 0) + { + return false; + } + ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.", + AttachmentPath, + Ec.message(), + RetriesLeft); + Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + RetriesLeft--; + return true; + }); ZEN_INFO("Saved temp attachment to '{}', {} ({})", AttachmentPath, NiceBytes(RawSize), @@ -2392,7 +2362,17 @@ SaveOplog(CidStore& ChunkStore, BlockPath.append(Block.BlockHash.ToHexString()); try { - IoBuffer BlockBuffer = remotestore_impl::WriteToTempFile(std::move(CompressedBlock), BlockPath); + uint32_t RetriesLeft = 3; + IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath, [&](std::error_code& Ec) { + if (RetriesLeft == 0) + { + return false; + } + ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", BlockPath, Ec.message(), RetriesLeft); + Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + RetriesLeft--; + return true; + }); RwLock::ExclusiveLockScope __(AttachmentsLock); CreatedBlocks.insert({Block.BlockHash, {.Payload = std::move(BlockBuffer), .Block = std::move(Block)}}); ZEN_DEBUG("Saved temp block to '{}', {}", AttachmentTempPath, NiceBytes(BlockBuffer.GetSize())); -- cgit v1.2.3 From 574c62a94f61265be5ae3d086834fa8b9d29eefc Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 24 Feb 2025 12:38:18 +0100 Subject: strip leading path separator when creating workspace shares (#285) --- src/zen/cmds/workspaces_cmd.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 05d3c573f..166d4218d 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -25,7 +25,7 @@ namespace { if (!Path.empty()) { std::u8string PathString = Path.u8string(); - if (PathString.ends_with(std::filesystem::path::preferred_separator)) + if (PathString.ends_with(std::filesystem::path::preferred_separator) || PathString.ends_with('/')) { PathString.pop_back(); Path = std::filesystem::path(PathString); @@ -42,6 +42,19 @@ namespace { } } + static void RemoveLeadingPathSeparator(std::filesystem::path& Path) + { + if (!Path.empty()) + { + std::u8string PathString = Path.u8string(); + if (PathString.starts_with(std::filesystem::path::preferred_separator) || PathString.starts_with('/')) + { + PathString.erase(PathString.begin()); + Path = std::filesystem::path(PathString); + } + } + } + void ShowShare(const Workspaces::WorkspaceShareConfiguration& Share, const Oid& WorkspaceId, std::string_view Prefix) { ZEN_CONSOLE("{}Id: {}", Prefix, Share.Id); @@ -451,6 +464,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } RemoveTrailingPathSeparator(m_SharePath); + RemoveLeadingPathSeparator(m_SharePath); if (m_ShareId.empty()) { -- cgit v1.2.3 From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 25 Feb 2025 15:48:43 +0100 Subject: improvements and infrastructure for upcoming builds api command line (#284) * add modification tick to filesystem traversal * add ShowDetails option to ProgressBar * log callstack if we terminate process * handle chunking if MaxSize > 1MB * BasicFile write helpers and WriteToTempFile simplifications * bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data revert of earlier optimization * faster compress/decompress for large disk-based files * enable progress feedback in IoHash::HashBuffer * add payload validation in HttpClient::Get * fix range requests (range is including end byte) * remove BuildPartId for blob/block related operations in builds api --- src/zen/cmds/copy_cmd.cpp | 6 +- src/zen/cmds/serve_cmd.cpp | 2 +- src/zen/zen.cpp | 20 +- src/zen/zen.h | 3 +- src/zencore/basicfile.cpp | 79 ++++++-- src/zencore/compositebuffer.cpp | 34 +--- src/zencore/compress.cpp | 214 ++++++++++++++++----- src/zencore/filesystem.cpp | 26 ++- src/zencore/include/zencore/basicfile.h | 6 +- src/zencore/include/zencore/filesystem.h | 20 +- src/zencore/include/zencore/iohash.h | 4 +- src/zencore/iohash.cpp | 28 ++- src/zenhttp/httpclient.cpp | 27 ++- src/zenserver/objectstore/objectstore.cpp | 2 +- .../projectstore/buildsremoteprojectstore.cpp | 31 ++- src/zenserver/projectstore/remoteprojectstore.cpp | 55 +----- src/zenstore/filecas.cpp | 4 +- src/zenutil/chunkedfile.cpp | 2 +- .../include/zenutil/jupiter/jupitersession.h | 13 +- src/zenutil/jupiter/jupitersession.cpp | 85 +++----- src/zenutil/logging.cpp | 9 +- 21 files changed, 399 insertions(+), 271 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index d42d3c107..cc6ddd505 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -120,7 +120,11 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { } - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, + const path_view& File, + uint64_t FileSize, + uint32_t, + uint64_t) override { ZEN_UNUSED(FileSize); std::error_code Ec; diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index 8e36e74ce..f87725e36 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -120,7 +120,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) struct FsVisitor : public FileSystemTraversal::TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::filesystem::path ServerPath = std::filesystem::relative(Parent / File, RootPath); std::string ServerPathString = reinterpret_cast(ServerPath.generic_u8string().c_str()); diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index fd58b024a..872ea8941 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -24,6 +24,7 @@ #include "cmds/vfs_cmd.h" #include "cmds/workspaces_cmd.h" +#include #include #include #include @@ -251,7 +252,10 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy); } -ProgressBar::ProgressBar(bool PlainProgress) : m_PlainProgress(PlainProgress), m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000) +ProgressBar::ProgressBar(bool PlainProgress, bool ShowDetails) +: m_PlainProgress(PlainProgress) +, m_ShowDetails(ShowDetails) +, m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000) { } @@ -270,6 +274,7 @@ ProgressBar::~ProgressBar() void ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) { + ZEN_ASSERT(NewState.TotalCount >= NewState.RemainingCount); if (DoLinebreak == false && m_State == NewState) { return; @@ -288,7 +293,8 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) if (m_PlainProgress) { - ZEN_CONSOLE("{} {}% ({})", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS)); + std::string Details = (m_ShowDetails && !NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + ZEN_CONSOLE("{} {}% ({}){}", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); } else { @@ -327,7 +333,7 @@ ProgressBar::ForceLinebreak() void ProgressBar::Finish() { - if (m_LastOutputLength > 0 && m_State.RemainingCount > 0) + if (m_LastOutputLength > 0) { State NewState = m_State; NewState.RemainingCount = 0; @@ -367,7 +373,13 @@ main(int argc, char** argv) // Set output mode to handle virtual terminal sequences zen::logging::EnableVTMode(); - std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); }); + std::set_terminate([]() { + void* Frames[8]; + uint32_t FrameCount = GetCallstack(2, 8, Frames); + CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); + ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, " ")); + FreeCallstack(Callstack); + }); LoggerRef DefaultLogger = zen::logging::Default(); auto& Sinks = DefaultLogger.SpdLogger->sinks(); diff --git a/src/zen/zen.h b/src/zen/zen.h index 9c9586050..835c2b6ac 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -84,7 +84,7 @@ public: uint64_t RemainingCount = 0; }; - explicit ProgressBar(bool PlainProgress); + explicit ProgressBar(bool PlainProgress, bool ShowDetails = true); ~ProgressBar(); void UpdateState(const State& NewState, bool DoLinebreak); @@ -95,6 +95,7 @@ public: private: const bool m_PlainProgress; + const bool m_ShowDetails; Stopwatch m_SW; uint64_t m_LastUpdateMS; State m_State; diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index b3bffd34d..6e879ca0d 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -280,6 +280,20 @@ BasicFile::StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& RetryCallback) +WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path) { - if (std::filesystem::is_regular_file(Path)) + TemporaryFile Temp; + std::error_code Ec; + Temp.CreateTemporary(Path.parent_path(), Ec); + if (Ec) { - IoBuffer ExistingTempFile = IoBuffer(IoBufferBuilder::MakeFromFile(Path)); - if (ExistingTempFile && ExistingTempFile.GetSize() == Buffer.GetSize()) - { - ExistingTempFile.SetDeleteOnClose(true); - return ExistingTempFile; - } + throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path)); } - BasicFile BlockFile; - BlockFile.Open(Path, BasicFile::Mode::kTruncateDelete, std::move(RetryCallback)); - uint64_t Offset = 0; + { + uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; - BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + // BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); for (const SharedBuffer& Segment : Buffer.GetSegments()) { size_t SegmentSize = Segment.GetSize(); @@ -859,22 +883,39 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path, std FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&BufferedOutput, &Offset](const void* Data, size_t Size) { - BufferedOutput.Write(Data, Size, Offset); + [&Temp, &Offset](const void* Data, size_t Size) { + Temp.Write(Data, Size, Offset); Offset += Size; }); } else { - BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); + Temp.Write(Segment.GetData(), SegmentSize, Offset); Offset += SegmentSize; } } } - void* FileHandle = BlockFile.Detach(); - IoBuffer BlockBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, Offset, /*IsWholeFile*/ true); - BlockBuffer.SetDeleteOnClose(true); - return BlockBuffer; + + Temp.MoveTemporaryIntoPlace(Path, Ec); + if (Ec) + { + IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + if (TmpBuffer) + { + IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); + const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); + if (ExistingHash == ExpectedHash) + { + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; + } + } + throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); + } + + IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp index 49870a304..252ac9045 100644 --- a/src/zencore/compositebuffer.cpp +++ b/src/zencore/compositebuffer.cpp @@ -275,36 +275,18 @@ CompositeBuffer::IterateRange(uint64_t Offset, Visitor(View, Segment); break; } - if (Offset < SegmentSize) + else if (Offset <= SegmentSize) { - if (Offset == 0 && Size >= SegmentSize) + const MemoryView View = Segment.GetView().Mid(Offset, Size); + Offset = 0; + if (Size == 0 || !View.IsEmpty()) { - const MemoryView View = Segment.GetView(); - if (!View.IsEmpty()) - { - Visitor(View, Segment); - } - Size -= View.GetSize(); - if (Size == 0) - { - break; - } + Visitor(View, Segment); } - else + Size -= View.GetSize(); + if (Size == 0) { - // If we only want a section of the segment, do a subrange so we don't have to materialize the entire iobuffer - IoBuffer SubRange(Segment.AsIoBuffer(), Offset, Min(Size, SegmentSize - Offset)); - const MemoryView View = SubRange.GetView(); - if (!View.IsEmpty()) - { - Visitor(View, Segment); - } - Size -= View.GetSize(); - if (Size == 0) - { - break; - } - Offset = 0; + break; } } else diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 29c1d9256..0e2ce2b54 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -314,37 +315,77 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) CompressedBlockSizes.reserve(BlockCount); uint64_t CompressedSize = 0; { - UniqueBuffer RawBlockCopy; MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize; - CompositeBuffer::Iterator It = RawData.GetIterator(0); - - for (uint64_t RawOffset = 0; RawOffset < RawSize;) + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) { - const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); - const MemoryView RawBlock = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy); - RawHash.Append(RawBlock); - - MutableMemoryView CompressedBlock = CompressedBlocksView; - if (!CompressBlock(CompressedBlock, RawBlock)) + ZEN_ASSERT(FileRef.FileHandle != nullptr); + UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize); + BasicFile Source; + Source.Attach(FileRef.FileHandle); + for (uint64_t RawOffset = 0; RawOffset < RawSize;) { - return CompositeBuffer(); - } + const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); + Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset); + const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize); + RawHash.Append(RawBlock); + MutableMemoryView CompressedBlock = CompressedBlocksView; + if (!CompressBlock(CompressedBlock, RawBlock)) + { + Source.Detach(); + return CompositeBuffer(); + } - uint64_t CompressedBlockSize = CompressedBlock.GetSize(); - if (RawBlockSize <= CompressedBlockSize) - { - CompressedBlockSize = RawBlockSize; - CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock); + uint64_t CompressedBlockSize = CompressedBlock.GetSize(); + if (RawBlockSize <= CompressedBlockSize) + { + CompressedBlockSize = RawBlockSize; + CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock); + } + else + { + CompressedBlocksView += CompressedBlockSize; + } + + CompressedBlockSizes.push_back(static_cast(CompressedBlockSize)); + CompressedSize += CompressedBlockSize; + RawOffset += RawBlockSize; } - else + Source.Detach(); + } + else + { + UniqueBuffer RawBlockCopy; + CompositeBuffer::Iterator It = RawData.GetIterator(0); + + for (uint64_t RawOffset = 0; RawOffset < RawSize;) { - CompressedBlocksView += CompressedBlockSize; - } + const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); + const MemoryView RawBlock = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy); + RawHash.Append(RawBlock); - CompressedBlockSizes.push_back(static_cast(CompressedBlockSize)); - CompressedSize += CompressedBlockSize; - RawOffset += RawBlockSize; + MutableMemoryView CompressedBlock = CompressedBlocksView; + if (!CompressBlock(CompressedBlock, RawBlock)) + { + return CompositeBuffer(); + } + + uint64_t CompressedBlockSize = CompressedBlock.GetSize(); + if (RawBlockSize <= CompressedBlockSize) + { + CompressedBlockSize = RawBlockSize; + CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock); + } + else + { + CompressedBlocksView += CompressedBlockSize; + } + + CompressedBlockSizes.push_back(static_cast(CompressedBlockSize)); + CompressedSize += CompressedBlockSize; + RawOffset += RawBlockSize; + } } } @@ -560,51 +601,118 @@ BlockDecoder::TryDecompressTo(const BufferHeader& Header, CompressedOffset += CompressedBlockSize; } - for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) { - const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; - const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); - const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; + ZEN_ASSERT(FileRef.FileHandle != nullptr); + BasicFile Source; + Source.Attach(FileRef.FileHandle); - const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock) - : zen::Min(RemainingRawSize, BlockSize); + for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) + { + const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; + const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); + const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; - MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy); + const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 + ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock) + : zen::Min(RemainingRawSize, BlockSize); - if (IsCompressed) - { - MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress); + if (CompressedBlockCopy.GetSize() < CompressedBlockSize) + { + CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize); + } + Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset); - const bool IsAligned = BytesToUncompress == UncompressedBlockSize; - if (!IsAligned) + MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize); + + if (IsCompressed) { - // Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries. - if (UncompressedBlockCopy.IsNull()) + MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress); + + const bool IsAligned = BytesToUncompress == UncompressedBlockSize; + if (!IsAligned) { - UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize); + // Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries. + if (UncompressedBlockCopy.IsNull()) + { + UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize); + } + UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize); } - UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize); - } - if (!DecompressBlock(UncompressedBlock, CompressedBlock)) - { - return false; - } + if (!DecompressBlock(UncompressedBlock, CompressedBlock)) + { + Source.Detach(); + return false; + } - if (!IsAligned) + if (!IsAligned) + { + RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); + } + } + else { - RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); + RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); } + + OffsetInFirstBlock = 0; + RemainingRawSize -= BytesToUncompress; + CompressedOffset += CompressedBlockSize; + RawView += BytesToUncompress; } - else + Source.Detach(); + } + else + { + for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) { - RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); - } + const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; + const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); + const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; - OffsetInFirstBlock = 0; - RemainingRawSize -= BytesToUncompress; - CompressedOffset += CompressedBlockSize; - RawView += BytesToUncompress; + const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 + ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock) + : zen::Min(RemainingRawSize, BlockSize); + + MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy); + + if (IsCompressed) + { + MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress); + + const bool IsAligned = BytesToUncompress == UncompressedBlockSize; + if (!IsAligned) + { + // Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries. + if (UncompressedBlockCopy.IsNull()) + { + UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize); + } + UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize); + } + + if (!DecompressBlock(UncompressedBlock, CompressedBlock)) + { + return false; + } + + if (!IsAligned) + { + RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); + } + } + else + { + RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress)); + } + + OffsetInFirstBlock = 0; + RemainingRawSize -= BytesToUncompress; + CompressedOffset += CompressedBlockSize; + RawView += BytesToUncompress; + } } return RemainingRawSize == 0; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index b8c35212f..5716d1255 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -683,7 +683,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { } - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::error_code Ec; const std::filesystem::path Relative = std::filesystem::relative(Parent, BasePath, Ec); @@ -1236,7 +1236,11 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else { - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart, gsl::narrow(DirInfo->FileAttributes)); + Visitor.VisitFile(RootDir, + FileName, + DirInfo->EndOfFile.QuadPart, + gsl::narrow(DirInfo->FileAttributes), + (uint64_t)DirInfo->LastWriteTime.QuadPart); } const uint64_t NextOffset = DirInfo->NextEntryOffset; @@ -1285,7 +1289,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else if (S_ISREG(Stat.st_mode)) { - Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow(Stat.st_mode)); + Visitor.VisitFile(RootDir, FileName, Stat.st_size, gsl::narrow(Stat.st_mode), gsl::narrow(Stat.st_mtime)); } else { @@ -1544,7 +1548,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) override + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) override { if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)) { @@ -1557,6 +1562,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags { Content.FileAttributes.push_back(NativeModeOrAttributes); } + if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick)) + { + Content.FileModificationTicks.push_back(NativeModificationTick); + } } } @@ -1612,7 +1621,8 @@ GetDirectoryContent(const std::filesystem::path& RootDir, virtual void VisitFile(const std::filesystem::path&, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) override + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) override { if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeFiles)) { @@ -1625,6 +1635,10 @@ GetDirectoryContent(const std::filesystem::path& RootDir, { Content.FileAttributes.push_back(NativeModeOrAttributes); } + if (EnumHasAnyFlags(Flags, DirectoryContentFlags::IncludeModificationTick)) + { + Content.FileModificationTicks.push_back(NativeModificationTick); + } } } @@ -1928,7 +1942,7 @@ TEST_CASE("filesystem") // Traversal struct : public FileSystemTraversal::TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t, uint32_t, uint64_t) override { bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); } diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 7edd40c9c..a78132879 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -60,6 +60,7 @@ public: void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(MemoryView Data, uint64_t FileOffset); void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec); + uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset); uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec); @@ -174,6 +175,7 @@ public: ~BasicFileWriter(); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); + void Write(const CompositeBuffer& Data, uint64_t FileOffset); void Flush(); private: @@ -184,9 +186,7 @@ private: uint64_t m_BufferEnd; }; -IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, - const std::filesystem::path& Path, - std::function&& RetryCallback); +IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path); ZENCORE_API void basicfile_forcelink(); diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index ca8682cd7..250745e86 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -203,7 +203,8 @@ public: virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) = 0; + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) = 0; // This should return true if we should recurse into the directory virtual bool VisitDirectory(const std::filesystem::path& Parent, @@ -216,13 +217,14 @@ public: enum class DirectoryContentFlags : uint8_t { - None = 0, - IncludeDirs = 1u << 0, - IncludeFiles = 1u << 1, - Recursive = 1u << 2, - IncludeFileSizes = 1u << 3, - IncludeAttributes = 1u << 4, - IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive + None = 0, + IncludeDirs = 1u << 0, + IncludeFiles = 1u << 1, + Recursive = 1u << 2, + IncludeFileSizes = 1u << 3, + IncludeAttributes = 1u << 4, + IncludeModificationTick = 1u << 5, + IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive }; ENUM_CLASS_FLAGS(DirectoryContentFlags) @@ -232,6 +234,7 @@ struct DirectoryContent std::vector Files; std::vector FileSizes; std::vector FileAttributes; + std::vector FileModificationTicks; std::vector Directories; std::vector DirectoryAttributes; }; @@ -246,6 +249,7 @@ public: std::vector FileNames; std::vector FileSizes; std::vector FileAttributes; + std::vector FileModificationTicks; std::vector DirectoryNames; std::vector DirectoryAttributes; }; diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h index 8871a5895..7443e17b7 100644 --- a/src/zencore/include/zencore/iohash.h +++ b/src/zencore/include/zencore/iohash.h @@ -47,8 +47,8 @@ struct IoHash static IoHash HashBuffer(const void* data, size_t byteCount); static IoHash HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); } - static IoHash HashBuffer(const CompositeBuffer& Buffer); - static IoHash HashBuffer(const IoBuffer& Buffer); + static IoHash HashBuffer(const CompositeBuffer& Buffer, std::atomic* ProcessedBytes = nullptr); + static IoHash HashBuffer(const IoBuffer& Buffer, std::atomic* ProcessedBytes = nullptr); static IoHash FromHexString(const char* string); static IoHash FromHexString(const std::string_view string); static bool TryParse(std::string_view Str, IoHash& Hash); diff --git a/src/zencore/iohash.cpp b/src/zencore/iohash.cpp index 7200e6e3f..3b2af0db4 100644 --- a/src/zencore/iohash.cpp +++ b/src/zencore/iohash.cpp @@ -30,7 +30,7 @@ IoHash::HashBuffer(const void* data, size_t byteCount) } IoHash -IoHash::HashBuffer(const CompositeBuffer& Buffer) +IoHash::HashBuffer(const CompositeBuffer& Buffer, std::atomic* ProcessedBytes) { IoHashStream Hasher; @@ -46,11 +46,21 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer) FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); }); + [&Hasher, ProcessedBytes](const void* Data, size_t Size) { + Hasher.Append(Data, Size); + if (ProcessedBytes != nullptr) + { + ProcessedBytes->fetch_add(Size); + } + }); } else { Hasher.Append(Segment.GetData(), SegmentSize); + if (ProcessedBytes != nullptr) + { + ProcessedBytes->fetch_add(SegmentSize); + } } } @@ -58,7 +68,7 @@ IoHash::HashBuffer(const CompositeBuffer& Buffer) } IoHash -IoHash::HashBuffer(const IoBuffer& Buffer) +IoHash::HashBuffer(const IoBuffer& Buffer, std::atomic* ProcessedBytes) { IoHashStream Hasher; @@ -71,11 +81,21 @@ IoHash::HashBuffer(const IoBuffer& Buffer) FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&Hasher](const void* Data, size_t Size) { Hasher.Append(Data, Size); }); + [&Hasher, ProcessedBytes](const void* Data, size_t Size) { + Hasher.Append(Data, Size); + if (ProcessedBytes != nullptr) + { + ProcessedBytes->fetch_add(Size); + } + }); } else { Hasher.Append(Buffer.GetData(), BufferSize); + if (ProcessedBytes != nullptr) + { + ProcessedBytes->fetch_add(BufferSize); + } } return Hasher.GetHash(); diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 211a5d05c..bb15a6fce 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -449,12 +449,27 @@ ValidatePayload(cpr::Response& Response, std::unique_ptr&& Func, uint8_t RetryCount) +DoWithRetry( + std::string_view SessionId, + std::function&& Func, + uint8_t RetryCount, + std::function&& Validate = [](cpr::Response&) { return true; }) { uint8_t Attempt = 0; cpr::Response Result = Func(); - while (Attempt < RetryCount && ShouldRetry(Result)) + while (Attempt < RetryCount) { + if (!ShouldRetry(Result)) + { + if (Result.error || !IsHttpSuccessCode(Result.status_code)) + { + break; + } + if (Validate(Result)) + { + break; + } + } Sleep(100 * (Attempt + 1)); Attempt++; ZEN_INFO("{} Attempt {}/{}", CommonResponse(SessionId, std::move(Result)).ErrorMessage("Retry"), Attempt, RetryCount + 1); @@ -881,7 +896,11 @@ HttpClient::Get(std::string_view Url, const KeyValueMap& AdditionalHeader, const m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, Parameters, m_SessionId, GetAccessToken()); return Sess.Get(); }, - m_ConnectionSettings.RetryCount)); + m_ConnectionSettings.RetryCount, + [](cpr::Response& Result) { + std::unique_ptr NoTempFile; + return ValidatePayload(Result, NoTempFile); + })); } HttpClient::Response @@ -1247,7 +1266,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold { uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); - std::string Range = fmt::format("bytes={}-{}", DownloadedSize, ContentLength.value()); + std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength.value() - 1); if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) { if (RangeIt->second == Range) diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index b0212ab07..5d96de225 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -376,7 +376,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Writer.BeginArray("Contents"sv); } - void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { const fs::path FullPath = Parent / fs::path(File); fs::path RelativePath = fs::relative(FullPath, BucketPath); diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index 412769174..e4e91104c 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -123,18 +123,17 @@ public: JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); JupiterResult PutResult = - Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, ZenContentType::kCompressedBinary, Payload); + Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); AddStats(PutResult); SaveAttachmentResult Result{ConvertResult(PutResult)}; if (Result.ErrorCode) { - Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed saving oplog attachment to {}/{}/{}/{}/{}. Reason: '{}'", m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, - m_OplogBuildPartId, RawHash, Result.Reason); return Result; @@ -147,18 +146,16 @@ public: IoBuffer MetaPayload = BuildChunkBlockDescription(Block, BlockMetaData.Save()).GetBuffer().AsIoBuffer(); MetaPayload.SetContentType(ZenContentType::kCbObject); - JupiterResult PutMetaResult = - Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, MetaPayload); + JupiterResult PutMetaResult = Session.PutBlockMetadata(m_Namespace, m_Bucket, m_BuildId, RawHash, MetaPayload); AddStats(PutMetaResult); RemoteProjectStore::Result MetaDataResult = ConvertResult(PutMetaResult); if (MetaDataResult.ErrorCode) { - ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}/{}. Reason: '{}'", + ZEN_WARN("Failed saving block attachment meta data to {}/{}/{}/{}/{}. Reason: '{}'", m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, - m_OplogBuildPartId, RawHash, MetaDataResult.Reason); } @@ -311,30 +308,28 @@ public: { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId); + JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; if (Result.ErrorCode) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, - m_OplogBuildPartId, Result.Reason); return Result; } if (ValidateCompactBinary(FindResult.Response.GetView(), CbValidateMode::Default) != CbValidateError::None) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a compact binary object"sv, + Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a compact binary object"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, - m_BuildId, - m_OplogBuildPartId); + m_BuildId); return Result; } std::optional> Blocks = @@ -342,12 +337,11 @@ public: if (!Blocks) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The block list {}/{}/{}/{} is not formatted as a list of blocks"sv, + Result.Reason = fmt::format("The block list {}/{}/{} is not formatted as a list of blocks"sv, m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, - m_BuildId, - m_OplogBuildPartId); + m_BuildId); return Result; } Result.Blocks = std::move(Blocks.value()); @@ -358,18 +352,17 @@ public: { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash, m_TempFilePath); + JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath); AddStats(GetResult); LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; if (GetResult.ErrorCode) { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}&{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}/{}. Reason: '{}'", m_JupiterClient->ServiceUrl(), m_Namespace, m_Bucket, m_BuildId, - m_OplogBuildPartId, RawHash, Result.Reason); } diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index 0285cc22f..b4b2c6fc4 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -1466,10 +1466,7 @@ BuildContainer(CidStore& ChunkStore, }; std::vector ChunkedFiles; - auto ChunkFile = [AttachmentTempPath](const IoHash& RawHash, - IoBuffer& RawData, - const IoBufferFileReference& FileRef, - JobContext*) -> ChunkedFile { + auto ChunkFile = [](const IoHash& RawHash, IoBuffer& RawData, const IoBufferFileReference& FileRef, JobContext*) -> ChunkedFile { ChunkedFile Chunked; Stopwatch Timer; @@ -1578,22 +1575,7 @@ BuildContainer(CidStore& ChunkStore, std::filesystem::path AttachmentPath = AttachmentTempPath; AttachmentPath.append(RawHash.ToHexString()); - uint32_t RetriesLeft = 3; - - IoBuffer TempAttachmentBuffer = - WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.", - AttachmentPath, - Ec.message(), - RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); + IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath); ZEN_INFO("Saved temp attachment to '{}', {} ({})", AttachmentPath, NiceBytes(RawSize), @@ -1612,34 +1594,21 @@ BuildContainer(CidStore& ChunkStore, std::filesystem::path AttachmentPath = AttachmentTempPath; AttachmentPath.append(RawHash.ToHexString()); - uint32_t RetriesLeft = 3; - IoBuffer TempAttachmentBuffer = - WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temporary attachment '{}': '{}', retries left: {}.", - AttachmentPath, - Ec.message(), - RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); + uint64_t CompressedSize = Compressed.GetCompressedSize(); + IoBuffer TempAttachmentBuffer = WriteToTempFile(std::move(Compressed).GetCompressed(), AttachmentPath); ZEN_INFO("Saved temp attachment to '{}', {} ({})", AttachmentPath, NiceBytes(RawSize), NiceBytes(TempAttachmentBuffer.GetSize())); - if (Compressed.GetCompressedSize() > MaxChunkEmbedSize) + if (CompressedSize > MaxChunkEmbedSize) { OnLargeAttachment(RawHash, [Data = std::move(TempAttachmentBuffer)](const IoHash&) { return Data; }); ResolveLock.WithExclusiveLock([RawHash, &LargeChunkHashes]() { LargeChunkHashes.insert(RawHash); }); } else { - UploadAttachment->Size = Compressed.GetCompressedSize(); + UploadAttachment->Size = CompressedSize; ResolveLock.WithExclusiveLock( [RawHash, RawSize, &LooseUploadAttachments, Data = std::move(TempAttachmentBuffer)]() { LooseUploadAttachments.insert_or_assign(RawHash, std::make_pair(RawSize, std::move(Data))); @@ -2362,17 +2331,7 @@ SaveOplog(CidStore& ChunkStore, BlockPath.append(Block.BlockHash.ToHexString()); try { - uint32_t RetriesLeft = 3; - IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath, [&](std::error_code& Ec) { - if (RetriesLeft == 0) - { - return false; - } - ZEN_WARN("Failed to create temporary oplog block '{}': '{}', retries left: {}.", BlockPath, Ec.message(), RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms - RetriesLeft--; - return true; - }); + IoBuffer BlockBuffer = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), BlockPath); RwLock::ExclusiveLockScope __(AttachmentsLock); CreatedBlocks.insert({Block.BlockHash, {.Payload = std::move(BlockBuffer), .Block = std::move(Block)}}); ZEN_DEBUG("Saved temp block to '{}', {}", AttachmentTempPath, NiceBytes(BlockBuffer.GetSize())); diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 14123528c..34db51aa9 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -185,7 +185,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN // in this folder as well struct Visitor : public FileSystemTraversal::TreeVisitor { - virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t, uint32_t) override + virtual void VisitFile(const std::filesystem::path&, const path_view&, uint64_t, uint32_t, uint64_t) override { // We don't care about files } @@ -1174,7 +1174,7 @@ FileCasStrategy::ScanFolderForCasFiles(const std::filesystem::path& RootDir) struct Visitor : public FileSystemTraversal::TreeVisitor { Visitor(const std::filesystem::path& RootDir, std::vector& Entries) : RootDirectory(RootDir), Entries(Entries) {} - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override { std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp index c08492eb0..3f3a6661c 100644 --- a/src/zenutil/chunkedfile.cpp +++ b/src/zenutil/chunkedfile.cpp @@ -121,7 +121,7 @@ ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Para Chunker.SetUseThreshold(Params.UseThreshold); Chunker.SetChunkSize(Params.MinSize, Params.MaxSize, Params.AvgSize); size_t End = Offset + Size; - const size_t ScanBufferSize = 1u * 1024 * 1024; // (Params.MaxSize * 9) / 3;//1 * 1024 * 1024; + const size_t ScanBufferSize = Max(1u * 1024 * 1024, Params.MaxSize); BasicFileBuffer RawBuffer(RawData, ScanBufferSize); MemoryView SliceView = RawBuffer.MakeView(Min(End - Offset, ScanBufferSize), Offset); ZEN_ASSERT(!SliceView.IsEmpty()); diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 075c35b40..852271868 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -116,21 +116,18 @@ public: JupiterResult PutBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, const CompositeBuffer& Payload); JupiterResult GetBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, std::filesystem::path TempFolderPath); JupiterResult PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, uint64_t PayloadSize, @@ -139,7 +136,6 @@ public: JupiterResult GetMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, uint64_t ChunkSize, std::function&& Receiver, @@ -147,7 +143,6 @@ public: JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, const IoBuffer& Payload); FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace, @@ -155,12 +150,8 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); - JupiterResult GetBlockMetadata(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - IoBuffer Payload); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); private: inline LoggerRef Log() { return m_Log; } diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index d56927b44..06ac6ae36 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -436,15 +436,14 @@ JupiterResult JupiterSession::PutBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, const CompositeBuffer& Payload) { - HttpClient::Response Response = m_HttpClient.Upload( - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()), - Payload, - ContentType); + HttpClient::Response Response = + m_HttpClient.Upload(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()), + Payload, + ContentType); return detail::ConvertResponse(Response, "JupiterSession::PutBuildBlob"sv); } @@ -452,7 +451,6 @@ JupiterResult JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, uint64_t PayloadSize, @@ -498,12 +496,8 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, StartMultipartPayloadWriter.AddInteger("blobLength"sv, PayloadSize); CbObject StartMultipartPayload = StartMultipartPayloadWriter.Save(); - std::string StartMultipartResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/startMultipartUpload", - Namespace, - BucketId, - BuildId, - PartId, - Hash.ToHexString()); + std::string StartMultipartResponseRequestString = + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/startMultipartUpload", Namespace, BucketId, BuildId, Hash.ToHexString()); // ZEN_INFO("POST: {}", StartMultipartResponseRequestString); HttpClient::Response StartMultipartResponse = m_HttpClient.Post(StartMultipartResponseRequestString, StartMultipartPayload, HttpClient::Accept(ZenContentType::kCbObject)); @@ -529,15 +523,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, for (size_t PartIndex = 0; PartIndex < Workload->PartDescription.Parts.size(); PartIndex++) { - OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, PartId, Hash, ContentType, Workload, PartIndex]( + OutWorkItems.emplace_back([this, Namespace, BucketId, BuildId, Hash, ContentType, Workload, PartIndex]( bool& OutIsComplete) -> JupiterResult { const MultipartUploadResponse::Part& Part = Workload->PartDescription.Parts[PartIndex]; IoBuffer PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte); - std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}", + std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}", Namespace, BucketId, BuildId, - PartId, Hash.ToHexString(), Part.QueryString); // ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString); @@ -571,12 +564,7 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, CbObject CompletePayload = CompletePayloadWriter.Save(); std::string MultipartEndResponseRequestString = - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/completeMultipart", - Namespace, - BucketId, - BuildId, - PartId, - Hash.ToHexString()); + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/completeMultipart", Namespace, BucketId, BuildId, Hash.ToHexString()); MultipartEndResponse = m_HttpClient.Post(MultipartEndResponseRequestString, CompletePayload, @@ -600,13 +588,12 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, size_t RetryPartIndex = PartNameToIndex.at(RetryPartId); const MultipartUploadResponse::Part& RetryPart = Workload->PartDescription.Parts[RetryPartIndex]; IoBuffer RetryPartPayload = - Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte); + Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1); std::string RetryMultipartUploadResponseRequestString = - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}/uploadMultipart{}", + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}", Namespace, BucketId, BuildId, - RetryPartId, Hash.ToHexString(), RetryPart.QueryString); @@ -642,14 +629,12 @@ JupiterResult JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, uint64_t ChunkSize, std::function&& Receiver, std::vector>& OutWorkItems) { - std::string RequestUrl = - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()); + std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); HttpClient::Response Response = m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}})); if (Response.IsSuccess()) @@ -683,17 +668,12 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespa { uint64_t PartSize = Min(ChunkSize, TotalSize - Offset); OutWorkItems.emplace_back( - [this, Namespace, BucketId, BuildId, PartId, Hash, TotalSize, Workload, Offset, PartSize]() - -> JupiterResult { - std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", - Namespace, - BucketId, - BuildId, - PartId, - Hash.ToHexString()); - HttpClient::Response Response = m_HttpClient.Get( - RequestUrl, - HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); + [this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult { + std::string RequestUrl = + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); + HttpClient::Response Response = m_HttpClient.Get( + RequestUrl, + HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); if (Response.IsSuccess()) { uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize()); @@ -717,13 +697,12 @@ JupiterResult JupiterSession::GetBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, std::filesystem::path TempFolderPath) { - HttpClient::Response Response = m_HttpClient.Download( - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blobs/{}", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()), - TempFolderPath); + HttpClient::Response Response = + m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()), + TempFolderPath); return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv); } @@ -732,14 +711,13 @@ JupiterResult JupiterSession::PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, const IoBuffer& Payload) { ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); - HttpClient::Response Response = m_HttpClient.Put( - fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, PartId, Hash.ToHexString()), - Payload); + HttpClient::Response Response = + m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/blocks/{}/metadata", Namespace, BucketId, BuildId, Hash.ToHexString()), + Payload); return detail::ConvertResponse(Response, "JupiterSession::PutBlockMetadata"sv); } @@ -772,24 +750,19 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace, } JupiterResult -JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId) +JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId) { - HttpClient::Response Response = - m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/listBlocks", Namespace, BucketId, BuildId, PartId), - HttpClient::Accept(ZenContentType::kCbObject)); + HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId), + HttpClient::Accept(ZenContentType::kCbObject)); return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv); } JupiterResult -JupiterSession::GetBlockMetadata(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - IoBuffer Payload) +JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload) { ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); HttpClient::Response Response = - m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId, PartId), + m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, BucketId, BuildId), Payload, HttpClient::Accept(ZenContentType::kCbObject)); return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv); diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index 6314c407f..0444fa2c4 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -10,6 +10,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +#include #include #include #include @@ -97,7 +98,13 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) } } - std::set_terminate([]() { ZEN_CRITICAL("Program exited abnormally via std::terminate()"); }); + std::set_terminate([]() { + void* Frames[8]; + uint32_t FrameCount = GetCallstack(2, 8, Frames); + CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); + ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, " ")); + FreeCallstack(Callstack); + }); // Default -- cgit v1.2.3 From 7d8fe45af3b49d800f84f0ddce051c0b3b2e837d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Feb 2025 15:10:14 +0100 Subject: builds upload command (#278) - Feature: **EXPERIMENTAL** New `zen builds` command to list, upload and download folders to Cloud Build API - `builds list` list available builds (**INCOMPLETE - FILTERING MISSING**) - `builds upload` upload a folder to Cloud Build API - `--local-path` source folder to upload - `--create-build` creates a new parent build object (using the object id), if omitted a parent build must exist and `--build-id` must be given - `--build-id` an Oid in hex form for the Build identifier to use - omit to have the id auto generated - `--build-part-id` and Oid in hex form for the Build Part identifier for the folder - omit to have the id auto generated - `--build-part-name` name of the build part - if omitted the name of the leaf folder name give in `--local-path` - `--metadata-path` path to a json formatted file with meta data information about the build. Meta-data must be provided if `--create-build` is set - `--metadata` key-value pairs separated by ';' with build meta data for the build. (key1=value1;key2=value2). Meta-data must be provided if `--create-build` is set - `--clean` ignore any existing blocks of chunk data and upload a fresh set of blocks - `--allow-multipart` enable usage of multi-part http upload requests - `--manifest-path` path to text file listing files to include in upload. Exclude to upload everything in `--local-path` - `builds download` download a folder from Cloud Build API (**INCOMPLETE - WILL WIPE UNTRACKED DATA FROM TARGET FOLDER**) - `--local-path` target folder to download to - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` a comma separated list of Oid in hex for the build part identifier(s) to download - mutually exclusive to `--build-part-name` - `--build-part-name` a comma separated list of names for the build part(s) to download - if omitted the name of the leaf folder name give in `--local-path` - `--clean` deletes all data in target folder before downloading (NON-CLEAN IS NOT IMPLEMENTED YET) - `--allow-multipart` enable usage of multi-part http download reqeusts - `builds diff` download a folder from Cloud Build API - `--local-path` target folder to download to - `--compare-path` folder to compare target with - `--only-chunked` compare only files that would be chunked - `builds fetch-blob` fetch and validate a blob from remote store - `--build-id` an Oid in hex form for the Build identifier to use - `--blob-hash` an IoHash in hex form identifying the blob to download - `builds validate part` fetch a build part and validate all referenced attachments - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` an Oid in hex for the build part identifier to validate - mutually exclusive to `--build-part-name` - `--build-part-name` a name for the build part to validate - mutually exclusive to `--build-part-id` - `builds test` a series of operation that uploads, downloads and test various aspects of incremental operations - `--local-path` source folder to upload - Options for Cloud Build API remote store (`list`, `upload`, `download`, `fetch-blob`, `validate-part`) - `--url` Cloud Builds URL - `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake - `--namespace` Builds Storage namespace - `--bucket` Builds Storage bucket - Authentication options for Cloud Build API - Auth token - `--access-token` http auth Cloud Storage access token - `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token - `--access-token-path` path to json file that holds the Http auth Cloud Storage access token - OpenId authentication - `--openid-provider-name` Open ID provider name - `--openid-provider-url` Open ID provider url - `--openid-client-id`Open ID client id - `--openid-refresh-token` Open ID refresh token - `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials - `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials - OAuth authentication - `--oauth-url` OAuth provier url - `--oauth-clientid` OAuth client id - `--oauth-clientsecret` OAuth client secret - Options for file based remote store used for for testing purposes (`list`, `upload`, `download`, `fetch-blob`, `validate-part`, `test`) - `--storage-path` path to folder to store builds data - `--json-metadata` enable json output in store for all compact binary objects (off by default) - Output options for all builds commands - `--plain-progress` use plain line-by-line progress output - `--verbose` --- src/zen/cmds/builds_cmd.cpp | 6106 ++++++++++++++++++++ src/zen/cmds/builds_cmd.h | 106 + src/zen/zen.cpp | 3 + src/zencore/filesystem.cpp | 140 + src/zencore/include/zencore/filesystem.h | 21 + .../projectstore/buildsremoteprojectstore.cpp | 7 +- .../projectstore/fileremoteprojectstore.cpp | 4 +- .../projectstore/jupiterremoteprojectstore.cpp | 2 +- src/zenserver/projectstore/projectstore.cpp | 6 +- src/zenserver/projectstore/remoteprojectstore.cpp | 72 +- src/zenserver/projectstore/remoteprojectstore.h | 6 +- src/zenutil/chunkblock.cpp | 94 +- src/zenutil/chunkedcontent.cpp | 865 +++ src/zenutil/chunkingcontroller.cpp | 265 + src/zenutil/filebuildstorage.cpp | 616 ++ src/zenutil/include/zenutil/buildstorage.h | 55 + src/zenutil/include/zenutil/chunkblock.h | 17 +- src/zenutil/include/zenutil/chunkedcontent.h | 256 + src/zenutil/include/zenutil/chunkingcontroller.h | 55 + src/zenutil/include/zenutil/filebuildstorage.h | 16 + .../include/zenutil/jupiter/jupiterbuildstorage.h | 17 + src/zenutil/include/zenutil/parallellwork.h | 69 + src/zenutil/jupiter/jupiterbuildstorage.cpp | 371 ++ 23 files changed, 9108 insertions(+), 61 deletions(-) create mode 100644 src/zen/cmds/builds_cmd.cpp create mode 100644 src/zen/cmds/builds_cmd.h create mode 100644 src/zenutil/chunkedcontent.cpp create mode 100644 src/zenutil/chunkingcontroller.cpp create mode 100644 src/zenutil/filebuildstorage.cpp create mode 100644 src/zenutil/include/zenutil/buildstorage.h create mode 100644 src/zenutil/include/zenutil/chunkedcontent.h create mode 100644 src/zenutil/include/zenutil/chunkingcontroller.h create mode 100644 src/zenutil/include/zenutil/filebuildstorage.h create mode 100644 src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h create mode 100644 src/zenutil/include/zenutil/parallellwork.h create mode 100644 src/zenutil/jupiter/jupiterbuildstorage.cpp (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp new file mode 100644 index 000000000..ececab29e --- /dev/null +++ b/src/zen/cmds/builds_cmd.cpp @@ -0,0 +1,6106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "builds_cmd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +# include +# include +# include +#endif + +#define EXTRA_VERIFY 0 + +namespace zen { +namespace { + using namespace std::literals; + + static const size_t DefaultMaxBlockSize = 64u * 1024u * 1024u; + static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u; + + struct ChunksBlockParameters + { + size_t MaxBlockSize = DefaultMaxBlockSize; + size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize; + }; + + const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, + .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; + const std::string ZenFolderName = ".zen"; + const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); + const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); + const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + const std::string ZenTempReuseFolderName = fmt::format("{}/reuse", ZenTempFolderName); + const std::string ZenTempStorageFolderName = fmt::format("{}/storage", ZenTempFolderName); + const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); + const std::string ZenTempChunkFolderName = fmt::format("{}/chunks", ZenTempFolderName); + const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; + + const std::string UnsyncFolderName = ".unsync"; + + const std::string UGSFolderName = ".ugs"; + const std::string LegacyZenTempFolderName = ".zen-tmp"; + + const std::vector DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); + const std::vector DefaultExcludeExtensions({}); + + static bool IsVerbose = false; + static bool UsePlainProgress = false; + +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ + } + + const std::string DefaultAccessTokenEnvVariableName( +#if ZEN_PLATFORM_WINDOWS + "UE-CloudDataCacheAccessToken"sv +#endif +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + "UE_CloudDataCacheAccessToken"sv +#endif + + ); + + template + std::string FormatArray(std::span Items, std::string_view Prefix) + { + ExtendableStringBuilder<512> SB; + for (const T& Item : Items) + { + SB.Append(fmt::format("{}{}", Prefix, Item)); + } + return SB.ToString(); + } + + void CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) + { + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + { + std::filesystem::remove(LocalFilePath); + } + + for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + { + bool Leave = false; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (LocalDirPath == (Path / ExcludeDirectory)) + { + Leave = true; + break; + } + } + if (!Leave) + { + zen::CleanDirectory(LocalDirPath); + std::filesystem::remove(LocalDirPath); + } + } + } + + std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) + { + if (!std::filesystem::is_regular_file(Path)) + { + throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); + } + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json TokenInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); + } + const std::string AuthToken = TokenInfo["Token"].string_value(); + if (AuthToken.empty()) + { + throw std::runtime_error(fmt::format("the json file '{}' does not contain a value for \"Token\"", Path)); + } + return AuthToken; + } + + CompositeBuffer WriteToTempFileIfNeeded(const CompositeBuffer& Buffer, const std::filesystem::path& TempFolderPath, const IoHash& Hash) + { + // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory + std::span Segments = Buffer.GetSegments(); + ZEN_ASSERT(Buffer.GetSegments().size() > 0); + size_t SegmentIndexToCheck = Segments.size() > 1 ? 1 : 0; + IoBufferFileReference FileRef; + if (Segments[SegmentIndexToCheck].GetFileReference(FileRef)) + { + return Buffer; + } + std::filesystem::path TempFilePath = (TempFolderPath / Hash.ToHexString()).make_preferred(); + return CompositeBuffer(WriteToTempFile(Buffer, TempFilePath)); + } + + class FilteredRate + { + public: + FilteredRate() {} + + void Start() + { + if (StartTimeUS == (uint64_t)-1) + { + uint64_t Expected = (uint64_t)-1; + if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs())) + { + LastTimeUS = StartTimeUS.load(); + } + } + } + void Stop() + { + if (EndTimeUS == (uint64_t)-1) + { + uint64_t Expected = (uint64_t)-1; + EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()); + } + } + + void Update(uint64_t Count) + { + if (LastTimeUS == (uint64_t)-1) + { + return; + } + uint64_t TimeUS = Timer.GetElapsedTimeUs(); + uint64_t TimeDeltaUS = TimeUS - LastTimeUS; + if (TimeDeltaUS >= 1000000) + { + uint64_t Delta = Count - LastCount; + uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; + + LastPerSecond = PerSecond; + + LastCount = Count; + + FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8; + + LastTimeUS = TimeUS; + } + } + + uint64_t GetCurrent() const + { + if (LastTimeUS == (uint64_t)-1) + { + return 0; + } + return FilteredPerSecond; + } + + uint64_t GetElapsedTime() const + { + if (StartTimeUS == (uint64_t)-1) + { + return 0; + } + if (EndTimeUS == (uint64_t)-1) + { + return 0; + } + uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS; + return TimeDeltaUS; + } + + bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); } + + private: + Stopwatch Timer; + std::atomic StartTimeUS = (uint64_t)-1; + std::atomic EndTimeUS = (uint64_t)-1; + std::atomic LastTimeUS = (uint64_t)-1; + uint64_t LastCount = 0; + uint64_t LastPerSecond = 0; + uint64_t FilteredPerSecond = 0; + }; + + uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count) + { + if (ElapsedWallTimeUS == 0) + { + return 0; + } + return Count * 1000000 / ElapsedWallTimeUS; + } + + ChunkedFolderContent ScanAndChunkFolder( + GetFolderContentStatistics& GetFolderContentStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + std::function&& IsAcceptedFolder, + std::function&& IsAcceptedFile, + ChunkingController& ChunkController, + std::atomic& AbortFlag) + { + FolderContent Content = GetFolderContent( + GetFolderContentStats, + Path, + std::move(IsAcceptedFolder), + std::move(IsAcceptedFile), + GetMediumWorkerPool(EWorkloadType::Burst), + UsePlainProgress ? 5000 : 200, + [](bool, std::ptrdiff_t) {}, + AbortFlag); + if (AbortFlag) + { + return {}; + } + + ProgressBar ProgressBar(UsePlainProgress); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent FolderContent = ChunkFolderContent( + ChunkingStats, + GetMediumWorkerPool(EWorkloadType::Burst), + Path, + Content, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + GetFolderContentStats.AcceptedFileCount.load(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(GetFolderContentStats.FoundFileByteCount), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .TotalCount = GetFolderContentStats.AcceptedFileByteCount, + .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + ChunkingStats.FilesProcessed.load(), + NiceBytes(ChunkingStats.BytesHashed.load()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + Path, + NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + return FolderContent; + }; + + struct DiskStatistics + { + std::atomic OpenReadCount = 0; + std::atomic OpenWriteCount = 0; + std::atomic ReadCount = 0; + std::atomic ReadByteCount = 0; + std::atomic WriteCount = 0; + std::atomic WriteByteCount = 0; + std::atomic CurrentOpenFileCount = 0; + }; + + struct FindBlocksStatistics + { + uint64_t FindBlockTimeMS = 0; + uint64_t PotentialChunkCount = 0; + uint64_t PotentialChunkByteCount = 0; + uint64_t FoundBlockCount = 0; + uint64_t FoundBlockChunkCount = 0; + uint64_t FoundBlockByteCount = 0; + uint64_t AcceptedBlockCount = 0; + uint64_t AcceptedChunkCount = 0; + uint64_t AcceptedByteCount = 0; + uint64_t RejectedBlockCount = 0; + uint64_t RejectedChunkCount = 0; + uint64_t RejectedByteCount = 0; + uint64_t AcceptedReduntantChunkCount = 0; + uint64_t AcceptedReduntantByteCount = 0; + uint64_t NewBlocksCount = 0; + uint64_t NewBlocksChunkCount = 0; + uint64_t NewBlocksChunkByteCount = 0; + }; + + struct UploadStatistics + { + std::atomic BlockCount = 0; + std::atomic BlocksBytes = 0; + std::atomic ChunkCount = 0; + std::atomic ChunksBytes = 0; + std::atomic ReadFromDiskBytes = 0; + std::atomic MultipartAttachmentCount = 0; + uint64_t ElapsedWallTimeUS = 0; + + UploadStatistics& operator+=(const UploadStatistics& Rhs) + { + BlockCount += Rhs.BlockCount; + BlocksBytes += Rhs.BlocksBytes; + ChunkCount += Rhs.ChunkCount; + ChunksBytes += Rhs.ChunksBytes; + ReadFromDiskBytes += Rhs.ReadFromDiskBytes; + MultipartAttachmentCount += Rhs.MultipartAttachmentCount; + ElapsedWallTimeUS += Rhs.ElapsedWallTimeUS; + return *this; + } + }; + + struct LooseChunksStatistics + { + uint64_t ChunkCount = 0; + uint64_t ChunkByteCount = 0; + std::atomic CompressedChunkCount = 0; + std::atomic CompressedChunkBytes = 0; + uint64_t CompressChunksElapsedWallTimeUS = 0; + + LooseChunksStatistics& operator+=(const LooseChunksStatistics& Rhs) + { + ChunkCount += Rhs.ChunkCount; + ChunkByteCount += Rhs.ChunkByteCount; + CompressedChunkCount += Rhs.CompressedChunkCount; + CompressedChunkBytes += Rhs.CompressedChunkBytes; + CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; + return *this; + } + }; + + struct GenerateBlocksStatistics + { + std::atomic GeneratedBlockByteCount = 0; + std::atomic GeneratedBlockCount = 0; + uint64_t GenerateBlocksElapsedWallTimeUS = 0; + + GenerateBlocksStatistics& operator+=(const GenerateBlocksStatistics& Rhs) + { + GeneratedBlockByteCount += Rhs.GeneratedBlockByteCount; + GeneratedBlockCount += Rhs.GeneratedBlockCount; + GenerateBlocksElapsedWallTimeUS += Rhs.GenerateBlocksElapsedWallTimeUS; + return *this; + } + }; + + std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, + const std::span LocalChunkOrder, + const tsl::robin_map& ChunkHashToLocalChunkIndex, + const std::span& LooseChunkIndexes, + const std::span& BlockDescriptions) + { +#if EXTRA_VERIFY + std::vector TmpAbsoluteChunkHashes; + TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size()); +#endif // EXTRA_VERIFY + std::vector LocalChunkIndexToAbsoluteChunkIndex; + LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1); + std::uint32_t AbsoluteChunkCount = 0; + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount; +#if EXTRA_VERIFY + TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]); +#endif // EXTRA_VERIFY + AbsoluteChunkCount++; + } + for (const ChunkBlockDescription& Block : BlockDescriptions) + { + for (const IoHash& ChunkHash : Block.ChunkRawHashes) + { + if (auto It = ChunkHashToLocalChunkIndex.find(ChunkHash); It != ChunkHashToLocalChunkIndex.end()) + { + const uint32_t LocalChunkIndex = It->second; + ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash); + LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount; + } +#if EXTRA_VERIFY + TmpAbsoluteChunkHashes.push_back(ChunkHash); +#endif // EXTRA_VERIFY + AbsoluteChunkCount++; + } + } + std::vector AbsoluteChunkOrder; + AbsoluteChunkOrder.reserve(LocalChunkHashes.size()); + for (const uint32_t LocalChunkIndex : LocalChunkOrder) + { + const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex]; +#if EXTRA_VERIFY + ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]); +#endif // EXTRA_VERIFY + AbsoluteChunkOrder.push_back(AbsoluteChunkIndex); + } +#if EXTRA_VERIFY + { + uint32_t OrderIndex = 0; + while (OrderIndex < LocalChunkOrder.size()) + { + const uint32_t LocalChunkIndex = LocalChunkOrder[OrderIndex]; + const IoHash& LocalChunkHash = LocalChunkHashes[LocalChunkIndex]; + const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrder[OrderIndex]; + const IoHash& AbsoluteChunkHash = TmpAbsoluteChunkHashes[AbsoluteChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + OrderIndex++; + } + } +#endif // EXTRA_VERIFY + return AbsoluteChunkOrder; + } + + void CalculateLocalChunkOrders(const std::span& AbsoluteChunkOrders, + const std::span LooseChunkHashes, + const std::span LooseChunkRawSizes, + const std::span& BlockDescriptions, + std::vector& OutLocalChunkHashes, + std::vector& OutLocalChunkRawSizes, + std::vector& OutLocalChunkOrders) + { + std::vector AbsoluteChunkHashes; + std::vector AbsoluteChunkRawSizes; + AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end()); + AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), LooseChunkRawSizes.begin(), LooseChunkRawSizes.end()); + for (const ChunkBlockDescription& Block : BlockDescriptions) + { + AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), Block.ChunkRawHashes.begin(), Block.ChunkRawHashes.end()); + AbsoluteChunkRawSizes.insert(AbsoluteChunkRawSizes.end(), Block.ChunkRawLengths.begin(), Block.ChunkRawLengths.end()); + } + OutLocalChunkHashes.reserve(AbsoluteChunkHashes.size()); + OutLocalChunkRawSizes.reserve(AbsoluteChunkRawSizes.size()); + OutLocalChunkOrders.reserve(AbsoluteChunkOrders.size()); + + tsl::robin_map ChunkHashToChunkIndex; + ChunkHashToChunkIndex.reserve(AbsoluteChunkHashes.size()); + + for (uint32_t AbsoluteChunkOrderIndex = 0; AbsoluteChunkOrderIndex < AbsoluteChunkOrders.size(); AbsoluteChunkOrderIndex++) + { + const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[AbsoluteChunkOrderIndex]; + const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; + const uint64_t AbsoluteChunkRawSize = AbsoluteChunkRawSizes[AbsoluteChunkIndex]; + + if (auto It = ChunkHashToChunkIndex.find(AbsoluteChunkHash); It != ChunkHashToChunkIndex.end()) + { + const uint32_t LocalChunkIndex = It->second; + OutLocalChunkOrders.push_back(LocalChunkIndex); + } + else + { + uint32_t LocalChunkIndex = gsl::narrow(OutLocalChunkHashes.size()); + OutLocalChunkHashes.push_back(AbsoluteChunkHash); + OutLocalChunkRawSizes.push_back(AbsoluteChunkRawSize); + OutLocalChunkOrders.push_back(LocalChunkIndex); + ChunkHashToChunkIndex.insert_or_assign(AbsoluteChunkHash, LocalChunkIndex); + } +#if EXTRA_VERIFY + const uint32_t LocalChunkIndex = OutLocalChunkOrders[AbsoluteChunkOrderIndex]; + const IoHash& LocalChunkHash = OutLocalChunkHashes[LocalChunkIndex]; + const uint64_t& LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + ZEN_ASSERT(LocalChunkRawSize == AbsoluteChunkRawSize); +#endif // EXTRA_VERIFY + } +#if EXTRA_VERIFY + for (uint32_t OrderIndex = 0; OrderIndex < OutLocalChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = OutLocalChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = OutLocalChunkHashes[LocalChunkIndex]; + uint64_t LocalChunkRawSize = OutLocalChunkRawSizes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = AbsoluteChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; + uint64_t VerifyChunkRawSize = AbsoluteChunkRawSizes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); + } +#endif // EXTRA_VERIFY + } + + void WriteBuildContentToCompactBinary(CbObjectWriter& PartManifestWriter, + const SourcePlatform Platform, + std::span Paths, + std::span RawHashes, + std::span RawSizes, + std::span Attributes, + std::span SequenceRawHashes, + std::span ChunkCounts, + std::span LocalChunkHashes, + std::span LocalChunkRawSizes, + std::vector AbsoluteChunkOrders, + const std::span LooseLocalChunkIndexes, + const std::span BlockHashes) + { + ZEN_ASSERT(Platform != SourcePlatform::_Count); + PartManifestWriter.AddString("platform"sv, ToString(Platform)); + + uint64_t TotalSize = 0; + for (const uint64_t Size : RawSizes) + { + TotalSize += Size; + } + PartManifestWriter.AddInteger("totalSize", TotalSize); + + PartManifestWriter.BeginObject("files"sv); + { + compactbinary_helpers::WriteArray(Paths, "paths"sv, PartManifestWriter); + compactbinary_helpers::WriteArray(RawHashes, "rawhashes"sv, PartManifestWriter); + compactbinary_helpers::WriteArray(RawSizes, "rawsizes"sv, PartManifestWriter); + if (Platform == SourcePlatform::Windows) + { + compactbinary_helpers::WriteArray(Attributes, "attributes"sv, PartManifestWriter); + } + if (Platform == SourcePlatform::Linux || Platform == SourcePlatform::MacOS) + { + compactbinary_helpers::WriteArray(Attributes, "mode"sv, PartManifestWriter); + } + } + PartManifestWriter.EndObject(); // files + + PartManifestWriter.BeginObject("chunkedContent"); + { + compactbinary_helpers::WriteArray(SequenceRawHashes, "sequenceRawHashes"sv, PartManifestWriter); + compactbinary_helpers::WriteArray(ChunkCounts, "chunkcounts"sv, PartManifestWriter); + compactbinary_helpers::WriteArray(AbsoluteChunkOrders, "chunkorders"sv, PartManifestWriter); + } + PartManifestWriter.EndObject(); // chunkedContent + + size_t LooseChunkCount = LooseLocalChunkIndexes.size(); + if (LooseChunkCount > 0) + { + PartManifestWriter.BeginObject("chunkAttachments"); + { + PartManifestWriter.BeginArray("rawHashes"sv); + for (uint32_t ChunkIndex : LooseLocalChunkIndexes) + { + PartManifestWriter.AddBinaryAttachment(LocalChunkHashes[ChunkIndex]); + } + PartManifestWriter.EndArray(); // rawHashes + + PartManifestWriter.BeginArray("chunkRawSizes"sv); + for (uint32_t ChunkIndex : LooseLocalChunkIndexes) + { + PartManifestWriter.AddInteger(LocalChunkRawSizes[ChunkIndex]); + } + PartManifestWriter.EndArray(); // chunkSizes + } + PartManifestWriter.EndObject(); // + } + + if (BlockHashes.size() > 0) + { + PartManifestWriter.BeginObject("blockAttachments"); + { + compactbinary_helpers::WriteBinaryAttachmentArray(BlockHashes, "rawHashes"sv, PartManifestWriter); + } + PartManifestWriter.EndObject(); // blocks + } + } + + void ReadBuildContentFromCompactBinary(CbObjectView BuildPartManifest, + SourcePlatform& OutPlatform, + std::vector& OutPaths, + std::vector& OutRawHashes, + std::vector& OutRawSizes, + std::vector& OutAttributes, + std::vector& OutSequenceRawHashes, + std::vector& OutChunkCounts, + std::vector& OutAbsoluteChunkOrders, + std::vector& OutLooseChunkHashes, + std::vector& OutLooseChunkRawSizes, + std::vector& OutBlockRawHashes) + { + OutPlatform = FromString(BuildPartManifest["platform"sv].AsString(), SourcePlatform::_Count); + + CbObjectView FilesObject = BuildPartManifest["files"sv].AsObjectView(); + + compactbinary_helpers::ReadArray("paths"sv, FilesObject, OutPaths); + compactbinary_helpers::ReadArray("rawhashes"sv, FilesObject, OutRawHashes); + compactbinary_helpers::ReadArray("rawsizes"sv, FilesObject, OutRawSizes); + + uint64_t PathCount = OutPaths.size(); + if (OutRawHashes.size() != PathCount) + { + throw std::runtime_error(fmt::format("Number of raw hashes entries does not match number of paths")); + } + if (OutRawSizes.size() != PathCount) + { + throw std::runtime_error(fmt::format("Number of raw sizes entries does not match number of paths")); + } + + std::vector ModeArray; + compactbinary_helpers::ReadArray("mode"sv, FilesObject, ModeArray); + if (ModeArray.size() != PathCount && ModeArray.size() != 0) + { + throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths")); + } + + std::vector AttributeArray; + compactbinary_helpers::ReadArray("attributes"sv, FilesObject, ModeArray); + if (AttributeArray.size() != PathCount && AttributeArray.size() != 0) + { + throw std::runtime_error(fmt::format("Number of attribute entries does not match number of paths")); + } + + if (ModeArray.size() > 0) + { + if (OutPlatform == SourcePlatform::_Count) + { + OutPlatform = SourcePlatform::Linux; // Best guess - under dev format + } + OutAttributes = std::move(ModeArray); + } + else if (AttributeArray.size() > 0) + { + if (OutPlatform == SourcePlatform::_Count) + { + OutPlatform = SourcePlatform::Windows; + } + OutAttributes = std::move(AttributeArray); + } + else + { + if (OutPlatform == SourcePlatform::_Count) + { + OutPlatform = GetSourceCurrentPlatform(); + } + } + + if (FilesObject["chunkcounts"sv]) + { + // Legacy style + + std::vector LegacyChunkCounts; + compactbinary_helpers::ReadArray("chunkcounts"sv, FilesObject, LegacyChunkCounts); + if (LegacyChunkCounts.size() != PathCount) + { + throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths")); + } + std::vector LegacyAbsoluteChunkOrders; + compactbinary_helpers::ReadArray("chunkorders"sv, FilesObject, LegacyAbsoluteChunkOrders); + + CbArrayView ChunkOrdersArray = BuildPartManifest["chunkorders"sv].AsArrayView(); + const uint64_t ChunkOrdersCount = ChunkOrdersArray.Num(); + + tsl::robin_set FoundRawHashes; + FoundRawHashes.reserve(PathCount); + + OutChunkCounts.reserve(PathCount); + OutAbsoluteChunkOrders.reserve(ChunkOrdersCount); + + uint32_t OrderIndexOffset = 0; + for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++) + { + const IoHash& PathRawHash = OutRawHashes[PathIndex]; + uint32_t LegacyChunkCount = LegacyChunkCounts[PathIndex]; + + if (FoundRawHashes.insert(PathRawHash).second) + { + OutSequenceRawHashes.push_back(PathRawHash); + OutChunkCounts.push_back(LegacyChunkCount); + std::span AbsoluteChunkOrder = + std::span(LegacyAbsoluteChunkOrders).subspan(OrderIndexOffset, LegacyChunkCount); + OutAbsoluteChunkOrders.insert(OutAbsoluteChunkOrders.end(), AbsoluteChunkOrder.begin(), AbsoluteChunkOrder.end()); + } + OrderIndexOffset += LegacyChunkCounts[PathIndex]; + } + } + if (CbObjectView ChunkContentView = BuildPartManifest["chunkedContent"sv].AsObjectView(); ChunkContentView) + { + compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkContentView, OutSequenceRawHashes); + compactbinary_helpers::ReadArray("chunkcounts"sv, ChunkContentView, OutChunkCounts); + if (OutChunkCounts.size() != OutSequenceRawHashes.size()) + { + throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths")); + } + compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders); + } + + CbObjectView ChunkAttachmentsView = BuildPartManifest["chunkAttachments"sv].AsObjectView(); + { + compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView, OutLooseChunkHashes); + compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkAttachmentsView, OutLooseChunkRawSizes); + if (OutLooseChunkHashes.size() != OutLooseChunkRawSizes.size()) + { + throw std::runtime_error( + fmt::format("Number of attachment chunk hashes does not match number of attachemnt chunk raw sizes")); + } + } + + CbObjectView BlocksView = BuildPartManifest["blockAttachments"sv].AsObjectView(); + { + compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlocksView, OutBlockRawHashes); + } + } + + bool ReadStateObject(CbObjectView StateView, + Oid& OutBuildId, + std::vector& BuildPartsIds, + std::vector& BuildPartsNames, + std::vector& OutPartContents, + FolderContent& OutLocalFolderState) + { + try + { + CbObjectView BuildView = StateView["builds"sv].AsArrayView().CreateViewIterator().AsObjectView(); + OutBuildId = BuildView["buildId"sv].AsObjectId(); + for (CbFieldView PartView : BuildView["parts"sv].AsArrayView()) + { + CbObjectView PartObjectView = PartView.AsObjectView(); + BuildPartsIds.push_back(PartObjectView["partId"sv].AsObjectId()); + BuildPartsNames.push_back(std::string(PartObjectView["partName"sv].AsString())); + OutPartContents.push_back(LoadChunkedFolderContentToCompactBinary(PartObjectView["content"sv].AsObjectView())); + } + OutLocalFolderState = LoadFolderContentToCompactBinary(StateView["localFolderState"sv].AsObjectView()); + return true; + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Unable to read local state: ", Ex.what()); + return false; + } + } + + CbObject CreateStateObject(const Oid& BuildId, + std::vector> AllBuildParts, + std::span PartContents, + const FolderContent& LocalFolderState) + { + CbObjectWriter CurrentStateWriter; + CurrentStateWriter.BeginArray("builds"sv); + { + CurrentStateWriter.BeginObject(); + { + CurrentStateWriter.AddObjectId("buildId"sv, BuildId); + CurrentStateWriter.BeginArray("parts"sv); + for (size_t PartIndex = 0; PartIndex < AllBuildParts.size(); PartIndex++) + { + const Oid BuildPartId = AllBuildParts[PartIndex].first; + CurrentStateWriter.BeginObject(); + { + CurrentStateWriter.AddObjectId("partId"sv, BuildPartId); + CurrentStateWriter.AddString("partName"sv, AllBuildParts[PartIndex].second); + CurrentStateWriter.BeginObject("content"); + { + SaveChunkedFolderContentToCompactBinary(PartContents[PartIndex], CurrentStateWriter); + } + CurrentStateWriter.EndObject(); + } + CurrentStateWriter.EndObject(); + } + CurrentStateWriter.EndArray(); // parts + } + CurrentStateWriter.EndObject(); + } + CurrentStateWriter.EndArray(); // builds + + CurrentStateWriter.BeginObject("localFolderState"sv); + { + SaveFolderContentToCompactBinary(LocalFolderState, CurrentStateWriter); + } + CurrentStateWriter.EndObject(); // localFolderState + + return CurrentStateWriter.Save(); + } + + class BufferedOpenFile + { + public: + BufferedOpenFile(const std::filesystem::path Path) : Source(Path, BasicFile::Mode::kRead), SourceSize(Source.FileSize()) {} + BufferedOpenFile() = delete; + BufferedOpenFile(const BufferedOpenFile&) = delete; + BufferedOpenFile(BufferedOpenFile&&) = delete; + BufferedOpenFile& operator=(BufferedOpenFile&&) = delete; + BufferedOpenFile& operator=(const BufferedOpenFile&) = delete; + + const uint64_t BlockSize = 256u * 1024u; + CompositeBuffer GetRange(uint64_t Offset, uint64_t Size) + { + ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); + auto _ = MakeGuard([&]() { ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); }); + + ZEN_ASSERT((Offset + Size) <= SourceSize); + const uint64_t BlockIndexStart = Offset / BlockSize; + const uint64_t BlockIndexEnd = (Offset + Size) / BlockSize; + + std::vector BufferRanges; + BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1); + + uint64_t ReadOffset = Offset; + for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++) + { + const uint64_t BlockStartOffset = BlockIndex * BlockSize; + if (CacheBlockIndex != BlockIndex) + { + uint64_t CacheSize = Min(BlockSize, SourceSize - BlockStartOffset); + ZEN_ASSERT(CacheSize > 0); + Cache = IoBuffer(CacheSize); + Source.Read(Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset); + CacheBlockIndex = BlockIndex; + } + + const uint64_t BytesRead = ReadOffset - Offset; + ZEN_ASSERT(BlockStartOffset <= ReadOffset); + const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset; + ZEN_ASSERT(OffsetIntoBlock < Cache.GetSize()); + const uint64_t BlockBytes = Min(Cache.GetSize() - OffsetIntoBlock, Size - BytesRead); + BufferRanges.emplace_back(SharedBuffer(IoBuffer(Cache, OffsetIntoBlock, BlockBytes))); + ReadOffset += BlockBytes; + } + CompositeBuffer Result(std::move(BufferRanges)); + ZEN_ASSERT(Result.GetSize() == Size); + return Result; + } + + private: + BasicFile Source; + const uint64_t SourceSize; + uint64_t CacheBlockIndex = (uint64_t)-1; + IoBuffer Cache; + }; + + class ReadFileCache + { + public: + // A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten + ReadFileCache(DiskStatistics& DiskStats, + const std::filesystem::path& Path, + const std::vector& Paths, + const std::vector& RawSizes, + size_t MaxOpenFileCount) + : m_Path(Path) + , m_Paths(Paths) + , m_RawSizes(RawSizes) + , m_DiskStats(DiskStats) + { + m_OpenFiles.reserve(MaxOpenFileCount); + } + ~ReadFileCache() + { + m_DiskStats.CurrentOpenFileCount -= m_OpenFiles.size(); + m_OpenFiles.clear(); + } + + CompositeBuffer GetRange(uint32_t PathIndex, uint64_t Offset, uint64_t Size) + { + auto CacheIt = + std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [PathIndex](const auto& Lhs) { return Lhs.first == PathIndex; }); + if (CacheIt != m_OpenFiles.end()) + { + if (CacheIt != m_OpenFiles.begin()) + { + auto CachedFile(std::move(CacheIt->second)); + m_OpenFiles.erase(CacheIt); + m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(PathIndex, std::move(CachedFile))); + } + CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); + m_DiskStats.ReadByteCount += Result.GetSize(); + return Result; + } + const std::filesystem::path AttachmentPath = (m_Path / m_Paths[PathIndex]).make_preferred(); + if (Size == m_RawSizes[PathIndex]) + { + IoBuffer Result = IoBufferBuilder::MakeFromFile(AttachmentPath); + m_DiskStats.OpenReadCount++; + m_DiskStats.ReadByteCount += Result.GetSize(); + return CompositeBuffer(SharedBuffer(Result)); + } + if (m_OpenFiles.size() == m_OpenFiles.capacity()) + { + m_OpenFiles.pop_back(); + m_DiskStats.CurrentOpenFileCount--; + } + m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(PathIndex, std::make_unique(AttachmentPath))); + CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); + m_DiskStats.ReadByteCount += Result.GetSize(); + m_DiskStats.OpenReadCount++; + m_DiskStats.CurrentOpenFileCount++; + return Result; + } + + private: + const std::filesystem::path m_Path; + const std::vector& m_Paths; + const std::vector& m_RawSizes; + std::vector>> m_OpenFiles; + DiskStatistics& m_DiskStats; + }; + + CompositeBuffer ValidateBlob(BuildStorage& Storage, + const Oid& BuildId, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) + { + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); + if (!Payload) + { + throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash)); + } + if (Payload.GetContentType() != ZenContentType::kCompressedBinary) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'", + BlobHash, + Payload.GetSize(), + ToString(Payload.GetContentType()))); + } + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) compressed header is invalid", BlobHash, Payload.GetSize())); + } + if (RawHash != BlobHash) + { + throw std::runtime_error( + fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash)); + } + SharedBuffer Decompressed = Compressed.Decompress(); + if (!Decompressed) + { + throw std::runtime_error( + fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize())); + } + IoHash ValidateRawHash = IoHash::HashBuffer(Decompressed); + if (ValidateRawHash != BlobHash) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information", + BlobHash, + Payload.GetSize(), + ValidateRawHash)); + } + CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite(); + if (!DecompressedComposite) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize())); + } + OutCompressedSize = Payload.GetSize(); + OutDecompressedSize = RawSize; + return DecompressedComposite; + } + + ChunkBlockDescription ValidateChunkBlock(BuildStorage& Storage, + const Oid& BuildId, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) + { + CompositeBuffer BlockBuffer = ValidateBlob(Storage, BuildId, BlobHash, OutCompressedSize, OutDecompressedSize); + return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); + } + + CompositeBuffer FetchChunk(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const IoHash& ChunkHash, + ReadFileCache& OpenFileCache) + { + auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end()); + uint32_t ChunkIndex = It->second; + std::span ChunkLocations = GetChunkLocations(Lookup, ChunkIndex); + ZEN_ASSERT(!ChunkLocations.empty()); + CompositeBuffer Chunk = + OpenFileCache.GetRange(ChunkLocations[0].PathIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + return Chunk; + }; + + CompressedBuffer GenerateBlock(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + const std::vector& ChunksInBlock, + ChunkBlockDescription& OutBlockDescription, + DiskStatistics& DiskStats) + { + ReadFileCache OpenFileCache(DiskStats, Path, Content.Paths, Content.RawSizes, 4); + + std::vector> BlockContent; + BlockContent.reserve(ChunksInBlock.size()); + for (uint32_t ChunkIndex : ChunksInBlock) + { + BlockContent.emplace_back(std::make_pair( + Content.ChunkedContent.ChunkHashes[ChunkIndex], + [&Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair { + CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache); + if (!Chunk) + { + ZEN_ASSERT(false); + } + uint64_t RawSize = Chunk.GetSize(); + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + })); + } + + return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription); + }; + + void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + uint64_t MaxBlockSize, + std::vector& ChunkIndexes, + std::vector>& OutBlocks) + { + std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { + const ChunkedContentLookup::ChunkLocation& LhsLocation = GetChunkLocations(Lookup, Lhs)[0]; + const ChunkedContentLookup::ChunkLocation& RhsLocation = GetChunkLocations(Lookup, Rhs)[0]; + if (LhsLocation.PathIndex < RhsLocation.PathIndex) + { + return true; + } + else if (LhsLocation.PathIndex > RhsLocation.PathIndex) + { + return false; + } + return LhsLocation.Offset < RhsLocation.Offset; + }); + + uint64_t MaxBlockSizeLowThreshold = MaxBlockSize - (MaxBlockSize / 16); + + uint64_t BlockSize = 0; + + uint32_t ChunkIndexStart = 0; + for (uint32_t ChunkIndexOffset = 0; ChunkIndexOffset < ChunkIndexes.size();) + { + const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset]; + const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + + if ((BlockSize + ChunkSize) > MaxBlockSize) + { + // Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break + // between source paths for chunks. Break the block at the last such break if any. + ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart); + + const uint32_t ChunkPathIndex = Lookup.ChunkLocations[Lookup.ChunkLocationOffset[ChunkIndex]].PathIndex; + + uint64_t ScanBlockSize = BlockSize; + + uint32_t ScanChunkIndexOffset = ChunkIndexOffset - 1; + while (ScanChunkIndexOffset > (ChunkIndexStart + 2)) + { + const uint32_t TestChunkIndex = ChunkIndexes[ScanChunkIndexOffset]; + const uint64_t TestChunkSize = Content.ChunkedContent.ChunkRawSizes[TestChunkIndex]; + if ((ScanBlockSize - TestChunkSize) < MaxBlockSizeLowThreshold) + { + break; + } + + const uint32_t TestPathIndex = Lookup.ChunkLocations[Lookup.ChunkLocationOffset[TestChunkIndex]].PathIndex; + if (ChunkPathIndex != TestPathIndex) + { + ChunkIndexOffset = ScanChunkIndexOffset + 1; + break; + } + + ScanBlockSize -= TestChunkSize; + ScanChunkIndexOffset--; + } + + std::vector ChunksInBlock; + ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart); + for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++) + { + const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; + ChunksInBlock.push_back(AddChunkIndex); + } + OutBlocks.emplace_back(std::move(ChunksInBlock)); + BlockSize = 0; + ChunkIndexStart = ChunkIndexOffset; + } + else + { + ChunkIndexOffset++; + BlockSize += ChunkSize; + } + } + if (ChunkIndexStart < ChunkIndexes.size()) + { + std::vector ChunksInBlock; + ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart); + for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++) + { + const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset]; + ChunksInBlock.push_back(AddChunkIndex); + } + OutBlocks.emplace_back(std::move(ChunksInBlock)); + } + } + + CompositeBuffer CompressChunk(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex, + const std::filesystem::path& TempFolderPath) + { + const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + + const ChunkedContentLookup::ChunkLocation& Source = GetChunkLocations(Lookup, ChunkIndex)[0]; + + IoBuffer RawSource = + IoBufferBuilder::MakeFromFile((Path / Content.Paths[Source.PathIndex]).make_preferred(), Source.Offset, ChunkSize); + if (!RawSource) + { + throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash)); + } + if (RawSource.GetSize() != ChunkSize) + { + throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash)); + } + ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); + + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(RawSource))); + if (!CompressedBlob) + { + throw std::runtime_error(fmt::format("Failed decompressing chunk {}", ChunkHash)); + } + if (TempFolderPath.empty()) + { + return CompressedBlob.GetCompressed().MakeOwned(); + } + else + { + CompositeBuffer TempPayload = WriteToTempFileIfNeeded(CompressedBlob.GetCompressed(), TempFolderPath, ChunkHash); + return CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)).GetCompressed(); + } + } + + struct GeneratedBlocks + { + std::vector BlockDescriptions; + std::vector BlockSizes; + std::vector BlockBuffers; + std::vector BlockMetaDatas; + std::vector MetaDataHasBeenUploaded; + tsl::robin_map BlockHashToBlockIndex; + }; + + void GenerateBuildBlocks(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + BuildStorage& Storage, + const Oid& BuildId, + std::atomic& AbortFlag, + const std::vector>& NewBlockChunks, + GeneratedBlocks& OutBlocks, + DiskStatistics& DiskStats, + UploadStatistics& UploadStats, + GenerateBlocksStatistics& GenerateBlocksStats) + { + const std::size_t NewBlockCount = NewBlockChunks.size(); + if (NewBlockCount > 0) + { + ProgressBar ProgressBar(UsePlainProgress); + + OutBlocks.BlockDescriptions.resize(NewBlockCount); + OutBlocks.BlockSizes.resize(NewBlockCount); + OutBlocks.BlockBuffers.resize(NewBlockCount); + OutBlocks.BlockMetaDatas.resize(NewBlockCount); + OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, false); + OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount); + + RwLock Lock; + + WorkerThreadPool& GenerateBlobsPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + + FilteredRate FilteredGeneratedBytesPerSecond; + FilteredRate FilteredUploadedBytesPerSecond; + + ParallellWork Work(AbortFlag); + + std::atomic PendingUploadCount(0); + + for (size_t BlockIndex = 0; BlockIndex < NewBlockCount; BlockIndex++) + { + if (Work.IsAborted()) + { + break; + } + const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; + Work.ScheduleWork( + GenerateBlobsPool, + [&, BlockIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + FilteredGeneratedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function + + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); + ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlock.GetCompressedSize()), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + + OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); + + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompressedBlock.GetCompressed(), + Path / ZenTempBlockFolderName, + OutBlocks.BlockDescriptions[BlockIndex].BlockHash); + { + CbObjectWriter Writer; + Writer.AddString("createdBy", "zen"); + OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); + } + GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; + GenerateBlocksStats.GeneratedBlockCount++; + + Lock.WithExclusiveLock([&]() { + OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockIndex); + }); + + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + { + FilteredGeneratedBytesPerSecond.Stop(); + } + + if (!AbortFlag) + { + PendingUploadCount++; + Work.ScheduleWork( + UploadBlocksPool, + [&, BlockIndex, Payload = std::move(Payload)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + { + FilteredUploadedBytesPerSecond.Stop(); + OutBlocks.BlockBuffers[BlockIndex] = std::move(Payload); + } + else + { + FilteredUploadedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function + + PendingUploadCount--; + + const CbObject BlockMetaData = + BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], + OutBlocks.BlockMetaDatas[BlockIndex]); + + const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + UploadStats.BlocksBytes += Payload.GetSize(); + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(Payload.GetSize()), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + + Storage.PutBlockMetadata(BuildId, + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(BlockMetaData.GetSize())); + + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + UploadStats.BlockCount++; + if (UploadStats.BlockCount == NewBlockCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } + } + } + }, + [&](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed uploading block. Reason: {}", Ex.what()); + AbortFlag = true; + }); + } + } + }, + [&](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed generating block. Reason: {}", Ex.what()); + AbortFlag = true; + }); + } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + + FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); + FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); + + std::string Details = fmt::format("Generated {}/{} ({}, {}B/s) and uploaded {}/{} ({}, {}bits/s) blocks", + GenerateBlocksStats.GeneratedBlockCount.load(), + NewBlockCount, + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()), + UploadStats.BlockCount.load(), + NewBlockCount, + NiceBytes(UploadStats.BlocksBytes.load()), + NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8)); + + ProgressBar.UpdateState( + {.Task = "Generating blocks", + .Details = Details, + .TotalCount = gsl::narrow(NewBlockCount), + .RemainingCount = gsl::narrow(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load())}, + false); + }); + + ProgressBar.Finish(); + + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTime(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTime(); + } + } + + void UploadPartBlobs(BuildStorage& Storage, + const Oid& BuildId, + const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::span RawHashes, + const std::vector>& NewBlockChunks, + GeneratedBlocks& NewBlocks, + std::span LooseChunkIndexes, + const std::uint64_t LargeAttachmentSize, + std::atomic& AbortFlag, + DiskStatistics& DiskStats, + UploadStatistics& UploadStats, + GenerateBlocksStatistics& GenerateBlocksStats, + LooseChunksStatistics& LooseChunksStats) + { + { + ProgressBar ProgressBar(UsePlainProgress); + + WorkerThreadPool& ReadChunkPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& UploadChunkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + + FilteredRate FilteredGenerateBlockBytesPerSecond; + FilteredRate FilteredCompressedBytesPerSecond; + FilteredRate FilteredUploadedBytesPerSecond; + + ParallellWork Work(AbortFlag); + + std::atomic UploadedBlockSize = 0; + std::atomic UploadedBlockCount = 0; + std::atomic UploadedChunkSize = 0; + std::atomic UploadedChunkCount = 0; + + tsl::robin_map ChunkIndexToLooseChunkOrderIndex; + ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size()); + for (uint32_t OrderIndex = 0; OrderIndex < LooseChunkIndexes.size(); OrderIndex++) + { + ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex); + } + + std::vector FoundChunkHashes; + FoundChunkHashes.reserve(RawHashes.size()); + + std::vector BlockIndexes; + std::vector LooseChunkOrderIndexes; + + uint64_t TotalChunksSize = 0; + uint64_t TotalBlocksSize = 0; + for (const IoHash& RawHash : RawHashes) + { + if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end()) + { + BlockIndexes.push_back(It->second); + TotalBlocksSize += NewBlocks.BlockSizes[It->second]; + } + if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = ChunkIndexIt->second; + if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex); + LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end()) + { + LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second); + TotalChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + } + } + uint64_t TotalSize = TotalChunksSize + TotalBlocksSize; + + const size_t UploadBlockCount = BlockIndexes.size(); + const uint32_t UploadChunkCount = gsl::narrow(LooseChunkOrderIndexes.size()); + + auto AsyncUploadBlock = [&](const size_t BlockIndex, const IoHash BlockHash, CompositeBuffer&& Payload) { + Work.ScheduleWork( + UploadChunkPool, + [&, BlockIndex, BlockHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic& AbortFlag) { + if (!AbortFlag) + { + FilteredUploadedBytesPerSecond.Start(); + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + + Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(Payload.GetSize()), + NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + UploadedBlockSize += Payload.GetSize(); + UploadStats.BlocksBytes += Payload.GetSize(); + + Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(BlockMetaData.GetSize())); + + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + + UploadStats.BlockCount++; + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + + UploadedBlockCount++; + if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } + } + }, + [&](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed uploading block. Reason: {}", Ex.what()); + AbortFlag = true; + }); + }; + + auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, CompositeBuffer&& Payload) { + Work.ScheduleWork( + UploadChunkPool, + [&, RawHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic& AbortFlag) { + if (!AbortFlag) + { + const uint64_t PayloadSize = Payload.GetSize(); + if (PayloadSize >= LargeAttachmentSize) + { + UploadStats.MultipartAttachmentCount++; + std::vector> MultipartWork = Storage.PutLargeBuildBlob( + BuildId, + RawHash, + ZenContentType::kCompressedBinary, + PayloadSize, + [Payload = std::move(Payload), &FilteredUploadedBytesPerSecond](uint64_t Offset, + uint64_t Size) -> IoBuffer { + FilteredUploadedBytesPerSecond.Start(); + + IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer(); + PartPayload.SetContentType(ZenContentType::kBinary); + return PartPayload; + }, + [&](uint64_t SentBytes, bool IsComplete) { + UploadStats.ChunksBytes += SentBytes; + UploadedChunkSize += SentBytes; + if (IsComplete) + { + UploadStats.ChunkCount++; + UploadedChunkCount++; + if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } + } + }); + for (auto& WorkPart : MultipartWork) + { + Work.ScheduleWork( + UploadChunkPool, + [Work = std::move(WorkPart)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + Work(); + } + }, + [&, RawHash](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed uploading multipart blob {}. Reason: {}", RawHash, Ex.what()); + AbortFlag = true; + }); + } + ZEN_CONSOLE_VERBOSE("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize)); + } + else + { + Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); + UploadStats.ChunksBytes += Payload.GetSize(); + UploadStats.ChunkCount++; + UploadedChunkSize += Payload.GetSize(); + UploadedChunkCount++; + if (UploadedChunkCount == UploadChunkCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } + } + } + }, + [&](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed uploading chunk. Reason: {}", Ex.what()); + AbortFlag = true; + }); + }; + + std::vector GenerateBlockIndexes; + + std::atomic GeneratedBlockCount = 0; + std::atomic GeneratedBlockByteCount = 0; + + // Start upload of any pre-built blocks + for (const size_t BlockIndex : BlockIndexes) + { + if (CompositeBuffer BlockPayload = std::move(NewBlocks.BlockBuffers[BlockIndex]); BlockPayload) + { + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + FoundChunkHashes.push_back(BlockHash); + if (!AbortFlag) + { + AsyncUploadBlock(BlockIndex, BlockHash, std::move(BlockPayload)); + } + // GeneratedBlockCount++; + } + else + { + GenerateBlockIndexes.push_back(BlockIndex); + } + } + + std::vector CompressLooseChunkOrderIndexes; + + // Start upload of any pre-compressed loose chunks + for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) + { + CompressLooseChunkOrderIndexes.push_back(LooseChunkOrderIndex); + } + + // Start generation of any non-prebuilt blocks and schedule upload + for (const size_t BlockIndex : GenerateBlockIndexes) + { + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + FoundChunkHashes.push_back(BlockHash); + if (!AbortFlag) + { + Work.ScheduleWork( + ReadChunkPool, + [&, BlockIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + FilteredGenerateBlockBytesPerSecond.Start(); + ChunkBlockDescription BlockDescription; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); + if (!CompressedBlock) + { + throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + } + ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); + + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompressedBlock.GetCompressed(), + Path / ZenTempBlockFolderName, + BlockDescription.BlockHash); + + GenerateBlocksStats.GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; + GenerateBlocksStats.GeneratedBlockCount++; + GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; + GeneratedBlockCount++; + if (GeneratedBlockCount == GenerateBlockIndexes.size()) + { + FilteredGenerateBlockBytesPerSecond.Stop(); + } + if (!AbortFlag) + { + AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload)); + } + ZEN_CONSOLE_VERBOSE("Regenerated block {} ({}) containing {} chunks", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlock.GetCompressedSize()), + NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + } + }, + [&](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed generating block. Reason: {}", Ex.what()); + AbortFlag = true; + }); + } + } + + std::atomic CompressedLooseChunkCount = 0; + std::atomic CompressedLooseChunkByteCount = 0; + + // Start compression of any non-precompressed loose chunks and schedule upload + for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) + { + const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; + Work.ScheduleWork( + ReadChunkPool, + [&, ChunkIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + FilteredCompressedBytesPerSecond.Start(); + CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName); + ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", + Content.ChunkedContent.ChunkHashes[ChunkIndex], + NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), + NiceBytes(Payload.GetSize())); + UploadStats.ReadFromDiskBytes += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + LooseChunksStats.CompressedChunkBytes += Payload.GetSize(); + LooseChunksStats.CompressedChunkCount++; + CompressedLooseChunkByteCount += Payload.GetSize(); + CompressedLooseChunkCount++; + if (CompressedLooseChunkCount == CompressLooseChunkOrderIndexes.size()) + { + FilteredCompressedBytesPerSecond.Stop(); + } + if (!AbortFlag) + { + AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], std::move(Payload)); + } + } + }, + [&, ChunkIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed compressing part blob {}. Reason: {}", + Content.ChunkedContent.ChunkHashes[ChunkIndex], + Ex.what()); + AbortFlag = true; + }); + } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + FilteredCompressedBytesPerSecond.Update(CompressedLooseChunkByteCount.load()); + FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); + FilteredUploadedBytesPerSecond.Update(UploadedChunkSize.load() + UploadedBlockSize.load()); + uint64_t UploadedSize = UploadedChunkSize.load() + UploadedBlockSize.load(); + ProgressBar.UpdateState({.Task = "Uploading blobs ", + .Details = fmt::format("Compressed {}/{} chunks. " + "Uploaded {}/{} blobs ({}/{} {}bits/s)", + CompressedLooseChunkCount.load(), + CompressLooseChunkOrderIndexes.size(), + + UploadedBlockCount.load() + UploadedChunkCount.load(), + UploadBlockCount + UploadChunkCount, + + NiceBytes(UploadedChunkSize.load() + UploadedBlockSize.load()), + NiceBytes(TotalSize), + NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())), + .TotalCount = gsl::narrow(TotalSize), + .RemainingCount = gsl::narrow(TotalSize - UploadedSize)}, + false); + }); + + ProgressBar.Finish(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTime(); + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTime(); + LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTime(); + } + } + + std::vector FindReuseBlocks(const std::vector& KnownBlocks, + std::span ChunkHashes, + std::span ChunkIndexes, + uint8_t MinPercentLimit, + std::vector& OutUnusedChunkIndexes, + FindBlocksStatistics& FindBlocksStats) + { + // Find all blocks with a usage level higher than MinPercentLimit + // Pick out the blocks with usage higher or equal to MinPercentLimit + // Sort them with highest size usage - most usage first + // Make a list of all chunks and mark them as not found + // For each block, recalculate the block has usage percent based on the chunks marked as not found + // If the block still reaches MinPercentLimit, keep it and remove the matching chunks from the not found list + // Repeat for following all remaining block that initially matched MinPercentLimit + + std::vector FilteredReuseBlockIndexes; + + uint32_t ChunkCount = gsl::narrow(ChunkHashes.size()); + std::vector ChunkFound(ChunkCount, false); + + if (ChunkCount > 0) + { + if (!KnownBlocks.empty()) + { + Stopwatch ReuseTimer; + + tsl::robin_map ChunkHashToChunkIndex; + ChunkHashToChunkIndex.reserve(ChunkIndexes.size()); + for (uint32_t ChunkIndex : ChunkIndexes) + { + ChunkHashToChunkIndex.insert_or_assign(ChunkHashes[ChunkIndex], ChunkIndex); + } + + std::vector BlockSizes(KnownBlocks.size(), 0); + std::vector BlockUseSize(KnownBlocks.size(), 0); + + std::vector ReuseBlockIndexes; + + for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++) + { + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + if (KnownBlock.BlockHash != IoHash::Zero && + KnownBlock.ChunkRawHashes.size() == KnownBlock.ChunkCompressedLengths.size()) + { + size_t BlockAttachmentCount = KnownBlock.ChunkRawHashes.size(); + if (BlockAttachmentCount == 0) + { + continue; + } + size_t ReuseSize = 0; + size_t BlockSize = 0; + size_t FoundAttachmentCount = 0; + size_t BlockChunkCount = KnownBlock.ChunkRawHashes.size(); + for (size_t BlockChunkIndex = 0; BlockChunkIndex < BlockChunkCount; BlockChunkIndex++) + { + const IoHash& BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex]; + const uint32_t BlockChunkSize = KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + BlockSize += BlockChunkSize; + if (ChunkHashToChunkIndex.contains(BlockChunkHash)) + { + ReuseSize += BlockChunkSize; + FoundAttachmentCount++; + } + } + + size_t ReusePercent = (ReuseSize * 100) / BlockSize; + + if (ReusePercent >= MinPercentLimit) + { + ZEN_CONSOLE_VERBOSE("Reusing block {}. {} attachments found, usage level: {}%", + KnownBlock.BlockHash, + FoundAttachmentCount, + ReusePercent); + ReuseBlockIndexes.push_back(KnownBlockIndex); + + BlockSizes[KnownBlockIndex] = BlockSize; + BlockUseSize[KnownBlockIndex] = ReuseSize; + } + else if (FoundAttachmentCount > 0) + { + ZEN_CONSOLE_VERBOSE("Skipping block {}. {} attachments found, usage level: {}%", + KnownBlock.BlockHash, + FoundAttachmentCount, + ReusePercent); + FindBlocksStats.RejectedBlockCount++; + FindBlocksStats.RejectedChunkCount += FoundAttachmentCount; + FindBlocksStats.RejectedByteCount += ReuseSize; + } + } + } + + if (!ReuseBlockIndexes.empty()) + { + std::sort(ReuseBlockIndexes.begin(), ReuseBlockIndexes.end(), [&](size_t Lhs, size_t Rhs) { + return BlockUseSize[Lhs] > BlockUseSize[Rhs]; + }); + + for (size_t KnownBlockIndex : ReuseBlockIndexes) + { + std::vector FoundChunkIndexes; + size_t BlockSize = 0; + size_t AdjustedReuseSize = 0; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + for (size_t BlockChunkIndex = 0; BlockChunkIndex < KnownBlock.ChunkRawHashes.size(); BlockChunkIndex++) + { + const IoHash& BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex]; + const uint32_t BlockChunkSize = KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + BlockSize += BlockChunkSize; + if (auto It = ChunkHashToChunkIndex.find(BlockChunkHash); It != ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = It->second; + if (!ChunkFound[ChunkIndex]) + { + FoundChunkIndexes.push_back(ChunkIndex); + AdjustedReuseSize += KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + } + } + } + + size_t ReusePercent = (AdjustedReuseSize * 100) / BlockSize; + + if (ReusePercent >= MinPercentLimit) + { + ZEN_CONSOLE_VERBOSE("Reusing block {}. {} attachments found, usage level: {}%", + KnownBlock.BlockHash, + FoundChunkIndexes.size(), + ReusePercent); + FilteredReuseBlockIndexes.push_back(KnownBlockIndex); + + for (uint32_t ChunkIndex : FoundChunkIndexes) + { + ChunkFound[ChunkIndex] = true; + } + FindBlocksStats.AcceptedChunkCount += FoundChunkIndexes.size(); + FindBlocksStats.AcceptedByteCount += AdjustedReuseSize; + FindBlocksStats.AcceptedReduntantChunkCount += KnownBlock.ChunkRawHashes.size() - FoundChunkIndexes.size(); + FindBlocksStats.AcceptedReduntantByteCount += BlockSize - AdjustedReuseSize; + } + else + { + ZEN_CONSOLE_VERBOSE("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent); + FindBlocksStats.RejectedBlockCount++; + FindBlocksStats.RejectedChunkCount += FoundChunkIndexes.size(); + FindBlocksStats.RejectedByteCount += AdjustedReuseSize; + } + } + } + } + OutUnusedChunkIndexes.reserve(ChunkIndexes.size() - FindBlocksStats.AcceptedChunkCount); + for (uint32_t ChunkIndex : ChunkIndexes) + { + if (!ChunkFound[ChunkIndex]) + { + OutUnusedChunkIndexes.push_back(ChunkIndex); + } + } + } + return FilteredReuseBlockIndexes; + }; + + bool UploadFolder(BuildStorage& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + const std::string_view BuildPartName, + const std::filesystem::path& Path, + const std::filesystem::path& ManifestPath, + const uint8_t BlockReuseMinPercentLimit, + bool AllowMultiparts, + const CbObject& MetaData, + bool CreateBuild, + bool IgnoreExistingBlocks) + { + Stopwatch ProcessTimer; + + std::atomic AbortFlag = false; + + const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + CreateDirectories(ZenTempFolder); + CleanDirectory(ZenTempFolder, {}); + auto _ = MakeGuard([&]() { + CleanDirectory(ZenTempFolder, {}); + std::filesystem::remove(ZenTempFolder); + }); + CreateDirectories(Path / ZenTempBlockFolderName); + CreateDirectories(Path / ZenTempChunkFolderName); + + CbObject ChunkerParameters; + + ChunkedFolderContent LocalContent; + + GetFolderContentStatistics LocalFolderScanStats; + ChunkingStatistics ChunkingStats; + { + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + auto IsAcceptedFile = [ExcludeExtensions = + DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { + for (const std::string_view& ExcludeExtension : ExcludeExtensions) + { + if (RelativePath.ends_with(ExcludeExtension)) + { + return false; + } + } + return true; + }; + + auto ParseManifest = [](const std::filesystem::path& Path, + const std::filesystem::path& ManifestPath) -> std::vector { + std::vector AssetPaths; + std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath; + IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); + std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); + std::string_view::size_type Offset = 0; + while (Offset < ManifestContent.GetSize()) + { + size_t PathBreakOffset = ManifestString.find_first_of("\t\r\n", Offset); + if (PathBreakOffset == std::string_view::npos) + { + PathBreakOffset = ManifestContent.GetSize(); + } + std::string_view AssetPath = ManifestString.substr(Offset, PathBreakOffset - Offset); + if (!AssetPath.empty()) + { + AssetPaths.emplace_back(std::filesystem::path(AssetPath)); + } + Offset = PathBreakOffset; + size_t EolOffset = ManifestString.find_first_of("\r\n", Offset); + if (EolOffset == std::string_view::npos) + { + break; + } + Offset = EolOffset; + size_t LineBreakOffset = ManifestString.find_first_not_of("\t\r\n", Offset); + if (LineBreakOffset == std::string_view::npos) + { + break; + } + Offset = LineBreakOffset; + } + return AssetPaths; + }; + + Stopwatch ScanTimer; + FolderContent Content; + if (ManifestPath.empty()) + { + std::filesystem::path ExcludeManifestPath = Path / ZenExcludeManifestName; + tsl::robin_set ExcludeAssetPaths; + if (std::filesystem::is_regular_file(ExcludeManifestPath)) + { + std::vector AssetPaths = ParseManifest(Path, ExcludeManifestPath); + ExcludeAssetPaths.reserve(AssetPaths.size()); + for (const std::filesystem::path& AssetPath : AssetPaths) + { + ExcludeAssetPaths.insert(AssetPath.generic_string()); + } + } + Content = GetFolderContent( + LocalFolderScanStats, + Path, + std::move(IsAcceptedFolder), + [&IsAcceptedFile, + &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool { + if (RelativePath == ZenExcludeManifestName) + { + return false; + } + if (!IsAcceptedFile(RelativePath, Size, Attributes)) + { + return false; + } + if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string())) + { + return false; + } + return true; + }, + GetMediumWorkerPool(EWorkloadType::Burst), + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + }, + AbortFlag); + } + else + { + Stopwatch ManifestParseTimer; + std::vector AssetPaths = ParseManifest(Path, ManifestPath); + for (const std::filesystem::path& AssetPath : AssetPaths) + { + Content.Paths.push_back(AssetPath); + Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath)); +#if ZEN_PLATFORM_WINDOWS + Content.Attributes.push_back(GetFileAttributes(Path / AssetPath)); +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + Content.Attributes.push_back(GetFileMode(Path / AssetPath)); +#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); + LocalFolderScanStats.AcceptedFileCount++; + } + if (ManifestPath.is_relative()) + { + Content.Paths.push_back(ManifestPath); + Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath)); +#if ZEN_PLATFORM_WINDOWS + Content.Attributes.push_back(GetFileAttributes(ManifestPath)); +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + Content.Attributes.push_back(GetFileMode(ManifestPath)); +#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + + LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); + LocalFolderScanStats.AcceptedFileCount++; + } + LocalFolderScanStats.FoundFileByteCount.store(LocalFolderScanStats.AcceptedFileByteCount); + LocalFolderScanStats.FoundFileCount.store(LocalFolderScanStats.AcceptedFileCount); + LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs(); + } + + std::unique_ptr ChunkController = CreateBasicChunkingController(); + { + CbObjectWriter ChunkParametersWriter; + ChunkParametersWriter.AddString("name"sv, ChunkController->GetName()); + ChunkParametersWriter.AddObject("parameters"sv, ChunkController->GetParameters()); + ChunkerParameters = ChunkParametersWriter.Save(); + } + + std::uint64_t TotalRawSize = 0; + for (uint64_t RawSize : Content.RawSizes) + { + TotalRawSize += RawSize; + } + { + ProgressBar ProgressBar(UsePlainProgress); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + LocalContent = ChunkFolderContent( + ChunkingStats, + GetMediumWorkerPool(EWorkloadType::Burst), + Path, + Content, + *ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + Content.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(TotalRawSize), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .TotalCount = TotalRawSize, + .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + } + + if (AbortFlag) + { + return true; + } + + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + LocalContent.Paths.size(), + NiceBytes(TotalRawSize), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + Path, + NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + } + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + if (CreateBuild) + { + Stopwatch PutBuildTimer; + CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); + ZEN_CONSOLE("PutBuild took {}. Payload size: {}", + NiceLatencyNs(PutBuildTimer.GetElapsedTimeUs() * 1000), + NiceBytes(MetaData.GetSize())); + PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(PreferredMultipartChunkSize); + } + else + { + Stopwatch GetBuildTimer; + CbObject Build = Storage.GetBuild(BuildId); + ZEN_CONSOLE("GetBuild took {}. Payload size: {}", + NiceLatencyNs(GetBuildTimer.GetElapsedTimeUs() * 1000), + NiceBytes(Build.GetSize())); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + PreferredMultipartChunkSize = ChunkSize; + } + else if (AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(PreferredMultipartChunkSize)); + } + } + + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + + FindBlocksStatistics FindBlocksStats; + GenerateBlocksStatistics GenerateBlocksStats; + LooseChunksStatistics LooseChunksStats; + + std::vector KnownBlocks; + std::vector ReuseBlockIndexes; + std::vector NewBlockChunkIndexes; + Stopwatch BlockArrangeTimer; + + std::vector LooseChunkIndexes; + { + bool EnableBlocks = true; + std::vector BlockChunkIndexes; + for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + { + const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > DefaultChunksBlockParams.MaxChunkEmbedSize) + { + LooseChunkIndexes.push_back(ChunkIndex); + LooseChunksStats.ChunkByteCount += ChunkRawSize; + } + else + { + BlockChunkIndexes.push_back(ChunkIndex); + FindBlocksStats.PotentialChunkByteCount += ChunkRawSize; + } + } + FindBlocksStats.PotentialChunkCount = BlockChunkIndexes.size(); + LooseChunksStats.ChunkCount = LooseChunkIndexes.size(); + + if (IgnoreExistingBlocks) + { + ZEN_CONSOLE("Ignoring any existing blocks in store"); + NewBlockChunkIndexes = std::move(BlockChunkIndexes); + } + else + { + Stopwatch KnownBlocksTimer; + KnownBlocks = Storage.FindBlocks(BuildId); + FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); + FindBlocksStats.FoundBlockCount = KnownBlocks.size(); + + ReuseBlockIndexes = FindReuseBlocks(KnownBlocks, + LocalContent.ChunkedContent.ChunkHashes, + BlockChunkIndexes, + BlockReuseMinPercentLimit, + NewBlockChunkIndexes, + FindBlocksStats); + FindBlocksStats.AcceptedBlockCount = ReuseBlockIndexes.size(); + + for (const ChunkBlockDescription& Description : KnownBlocks) + { + for (uint32_t ChunkRawLength : Description.ChunkRawLengths) + { + FindBlocksStats.FoundBlockByteCount += ChunkRawLength; + } + FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size(); + } + } + } + + std::vector> NewBlockChunks; + ArrangeChunksIntoBlocks(LocalContent, LocalLookup, DefaultChunksBlockParams.MaxBlockSize, NewBlockChunkIndexes, NewBlockChunks); + + FindBlocksStats.NewBlocksCount = NewBlockChunks.size(); + for (uint32_t ChunkIndex : NewBlockChunkIndexes) + { + FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size(); + + const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * FindBlocksStats.AcceptedByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; + + const double AcceptedReduntantByteCountPercent = + FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / + (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) + : 0.0; + ZEN_CONSOLE( + "Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n" + " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" + " Accepting {} ({}) redundant chunks ({:.1f}%)\n" + " Rejected {} ({}) chunks in {} blocks\n" + " Arranged {} ({}) chunks in {} new blocks\n" + " Keeping {} ({}) chunks as loose chunks\n" + " Discovery completed in {}", + FindBlocksStats.FoundBlockChunkCount, + FindBlocksStats.FoundBlockCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + + FindBlocksStats.AcceptedChunkCount, + NiceBytes(FindBlocksStats.AcceptedByteCount), + FindBlocksStats.AcceptedBlockCount, + AcceptedByteCountPercent, + + FindBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), + AcceptedReduntantByteCountPercent, + + FindBlocksStats.RejectedChunkCount, + NiceBytes(FindBlocksStats.RejectedByteCount), + FindBlocksStats.RejectedBlockCount, + + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), + FindBlocksStats.NewBlocksCount, + + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), + + NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs())); + + DiskStatistics DiskStats; + UploadStatistics UploadStats; + GeneratedBlocks NewBlocks; + + if (!NewBlockChunks.empty()) + { + Stopwatch GenerateBuildBlocksTimer; + auto __ = MakeGuard([&]() { + uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs(); + ZEN_CONSOLE("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.", + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount), + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + NiceTimeSpanMs(BlockGenerateTimeUs / 1000), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + GenerateBlocksStats.GeneratedBlockByteCount)), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8))); + }); + GenerateBuildBlocks(Path, + LocalContent, + LocalLookup, + Storage, + BuildId, + AbortFlag, + NewBlockChunks, + NewBlocks, + DiskStats, + UploadStats, + GenerateBlocksStats); + } + + if (AbortFlag) + { + return true; + } + + CbObject PartManifest; + { + CbObjectWriter PartManifestWriter; + Stopwatch ManifestGenerationTimer; + auto __ = MakeGuard([&]() { + ZEN_CONSOLE("Generated build part manifest in {} ({})", + NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()), + NiceBytes(PartManifestWriter.GetSaveSize())); + }); + PartManifestWriter.AddObject("chunker"sv, ChunkerParameters); + + std::vector AllChunkBlockHashes; + std::vector AllChunkBlockDescriptions; + AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); + for (size_t ReuseBlockIndex : ReuseBlockIndexes) + { + AllChunkBlockDescriptions.push_back(KnownBlocks[ReuseBlockIndex]); + AllChunkBlockHashes.push_back(KnownBlocks[ReuseBlockIndex].BlockHash); + } + AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(), + NewBlocks.BlockDescriptions.begin(), + NewBlocks.BlockDescriptions.end()); + for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions) + { + AllChunkBlockHashes.push_back(BlockDescription.BlockHash); + } +#if EXTRA_VERIFY + tsl::robin_map ChunkHashToAbsoluteChunkIndex; + std::vector AbsoluteChunkHashes; + AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size()); + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions) + { + for (const IoHash& ChunkHash : Block.ChunkHashes) + { + ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()}); + AbsoluteChunkHashes.push_back(ChunkHash); + } + } + for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash); + } + for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders) + { + ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] == + LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex); + } +#endif // EXTRA_VERIFY + std::vector AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkOrders, + LocalLookup.ChunkHashToChunkIndex, + LooseChunkIndexes, + AllChunkBlockDescriptions); + +#if EXTRA_VERIFY + for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex]; + uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex]; + const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex]; + ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash); + } +#endif // EXTRA_VERIFY + + WriteBuildContentToCompactBinary(PartManifestWriter, + LocalContent.Platform, + LocalContent.Paths, + LocalContent.RawHashes, + LocalContent.RawSizes, + LocalContent.Attributes, + LocalContent.ChunkedContent.SequenceRawHashes, + LocalContent.ChunkedContent.ChunkCounts, + LocalContent.ChunkedContent.ChunkHashes, + LocalContent.ChunkedContent.ChunkRawSizes, + AbsoluteChunkOrders, + LooseChunkIndexes, + AllChunkBlockHashes); + +#if EXTRA_VERIFY + { + ChunkedFolderContent VerifyFolderContent; + + std::vector OutAbsoluteChunkOrders; + std::vector OutLooseChunkHashes; + std::vector OutLooseChunkRawSizes; + std::vector OutBlockRawHashes; + + ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), + VerifyFolderContent.Platform, + VerifyFolderContent.Paths, + VerifyFolderContent.RawHashes, + VerifyFolderContent.RawSizes, + VerifyFolderContent.Attributes, + VerifyFolderContent.ChunkedContent.SequenceRawHashes, + VerifyFolderContent.ChunkedContent.ChunkCounts, + OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + OutBlockRawHashes); + ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes); + + for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + } + + CalculateLocalChunkOrders(OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + AllChunkBlockDescriptions, + VerifyFolderContent.ChunkedContent.ChunkHashes, + VerifyFolderContent.ChunkedContent.ChunkRawSizes, + VerifyFolderContent.ChunkedContent.ChunkOrders); + + ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths); + ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes); + ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes); + ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes); + ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes); + ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts); + + for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++) + { + uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + + uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex]; + const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex]; + uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex]; + + ZEN_ASSERT(LocalChunkHash == VerifyChunkHash); + ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize); + } + } +#endif // EXTRA_VERIFY + PartManifest = PartManifestWriter.Save(); + } + + Stopwatch PutBuildPartResultTimer; + std::pair> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); + ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are missing.", + NiceLatencyNs(PutBuildPartResultTimer.GetElapsedTimeUs() * 1000), + NiceBytes(PartManifest.GetSize()), + PutBuildPartResult.second.size()); + IoHash PartHash = PutBuildPartResult.first; + + auto UploadAttachments = [&](std::span RawHashes) { + if (!AbortFlag) + { + ZEN_CONSOLE_VERBOSE("Uploading attachments: {}", FormatArray(RawHashes, "\n "sv)); + + UploadStatistics TempUploadStats; + GenerateBlocksStatistics TempGenerateBlocksStats; + LooseChunksStatistics TempLooseChunksStats; + + Stopwatch TempUploadTimer; + auto __ = MakeGuard([&]() { + uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); + ZEN_CONSOLE( + "Generated {} ({} {}B/s) and uploaded {} ({}) blocks. " + "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " + "Transferred {} ({}B/s) in {}", + TempGenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(TempGenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(GetBytesPerSecond(TempGenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, + TempGenerateBlocksStats.GeneratedBlockByteCount)), + TempUploadStats.BlockCount.load(), + NiceBytes(TempUploadStats.BlocksBytes), + + TempLooseChunksStats.CompressedChunkCount.load(), + NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, + TempLooseChunksStats.CompressedChunkBytes)), + TempUploadStats.ChunkCount.load(), + NiceBytes(TempUploadStats.ChunksBytes), + + NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)), + NiceTimeSpanMs(TempChunkUploadTimeUs / 1000)); + }); + UploadPartBlobs(Storage, + BuildId, + Path, + LocalContent, + LocalLookup, + RawHashes, + NewBlockChunks, + NewBlocks, + LooseChunkIndexes, + LargeAttachmentSize, + AbortFlag, + DiskStats, + TempUploadStats, + TempGenerateBlocksStats, + TempLooseChunksStats); + UploadStats += TempUploadStats; + LooseChunksStats += TempLooseChunksStats; + GenerateBlocksStats += TempGenerateBlocksStats; + } + }; + if (IgnoreExistingBlocks) + { + ZEN_CONSOLE_VERBOSE("PutBuildPart uploading all attachments, needs are: {}", + FormatArray(PutBuildPartResult.second, "\n "sv)); + + std::vector ForceUploadChunkHashes; + ForceUploadChunkHashes.reserve(LooseChunkIndexes.size()); + + for (uint32_t ChunkIndex : LooseChunkIndexes) + { + ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); + } + + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockBuffers.size(); BlockIndex++) + { + if (NewBlocks.BlockBuffers[BlockIndex]) + { + // Block was not uploaded during generation + ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); + } + } + UploadAttachments(ForceUploadChunkHashes); + } + else if (!PutBuildPartResult.second.empty()) + { + ZEN_CONSOLE_VERBOSE("PutBuildPart needs attachments: {}", FormatArray(PutBuildPartResult.second, "\n "sv)); + UploadAttachments(PutBuildPartResult.second); + } + + while (true) + { + Stopwatch FinalizeBuildPartTimer; + std::vector Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); + ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", + NiceLatencyNs(FinalizeBuildPartTimer.GetElapsedTimeUs() * 1000), + Needs.size()); + if (Needs.empty()) + { + break; + } + if (AbortFlag) + { + return true; + } + ZEN_CONSOLE_VERBOSE("FinalizeBuildPart needs attachments: {}", FormatArray(Needs, "\n "sv)); + UploadAttachments(Needs); + if (AbortFlag) + { + return true; + } + } + + if (AbortFlag) + { + return true; + } + + if (CreateBuild) + { + Stopwatch FinalizeBuildTimer; + Storage.FinalizeBuild(BuildId); + ZEN_CONSOLE("FinalizeBuild took {}", NiceLatencyNs(FinalizeBuildTimer.GetElapsedTimeUs() * 1000)); + } + + if (!NewBlocks.BlockDescriptions.empty()) + { + uint64_t UploadBlockMetadataCount = 0; + std::vector BlockHashes; + BlockHashes.reserve(NewBlocks.BlockDescriptions.size()); + Stopwatch UploadBlockMetadataTimer; + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) + { + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + { + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadBlockMetadataCount++; + } + BlockHashes.push_back(BlockHash); + } + if (UploadBlockMetadataCount > 0) + { + uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs(); + UploadStats.ElapsedWallTimeUS += ElapsedUS; + ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); + } + + std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockHashes); + if (VerifyBlockDescriptions.size() != BlockHashes.size()) + { + ZEN_CONSOLE("Uploaded blocks could not all be found, {} blocks are missing", + BlockHashes.size() - VerifyBlockDescriptions.size()); + return true; + } + } + + const double DeltaByteCountPercent = + ChunkingStats.BytesHashed > 0 + ? (100.0 * (FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes)) / (ChunkingStats.BytesHashed) + : 0.0; + + const std::string LargeAttachmentStats = + (LargeAttachmentSize != (uint64_t)-1) ? fmt::format(" ({} as multipart)", UploadStats.MultipartAttachmentCount.load()) : ""; + + ZEN_CONSOLE_VERBOSE( + "Folder scanning stats:" + "\n FoundFileCount: {}" + "\n FoundFileByteCount: {}" + "\n AcceptedFileCount: {}" + "\n AcceptedFileByteCount: {}" + "\n ElapsedWallTimeUS: {}", + LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), + LocalFolderScanStats.AcceptedFileCount.load(), + NiceBytes(LocalFolderScanStats.AcceptedFileByteCount.load()), + NiceLatencyNs(LocalFolderScanStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Chunking stats:" + "\n FilesProcessed: {}" + "\n FilesChunked: {}" + "\n BytesHashed: {}" + "\n UniqueChunksFound: {}" + "\n UniqueSequencesFound: {}" + "\n UniqueBytesFound: {}" + "\n ElapsedWallTimeUS: {}", + ChunkingStats.FilesProcessed.load(), + ChunkingStats.FilesChunked.load(), + NiceBytes(ChunkingStats.BytesHashed.load()), + ChunkingStats.UniqueChunksFound.load(), + ChunkingStats.UniqueSequencesFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load()), + NiceLatencyNs(ChunkingStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Find block stats:" + "\n FindBlockTimeMS: {}" + "\n PotentialChunkCount: {}" + "\n PotentialChunkByteCount: {}" + "\n FoundBlockCount: {}" + "\n FoundBlockChunkCount: {}" + "\n FoundBlockByteCount: {}" + "\n AcceptedBlockCount: {}" + "\n AcceptedChunkCount: {}" + "\n AcceptedByteCount: {}" + "\n RejectedBlockCount: {}" + "\n RejectedChunkCount: {}" + "\n RejectedByteCount: {}" + "\n AcceptedReduntantChunkCount: {}" + "\n AcceptedReduntantByteCount: {}" + "\n NewBlocksCount: {}" + "\n NewBlocksChunkCount: {}" + "\n NewBlocksChunkByteCount: {}", + NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), + FindBlocksStats.PotentialChunkCount, + NiceBytes(FindBlocksStats.PotentialChunkByteCount), + FindBlocksStats.FoundBlockCount, + FindBlocksStats.FoundBlockChunkCount, + NiceBytes(FindBlocksStats.FoundBlockByteCount), + FindBlocksStats.AcceptedBlockCount, + FindBlocksStats.AcceptedChunkCount, + NiceBytes(FindBlocksStats.AcceptedByteCount), + FindBlocksStats.RejectedBlockCount, + FindBlocksStats.RejectedChunkCount, + NiceBytes(FindBlocksStats.RejectedByteCount), + FindBlocksStats.AcceptedReduntantChunkCount, + NiceBytes(FindBlocksStats.AcceptedReduntantByteCount), + FindBlocksStats.NewBlocksCount, + FindBlocksStats.NewBlocksChunkCount, + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount)); + + ZEN_CONSOLE_VERBOSE( + "Generate blocks stats:" + "\n GeneratedBlockByteCount: {}" + "\n GeneratedBlockCount: {}" + "\n GenerateBlocksElapsedWallTimeUS: {}", + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceLatencyNs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Generate blocks stats:" + "\n ChunkCount: {}" + "\n ChunkByteCount: {}" + "\n CompressedChunkCount: {}" + "\n CompressChunksElapsedWallTimeUS: {}", + LooseChunksStats.ChunkCount, + NiceBytes(LooseChunksStats.ChunkByteCount), + LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), + NiceLatencyNs(LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Disk stats:" + "\n OpenReadCount: {}" + "\n OpenWriteCount: {}" + "\n ReadCount: {}" + "\n ReadByteCount: {}" + "\n WriteCount: {}" + "\n WriteByteCount: {}" + "\n CurrentOpenFileCount: {}", + DiskStats.OpenReadCount.load(), + DiskStats.OpenWriteCount.load(), + DiskStats.ReadCount.load(), + NiceBytes(DiskStats.ReadByteCount.load()), + DiskStats.WriteCount.load(), + NiceBytes(DiskStats.WriteByteCount.load()), + DiskStats.CurrentOpenFileCount.load()); + + ZEN_CONSOLE_VERBOSE( + "Upload stats:" + "\n BlockCount: {}" + "\n BlocksBytes: {}" + "\n ChunkCount: {}" + "\n ChunksBytes: {}" + "\n ReadFromDiskBytes: {}" + "\n MultipartAttachmentCount: {}" + "\n ElapsedWallTimeUS: {}", + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes.load()), + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.ChunksBytes.load()), + NiceBytes(UploadStats.ReadFromDiskBytes.load()), + UploadStats.MultipartAttachmentCount.load(), + NiceLatencyNs(UploadStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE( + "Uploaded {}\n" + " Delta: {}/{} ({:.1f}%)\n" + " Blocks: {} ({})\n" + " Chunks: {} ({}){}\n" + " Rate: {}bits/sec", + NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), + + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), + NiceBytes(ChunkingStats.BytesHashed), + DeltaByteCountPercent, + + UploadStats.BlockCount.load(), + NiceBytes(UploadStats.BlocksBytes), + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.ChunksBytes), + LargeAttachmentStats, + + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8)))); + + ZEN_CONSOLE("Uploaded ({}) build {} part {} ({}) in {}", + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), + BuildId, + BuildPartName, + BuildPartId, + NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs())); + return false; + } + + void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, std::atomic& AbortFlag) + { + ProgressBar ProgressBar(UsePlainProgress); + std::atomic FilesVerified(0); + std::atomic FilesFailed(0); + std::atomic ReadBytes(0); + + WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + + ParallellWork Work(AbortFlag); + + const uint32_t PathCount = gsl::narrow(Content.Paths.size()); + + RwLock ErrorLock; + std::vector Errors; + + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + const ChunkedContentLookup Lookup = BuildChunkedContentLookup(Content); + + for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++) + { + if (Work.IsAborted()) + { + break; + } + + Work.ScheduleWork( + VerifyPool, + [&, PathIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + // TODO: Convert ScheduleWork body to function + + const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); + if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) + { + const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; + if (!std::filesystem::exists(TargetPath)) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); + }); + FilesFailed++; + } + else + { + std::error_code Ec; + uint64_t SizeOnDisk = gsl::narrow(std::filesystem::file_size(TargetPath, Ec)); + if (Ec) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back( + fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); + }); + FilesFailed++; + } + else if (SizeOnDisk < ExpectedSize) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}", + TargetPath, + ExpectedSize, + SizeOnDisk)); + }); + FilesFailed++; + } + else if (SizeOnDisk > ExpectedSize) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}", + TargetPath, + ExpectedSize, + SizeOnDisk)); + }); + FilesFailed++; + } + else if (SizeOnDisk > 0) + { + const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; + IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); + IoHash RawHash = IoHash::HashBuffer(Buffer); + if (RawHash != ExpectedRawHash) + { + uint64_t FileOffset = 0; + const uint32_t SequenceRawHashesIndex = Lookup.RawHashToSequenceRawHashIndex.at(ExpectedRawHash); + const uint32_t OrderOffset = Lookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashesIndex]; + for (uint32_t OrderIndex = OrderOffset; + OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceRawHashesIndex]; + OrderIndex++) + { + uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; + uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize); + if (IoHash::HashBuffer(FileChunk) != ChunkHash) + { + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format( + "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. " + "Mismatch at chunk {}", + TargetPath, + ExpectedRawHash, + RawHash, + OrderIndex - OrderOffset)); + }); + break; + } + FileOffset += ChunkSize; + } + FilesFailed++; + } + ReadBytes += SizeOnDisk; + } + } + } + FilesVerified++; + } + }, + [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_UNUSED(AbortFlag); + + ErrorLock.WithExclusiveLock([&]() { + Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", + (Path / Content.Paths[PathIndex]).make_preferred(), + Ex.what())); + }); + FilesFailed++; + }); + } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + ProgressBar.UpdateState({.Task = "Verifying files ", + .Details = fmt::format("Verified {} files out of {}. Verfied: {}. Failed files: {}", + FilesVerified.load(), + PathCount, + NiceBytes(ReadBytes.load()), + FilesFailed.load()), + .TotalCount = gsl::narrow(PathCount), + .RemainingCount = gsl::narrow(PathCount - FilesVerified.load())}, + false); + }); + ProgressBar.Finish(); + for (const std::string& Error : Errors) + { + ZEN_CONSOLE("{}", Error); + } + if (!Errors.empty()) + { + throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size())); + } + } + + class WriteFileCache + { + public: + WriteFileCache() {} + ~WriteFileCache() { Flush(); } + + template + void WriteToFile(uint32_t TargetIndex, + std::function&& GetTargetPath, + const TBufferType& Buffer, + uint64_t FileOffset, + uint64_t TargetFinalSize) + { + if (!SeenTargetIndexes.empty() && SeenTargetIndexes.back() == TargetIndex) + { + ZEN_ASSERT(OpenFileWriter); + OpenFileWriter->Write(Buffer, FileOffset); + } + else + { + Flush(); + const std::filesystem::path& TargetPath = GetTargetPath(TargetIndex); + CreateDirectories(TargetPath.parent_path()); + uint32_t Tries = 5; + std::unique_ptr NewOutputFile( + std::make_unique(TargetPath, BasicFile::Mode::kWrite, [&Tries, TargetPath](std::error_code& Ec) { + if (Tries < 3) + { + ZEN_CONSOLE("Failed opening file '{}': {}{}", TargetPath, Ec.message(), Tries > 1 ? " Retrying"sv : ""sv); + } + if (Tries > 1) + { + Sleep(100); + } + return --Tries > 0; + })); + + const bool CacheWriter = TargetFinalSize > Buffer.GetSize(); + if (CacheWriter) + { + ZEN_ASSERT(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); + + OutputFile = std::move(NewOutputFile); + OpenFileWriter = std::make_unique(*OutputFile, Min(TargetFinalSize, 256u * 1024u)); + OpenFileWriter->Write(Buffer, FileOffset); + SeenTargetIndexes.push_back(TargetIndex); + } + else + { + NewOutputFile->Write(Buffer, FileOffset); + } + } + } + + void Flush() + { + OpenFileWriter = {}; + OutputFile = {}; + } + std::vector SeenTargetIndexes; + std::unique_ptr OutputFile; + std::unique_ptr OpenFileWriter; + }; + + std::vector GetRemainingChunkTargets( + const std::vector& RemotePathIndexWantsCopyFromCacheFlags, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) + { + std::span ChunkSources = GetChunkLocations(Lookup, ChunkIndex); + std::vector ChunkTargetPtrs; + if (!ChunkSources.empty()) + { + ChunkTargetPtrs.reserve(ChunkSources.size()); + for (const ChunkedContentLookup::ChunkLocation& Source : ChunkSources) + { + if (!RemotePathIndexWantsCopyFromCacheFlags[Source.PathIndex]) + { + ChunkTargetPtrs.push_back(&Source); + } + } + } + return ChunkTargetPtrs; + }; + + bool WriteBlockToDisk(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const std::vector& RemotePathIndexWantsCopyFromCacheFlags, + const CompositeBuffer& DecompressedBlockBuffer, + const ChunkedContentLookup& Lookup, + std::atomic* RemoteChunkIndexNeedsCopyFromSourceFlags, + uint32_t& OutChunksComplete, + uint64_t& OutBytesWritten) + { + std::vector ChunkBuffers; + struct WriteOpData + { + const ChunkedContentLookup::ChunkLocation* Target; + size_t ChunkBufferIndex; + }; + std::vector WriteOps; + + SharedBuffer BlockBuffer = DecompressedBlockBuffer.Flatten(); + uint64_t HeaderSize = 0; + if (IterateChunkBlock( + BlockBuffer, + [&](CompressedBuffer&& Chunk, const IoHash& ChunkHash) { + if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = It->second; + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, Lookup, ChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + bool NeedsWrite = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) + { + CompositeBuffer Decompressed = Chunk.DecompressToComposite(); + if (!Decompressed) + { + throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); + } + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + ZEN_ASSERT(Decompressed.GetSize() == Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + for (const ChunkedContentLookup::ChunkLocation* Target : ChunkTargetPtrs) + { + WriteOps.push_back(WriteOpData{.Target = Target, .ChunkBufferIndex = ChunkBuffers.size()}); + } + ChunkBuffers.emplace_back(std::move(Decompressed)); + } + } + } + }, + HeaderSize)) + { + if (!WriteOps.empty()) + { + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOpData& Lhs, const WriteOpData& Rhs) { + if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + { + return true; + } + if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + + WriteFileCache OpenFileCache; + for (const WriteOpData& WriteOp : WriteOps) + { + const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t PathIndex = WriteOp.Target->PathIndex; + const uint64_t ChunkSize = Chunk.GetSize(); + const uint64_t FileOffset = WriteOp.Target->Offset; + ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); + + OpenFileCache.WriteToFile( + PathIndex, + [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, + Chunk, + FileOffset, + Content.RawSizes[PathIndex]); + OutBytesWritten += ChunkSize; + } + OutChunksComplete += gsl::narrow(ChunkBuffers.size()); + } + return true; + } + return false; + } + + SharedBuffer Decompress(const IoBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(CompressedChunk), RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Invalid build blob format for chunk {}", ChunkHash)); + } + if (RawHash != ChunkHash) + { + throw std::runtime_error(fmt::format("Mismatching build blob {}, but compressed header rawhash is {}", ChunkHash, RawHash)); + } + if (RawSize != ChunkRawSize) + { + throw std::runtime_error( + fmt::format("Mismatching build blob {}, expected raw size {} but recevied raw size {}", ChunkHash, ChunkRawSize, RawSize)); + } + if (!Compressed) + { + throw std::runtime_error(fmt::format("Invalid build blob {}, not a compressed buffer", ChunkHash)); + } + + SharedBuffer Decompressed = Compressed.Decompress(); + + if (!Decompressed) + { + throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); + } + return Decompressed; + } + + void WriteChunkToDisk(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + std::span ChunkTargets, + const CompositeBuffer& ChunkData, + WriteFileCache& OpenFileCache, + uint64_t& OutBytesWritten) + { + for (const ChunkedContentLookup::ChunkLocation* TargetPtr : ChunkTargets) + { + const auto& Target = *TargetPtr; + const uint64_t FileOffset = Target.Offset; + + OpenFileCache.WriteToFile( + Target.PathIndex, + [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, + ChunkData, + FileOffset, + Content.RawSizes[Target.PathIndex]); + OutBytesWritten += ChunkData.GetSize(); + } + } + + void DownloadLargeBlob(BuildStorage& Storage, + const std::filesystem::path& Path, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + const std::vector& ChunkTargetPtrs, + ParallellWork& Work, + WorkerThreadPool& WritePool, + WorkerThreadPool& NetworkPool, + std::atomic& AbortFlag, + std::atomic& BytesWritten, + std::atomic& WriteToDiskBytes, + std::atomic& BytesDownloaded, + std::atomic& LooseChunksBytes, + std::atomic& DownloadedChunks, + std::atomic& ChunksComplete, + std::atomic& MultipartAttachmentCount) + { + struct WorkloadData + { + TemporaryFile TempFile; + }; + std::shared_ptr Workload(std::make_shared()); + + std::error_code Ec; + Workload->TempFile.CreateTemporary(Path / ZenTempChunkFolderName, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); + } + std::vector> WorkItems = Storage.GetLargeBuildBlob( + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + [&Path, + &RemoteContent, + &RemoteLookup, + &Work, + &WritePool, + Workload, + ChunkHash, + &BytesDownloaded, + &LooseChunksBytes, + &BytesWritten, + &WriteToDiskBytes, + &DownloadedChunks, + &ChunksComplete, + ChunkTargetPtrs = std::vector(ChunkTargetPtrs), + &AbortFlag](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { + BytesDownloaded += Chunk.GetSize(); + LooseChunksBytes += Chunk.GetSize(); + + if (!AbortFlag.load()) + { + Workload->TempFile.Write(Chunk.GetView(), Offset); + if (Chunk.GetSize() == BytesRemaining) + { + DownloadedChunks++; + + Work.ScheduleWork( + WritePool, + [&Path, + &RemoteContent, + &RemoteLookup, + ChunkHash, + Workload, + Offset, + BytesRemaining, + &ChunksComplete, + &BytesWritten, + &WriteToDiskBytes, + ChunkTargetPtrs](std::atomic& AbortFlag) { + if (!AbortFlag) + { + uint64_t CompressedSize = Workload->TempFile.FileSize(); + void* FileHandle = Workload->TempFile.Detach(); + IoBuffer CompressedPart = IoBuffer(IoBuffer::File, + FileHandle, + 0, + CompressedSize, + /*IsWholeFile*/ true); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Multipart build blob {} is not a compressed buffer", ChunkHash)); + } + CompressedPart.SetDeleteOnClose(true); + + uint64_t TotalBytesWritten = 0; + + uint32_t ChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); + + SharedBuffer Chunk = + Decompress(CompressedPart, ChunkHash, RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + // ZEN_ASSERT_SLOW(ChunkHash == + // IoHash::HashBuffer(Chunk.AsIoBuffer())); + + { + WriteFileCache OpenFileCache; + + WriteChunkToDisk(Path, + RemoteContent, + ChunkTargetPtrs, + CompositeBuffer(Chunk), + OpenFileCache, + TotalBytesWritten); + ChunksComplete++; + BytesWritten += TotalBytesWritten; + WriteToDiskBytes += TotalBytesWritten; + } + } + }, + [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed writing chunk {}. Reason: {}", ChunkHash, Ex.what()); + AbortFlag = true; + }); + } + } + }); + if (!WorkItems.empty()) + { + MultipartAttachmentCount++; + } + for (auto& WorkItem : WorkItems) + { + Work.ScheduleWork( + NetworkPool, + [WorkItem = std::move(WorkItem)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + WorkItem(); + } + }, + [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed uploading multipart blob {}. Reason: {}", ChunkHash, Ex.what()); + AbortFlag = true; + }); + } + } + + bool UpdateFolder(BuildStorage& Storage, + const Oid& BuildId, + const std::filesystem::path& Path, + const std::uint64_t LargeAttachmentSize, + const std::uint64_t PreferredMultipartChunkSize, + const ChunkedFolderContent& LocalContent, + const ChunkedFolderContent& RemoteContent, + const std::vector& BlockDescriptions, + const std::vector& LooseChunkHashes, + bool WipeTargetFolder, + std::atomic& AbortFlag, + FolderContent& OutLocalFolderState) + { + std::atomic DownloadedBlocks = 0; + std::atomic BlockBytes = 0; + std::atomic DownloadedChunks = 0; + std::atomic LooseChunksBytes = 0; + std::atomic WriteToDiskBytes = 0; + std::atomic MultipartAttachmentCount = 0; + + DiskStatistics DiskStats; + + Stopwatch IndexTimer; + + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); + + const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + + ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + + const std::filesystem::path CacheFolderPath = Path / ZenTempReuseFolderName; + + tsl::robin_map LocalRawHashToPathIndex; + + if (!WipeTargetFolder) + { + Stopwatch CacheTimer; + + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + { + if (LocalContent.RawSizes[LocalPathIndex] > 0) + { + const uint32_t SequenceRawHashIndex = + LocalLookup.RawHashToSequenceRawHashIndex.at(LocalContent.RawHashes[LocalPathIndex]); + uint32_t ChunkCount = LocalContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + if (ChunkCount > 0) + { + const IoHash LocalRawHash = LocalContent.RawHashes[LocalPathIndex]; + if (!LocalRawHashToPathIndex.contains(LocalRawHash)) + { + LocalRawHashToPathIndex.insert_or_assign(LocalRawHash, LocalPathIndex); + } + } + } + } + + { + std::vector IncludeLocalFiles(LocalContent.Paths.size(), false); + + for (const IoHash& ChunkHash : RemoteContent.ChunkedContent.ChunkHashes) + { + if (auto It = LocalLookup.ChunkHashToChunkIndex.find(ChunkHash); It != LocalLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t LocalChunkIndex = It->second; + std::span LocalChunkTargetRange = + GetChunkLocations(LocalLookup, LocalChunkIndex); + if (!LocalChunkTargetRange.empty()) + { + std::uint32_t LocalPathIndex = LocalChunkTargetRange[0].PathIndex; + IncludeLocalFiles[LocalPathIndex] = true; + } + } + } + for (const IoHash& RawHash : RemoteContent.RawHashes) + { + if (auto It = LocalRawHashToPathIndex.find(RawHash); It != LocalRawHashToPathIndex.end()) + { + uint32_t LocalPathIndex = It->second; + IncludeLocalFiles[LocalPathIndex] = true; + } + } + + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + { + if (!IncludeLocalFiles[LocalPathIndex]) + { + LocalRawHashToPathIndex.erase(LocalContent.RawHashes[LocalPathIndex]); + } + } + } + + uint64_t CachedBytes = 0; + CreateDirectories(CacheFolderPath); + for (auto& CachedLocalFile : LocalRawHashToPathIndex) + { + const IoHash& LocalRawHash = CachedLocalFile.first; + const uint32_t LocalPathIndex = CachedLocalFile.second; + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const std::filesystem::path CacheFilePath = (CacheFolderPath / LocalRawHash.ToHexString()).make_preferred(); + + SetFileReadOnly(LocalFilePath, false); + + std::filesystem::rename(LocalFilePath, CacheFilePath); + CachedBytes += std::filesystem::file_size(CacheFilePath); + } + + ZEN_CONSOLE("Cached {} ({}) local files in {}", + LocalRawHashToPathIndex.size(), + NiceBytes(CachedBytes), + NiceTimeSpanMs(CacheTimer.GetElapsedTimeMs())); + } + + if (AbortFlag) + { + return true; + } + + CleanDirectory(Path, DefaultExcludeFolders); + + Stopwatch CacheMappingTimer; + + std::atomic BytesWritten = 0; + uint64_t CacheMappedBytesForReuse = 0; + + std::vector RemotePathIndexWantsCopyFromCacheFlags(RemoteContent.Paths.size(), false); + std::vector> RemoteChunkIndexWantsCopyFromCacheFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) + std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + + struct CacheCopyData + { + std::filesystem::path OriginalSourceFileName; + IoHash LocalFileRawHash; + uint64_t LocalFileRawSize = 0; + std::vector RemotePathIndexes; + std::vector ChunkSourcePtrs; + struct ChunkTarget + { + uint32_t ChunkSourceCount; + uint64_t ChunkRawSize; + uint64_t LocalFileOffset; + }; + std::vector ChunkTargets; + }; + + tsl::robin_map RawHashToCacheCopyDataIndex; + std::vector CacheCopyDatas; + uint32_t ChunkCountToWrite = 0; + + // Pick up all whole files to copy and/or move + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + { + const IoHash& RemoteRawHash = RemoteContent.RawHashes[RemotePathIndex]; + if (auto It = LocalRawHashToPathIndex.find(RemoteRawHash); It != LocalRawHashToPathIndex.end()) + { + if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(RemoteRawHash); CopySourceIt != RawHashToCacheCopyDataIndex.end()) + { + CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; + Data.RemotePathIndexes.push_back(RemotePathIndex); + } + else + { + const uint32_t LocalPathIndex = It->second; + ZEN_ASSERT(LocalContent.RawSizes[LocalPathIndex] == RemoteContent.RawSizes[RemotePathIndex]); + ZEN_ASSERT(LocalContent.RawHashes[LocalPathIndex] == RemoteContent.RawHashes[RemotePathIndex]); + RawHashToCacheCopyDataIndex.insert_or_assign(RemoteRawHash, CacheCopyDatas.size()); + CacheCopyDatas.push_back(CacheCopyData{.OriginalSourceFileName = LocalContent.Paths[LocalPathIndex], + .LocalFileRawHash = RemoteRawHash, + .LocalFileRawSize = LocalContent.RawSizes[LocalPathIndex], + .RemotePathIndexes = {RemotePathIndex}}); + CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; + ChunkCountToWrite++; + } + RemotePathIndexWantsCopyFromCacheFlags[RemotePathIndex] = true; + } + } + + // Pick up all chunks in cached files and make sure we block moving of cache files if we need part of them + for (auto& CachedLocalFile : LocalRawHashToPathIndex) + { + const IoHash& LocalFileRawHash = CachedLocalFile.first; + const uint32_t LocalPathIndex = CachedLocalFile.second; + const uint32_t LocalSequenceRawHashIndex = LocalLookup.RawHashToSequenceRawHashIndex.at(LocalFileRawHash); + const uint32_t LocalOrderOffset = + LocalLookup.SequenceRawHashIndexChunkOrderOffset[LocalSequenceRawHashIndex]; // CachedLocalFile.second.ChunkOrderOffset; + + { + uint64_t SourceOffset = 0; + const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceRawHashIndex]; + for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) + { + const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; + const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); + RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = RemoteChunkIt->second; + if (!RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + CacheCopyData::ChunkTarget Target = {.ChunkSourceCount = gsl::narrow(ChunkTargetPtrs.size()), + .ChunkRawSize = LocalChunkRawSize, + .LocalFileOffset = SourceOffset}; + if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalFileRawHash); + CopySourceIt != RawHashToCacheCopyDataIndex.end()) + { + CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; + Data.ChunkSourcePtrs.insert(Data.ChunkSourcePtrs.end(), ChunkTargetPtrs.begin(), ChunkTargetPtrs.end()); + Data.ChunkTargets.push_back(Target); + } + else + { + RawHashToCacheCopyDataIndex.insert_or_assign(LocalFileRawHash, CacheCopyDatas.size()); + CacheCopyDatas.push_back( + CacheCopyData{.OriginalSourceFileName = LocalContent.Paths[LocalPathIndex], + .LocalFileRawHash = LocalFileRawHash, + .LocalFileRawSize = LocalContent.RawSizes[LocalPathIndex], + .RemotePathIndexes = {}, + .ChunkSourcePtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + CacheMappedBytesForReuse += LocalChunkRawSize; + } + RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex] = true; + } + } + SourceOffset += LocalChunkRawSize; + } + } + } + + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + { + if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + { + ChunkCountToWrite++; + } + else + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + if (!ChunkTargetPtrs.empty()) + { + RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; + ChunkCountToWrite++; + } + } + } + std::atomic ChunkCountWritten = 0; + + ZEN_CONSOLE("Mapped {} cached data for reuse in {}", + NiceBytes(CacheMappedBytesForReuse), + NiceTimeSpanMs(CacheMappingTimer.GetElapsedTimeMs())); + + auto CopyChunksFromCacheFile = [](const std::filesystem::path& Path, + BufferedOpenFile& SourceFile, + WriteFileCache& OpenFileCache, + const ChunkedFolderContent& RemoteContent, + const uint64_t LocalFileSourceOffset, + const uint64_t LocalChunkRawSize, + std::span ChunkTargetPtrs, + uint64_t& OutBytesWritten) { + CompositeBuffer Chunk = SourceFile.GetRange(LocalFileSourceOffset, LocalChunkRawSize); + uint64_t TotalBytesWritten = 0; + + WriteChunkToDisk(Path, RemoteContent, ChunkTargetPtrs, Chunk, OpenFileCache, TotalBytesWritten); + OutBytesWritten += TotalBytesWritten; + }; + + auto CloneFullFileFromCache = [](const std::filesystem::path& Path, + const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const IoHash& FileRawHash, + const uint64_t FileRawSize, + std::span FullCloneRemotePathIndexes, + bool CanMove, + uint64_t& OutBytesWritten) { + const std::filesystem::path CacheFilePath = (CacheFolderPath / FileRawHash.ToHexString()).make_preferred(); + + size_t CopyCount = FullCloneRemotePathIndexes.size(); + if (CanMove) + { + // If every reference to this chunk has are full files we can move the cache file to the last target + CopyCount--; + } + + for (uint32_t RemotePathIndex : FullCloneRemotePathIndexes) + { + const std::filesystem::path TargetPath = (Path / RemoteContent.Paths[RemotePathIndex]).make_preferred(); + CreateDirectories(TargetPath.parent_path()); + + if (CopyCount == 0) + { + std::filesystem::rename(CacheFilePath, TargetPath); + } + else + { + CopyFile(CacheFilePath, TargetPath, {.EnableClone = false}); + ZEN_ASSERT(CopyCount > 0); + CopyCount--; + } + OutBytesWritten += FileRawSize; + } + }; + + WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + + ProgressBar WriteProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); + + std::atomic BytesDownloaded = 0; + + for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) + { + if (AbortFlag) + { + break; + } + + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(),// + [&, CopyDataIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + const std::filesystem::path CacheFilePath = + (CacheFolderPath / CopyData.LocalFileRawHash.ToHexString()).make_preferred(); + + if (!CopyData.ChunkSourcePtrs.empty()) + { + uint64_t CacheLocalFileBytesRead = 0; + + size_t TargetStart = 0; + const std::span AllTargets(CopyData.ChunkSourcePtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkLocation* Target; + uint64_t LocalFileOffset; + uint64_t ChunkSize; + }; + + std::vector WriteOps; + WriteOps.reserve(CopyData.ChunkSourcePtrs.size()); + + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + { + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.ChunkSourceCount); + for (const ChunkedContentLookup::ChunkLocation* Target : TargetRange) + { + WriteOps.push_back(WriteOp{.Target = Target, + .LocalFileOffset = ChunkTarget.LocalFileOffset, + .ChunkSize = ChunkTarget.ChunkRawSize}); + } + TargetStart += ChunkTarget.ChunkSourceCount; + } + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + { + return true; + } + else if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); + + BufferedOpenFile SourceFile(CacheFilePath); + WriteFileCache OpenFileCache; + for (const WriteOp& Op : WriteOps) + { + const uint32_t RemotePathIndex = Op.Target->PathIndex; + const uint64_t ChunkSize = Op.ChunkSize; + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.LocalFileOffset, ChunkSize); + + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + + OpenFileCache.WriteToFile( + RemotePathIndex, + [&Path, &RemoteContent](uint32_t TargetIndex) { + return (Path / RemoteContent.Paths[TargetIndex]).make_preferred(); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); + BytesWritten += ChunkSize; + WriteToDiskBytes += ChunkSize; + CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? + } + ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); + ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), CopyData.OriginalSourceFileName); + } + + if (CopyData.RemotePathIndexes.empty()) + { + std::filesystem::remove(CacheFilePath); + } + else + { + uint64_t LocalBytesWritten = 0; + CloneFullFileFromCache(Path, + CacheFolderPath, + RemoteContent, + CopyData.LocalFileRawHash, + CopyData.LocalFileRawSize, + CopyData.RemotePathIndexes, + true, + LocalBytesWritten); + // CacheLocalFileBytesRead += CopyData.LocalFileRawSize; + BytesWritten += LocalBytesWritten; + WriteToDiskBytes += LocalBytesWritten; + ChunkCountWritten++; + + ZEN_DEBUG("Used full cached file {} ({}) for {} ({}) targets", + CopyData.OriginalSourceFileName, + NiceBytes(CopyData.LocalFileRawSize), + CopyData.RemotePathIndexes.size(), + NiceBytes(LocalBytesWritten)); + } + } + }, + [&, CopyDataIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed reading cached file {}. Reason: {}", + CacheCopyDatas[CopyDataIndex].OriginalSourceFileName, + Ex.what()); + AbortFlag = true; + }); + } + + for (const IoHash ChunkHash : LooseChunkHashes) + { + if (AbortFlag) + { + break; + } + + uint32_t RemoteChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); + if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + { + ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + continue; + } + bool NeedsCopy = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + + if (ChunkTargetPtrs.empty()) + { + ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic& AbortFlag) { + if (!AbortFlag) + { + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + { + DownloadLargeBlob(Storage, + Path, + RemoteContent, + RemoteLookup, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + ChunkTargetPtrs, + Work, + WritePool, + NetworkPool, + AbortFlag, + BytesWritten, + WriteToDiskBytes, + BytesDownloaded, + LooseChunksBytes, + DownloadedChunks, + ChunkCountWritten, + MultipartAttachmentCount); + } + else + { + IoBuffer CompressedPart = Storage.GetBuildBlob(BuildId, ChunkHash); + if (!CompressedPart) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + BytesDownloaded += CompressedPart.GetSize(); + LooseChunksBytes += CompressedPart.GetSize(); + DownloadedChunks++; + + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(CompressedPart)]( + std::atomic& AbortFlag) { + if (!AbortFlag) + { + uint64_t TotalBytesWritten = 0; + SharedBuffer Chunk = + Decompress(CompressedPart, + ChunkHash, + RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); + WriteFileCache OpenFileCache; + + WriteChunkToDisk(Path, + RemoteContent, + ChunkTargetPtrs, + CompositeBuffer(Chunk), + OpenFileCache, + TotalBytesWritten); + ChunkCountWritten++; + BytesWritten += TotalBytesWritten; + WriteToDiskBytes += TotalBytesWritten; + } + }, + [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed writing chunk {}. Reason: {}", ChunkHash, Ex.what()); + AbortFlag = true; + }); + } + } + } + }, + [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed downloading chunk {}. Reason: {}", ChunkHash, Ex.what()); + AbortFlag = true; + }); + } + } + } + + size_t BlockCount = BlockDescriptions.size(); + std::atomic BlocksComplete = 0; + + auto IsBlockNeeded = [&RemoteContent, &RemoteLookup, &RemoteChunkIndexNeedsCopyFromSourceFlags]( + const ChunkBlockDescription& BlockDescription) -> bool { + for (const IoHash& ChunkHash : BlockDescription.ChunkRawHashes) + { + if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + { + return true; + } + } + } + return false; + }; + + for (size_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) + { + if (Work.IsAborted()) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, BlockIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + if (IsBlockNeeded(BlockDescriptions[BlockIndex])) + { + Work.ScheduleWork( + NetworkPool, + [&, BlockIndex](std::atomic& AbortFlag) { + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); + } + BytesDownloaded += BlockBuffer.GetSize(); + BlockBytes += BlockBuffer.GetSize(); + DownloadedBlocks++; + + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockBuffer = std::move(BlockBuffer)](std::atomic& AbortFlag) { + if (!AbortFlag) + { + IoHash BlockRawHash; + uint64_t BlockRawSize; + CompressedBuffer CompressedBlockBuffer = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(BlockBuffer)), + BlockRawHash, + BlockRawSize); + if (!CompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", + BlockDescriptions[BlockIndex].BlockHash)); + } + + if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) + { + throw std::runtime_error( + fmt::format("Block {} header has a mismatching raw hash {}", + BlockDescriptions[BlockIndex].BlockHash, + BlockRawHash)); + } + + CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); + if (!DecompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} failed to decompress", + BlockDescriptions[BlockIndex].BlockHash)); + } + + ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == + IoHash::HashBuffer(DecompressedBlockBuffer)); + + uint64_t BytesWrittenToDisk = 0; + uint32_t ChunksReadFromBlock = 0; + if (WriteBlockToDisk(Path, + RemoteContent, + RemotePathIndexWantsCopyFromCacheFlags, + DecompressedBlockBuffer, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags.data(), + ChunksReadFromBlock, + BytesWrittenToDisk)) + { + BytesWritten += BytesWrittenToDisk; + WriteToDiskBytes += BytesWrittenToDisk; + ChunkCountWritten += ChunksReadFromBlock; + } + else + { + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescriptions[BlockIndex].BlockHash)); + } + BlocksComplete++; + } + }, + [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed writing block {}. Reason: {}", + BlockDescriptions[BlockIndex].BlockHash, + Ex.what()); + AbortFlag = true; + }); + } + }, + [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed downloading block {}. Reason: {}", + BlockDescriptions[BlockIndex].BlockHash, + Ex.what()); + AbortFlag = true; + }); + } + else + { + ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + BlocksComplete++; + } + } + }, + [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed determning if block {} is needed. Reason: {}", BlockDescriptions[BlockIndex].BlockHash, Ex.what()); + AbortFlag = true; + }); + } + for (uint32_t PathIndex = 0; PathIndex < RemoteContent.Paths.size(); PathIndex++) + { + if (Work.IsAborted()) + { + break; + } + if (RemoteContent.RawSizes[PathIndex] == 0) + { + Work.ScheduleWork( + WritePool, + [&, PathIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + const std::filesystem::path TargetPath = (Path / RemoteContent.Paths[PathIndex]).make_preferred(); + CreateDirectories(TargetPath.parent_path()); + BasicFile OutputFile; + OutputFile.Open(TargetPath, BasicFile::Mode::kTruncate); + } + }, + [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed creating file at {}. Reason: {}", RemoteContent.Paths[PathIndex], Ex.what()); + AbortFlag = true; + }); + } + } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); + WriteProgressBar.UpdateState( + {.Task = "Writing chunks ", + .Details = fmt::format("Written {} chunks out of {}. {} ouf of {} blocks complete. Downloaded: {}. Written: {}", + ChunkCountWritten.load(), + ChunkCountToWrite, + BlocksComplete.load(), + BlockCount, + NiceBytes(BytesDownloaded.load()), + NiceBytes(BytesWritten.load())), + .TotalCount = gsl::narrow(ChunkCountToWrite), + .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, + false); + }); + WriteProgressBar.Finish(); + + { + ProgressBar PremissionsProgressBar(false); + if (!RemoteContent.Attributes.empty()) + { + auto SetNativeFileAttributes = + [](const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) -> uint32_t { +#if ZEN_PLATFORM_WINDOWS + if (SourcePlatform == SourcePlatform::Windows) + { + SetFileAttributes(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentAttributes = GetFileAttributes(FilePath); + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, IsFileModeReadOnly(Attributes)); + if (CurrentAttributes != NewAttributes) + { + SetFileAttributes(FilePath, NewAttributes); + } + return NewAttributes; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + if (SourcePlatform != SourcePlatform::Windows) + { + SetFileMode(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentMode = GetFileMode(FilePath); + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, IsFileAttributeReadOnly(Attributes)); + if (CurrentMode != NewMode) + { + SetFileMode(FilePath, NewMode); + } + return NewMode; + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + }; + + OutLocalFolderState.Paths.reserve(RemoteContent.Paths.size()); + OutLocalFolderState.RawSizes.reserve(RemoteContent.Paths.size()); + OutLocalFolderState.Attributes.reserve(RemoteContent.Paths.size()); + OutLocalFolderState.ModificationTicks.reserve(RemoteContent.Paths.size()); + for (uint32_t PathIndex = 0; PathIndex < RemoteContent.Paths.size(); PathIndex++) + { + const std::filesystem::path LocalFilePath = (Path / RemoteContent.Paths[PathIndex]); + const uint32_t CurrentPlatformAttributes = + SetNativeFileAttributes(LocalFilePath, RemoteContent.Platform, RemoteContent.Attributes[PathIndex]); + + OutLocalFolderState.Paths.push_back(RemoteContent.Paths[PathIndex]); + OutLocalFolderState.RawSizes.push_back(RemoteContent.RawSizes[PathIndex]); + OutLocalFolderState.Attributes.push_back(CurrentPlatformAttributes); + OutLocalFolderState.ModificationTicks.push_back(GetModificationTickFromPath(LocalFilePath)); + + PremissionsProgressBar.UpdateState( + {.Task = "Set permissions ", + .Details = fmt::format("Updated {} files out of {}", PathIndex, RemoteContent.Paths.size()), + .TotalCount = RemoteContent.Paths.size(), + .RemainingCount = (RemoteContent.Paths.size() - PathIndex)}, + false); + } + } + PremissionsProgressBar.Finish(); + } + + return false; + } + + std::vector> ResolveBuildPartNames(BuildStorage& Storage, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + std::uint64_t& OutPreferredMultipartChunkSize) + { + std::vector> Result; + { + Stopwatch GetBuildTimer; + + std::vector> AvailableParts; + + CbObject BuildObject = Storage.GetBuild(BuildId); + ZEN_CONSOLE("GetBuild took {}. Payload size: {}", + NiceLatencyNs(GetBuildTimer.GetElapsedTimeUs() * 1000), + NiceBytes(BuildObject.GetSize())); + + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + if (!PartsObject) + { + throw std::runtime_error("Build object does not have a 'parts' object"); + } + + OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + + for (CbFieldView PartView : PartsObject) + { + const std::string BuildPartName = std::string(PartView.GetName()); + const Oid BuildPartId = PartView.AsObjectId(); + if (BuildPartId == Oid::Zero) + { + ExtendableStringBuilder<128> SB; + for (CbFieldView ScanPartView : PartsObject) + { + SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId())); + } + throw std::runtime_error( + fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView())); + } + AvailableParts.push_back({BuildPartId, BuildPartName}); + } + + if (BuildPartIds.empty() && BuildPartNames.empty()) + { + Result = AvailableParts; + } + else + { + for (const std::string& BuildPartName : BuildPartNames) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName)); + } + } + for (const Oid& BuildPartId : BuildPartIds) + { + if (auto It = std::find_if(AvailableParts.begin(), + AvailableParts.end(), + [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; }); + It != AvailableParts.end()) + { + Result.push_back(*It); + } + else + { + throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId)); + } + } + } + + if (Result.empty()) + { + throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId)); + } + } + return Result; + } + + ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, + const Oid& BuildId, + const std::vector>& BuildParts, + std::unique_ptr& OutChunkController, + std::vector& OutPartContents, + std::vector& OutBlockDescriptions, + std::vector& OutLooseChunkHashes) + { + Stopwatch GetBuildPartTimer; + CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildParts[0].first); + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + BuildParts[0].first, + BuildParts[0].second, + NiceLatencyNs(GetBuildPartTimer.GetElapsedTimeUs() * 1000), + NiceBytes(BuildPartManifest.GetSize())); + + { + CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); + std::string_view ChunkerName = Chunker["name"sv].AsString(); + CbObjectView Parameters = Chunker["parameters"sv].AsObjectView(); + OutChunkController = CreateChunkingController(ChunkerName, Parameters); + } + + auto ParseBuildPartManifest = [](BuildStorage& Storage, + const Oid& BuildId, + const Oid& BuildPartId, + CbObject BuildPartManifest, + ChunkedFolderContent& OutRemoteContent, + std::vector& OutBlockDescriptions, + std::vector& OutLooseChunkHashes) { + std::vector AbsoluteChunkOrders; + std::vector LooseChunkRawSizes; + std::vector BlockRawHashes; + + ReadBuildContentFromCompactBinary(BuildPartManifest, + OutRemoteContent.Platform, + OutRemoteContent.Paths, + OutRemoteContent.RawHashes, + OutRemoteContent.RawSizes, + OutRemoteContent.Attributes, + OutRemoteContent.ChunkedContent.SequenceRawHashes, + OutRemoteContent.ChunkedContent.ChunkCounts, + AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + BlockRawHashes); + + // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them + + Stopwatch GetBlockMetadataTimer; + OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes); + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceLatencyNs(GetBlockMetadataTimer.GetElapsedTimeUs() * 1000), + OutBlockDescriptions.size()); + + if (OutBlockDescriptions.size() != BlockRawHashes.size()) + { + bool AttemptFallback = false; + std::string ErrorDescription = + fmt::format("All required blocks could not be found, {} blocks does not have metadata in this context.", + BlockRawHashes.size() - OutBlockDescriptions.size()); + if (AttemptFallback) + { + ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); + std::vector AugmentedBlockDescriptions; + AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); + std::vector FoundBlocks = Storage.FindBlocks(BuildId); + + for (const IoHash& BlockHash : BlockRawHashes) + { + if (auto It = std::find_if( + OutBlockDescriptions.begin(), + OutBlockDescriptions.end(), + [BlockHash](const ChunkBlockDescription& Description) { return Description.BlockHash == BlockHash; }); + It != OutBlockDescriptions.end()) + { + AugmentedBlockDescriptions.emplace_back(std::move(*It)); + } + else if (auto ListBlocksIt = std::find_if( + FoundBlocks.begin(), + FoundBlocks.end(), + [BlockHash](const ChunkBlockDescription& Description) { return Description.BlockHash == BlockHash; }); + ListBlocksIt != FoundBlocks.end()) + { + ZEN_CONSOLE("Found block {} via context find successfully", BlockHash); + AugmentedBlockDescriptions.emplace_back(std::move(*ListBlocksIt)); + } + else + { + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockHash); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} could not be found", BlockHash)); + } + IoHash BlockRawHash; + uint64_t BlockRawSize; + CompressedBuffer CompressedBlockBuffer = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(BlockBuffer)), BlockRawHash, BlockRawSize); + if (!CompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", BlockHash)); + } + + if (BlockRawHash != BlockHash) + { + throw std::runtime_error( + fmt::format("Block {} header has a mismatching raw hash {}", BlockHash, BlockRawHash)); + } + + CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); + if (!DecompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} failed to decompress", BlockHash)); + } + + ChunkBlockDescription MissingChunkDescription = + GetChunkBlockDescription(DecompressedBlockBuffer.Flatten(), BlockHash); + AugmentedBlockDescriptions.emplace_back(std::move(MissingChunkDescription)); + } + } + OutBlockDescriptions.swap(AugmentedBlockDescriptions); + } + else + { + throw std::runtime_error(ErrorDescription); + } + } + + CalculateLocalChunkOrders(AbsoluteChunkOrders, + OutLooseChunkHashes, + LooseChunkRawSizes, + OutBlockDescriptions, + OutRemoteContent.ChunkedContent.ChunkHashes, + OutRemoteContent.ChunkedContent.ChunkRawSizes, + OutRemoteContent.ChunkedContent.ChunkOrders); + }; + + OutPartContents.resize(1); + ParseBuildPartManifest(Storage, + BuildId, + BuildParts[0].first, + BuildPartManifest, + OutPartContents[0], + OutBlockDescriptions, + OutLooseChunkHashes); + ChunkedFolderContent RemoteContent; + if (BuildParts.size() > 1) + { + std::vector OverlayBlockDescriptions; + std::vector OverlayLooseChunkHashes; + for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++) + { + const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; + const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; + Stopwatch GetOverlayBuildPartTimer; + CbObject OverlayBuildPartManifest = Storage.GetBuildPart(BuildId, OverlayBuildPartId); + ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", + OverlayBuildPartId, + OverlayBuildPartName, + NiceLatencyNs(GetOverlayBuildPartTimer.GetElapsedTimeUs() * 1000), + NiceBytes(OverlayBuildPartManifest.GetSize())); + + ChunkedFolderContent OverlayPartContent; + std::vector OverlayPartBlockDescriptions; + std::vector OverlayPartLooseChunkHashes; + + ParseBuildPartManifest(Storage, + BuildId, + OverlayBuildPartId, + OverlayBuildPartManifest, + OverlayPartContent, + OverlayPartBlockDescriptions, + OverlayPartLooseChunkHashes); + OutPartContents.push_back(OverlayPartContent); + OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(), + OverlayPartBlockDescriptions.begin(), + OverlayPartBlockDescriptions.end()); + OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(), + OverlayPartLooseChunkHashes.begin(), + OverlayPartLooseChunkHashes.end()); + } + + RemoteContent = + MergeChunkedFolderContents(OutPartContents[0], std::span(OutPartContents).subspan(1)); + { + tsl::robin_set AllBlockHashes; + for (const ChunkBlockDescription& Description : OutBlockDescriptions) + { + AllBlockHashes.insert(Description.BlockHash); + } + for (const ChunkBlockDescription& Description : OverlayBlockDescriptions) + { + if (!AllBlockHashes.contains(Description.BlockHash)) + { + AllBlockHashes.insert(Description.BlockHash); + OutBlockDescriptions.push_back(Description); + } + } + } + { + tsl::robin_set AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end()); + for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes) + { + if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash)) + { + AllLooseChunkHashes.insert(OverlayLooseChunkHash); + OutLooseChunkHashes.push_back(OverlayLooseChunkHash); + } + } + } + } + else + { + RemoteContent = OutPartContents[0]; + } + return RemoteContent; + } + + ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + ChunkingController& ChunkController, + std::atomic& AbortFlag) + { + ChunkedFolderContent LocalContent; + + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + auto IsAcceptedFile = [ExcludeExtensions = + DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { + for (const std::string_view& ExcludeExtension : ExcludeExtensions) + { + if (RelativePath.ends_with(ExcludeExtension)) + { + return false; + } + } + return true; + }; + + FolderContent CurrentLocalFolderContent = GetFolderContent( + LocalFolderScanStats, + Path, + std::move(IsAcceptedFolder), + std::move(IsAcceptedFile), + GetMediumWorkerPool(EWorkloadType::Burst), + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, + AbortFlag); + if (AbortFlag) + { + return {}; + } + + FolderContent LocalFolderState; + + bool ScanContent = true; + std::vector PathIndexesOufOfDate; + if (std::filesystem::is_regular_file(Path / ZenStateFilePath)) + { + try + { + Stopwatch ReadStateTimer; + CbObject CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object; + if (CurrentStateObject) + { + Oid CurrentBuildId; + std::vector SavedBuildPartIds; + std::vector SavedBuildPartsNames; + std::vector SavedPartContents; + if (ReadStateObject(CurrentStateObject, + CurrentBuildId, + SavedBuildPartIds, + SavedBuildPartsNames, + SavedPartContents, + LocalFolderState)) + { + if (!SavedPartContents.empty()) + { + if (SavedPartContents.size() == 1) + { + LocalContent = std::move(SavedPartContents[0]); + } + else + { + LocalContent = + MergeChunkedFolderContents(SavedPartContents[0], + std::span(SavedPartContents).subspan(1)); + } + + if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) + { + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } + + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated", + DeletedPaths.size(), + UpdatedContent.Paths.size()); + if (UpdatedContent.Paths.size() > 0) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : UpdatedContent.RawSizes) + { + ByteCountToScan += RawSize; + } + ProgressBar ProgressBar(false); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetMediumWorkerPool(EWorkloadType::Burst), + Path, + UpdatedContent, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + + ProgressBar.UpdateState( + {.Task = "Scanning files ", + .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + if (AbortFlag) + { + return {}; + } + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); + } + } + else + { + ZEN_CONSOLE("Using cached local state"); + } + ZEN_CONSOLE("Read local state in {}", NiceLatencyNs(ReadStateTimer.GetElapsedTimeUs() * 1000)); + ScanContent = false; + } + } + } + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + } + } + + if (ScanContent) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : CurrentLocalFolderContent.RawSizes) + { + ByteCountToScan += RawSize; + } + ProgressBar ProgressBar(false); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetMediumWorkerPool(EWorkloadType::Burst), + Path, + CurrentLocalFolderContent, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + CurrentLocalFolderContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + ByteCountToScan, + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .TotalCount = ByteCountToScan, + .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load())}, + false); + }, + AbortFlag); + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + + if (AbortFlag) + { + return {}; + } + } + return LocalContent; + } + + bool DownloadFolder(BuildStorage& Storage, + const Oid& BuildId, + const std::vector& BuildPartIds, + std::span BuildPartNames, + const std::filesystem::path& Path, + bool AllowMultiparts, + bool WipeTargetFolder) + { + Stopwatch DownloadTimer; + std::atomic AbortFlag(false); + + const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + CreateDirectories(ZenTempFolder); + auto _ = MakeGuard([&]() { + CleanDirectory(ZenTempFolder, {}); + std::filesystem::remove(ZenTempFolder); + }); + CreateDirectories(Path / ZenTempBlockFolderName); + CreateDirectories(Path / ZenTempChunkFolderName); + CreateDirectories(Path / ZenTempReuseFolderName); + + std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; + + std::vector> AllBuildParts = + ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + + std::vector PartContents; + + std::unique_ptr ChunkController; + + std::vector BlockDescriptions; + std::vector LooseChunkHashes; + + ChunkedFolderContent RemoteContent = + GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); + + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + if (!ChunkController) + { + ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ChunkController = CreateBasicChunkingController(); + } + + GetFolderContentStatistics LocalFolderScanStats; + ChunkingStatistics ChunkingStats; + ChunkedFolderContent LocalContent; + if (std::filesystem::is_directory(Path)) + { + if (!WipeTargetFolder) + { + LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController, AbortFlag); + } + } + else + { + CreateDirectories(Path); + } + if (AbortFlag.load()) + { + return true; + } + + auto CompareContent = [](const ChunkedFolderContent& Lsh, const ChunkedFolderContent& Rhs) { + tsl::robin_map RhsPathToIndex; + const size_t RhsPathCount = Rhs.Paths.size(); + RhsPathToIndex.reserve(RhsPathCount); + for (size_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++) + { + RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex}); + } + const size_t LhsPathCount = Lsh.Paths.size(); + for (size_t LhsPathIndex = 0; LhsPathIndex < LhsPathCount; LhsPathIndex++) + { + if (auto It = RhsPathToIndex.find(Lsh.Paths[LhsPathIndex].generic_string()); It != RhsPathToIndex.end()) + { + const size_t RhsPathIndex = It->second; + if ((Lsh.RawHashes[LhsPathIndex] != Rhs.RawHashes[RhsPathIndex]) || + (!FolderContent::AreFileAttributesEqual(Lsh.Attributes[LhsPathIndex], Rhs.Attributes[RhsPathIndex]))) + { + return false; + } + } + else + { + return false; + } + } + return true; + }; + + if (CompareContent(RemoteContent, LocalContent)) + { + ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", + NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + } + else + { + ExtendableStringBuilder<128> SB; + for (const std::pair& BuildPart : AllBuildParts) + { + SB.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); + } + ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, SB.ToView()); + FolderContent LocalFolderState; + if (UpdateFolder(Storage, + BuildId, + Path, + LargeAttachmentSize, + PreferredMultipartChunkSize, + LocalContent, + RemoteContent, + BlockDescriptions, + LooseChunkHashes, + WipeTargetFolder, + AbortFlag, + LocalFolderState)) + { + AbortFlag = true; + } + if (!AbortFlag) + { + VerifyFolder(RemoteContent, Path, AbortFlag); + } + + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + + CreateDirectories((Path / ZenStateFilePath).parent_path()); + TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceLatencyNs(WriteStateTimer.GetElapsedTimeUs() * 1000)); + +#if 0 + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); +#endif // 0 + + ZEN_CONSOLE("Downloaded build in {}.", NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + } + + return AbortFlag.load(); + } + + bool DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) + { + std::atomic AbortFlag(false); + + ChunkedFolderContent BaseFolderContent; + ChunkedFolderContent CompareFolderContent; + + { + std::unique_ptr ChunkController = CreateBasicChunkingController(); + std::vector ExcludeExtensions = DefaultExcludeExtensions; + if (OnlyChunked) + { + ExcludeExtensions.insert(ExcludeExtensions.end(), + DefaultChunkingExcludeExtensions.begin(), + DefaultChunkingExcludeExtensions.end()); + } + + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { + for (const std::string_view& ExcludeExtension : ExcludeExtensions) + { + if (RelativePath.ends_with(ExcludeExtension)) + { + return false; + } + } + return true; + }; + + GetFolderContentStatistics BaseGetFolderContentStats; + ChunkingStatistics BaseChunkingStats; + BaseFolderContent = ScanAndChunkFolder(BaseGetFolderContentStats, + BaseChunkingStats, + BasePath, + IsAcceptedFolder, + IsAcceptedFile, + *ChunkController, + AbortFlag); + + GetFolderContentStatistics CompareGetFolderContentStats; + ChunkingStatistics CompareChunkingStats; + CompareFolderContent = ScanAndChunkFolder(CompareGetFolderContentStats, + CompareChunkingStats, + ComparePath, + IsAcceptedFolder, + IsAcceptedFile, + *ChunkController, + AbortFlag); + } + + std::vector AddedHashes; + std::vector RemovedHashes; + uint64_t RemovedSize = 0; + uint64_t AddedSize = 0; + + tsl::robin_map BaseRawHashLookup; + for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++) + { + const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; + BaseRawHashLookup.insert_or_assign(RawHash, PathIndex); + } + tsl::robin_map CompareRawHashLookup; + for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++) + { + const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex]; + if (!BaseRawHashLookup.contains(RawHash)) + { + AddedHashes.push_back(RawHash); + AddedSize += CompareFolderContent.RawSizes[PathIndex]; + } + CompareRawHashLookup.insert_or_assign(RawHash, PathIndex); + } + for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) + { + const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex]; + if (!CompareRawHashLookup.contains(RawHash)) + { + RemovedHashes.push_back(RawHash); + RemovedSize += BaseFolderContent.RawSizes[PathIndex]; + } + } + + uint64_t BaseTotalRawSize = 0; + for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++) + { + BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex]; + } + + double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; + + ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept", + RemovedHashes.size(), + NiceBytes(RemovedSize), + AddedHashes.size(), + NiceBytes(AddedSize), + BaseFolderContent.Paths.size() - RemovedHashes.size(), + NiceBytes(BaseTotalRawSize - RemovedSize), + KeptPercent); + + uint64_t CompareTotalRawSize = 0; + + uint64_t FoundChunkCount = 0; + uint64_t FoundChunkSize = 0; + uint64_t NewChunkCount = 0; + uint64_t NewChunkSize = 0; + const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent); + for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++) + { + const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex]; + if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash)) + { + FoundChunkCount++; + FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + else + { + NewChunkCount++; + NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + + double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; + double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; + + ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", + FoundChunkCount, + NiceBytes(FoundChunkSize), + FoundPercent, + CompareFolderContent.ChunkedContent.ChunkHashes.size(), + NiceBytes(CompareTotalRawSize), + BaseFolderContent.ChunkedContent.ChunkHashes.size(), + NiceBytes(BaseTotalRawSize), + NewChunkCount, + NiceBytes(NewChunkSize), + NewPercent); + + return false; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////////////////////////////////// + +BuildsCommand::BuildsCommand() +{ + m_Options.add_options()("h,help", "Print help"); + + auto AddAuthOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + + // Direct access token (may expire) + Ops.add_option("auth-token", + "", + "access-token", + "Cloud/Builds Storage access token", + cxxopts::value(m_AccessToken), + ""); + Ops.add_option("auth-token", + "", + "access-token-env", + "Name of environment variable that holds the cloud/builds Storage access token", + cxxopts::value(m_AccessTokenEnv)->default_value(DefaultAccessTokenEnvVariableName), + ""); + Ops.add_option("auth-token", + "", + "access-token-path", + "Path to json file that holds the cloud/builds Storage access token", + cxxopts::value(m_AccessTokenPath), + ""); + + // Auth manager token encryption + Ops.add_option("security", + "", + "encryption-aes-key", + "256 bit AES encryption key", + cxxopts::value(m_EncryptionKey), + ""); + Ops.add_option("security", + "", + "encryption-aes-iv", + "128 bit AES encryption initialization vector", + cxxopts::value(m_EncryptionIV), + ""); + + // OpenId acccess token + Ops.add_option("openid", + "", + "openid-provider-name", + "Open ID provider name", + cxxopts::value(m_OpenIdProviderName), + "Default"); + Ops.add_option("openid", "", "openid-provider-url", "Open ID provider url", cxxopts::value(m_OpenIdProviderUrl), ""); + Ops.add_option("openid", "", "openid-client-id", "Open ID client id", cxxopts::value(m_OpenIdClientId), ""); + Ops.add_option("openid", + "", + "openid-refresh-token", + "Open ID refresh token", + cxxopts::value(m_OpenIdRefreshToken), + ""); + + // OAuth acccess token + Ops.add_option("oauth", "", "oauth-url", "OAuth provier url", cxxopts::value(m_OAuthUrl)->default_value(""), ""); + Ops.add_option("oauth", + "", + "oauth-clientid", + "OAuth client id", + cxxopts::value(m_OAuthClientId)->default_value(""), + ""); + Ops.add_option("oauth", + "", + "oauth-clientsecret", + "OAuth client secret", + cxxopts::value(m_OAuthClientSecret)->default_value(""), + ""); + }; + + auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { + AddAuthOptions(Ops); + + Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), ""); + Ops.add_option("cloud build", + "", + "assume-http2", + "Assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake", + cxxopts::value(m_AssumeHttp2), + ""); + + Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), ""); + Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), ""); + }; + + auto AddFileOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("filestorage", "", "storage-path", "Builds Storage Path", cxxopts::value(m_StoragePath), ""); + Ops.add_option("filestorage", + "", + "json-metadata", + "Write build, part and block metadata as .json files in addition to .cb files", + cxxopts::value(m_WriteMetadataAsJson), + ""); + }; + + auto AddOutputOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); + Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); + }; + + m_Options.add_option("", "v", "verb", "Verb for build - list, upload, download, diff", cxxopts::value(m_Verb), ""); + m_Options.parse_positional({"verb"}); + m_Options.positional_help("verb"); + + // list + AddCloudOptions(m_ListOptions); + AddFileOptions(m_ListOptions); + AddOutputOptions(m_ListOptions); + m_ListOptions.add_options()("h,help", "Print help"); + + // upload + AddCloudOptions(m_UploadOptions); + AddFileOptions(m_UploadOptions); + AddOutputOptions(m_UploadOptions); + m_UploadOptions.add_options()("h,help", "Print help"); + m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); + m_UploadOptions.add_option("", + "", + "create-build", + "Set to true to create the containing build, if unset a builds-id must be given and the build already exist", + cxxopts::value(m_CreateBuild), + ""); + m_UploadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); + m_UploadOptions.add_option("", + "", + "build-part-id", + "Build part Id, if not given it will be auto generated", + cxxopts::value(m_BuildPartId), + ""); + m_UploadOptions.add_option("", + "", + "build-part-name", + "Name of the build part, if not given it will be be named after the directory name at end of local-path", + cxxopts::value(m_BuildPartName), + ""); + m_UploadOptions.add_option("", + "", + "metadata-path", + "Path to json file that holds the metadata for the build. Requires the create-build option to be set", + cxxopts::value(m_BuildMetadataPath), + ""); + m_UploadOptions.add_option( + "", + "", + "metadata", + "Key-value pairs separated by ';' with build meta data. (key1=value1;key2=value2). Requires the create-build option to be set", + cxxopts::value(m_BuildMetadata), + ""); + m_UploadOptions.add_option("", "", "clean", "Ignore existing blocks", cxxopts::value(m_Clean), ""); + m_UploadOptions.add_option("", + "", + "block-min-reuse", + "Percent of an existing block that must be relevant for it to be resused. Defaults to 85.", + cxxopts::value(m_BlockReuseMinPercentLimit), + ""); + m_UploadOptions.add_option("", + "", + "allow-multipart", + "Allow large attachments to be transfered using multipart protocol. Defaults to true.", + cxxopts::value(m_AllowMultiparts), + ""); + m_UploadOptions.add_option("", + "", + "manifest-path", + "Path to a text file with one line of [TAB] per file to include.", + cxxopts::value(m_ManifestPath), + ""); + + m_UploadOptions.parse_positional({"local-path", "build-id"}); + m_UploadOptions.positional_help("local-path build-id"); + + // download + AddCloudOptions(m_DownloadOptions); + AddFileOptions(m_DownloadOptions); + AddOutputOptions(m_DownloadOptions); + m_DownloadOptions.add_options()("h,help", "Print help"); + m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); + m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); + m_DownloadOptions.add_option( + "", + "", + "build-part-id", + "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartIds), + ""); + m_DownloadOptions.add_option( + "", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + ""); + m_DownloadOptions + .add_option("", "", "clean", "Delete all data in target folder before downloading", cxxopts::value(m_Clean), ""); + m_DownloadOptions.add_option("", + "", + "allow-multipart", + "Allow large attachments to be transfered using multipart protocol. Defaults to true.", + cxxopts::value(m_AllowMultiparts), + ""); + m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); + m_DownloadOptions.positional_help("local-path build-id build-part-name"); + + AddOutputOptions(m_DiffOptions); + m_DiffOptions.add_options()("h,help", "Print help"); + m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); + m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), ""); + m_DiffOptions.add_option("", + "", + "only-chunked", + "Skip files from diff summation that are not processed with chunking", + cxxopts::value(m_OnlyChunked), + ""); + m_DiffOptions.parse_positional({"local-path", "compare-path"}); + m_DiffOptions.positional_help("local-path compare-path"); + + AddCloudOptions(m_TestOptions); + AddFileOptions(m_TestOptions); + AddOutputOptions(m_TestOptions); + m_TestOptions.add_options()("h,help", "Print help"); + m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); + m_TestOptions.parse_positional({"local-path"}); + m_TestOptions.positional_help("local-path"); + + AddCloudOptions(m_FetchBlobOptions); + AddFileOptions(m_FetchBlobOptions); + AddOutputOptions(m_FetchBlobOptions); + m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); + m_FetchBlobOptions + .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); + m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"}); + m_FetchBlobOptions.positional_help("build-id blob-hash"); + + AddCloudOptions(m_ValidateBuildPartOptions); + AddFileOptions(m_ValidateBuildPartOptions); + AddOutputOptions(m_ValidateBuildPartOptions); + m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); + m_ValidateBuildPartOptions.add_option("", + "", + "build-part-id", + "Build part Id, if not given it will be auto generated", + cxxopts::value(m_BuildPartId), + ""); + m_ValidateBuildPartOptions.add_option( + "", + "", + "build-part-name", + "Name of the build part, if not given it will be be named after the directory name at end of local-path", + cxxopts::value(m_BuildPartName), + ""); + m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); + m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); +} + +BuildsCommand::~BuildsCommand() = default; + +int +BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + using namespace std::literals; + + std::vector SubCommandArguments; + cxxopts::Options* SubOption = nullptr; + int ParentCommandArgCount = GetSubCommand(m_Options, argc, argv, m_SubCommands, SubOption, SubCommandArguments); + if (!ParseOptions(ParentCommandArgCount, argv)) + { + return 0; + } + + if (SubOption == nullptr) + { + throw zen::OptionParseException("command verb is missing"); + } + + if (!ParseOptions(*SubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) + { + return 0; + } + + auto ParseStorageOptions = [&]() { + if (!m_BuildsUrl.empty()) + { + if (!m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help())); + } + if (m_Namespace.empty() || m_Bucket.empty()) + { + throw zen::OptionParseException( + fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); + } + } + }; + + std::unique_ptr Auth; + HttpClientSettings ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2}; + + auto CreateAuthMgr = [&]() { + if (!Auth) + { + std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : m_SystemRootDir; + + if (m_EncryptionKey.empty()) + { + m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; + ZEN_CONSOLE("Warning: Using default encryption key"); + } + + if (m_EncryptionIV.empty()) + { + m_EncryptionIV = "0123456789abcdef"; + ZEN_CONSOLE("Warning: Using default encryption initialization vector"); + } + + AuthConfig AuthMgrConfig = {.RootDirectory = DataRoot / "auth", + .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), + .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; + if (!AuthMgrConfig.EncryptionKey.IsValid()) + { + throw zen::OptionParseException("Invalid AES encryption key"); + } + if (!AuthMgrConfig.EncryptionIV.IsValid()) + { + throw zen::OptionParseException("Invalid AES initialization vector"); + } + Auth = AuthMgr::Create(AuthMgrConfig); + } + }; + + auto ParseAuthOptions = [&]() { + if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) + { + CreateAuthMgr(); + std::string ProviderName = m_OpenIdProviderName.empty() ? "Default" : m_OpenIdProviderName; + Auth->AddOpenIdProvider({.Name = ProviderName, .Url = m_OpenIdProviderUrl, .ClientId = m_OpenIdClientId}); + if (!m_OpenIdRefreshToken.empty()) + { + Auth->AddOpenIdToken({.ProviderName = ProviderName, .RefreshToken = m_OpenIdRefreshToken}); + } + } + + if (!m_AccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); + } + else if (!m_AccessTokenPath.empty()) + { + std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); + if (!ResolvedAccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); + } + } + else if (!m_AccessTokenEnv.empty()) + { + std::string ResolvedAccessToken = GetEnvVariable(m_AccessTokenEnv); + if (!ResolvedAccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); + } + } + else if (!m_OAuthUrl.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials( + {.Url = m_OAuthUrl, .ClientId = m_OAuthClientId, .ClientSecret = m_OAuthClientSecret}); + } + else if (!m_OpenIdProviderName.empty()) + { + CreateAuthMgr(); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName); + } + else + { + CreateAuthMgr(); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); + } + + if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider) + { + ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials."); + } + }; + + auto ParseOutputOptions = [&]() { + IsVerbose = m_Verbose; + UsePlainProgress = IsVerbose || m_PlainProgress; + }; + ParseOutputOptions(); + + if (SubOption == &m_ListOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + CbObjectWriter QueryWriter; + QueryWriter.BeginObject("query"); + { + // QueryWriter.BeginObject("platform"); + // { + // QueryWriter.AddString("$eq", "Windows"); + // } + // QueryWriter.EndObject(); // changelist + } + QueryWriter.EndObject(); // query + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Querying builds in folder '{}'.", m_StoragePath); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + CbObject Response = Storage->ListBuilds(QueryWriter.Save()); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + return 0; + } + + if (SubOption == &m_UploadOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); + } + + if (m_CreateBuild) + { + if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); + } + } + else + { + if (!m_BuildMetadataPath.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadata.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + } + + if (m_BuildPartName.empty()) + { + m_BuildPartName = m_Path.filename().string(); + } + + const bool GeneratedBuildId = m_BuildId.empty(); + if (GeneratedBuildId) + { + m_BuildId = Oid::NewOid().ToString(); + } + else if (m_BuildId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid::FromHexString(m_BuildId) == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + + const bool GeneratedBuildPartId = m_BuildPartId.empty(); + if (GeneratedBuildPartId) + { + m_BuildPartId = Oid::NewOid().ToString(); + } + else if (m_BuildPartId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } + + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + std::unique_ptr Storage; + std::string StorageName; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", + m_BuildPartName, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + GeneratedBuildId ? "Generated " : "", + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", + m_BuildPartName, + m_Path, + m_StoragePath, + GeneratedBuildId ? "Generated " : "", + BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + CbObject MetaData; + if (m_CreateBuild) + { + if (!m_BuildMetadataPath.empty()) + { + std::filesystem::path MetadataPath(m_BuildMetadataPath); + IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); + std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + } + } + if (!m_BuildMetadata.empty()) + { + CbObjectWriter MetaDataWriter(1024); + ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { + size_t SplitPos = Pair.find('='); + if (SplitPos == std::string::npos || SplitPos == 0) + { + throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); + } + MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); + return true; + }); + MetaData = MetaDataWriter.Save(); + } + } + + bool Aborted = UploadFolder(*Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + m_ManifestPath, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + m_CreateBuild, + m_Clean); + if (Aborted) + { + ZEN_CONSOLE("Upload failed."); + } + + if (false) + { + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } + return Aborted ? 11 : 0; + } + + if (SubOption == &m_DownloadOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } + if (m_BuildId.empty()) + { + throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); + } + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); + } + + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + } + + std::vector BuildPartIds; + for (const std::string& BuildPartId : m_BuildPartIds) + { + BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); + if (BuildPartIds.back() == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + } + } + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + std::string StorageName; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + BuildId, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, m_Path, m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + bool Aborted = DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean); + if (Aborted) + { + ZEN_CONSOLE("Download failed."); + } + if (false) + { + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } + + return Aborted ? 11 : 0; + } + if (SubOption == &m_DiffOptions) + { + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } + if (m_DiffPath.empty()) + { + throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); + } + bool Aborted = DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); + return Aborted ? 11 : 0; + } + + if (SubOption == &m_TestOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } + + m_BuildId = Oid::NewOid().ToString(); + m_BuildPartName = m_Path.filename().string(); + m_BuildPartId = Oid::NewOid().ToString(); + m_CreateBuild = true; + + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + std::unique_ptr Storage; + std::string StorageName; + + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + m_StoragePath = GetRunningExecutablePath().parent_path() / ".tmpstore"; + CreateDirectories(m_StoragePath); + CleanDirectory(m_StoragePath); + } + auto _ = MakeGuard([&]() { + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + DeleteDirectories(m_StoragePath); + } + }); + + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", + m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, + m_Path, + m_StoragePath, + BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + auto MakeMetaData = [](const Oid& BuildId) -> CbObject { + CbObjectWriter BuildMetaDataWriter; + { + const uint32_t CL = BuildId.OidBits[2]; + BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); + BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("platform", "Windows"); + BuildMetaDataWriter.AddString("project", "Test"); + BuildMetaDataWriter.AddInteger("changelist", CL); + BuildMetaDataWriter.AddString("buildType", "test-folder"); + } + return BuildMetaDataWriter.Save(); + }; + CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); + } + + bool Aborted = UploadFolder(*Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + {}, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + m_CreateBuild, + m_Clean); + if (Aborted) + { + ZEN_CONSOLE("Upload failed."); + return 11; + } + + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true); + if (Aborted) + { + ZEN_CONSOLE("Download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (Aborted) + { + ZEN_CONSOLE("Re-download failed. (identical target)"); + return 11; + } + + auto ScrambleDir = [](const std::filesystem::path& Path) { + ZEN_CONSOLE("\nScrambling '{}'", Path); + Stopwatch Timer; + DirectoryContent DownloadContent; + GetDirectoryContent( + Path, + DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + DownloadContent); + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { + std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); + for (const std::string_view& ExcludeFolder : ExcludeFolders) + { + if (RelativePath.starts_with(ExcludeFolder)) + { + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } + } + } + return true; + }; + + std::atomic AbortFlag = false; + ParallellWork Work(AbortFlag); + + uint32_t Randomizer = 0; + auto FileSizeIt = DownloadContent.FileSizes.begin(); + for (const std::filesystem::path& FilePath : DownloadContent.Files) + { + if (IsAcceptedFolder(FilePath)) + { + uint32_t Case = (Randomizer++) % 7; + switch (Case) + { + case 0: + { + uint64_t SourceSize = *FileSizeIt; + if (SourceSize > 0) + { + Work.ScheduleWork( + GetMediumWorkerPool(EWorkloadType::Burst), + [SourceSize, FilePath](std::atomic& AbortFlag) { + if (!AbortFlag) + { + IoBuffer Scrambled(SourceSize); + { + IoBuffer Source = IoBufferBuilder::MakeFromFile(FilePath); + Scrambled.GetMutableView().CopyFrom( + Source.GetView().Mid(SourceSize / 3, SourceSize / 3)); + Scrambled.GetMutableView() + .Mid(SourceSize / 3) + .CopyFrom(Source.GetView().Mid(0, SourceSize / 3)); + Scrambled.GetMutableView() + .Mid((SourceSize / 3) * 2) + .CopyFrom(Source.GetView().Mid(SourceSize / 2, SourceSize / 3)); + } + bool IsReadOnly = SetFileReadOnly(FilePath, false); + WriteFile(FilePath, Scrambled); + if (IsReadOnly) + { + SetFileReadOnly(FilePath, true); + } + } + }, + [FilePath](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed scrambling file {}. Reason: {}", FilePath, Ex.what()); + AbortFlag = true; + }); + } + } + break; + case 1: + std::filesystem::remove(FilePath); + break; + default: + break; + } + } + FileSizeIt++; + } + Work.Wait(5000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted); + ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); + }); + ZEN_ASSERT(!AbortFlag.load()); + ZEN_CONSOLE("Scrambled files in {}", NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + }; + + ScrambleDir(DownloadPath); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (Aborted) + { + ZEN_CONSOLE("Re-download failed. (scrambled target)"); + return 11; + } + + ScrambleDir(DownloadPath); + + Oid BuildId2 = Oid::NewOid(); + Oid BuildPartId2 = Oid::NewOid(); + + CbObject MetaData2 = MakeMetaData(BuildId2); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); + } + + Aborted = UploadFolder(*Storage, + BuildId2, + BuildPartId2, + m_BuildPartName, + DownloadPath, + {}, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData2, + true, + false); + if (Aborted) + { + ZEN_CONSOLE("Upload of scrambled failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (Aborted) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + if (Aborted) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + Aborted = DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + if (Aborted) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } + + return 0; + } + + if (SubOption == &m_FetchBlobOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + if (m_BlobHash.empty()) + { + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + } + + IoHash BlobHash; + if (!IoHash::TryParse(m_BlobHash, BlobHash)) + { + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + } + + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } + + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + std::unique_ptr Storage; + std::string StorageName; + + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); + return 0; + } + + if (SubOption == &m_ValidateBuildPartOptions) + { + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } + + if (m_BuildId.empty()) + { + throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); + } + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); + } + + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + } + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + std::string StorageName; + + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); + CbObject Build = Storage->GetBuild(BuildId); + if (!m_BuildPartName.empty()) + { + BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); + if (BuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); + } + } + CbObject BuildPart = Storage->GetBuildPart(BuildId, BuildPartId); + ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + std::vector ChunkAttachments; + for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) + { + ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); + } + std::vector BlockAttachments; + for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) + { + BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); + } + + for (const IoHash& ChunkAttachment : ChunkAttachments) + { + uint64_t CompressedSize; + uint64_t DecompressedSize; + try + { + ValidateBlob(*Storage, BuildId, ChunkAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE("Chunk attachment {} ({} -> {}) is valid", + ChunkAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed validating chunk attachment {}: {}", ChunkAttachment, Ex.what()); + } + } + + for (const IoHash& BlockAttachment : BlockAttachments) + { + uint64_t CompressedSize; + uint64_t DecompressedSize; + try + { + ValidateChunkBlock(*Storage, BuildId, BlockAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE("Block attachment {} ({} -> {}) is valid", + BlockAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed validating block attachment {}: {}", BlockAttachment, Ex.what()); + } + } + + return 0; + } + + ZEN_ASSERT(false); +} + +} // namespace zen diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h new file mode 100644 index 000000000..fa223943b --- /dev/null +++ b/src/zen/cmds/builds_cmd.h @@ -0,0 +1,106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include +#include +#include + +namespace zen { + +class BuildsCommand : public CacheStoreCommand +{ +public: + static constexpr char Name[] = "builds"; + static constexpr char Description[] = "Manage builds - list, upload, download, diff"; + + BuildsCommand(); + ~BuildsCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + + std::filesystem::path m_SystemRootDir; + + bool m_PlainProgress = false; + bool m_Verbose = false; + + // cloud builds + std::string m_BuildsUrl; + bool m_AssumeHttp2 = false; + std::string m_Namespace; + std::string m_Bucket; + + // file storage + std::filesystem::path m_StoragePath; + bool m_WriteMetadataAsJson = false; + + std::string m_BuildId; + bool m_CreateBuild = false; + std::string m_BuildMetadataPath; + std::string m_BuildMetadata; + std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path + std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_AllowMultiparts = true; + std::filesystem::path m_ManifestPath; + + // Direct access token (may expire) + std::string m_AccessToken; + std::string m_AccessTokenEnv; + std::string m_AccessTokenPath; + + // Auth manager token encryption + std::string m_EncryptionKey; // 256 bit AES encryption key + std::string m_EncryptionIV; // 128 bit AES initialization vector + + // OpenId acccess token + std::string m_OpenIdProviderName; + std::string m_OpenIdProviderUrl; + std::string m_OpenIdClientId; + std::string m_OpenIdRefreshToken; + + // OAuth acccess token + std::string m_OAuthUrl; + std::string m_OAuthClientId; + std::string m_OAuthClientSecret; + + std::string m_Verb; // list, upload, download + + cxxopts::Options m_ListOptions{"list", "List available builds"}; + + std::filesystem::path m_Path; + + cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; + + cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; + std::vector m_BuildPartNames; + std::vector m_BuildPartIds; + + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; + std::filesystem::path m_DiffPath; + bool m_OnlyChunked = false; + + cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; + + cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; + std::string m_BlobHash; + + cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; + + cxxopts::Options* m_SubCommands[7] = {&m_ListOptions, + &m_UploadOptions, + &m_DownloadOptions, + &m_DiffOptions, + &m_TestOptions, + &m_FetchBlobOptions, + &m_ValidateBuildPartOptions}; +}; + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 872ea8941..2e230ed53 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -7,6 +7,7 @@ #include "cmds/admin_cmd.h" #include "cmds/bench_cmd.h" +#include "cmds/builds_cmd.h" #include "cmds/cache_cmd.h" #include "cmds/copy_cmd.h" #include "cmds/dedup_cmd.h" @@ -396,6 +397,7 @@ main(int argc, char** argv) AttachCommand AttachCmd; BenchCommand BenchCmd; + BuildsCommand BuildsCmd; CacheDetailsCommand CacheDetailsCmd; CacheGetCommand CacheGetCmd; CacheGenerateCommand CacheGenerateCmd; @@ -451,6 +453,7 @@ main(int argc, char** argv) // clang-format off {"attach", &AttachCmd, "Add a sponsor process to a running zen service"}, {"bench", &BenchCmd, "Utility command for benchmarking"}, + {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, {"cache-details", &CacheDetailsCmd, "Details on cache"}, {"cache-info", &CacheInfoCmd, "Info on cache, namespace or bucket"}, {CacheGetCommand::Name, &CacheGetCmd, CacheGetCommand::Description}, diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 5716d1255..8279fb952 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1469,6 +1469,36 @@ GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec) return 0; } +uint64_t +GetModificationTickFromPath(const std::filesystem::path& Filename) +{ + // PathFromHandle + void* Handle; +#if ZEN_PLATFORM_WINDOWS + Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); +#else + int Fd = open(Filename.c_str(), O_RDONLY | O_CLOEXEC); + if (Fd <= 9) + { + ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); + } + Handle = (void*)uintptr_t(Fd); + auto _ = MakeGuard([Handle]() { close(int(uintptr_t(Handle))); }); +#endif + std::error_code Ec; + uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); + if (Ec) + { + ThrowSystemError(Ec.value(), Ec.message()); + } + return ModificatonTick; +} + std::filesystem::path GetRunningExecutablePath() { @@ -1895,6 +1925,116 @@ PickDefaultSystemRootDirectory() #endif // ZEN_PLATFORM_WINDOWS } +#if ZEN_PLATFORM_WINDOWS + +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + } + return (uint32_t)Attributes; +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) + { + ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + } +} + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +uint32_t +GetFileMode(const std::filesystem::path& Filename) +{ + struct stat Stat; + int err = stat(Filename.native().c_str(), &Stat); + if (err) + { + ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + } + return (uint32_t)Stat.st_mode; +} + +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +{ + int err = chmod(Filename.native().c_str(), (mode_t)Attributes); + if (err) + { + ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + } +} + +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; +#endif // ZEN_PLATFORM_WINDOWS + +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsFileModeReadOnly(uint32_t FileMode) +{ + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ +#if ZEN_PLATFORM_WINDOWS + uint32_t CurrentAttributes = GetFileAttributes(Filename); + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + if (CurrentAttributes != NewAttributes) + { + SetFileAttributes(Filename, NewAttributes); + return true; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + uint32_t CurrentMode = GetFileMode(Filename); + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + if (CurrentMode != NewMode) + { + SetFileMode(Filename, NewMode); + return true; + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + return false; +} + ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 250745e86..20f6dc56c 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -52,6 +52,10 @@ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); */ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +/** Get a native time tick of last modification time + */ +ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -271,6 +275,23 @@ std::error_code RotateDirectories(const std::filesystem::path& DirectoryName, st std::filesystem::path PickDefaultSystemRootDirectory(); +#if ZEN_PLATFORM_WINDOWS +uint32_t GetFileAttributes(const std::filesystem::path& Filename); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +uint32_t GetFileMode(const std::filesystem::path& Filename); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +bool IsFileAttributeReadOnly(uint32_t FileAttributes); +bool IsFileModeReadOnly(uint32_t FileMode); +uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); +uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); + +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index e4e91104c..fbb9bc344 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -344,7 +344,12 @@ public: m_BuildId); return Result; } - Result.Blocks = std::move(Blocks.value()); + Result.Blocks.reserve(Blocks.value().size()); + for (ChunkBlockDescription& BlockDescription : Blocks.value()) + { + Result.Blocks.push_back(ThinChunkBlockDescription{.BlockHash = BlockDescription.BlockHash, + .ChunkRawHashes = std::move(BlockDescription.ChunkRawHashes)}); + } return Result; } diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 5a21a7540..98e292d91 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -192,8 +192,8 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; } - std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); - GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; + std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; Result.Blocks = std::move(KnownBlocks); return Result; } diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index 2b6a437d1..e5839ad3b 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -193,7 +193,7 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds}}; } - std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); + std::vector KnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); GetKnownBlocksResult Result{ {.ElapsedSeconds = LoadResult.ElapsedSeconds + ExistsResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000.0}}; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index f6f7eba99..53df12b14 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -8628,7 +8628,11 @@ TEST_CASE("project.store.block") } ChunkBlockDescription Block; CompressedBuffer BlockBuffer = GenerateChunkBlock(std::move(Chunks), Block); - CHECK(IterateChunkBlock(BlockBuffer.Decompress(), [](CompressedBuffer&&, const IoHash&) {})); + uint64_t HeaderSize; + CHECK(IterateChunkBlock( + BlockBuffer.Decompress(), + [](CompressedBuffer&&, const IoHash&) {}, + HeaderSize)); } TEST_CASE("project.store.iterateoplog") diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index b4b2c6fc4..a7263da83 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -516,21 +516,23 @@ namespace remotestore_impl { return; } - bool StoreChunksOK = IterateChunkBlock( - BlockPayload, - [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk, - const IoHash& AttachmentRawHash) { - if (WantedChunks.contains(AttachmentRawHash)) - { - WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); - IoHash RawHash; - uint64_t RawSize; - ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize)); - ZEN_ASSERT(RawHash == AttachmentRawHash); - WriteRawHashes.emplace_back(AttachmentRawHash); - WantedChunks.erase(AttachmentRawHash); - } - }); + uint64_t BlockHeaderSize = 0; + bool StoreChunksOK = IterateChunkBlock( + BlockPayload, + [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info](CompressedBuffer&& Chunk, + const IoHash& AttachmentRawHash) { + if (WantedChunks.contains(AttachmentRawHash)) + { + WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + IoHash RawHash; + uint64_t RawSize; + ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), RawHash, RawSize)); + ZEN_ASSERT(RawHash == AttachmentRawHash); + WriteRawHashes.emplace_back(AttachmentRawHash); + WantedChunks.erase(AttachmentRawHash); + } + }, + BlockHeaderSize); if (!StoreChunksOK) { @@ -1101,11 +1103,11 @@ GetBlockHashesFromOplog(CbObjectView ContainerObject) return BlockHashes; } -std::vector +std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes) { using namespace std::literals; - std::vector Result; + std::vector Result; CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); tsl::robin_set IncludeSet; IncludeSet.insert(IncludeBlockHashes.begin(), IncludeBlockHashes.end()); @@ -1128,7 +1130,7 @@ GetBlocksFromOplog(CbObjectView ContainerObject, std::span Include { ChunkHashes.push_back(ChunkField.AsHash()); } - Result.push_back({.BlockHash = BlockHash, .ChunkHashes = std::move(ChunkHashes)}); + Result.push_back(ThinChunkBlockDescription{.BlockHash = BlockHash, .ChunkRawHashes = std::move(ChunkHashes)}); } } return Result; @@ -1144,7 +1146,7 @@ BuildContainer(CidStore& ChunkStore, bool BuildBlocks, bool IgnoreMissingAttachments, bool AllowChunking, - const std::vector& KnownBlocks, + const std::vector& KnownBlocks, WorkerThreadPool& WorkerPool, const std::function& AsyncOnBlock, const std::function& OnLargeAttachment, @@ -1386,7 +1388,7 @@ BuildContainer(CidStore& ChunkStore, return {}; } - auto FindReuseBlocks = [](const std::vector& KnownBlocks, + auto FindReuseBlocks = [](const std::vector& KnownBlocks, const std::unordered_set& Attachments, JobContext* OptionalContext) -> std::vector { std::vector ReuseBlockIndexes; @@ -1399,14 +1401,14 @@ BuildContainer(CidStore& ChunkStore, for (size_t KnownBlockIndex = 0; KnownBlockIndex < KnownBlocks.size(); KnownBlockIndex++) { - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; - size_t BlockAttachmentCount = KnownBlock.ChunkHashes.size(); + const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + size_t BlockAttachmentCount = KnownBlock.ChunkRawHashes.size(); if (BlockAttachmentCount == 0) { continue; } size_t FoundAttachmentCount = 0; - for (const IoHash& KnownHash : KnownBlock.ChunkHashes) + for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes) { if (Attachments.contains(KnownHash)) { @@ -1447,8 +1449,8 @@ BuildContainer(CidStore& ChunkStore, std::vector ReusedBlockIndexes = FindReuseBlocks(KnownBlocks, FoundHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; - for (const IoHash& KnownHash : KnownBlock.ChunkHashes) + const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes) { if (UploadAttachments.erase(KnownHash) == 1) { @@ -1784,8 +1786,8 @@ BuildContainer(CidStore& ChunkStore, std::vector ReusedBlockFromChunking = FindReuseBlocks(KnownBlocks, ChunkedHashes, OptionalContext); for (size_t KnownBlockIndex : ReusedBlockIndexes) { - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; - for (const IoHash& KnownHash : KnownBlock.ChunkHashes) + const ThinChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + for (const IoHash& KnownHash : KnownBlock.ChunkRawHashes) { if (ChunkedHashes.erase(KnownHash) == 1) { @@ -1803,7 +1805,7 @@ BuildContainer(CidStore& ChunkStore, Blocks.reserve(ReuseBlockCount); for (auto It = ReusedBlockIndexes.begin(); It != UniqueKnownBlocksEnd; It++) { - Blocks.push_back(KnownBlocks[*It]); + Blocks.push_back({KnownBlocks[*It]}); } remotestore_impl::ReportMessage(OptionalContext, fmt::format("Reused {} attachments from {} blocks", ReusedAttachmentCount, ReuseBlockCount)); @@ -1919,9 +1921,9 @@ BuildContainer(CidStore& ChunkStore, { // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index RwLock::SharedLockScope _(BlocksLock); - Blocks[BlockIndex].ChunkHashes.insert(Blocks[BlockIndex].ChunkHashes.end(), - BlockAttachmentHashes.begin(), - BlockAttachmentHashes.end()); + Blocks[BlockIndex].ChunkRawHashes.insert(Blocks[BlockIndex].ChunkRawHashes.end(), + BlockAttachmentHashes.begin(), + BlockAttachmentHashes.end()); } uint64_t NowMS = Timer.GetElapsedTimeMs(); ZEN_INFO("Assembled block {} with {} chunks in {} ({})", @@ -2167,7 +2169,7 @@ BuildContainer(CidStore& ChunkStore, { for (const ChunkBlockDescription& B : Blocks) { - ZEN_ASSERT(!B.ChunkHashes.empty()); + ZEN_ASSERT(!B.ChunkRawHashes.empty()); if (BuildBlocks) { ZEN_ASSERT(B.BlockHash != IoHash::Zero); @@ -2177,7 +2179,7 @@ BuildContainer(CidStore& ChunkStore, OplogContinerWriter.AddBinaryAttachment("rawhash"sv, B.BlockHash); OplogContinerWriter.BeginArray("chunks"sv); { - for (const IoHash& RawHash : B.ChunkHashes) + for (const IoHash& RawHash : B.ChunkRawHashes) { OplogContinerWriter.AddHash(RawHash); } @@ -2193,7 +2195,7 @@ BuildContainer(CidStore& ChunkStore, { OplogContinerWriter.BeginArray("chunks"sv); { - for (const IoHash& RawHash : B.ChunkHashes) + for (const IoHash& RawHash : B.ChunkRawHashes) { OplogContinerWriter.AddBinaryAttachment(RawHash); } @@ -2389,7 +2391,7 @@ SaveOplog(CidStore& ChunkStore, OnBlock = UploadBlock; } - std::vector KnownBlocks; + std::vector KnownBlocks; uint64_t TransferWallTimeMS = 0; diff --git a/src/zenserver/projectstore/remoteprojectstore.h b/src/zenserver/projectstore/remoteprojectstore.h index 1ef0416b7..1210afc7c 100644 --- a/src/zenserver/projectstore/remoteprojectstore.h +++ b/src/zenserver/projectstore/remoteprojectstore.h @@ -66,7 +66,7 @@ public: struct GetKnownBlocksResult : public Result { - std::vector Blocks; + std::vector Blocks; }; struct RemoteStoreInfo @@ -166,7 +166,7 @@ RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, bool CleanOplog, JobContext* OptionalContext); -std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); -std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); +std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); +std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); } // namespace zen diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index 6dae5af11..a19cf5c1b 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -18,20 +19,27 @@ ParseChunkBlockDescription(const CbObjectView& BlockObject) Result.BlockHash = BlockObject["rawHash"sv].AsHash(); if (Result.BlockHash != IoHash::Zero) { + Result.HeaderSize = BlockObject["headerSize"sv].AsUInt64(); CbArrayView ChunksArray = BlockObject["rawHashes"sv].AsArrayView(); - Result.ChunkHashes.reserve(ChunksArray.Num()); + Result.ChunkRawHashes.reserve(ChunksArray.Num()); for (CbFieldView ChunkView : ChunksArray) { - Result.ChunkHashes.push_back(ChunkView.AsHash()); + Result.ChunkRawHashes.push_back(ChunkView.AsHash()); } - CbArrayView ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView(); - std::vector ChunkLengths; + CbArrayView ChunkRawLengthsArray = BlockObject["chunkRawLengths"sv].AsArrayView(); Result.ChunkRawLengths.reserve(ChunkRawLengthsArray.Num()); for (CbFieldView ChunkView : ChunkRawLengthsArray) { Result.ChunkRawLengths.push_back(ChunkView.AsUInt32()); } + + CbArrayView ChunkCompressedLengthsArray = BlockObject["chunkCompressedLengths"sv].AsArrayView(); + Result.ChunkCompressedLengths.reserve(ChunkCompressedLengthsArray.Num()); + for (CbFieldView ChunkView : ChunkCompressedLengthsArray) + { + Result.ChunkCompressedLengths.push_back(ChunkView.AsUInt32()); + } } return Result; } @@ -57,18 +65,23 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData) { - ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkHashes.size()); + ZEN_ASSERT(Block.BlockHash != IoHash::Zero); + ZEN_ASSERT(Block.HeaderSize > 0); + ZEN_ASSERT(Block.ChunkRawLengths.size() == Block.ChunkRawHashes.size()); + ZEN_ASSERT(Block.ChunkCompressedLengths.size() == Block.ChunkRawHashes.size()); CbObjectWriter Writer; Writer.AddHash("rawHash"sv, Block.BlockHash); + Writer.AddInteger("headerSize"sv, Block.HeaderSize); Writer.BeginArray("rawHashes"sv); { - for (const IoHash& ChunkHash : Block.ChunkHashes) + for (const IoHash& ChunkHash : Block.ChunkRawHashes) { Writer.AddHash(ChunkHash); } } Writer.EndArray(); + Writer.BeginArray("chunkRawLengths"); { for (uint32_t ChunkSize : Block.ChunkRawLengths) @@ -78,11 +91,58 @@ BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView Meta } Writer.EndArray(); + Writer.BeginArray("chunkCompressedLengths"); + { + for (uint32_t ChunkSize : Block.ChunkCompressedLengths) + { + Writer.AddInteger(ChunkSize); + } + } + Writer.EndArray(); + Writer.AddObject("metadata", MetaData); return Writer.Save(); } +ChunkBlockDescription +GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash) +{ + ChunkBlockDescription BlockDescription = {{.BlockHash = IoHash::HashBuffer(BlockPayload)}}; + if (BlockDescription.BlockHash != RawHash) + { + throw std::runtime_error(fmt::format("Block {} content hash {} does not match block hash", RawHash, BlockDescription.BlockHash)); + } + if (IterateChunkBlock( + BlockPayload, + [&BlockDescription, RawHash](CompressedBuffer&& Chunk, const IoHash& AttachmentHash) { + if (CompositeBuffer Decompressed = Chunk.DecompressToComposite(); Decompressed) + { + IoHash ChunkHash = IoHash::HashBuffer(Decompressed.Flatten()); + if (ChunkHash != AttachmentHash) + { + throw std::runtime_error( + fmt::format("Chunk {} in block {} content hash {} does not match chunk", AttachmentHash, RawHash, ChunkHash)); + } + BlockDescription.ChunkRawHashes.push_back(AttachmentHash); + BlockDescription.ChunkRawLengths.push_back(gsl::narrow(Decompressed.GetSize())); + BlockDescription.ChunkCompressedLengths.push_back(gsl::narrow(Chunk.GetCompressedSize())); + } + else + { + throw std::runtime_error(fmt::format("Chunk {} in block {} is not a compressed buffer", AttachmentHash, RawHash)); + } + }, + BlockDescription.HeaderSize)) + { + return BlockDescription; + } + else + { + throw std::runtime_error(fmt::format("Block {} is malformed", RawHash)); + } +} + CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock) { @@ -91,8 +151,9 @@ GenerateChunkBlock(std::vector>&& FetchChunks, std::vector ChunkSegments; ChunkSegments.resize(1); ChunkSegments.reserve(1 + ChunkCount); - OutBlock.ChunkHashes.reserve(ChunkCount); + OutBlock.ChunkRawHashes.reserve(ChunkCount); OutBlock.ChunkRawLengths.reserve(ChunkCount); + OutBlock.ChunkCompressedLengths.reserve(ChunkCount); { IoBuffer TempBuffer(ChunkCount * 9); MutableMemoryView View = TempBuffer.GetMutableView(); @@ -106,16 +167,19 @@ GenerateChunkBlock(std::vector>&& FetchChunks, std::span Segments = Chunk.second.GetCompressed().GetSegments(); for (const SharedBuffer& Segment : Segments) { + ZEN_ASSERT(Segment.IsOwned()); ChunkSize += Segment.GetSize(); ChunkSegments.push_back(Segment); } BufferEndPtr += WriteVarUInt(ChunkSize, BufferEndPtr); - OutBlock.ChunkHashes.push_back(It.first); + OutBlock.ChunkRawHashes.push_back(It.first); OutBlock.ChunkRawLengths.push_back(gsl::narrow(Chunk.first)); + OutBlock.ChunkCompressedLengths.push_back(gsl::narrow(ChunkSize)); } ZEN_ASSERT(BufferEndPtr <= View.GetDataEnd()); ptrdiff_t TempBufferLength = std::distance(BufferStartPtr, BufferEndPtr); ChunkSegments[0] = SharedBuffer(IoBuffer(TempBuffer, 0, gsl::narrow(TempBufferLength))); + OutBlock.HeaderSize = TempBufferLength; } CompressedBuffer CompressedBlock = CompressedBuffer::Compress(CompositeBuffer(std::move(ChunkSegments)), OodleCompressor::Mermaid, OodleCompressionLevel::None); @@ -124,7 +188,9 @@ GenerateChunkBlock(std::vector>&& FetchChunks, } bool -IterateChunkBlock(const SharedBuffer& BlockPayload, std::function Visitor) +IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor, + uint64_t& OutHeaderSize) { ZEN_ASSERT(BlockPayload); if (BlockPayload.GetSize() < 1) @@ -144,21 +210,23 @@ IterateChunkBlock(const SharedBuffer& BlockPayload, std::function + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +namespace { + void AddCunkSequence(ChunkingStatistics& Stats, + ChunkedContentData& InOutChunkedContent, + tsl::robin_map& ChunkHashToChunkIndex, + const IoHash& RawHash, + std::span ChunkSequence, + std::span ChunkHashes, + std::span ChunkRawSizes) + { + ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size()); + InOutChunkedContent.ChunkCounts.push_back(gsl::narrow(ChunkSequence.size())); + InOutChunkedContent.ChunkOrders.reserve(InOutChunkedContent.ChunkOrders.size() + ChunkSequence.size()); + + for (uint32_t ChunkedSequenceIndex : ChunkSequence) + { + const IoHash& ChunkHash = ChunkHashes[ChunkedSequenceIndex]; + if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end()) + { + uint32_t ChunkIndex = gsl::narrow(It->second); + InOutChunkedContent.ChunkOrders.push_back(ChunkIndex); + } + else + { + uint32_t ChunkIndex = gsl::narrow(InOutChunkedContent.ChunkHashes.size()); + ChunkHashToChunkIndex.insert_or_assign(ChunkHash, ChunkIndex); + InOutChunkedContent.ChunkHashes.push_back(ChunkHash); + InOutChunkedContent.ChunkRawSizes.push_back(ChunkRawSizes[ChunkedSequenceIndex]); + InOutChunkedContent.ChunkOrders.push_back(ChunkIndex); + Stats.UniqueChunksFound++; + Stats.UniqueBytesFound += ChunkRawSizes[ChunkedSequenceIndex]; + } + } + InOutChunkedContent.SequenceRawHashes.push_back(RawHash); + Stats.UniqueSequencesFound++; + } + + void AddCunkSequence(ChunkingStatistics& Stats, + ChunkedContentData& InOutChunkedContent, + tsl::robin_map& ChunkHashToChunkIndex, + const IoHash& RawHash, + const uint64_t RawSize) + { + InOutChunkedContent.ChunkCounts.push_back(1); + + if (auto It = ChunkHashToChunkIndex.find(RawHash); It != ChunkHashToChunkIndex.end()) + { + uint32_t ChunkIndex = gsl::narrow(It->second); + InOutChunkedContent.ChunkOrders.push_back(ChunkIndex); + } + else + { + uint32_t ChunkIndex = gsl::narrow(InOutChunkedContent.ChunkHashes.size()); + ChunkHashToChunkIndex.insert_or_assign(RawHash, ChunkIndex); + InOutChunkedContent.ChunkHashes.push_back(RawHash); + InOutChunkedContent.ChunkRawSizes.push_back(RawSize); + InOutChunkedContent.ChunkOrders.push_back(ChunkIndex); + Stats.UniqueChunksFound++; + Stats.UniqueBytesFound += RawSize; + } + InOutChunkedContent.SequenceRawHashes.push_back(RawHash); + Stats.UniqueSequencesFound++; + } + + IoHash HashOneFile(ChunkingStatistics& Stats, + const ChunkingController& InChunkingController, + ChunkedFolderContent& OutChunkedContent, + tsl::robin_map& ChunkHashToChunkIndex, + tsl::robin_map& RawHashToSequenceRawHashIndex, + RwLock& Lock, + const std::filesystem::path& FolderPath, + uint32_t PathIndex) + { + const uint64_t RawSize = OutChunkedContent.RawSizes[PathIndex]; + const std::filesystem::path& Path = OutChunkedContent.Paths[PathIndex]; + + if (RawSize == 0) + { + return IoHash::Zero; + } + else + { + ChunkedInfoWithSource Chunked; + const bool DidChunking = + InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed); + if (DidChunking) + { + Lock.WithExclusiveLock([&]() { + if (!RawHashToSequenceRawHashIndex.contains(Chunked.Info.RawHash)) + { + RawHashToSequenceRawHashIndex.insert( + {Chunked.Info.RawHash, gsl::narrow(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); + std::vector ChunkSizes; + ChunkSizes.reserve(Chunked.ChunkSources.size()); + for (const ChunkSource& Source : Chunked.ChunkSources) + { + ChunkSizes.push_back(Source.Size); + } + AddCunkSequence(Stats, + OutChunkedContent.ChunkedContent, + ChunkHashToChunkIndex, + Chunked.Info.RawHash, + Chunked.Info.ChunkSequence, + Chunked.Info.ChunkHashes, + ChunkSizes); + Stats.UniqueSequencesFound++; + } + }); + Stats.FilesChunked++; + return Chunked.Info.RawHash; + } + else + { + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); + const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + + Lock.WithExclusiveLock([&]() { + if (!RawHashToSequenceRawHashIndex.contains(Hash)) + { + RawHashToSequenceRawHashIndex.insert( + {Hash, gsl::narrow(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); + AddCunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize); + Stats.UniqueSequencesFound++; + } + }); + return Hash; + } + } + } + + std::string PathCompareString(const std::filesystem::path& Path) { return ToLower(Path.generic_string()); } + +} // namespace + +std::string_view FolderContentSourcePlatformNames[(size_t)SourcePlatform::_Count] = {"Windows"sv, "Linux"sv, "MacOS"sv}; + +std::string_view +ToString(SourcePlatform Platform) +{ + return FolderContentSourcePlatformNames[(size_t)Platform]; +} + +SourcePlatform +FromString(std::string_view Platform, SourcePlatform Default) +{ + for (size_t Index = 0; Index < (size_t)SourcePlatform::_Count; Index++) + { + if (Platform == FolderContentSourcePlatformNames[Index]) + { + return (SourcePlatform)Index; + } + } + return Default; +} + +SourcePlatform +GetSourceCurrentPlatform() +{ +#if ZEN_PLATFORM_WINDOWS + return SourcePlatform::Windows; +#endif +#if ZEN_PLATFORM_MAC + return SourcePlatform::MacOS; +#endif +#if ZEN_PLATFORM_LINUX + return SourcePlatform::Linux; +#endif +} + +bool +FolderContent::AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs) +{ +#if ZEN_PLATFORM_WINDOWS + return (Lhs & 0xff) == (Rhs & 0xff); +#endif +#if ZEN_PLATFORM_MAC + return Lhs == Rhs; +#endif +#if ZEN_PLATFORM_LINUX + return Lhs == Rhs; +#endif +} + +bool +FolderContent::operator==(const FolderContent& Rhs) const +{ + if ((Platform == Rhs.Platform) && (RawSizes == Rhs.RawSizes) && (Attributes == Rhs.Attributes) && + (ModificationTicks == Rhs.ModificationTicks) && (Paths.size() == Rhs.Paths.size())) + { + size_t PathCount = 0; + for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++) + { + if (Paths[PathIndex].generic_string() != Rhs.Paths[PathIndex].generic_string()) + { + return false; + } + } + return true; + } + return false; +} + +bool +FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const +{ + tsl::robin_map RhsPathToIndex; + const size_t RhsPathCount = Rhs.Paths.size(); + RhsPathToIndex.reserve(RhsPathCount); + for (size_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++) + { + RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex}); + } + const size_t PathCount = Paths.size(); + for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++) + { + if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end()) + { + const size_t RhsPathIndex = It->second; + if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) || + (!AreFileAttributesEqual(Attributes[PathIndex], Rhs.Attributes[RhsPathIndex])) || + (ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex])) + { + return false; + } + } + else + { + return false; + } + } + return true; +} + +void +FolderContent::UpdateState(const FolderContent& Rhs, std::vector& OutPathIndexesOufOfDate) +{ + tsl::robin_map RhsPathToIndex; + const uint32_t RhsPathCount = gsl::narrow(Rhs.Paths.size()); + RhsPathToIndex.reserve(RhsPathCount); + for (uint32_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++) + { + RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex}); + } + uint32_t PathCount = gsl::narrow(Paths.size()); + for (uint32_t PathIndex = 0; PathIndex < PathCount;) + { + if (auto It = RhsPathToIndex.find(Paths[PathIndex].generic_string()); It != RhsPathToIndex.end()) + { + const uint32_t RhsPathIndex = It->second; + + if ((RawSizes[PathIndex] != Rhs.RawSizes[RhsPathIndex]) || + (ModificationTicks[PathIndex] != Rhs.ModificationTicks[RhsPathIndex])) + { + RawSizes[PathIndex] = Rhs.RawSizes[RhsPathIndex]; + ModificationTicks[PathIndex] = Rhs.ModificationTicks[RhsPathIndex]; + OutPathIndexesOufOfDate.push_back(PathIndex); + } + Attributes[PathIndex] = Rhs.Attributes[RhsPathIndex]; + PathIndex++; + } + else + { + Paths.erase(Paths.begin() + PathIndex); + RawSizes.erase(RawSizes.begin() + PathIndex); + Attributes.erase(Attributes.begin() + PathIndex); + ModificationTicks.erase(ModificationTicks.begin() + PathIndex); + PathCount--; + } + } +} + +FolderContent +GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) +{ + FolderContent Result = {.Platform = Old.Platform}; + tsl::robin_map NewPathToIndex; + const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); + NewPathToIndex.reserve(NewPathCount); + for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++) + { + NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex}); + } + uint32_t OldPathCount = gsl::narrow(Old.Paths.size()); + for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++) + { + if (auto It = NewPathToIndex.find(Old.Paths[OldPathIndex].generic_string()); It != NewPathToIndex.end()) + { + const uint32_t NewPathIndex = It->second; + + if ((Old.RawSizes[OldPathIndex] != New.RawSizes[NewPathIndex]) || + (Old.ModificationTicks[OldPathIndex] != New.ModificationTicks[NewPathIndex])) + { + Result.Paths.push_back(New.Paths[NewPathIndex]); + Result.RawSizes.push_back(New.RawSizes[NewPathIndex]); + Result.Attributes.push_back(New.Attributes[NewPathIndex]); + Result.ModificationTicks.push_back(New.ModificationTicks[NewPathIndex]); + } + } + else + { + OutDeletedPathIndexes.push_back(Old.Paths[OldPathIndex]); + } + } + return Result; +} + +void +SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output) +{ + Output.AddString("platform"sv, ToString(Content.Platform)); + compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output); + compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output); + compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output); + compactbinary_helpers::WriteArray(Content.ModificationTicks, "modificationTimes"sv, Output); +} + +FolderContent +LoadFolderContentToCompactBinary(CbObjectView Input) +{ + FolderContent Content; + Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform()); + compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths); + compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes); + compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes); + compactbinary_helpers::ReadArray("modificationTimes"sv, Input, Content.ModificationTicks); + return Content; +} + +FolderContent +GetFolderContent(GetFolderContentStatistics& Stats, + const std::filesystem::path& RootPath, + std::function&& AcceptDirectory, + std::function&& AcceptFile, + WorkerThreadPool& WorkerPool, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag) +{ + Stopwatch Timer; + auto _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + FolderContent Content; + struct AsyncVisitor : public GetDirectoryContentVisitor + { + AsyncVisitor(GetFolderContentStatistics& Stats, + std::atomic& AbortFlag, + FolderContent& Content, + std::function&& AcceptDirectory, + std::function&& AcceptFile) + : m_Stats(Stats) + , m_AbortFlag(AbortFlag) + , m_FoundContent(Content) + , m_AcceptDirectory(std::move(AcceptDirectory)) + , m_AcceptFile(std::move(AcceptFile)) + { + } + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override + { + if (!m_AbortFlag) + { + m_Stats.FoundFileCount += Content.FileNames.size(); + for (uint64_t FileSize : Content.FileSizes) + { + m_Stats.FoundFileByteCount += FileSize; + } + std::string RelativeDirectoryPath = RelativeRoot.generic_string(); + if (m_AcceptDirectory(RelativeDirectoryPath)) + { + std::vector Paths; + std::vector RawSizes; + std::vector Attributes; + std::vector ModificatonTicks; + Paths.reserve(Content.FileNames.size()); + RawSizes.reserve(Content.FileNames.size()); + Attributes.reserve(Content.FileNames.size()); + ModificatonTicks.reserve(Content.FileModificationTicks.size()); + + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + std::string RelativePath = (RelativeRoot / FileName).generic_string(); + std::replace(RelativePath.begin(), RelativePath.end(), '\\', '/'); + if (m_AcceptFile(RelativePath, Content.FileSizes[FileIndex], Content.FileAttributes[FileIndex])) + { + Paths.emplace_back(std::move(RelativePath)); + RawSizes.emplace_back(Content.FileSizes[FileIndex]); + Attributes.emplace_back(Content.FileAttributes[FileIndex]); + ModificatonTicks.emplace_back(Content.FileModificationTicks[FileIndex]); + + m_Stats.AcceptedFileCount++; + m_Stats.AcceptedFileByteCount += Content.FileSizes[FileIndex]; + } + } + m_Lock.WithExclusiveLock([&]() { + m_FoundContent.Paths.insert(m_FoundContent.Paths.end(), Paths.begin(), Paths.end()); + m_FoundContent.RawSizes.insert(m_FoundContent.RawSizes.end(), RawSizes.begin(), RawSizes.end()); + m_FoundContent.Attributes.insert(m_FoundContent.Attributes.end(), Attributes.begin(), Attributes.end()); + m_FoundContent.ModificationTicks.insert(m_FoundContent.ModificationTicks.end(), + ModificatonTicks.begin(), + ModificatonTicks.end()); + }); + } + } + } + + GetFolderContentStatistics& m_Stats; + std::atomic& m_AbortFlag; + RwLock m_Lock; + FolderContent& m_FoundContent; + std::function m_AcceptDirectory; + std::function m_AcceptFile; + } Visitor(Stats, AbortFlag, Content, std::move(AcceptDirectory), std::move(AcceptFile)); + + Latch PendingWork(1); + GetDirectoryContent(RootPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes | + DirectoryContentFlags::IncludeAttributes | DirectoryContentFlags::IncludeModificationTick, + Visitor, + WorkerPool, + PendingWork); + PendingWork.CountDown(); + while (!PendingWork.Wait(UpdateInteralMS)) + { + UpdateCallback(AbortFlag.load(), PendingWork.Remaining()); + } + std::vector Order; + size_t PathCount = Content.Paths.size(); + Order.resize(Content.Paths.size()); + std::vector Parents; + Parents.reserve(PathCount); + std::vector Filenames; + Filenames.reserve(PathCount); + for (size_t OrderIndex = 0; OrderIndex < PathCount; OrderIndex++) + { + Order[OrderIndex] = OrderIndex; + Parents.emplace_back(Content.Paths[OrderIndex].parent_path().generic_string()); + Filenames.emplace_back(Content.Paths[OrderIndex].filename().generic_string()); + } + std::sort(Order.begin(), Order.end(), [&Parents, &Filenames](size_t Lhs, size_t Rhs) { + const std::string& LhsParent = Parents[Lhs]; + const std::string& RhsParent = Parents[Rhs]; + if (LhsParent < RhsParent) + { + return true; + } + else if (LhsParent > RhsParent) + { + return false; + } + return Filenames[Lhs] < Filenames[Rhs]; + }); + FolderContent OrderedContent; + OrderedContent.Paths.reserve(PathCount); + OrderedContent.RawSizes.reserve(PathCount); + OrderedContent.Attributes.reserve(PathCount); + OrderedContent.ModificationTicks.reserve(PathCount); + for (size_t OrderIndex : Order) + { + OrderedContent.Paths.emplace_back(std::move(Content.Paths[OrderIndex])); + OrderedContent.RawSizes.emplace_back(Content.RawSizes[OrderIndex]); + OrderedContent.Attributes.emplace_back(Content.Attributes[OrderIndex]); + OrderedContent.ModificationTicks.emplace_back(Content.ModificationTicks[OrderIndex]); + } + return OrderedContent; +} + +void +SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output) +{ + Output.AddString("platform"sv, ToString(Content.Platform)); + compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output); + compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output); + compactbinary_helpers::WriteArray(Content.Attributes, "attributes"sv, Output); + compactbinary_helpers::WriteArray(Content.RawHashes, "rawHashes"sv, Output); + + Output.BeginObject("chunkedContent"); + compactbinary_helpers::WriteArray(Content.ChunkedContent.SequenceRawHashes, "sequenceRawHashes"sv, Output); + compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkCounts, "chunkCounts"sv, Output); + compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkOrders, "chunkOrders"sv, Output); + compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkHashes, "chunkHashes"sv, Output); + compactbinary_helpers::WriteArray(Content.ChunkedContent.ChunkRawSizes, "chunkRawSizes"sv, Output); + Output.EndObject(); // chunkedContent +} + +ChunkedFolderContent +LoadChunkedFolderContentToCompactBinary(CbObjectView Input) +{ + ChunkedFolderContent Content; + Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform()); + compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths); + compactbinary_helpers::ReadArray("rawSizes"sv, Input, Content.RawSizes); + compactbinary_helpers::ReadArray("attributes"sv, Input, Content.Attributes); + compactbinary_helpers::ReadArray("rawHashes"sv, Input, Content.RawHashes); + + CbObjectView ChunkedContentView = Input["chunkedContent"sv].AsObjectView(); + compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkedContentView, Content.ChunkedContent.SequenceRawHashes); + compactbinary_helpers::ReadArray("chunkCounts"sv, ChunkedContentView, Content.ChunkedContent.ChunkCounts); + compactbinary_helpers::ReadArray("chunkOrders"sv, ChunkedContentView, Content.ChunkedContent.ChunkOrders); + compactbinary_helpers::ReadArray("chunkHashes"sv, ChunkedContentView, Content.ChunkedContent.ChunkHashes); + compactbinary_helpers::ReadArray("chunkRawSizes"sv, ChunkedContentView, Content.ChunkedContent.ChunkRawSizes); + return Content; +} + +ChunkedFolderContent +MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays) +{ + ZEN_ASSERT(!Overlays.empty()); + + ChunkedFolderContent Result; + const size_t BasePathCount = Base.Paths.size(); + Result.Paths.reserve(BasePathCount); + Result.RawSizes.reserve(BasePathCount); + Result.Attributes.reserve(BasePathCount); + Result.RawHashes.reserve(BasePathCount); + + const size_t BaseChunkCount = Base.ChunkedContent.ChunkHashes.size(); + Result.ChunkedContent.SequenceRawHashes.reserve(Base.ChunkedContent.SequenceRawHashes.size()); + Result.ChunkedContent.ChunkCounts.reserve(BaseChunkCount); + Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount); + Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount); + Result.ChunkedContent.ChunkOrders.reserve(Base.ChunkedContent.ChunkOrders.size()); + + tsl::robin_map GenericPathToActualPath; + for (const std::filesystem::path& Path : Base.Paths) + { + GenericPathToActualPath.insert({PathCompareString(Path), Path}); + } + for (const ChunkedFolderContent& Overlay : Overlays) + { + for (const std::filesystem::path& Path : Overlay.Paths) + { + GenericPathToActualPath.insert({PathCompareString(Path), Path}); + } + } + + tsl::robin_map RawHashToSequenceRawHashIndex; + + auto BuildOverlayPaths = [](std::span Overlays) -> tsl::robin_set { + tsl::robin_set Result; + for (const ChunkedFolderContent& OverlayContent : Overlays) + { + for (const std::filesystem::path& Path : OverlayContent.Paths) + { + Result.insert(PathCompareString(Path)); + } + } + return Result; + }; + + auto AddContent = [&BuildOverlayPaths](ChunkedFolderContent& Result, + const ChunkedFolderContent& OverlayContent, + tsl::robin_map& ChunkHashToChunkIndex, + tsl::robin_map& RawHashToSequenceRawHashIndex, + const tsl::robin_map& GenericPathToActualPath, + std::span Overlays) { + const ChunkedContentLookup OverlayLookup = BuildChunkedContentLookup(OverlayContent); + tsl::robin_set BaseOverlayPaths = BuildOverlayPaths(Overlays); + for (uint32_t PathIndex = 0; PathIndex < OverlayContent.Paths.size(); PathIndex++) + { + std::string GenericPath = PathCompareString(OverlayContent.Paths[PathIndex]); + if (!BaseOverlayPaths.contains(GenericPath)) + { + // This asset will not be overridden by a later layer - add it + + const std::filesystem::path OriginalPath = GenericPathToActualPath.at(GenericPath); + Result.Paths.push_back(OriginalPath); + const IoHash& RawHash = OverlayContent.RawHashes[PathIndex]; + Result.RawSizes.push_back(OverlayContent.RawSizes[PathIndex]); + Result.Attributes.push_back(OverlayContent.Attributes[PathIndex]); + Result.RawHashes.push_back(RawHash); + + if (OverlayContent.RawSizes[PathIndex] > 0) + { + if (!RawHashToSequenceRawHashIndex.contains(RawHash)) + { + RawHashToSequenceRawHashIndex.insert( + {RawHash, gsl::narrow(Result.ChunkedContent.SequenceRawHashes.size())}); + const uint32_t SequenceRawHashIndex = OverlayLookup.RawHashToSequenceRawHashIndex.at(RawHash); + const uint32_t OrderIndexOffset = OverlayLookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = OverlayContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + ChunkingStatistics Stats; + std::span OriginalChunkOrder = + std::span(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); + AddCunkSequence(Stats, + Result.ChunkedContent, + ChunkHashToChunkIndex, + RawHash, + OriginalChunkOrder, + OverlayContent.ChunkedContent.ChunkHashes, + OverlayContent.ChunkedContent.ChunkRawSizes); + Stats.UniqueSequencesFound++; + } + } + } + } + }; + + tsl::robin_map MergedChunkHashToChunkIndex; + AddContent(Result, Base, MergedChunkHashToChunkIndex, RawHashToSequenceRawHashIndex, GenericPathToActualPath, Overlays); + for (uint32_t OverlayIndex = 0; OverlayIndex < Overlays.size(); OverlayIndex++) + { + AddContent(Result, + Overlays[OverlayIndex], + MergedChunkHashToChunkIndex, + RawHashToSequenceRawHashIndex, + GenericPathToActualPath, + Overlays.subspan(OverlayIndex + 1)); + } + return Result; +} + +ChunkedFolderContent +DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span DeletedPaths) +{ + ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size()); + ChunkedFolderContent Result = {.Platform = BaseContent.Platform}; + if (DeletedPaths.size() < BaseContent.Paths.size()) + { + tsl::robin_set DeletedPathSet; + DeletedPathSet.reserve(DeletedPaths.size()); + for (const std::filesystem::path& DeletedPath : DeletedPaths) + { + DeletedPathSet.insert(PathCompareString(DeletedPath)); + } + const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); + tsl::robin_map ChunkHashToChunkIndex; + + tsl::robin_map RawHashToSequenceRawHashIndex; + for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) + { + const std::filesystem::path& Path = BaseContent.Paths[PathIndex]; + if (!DeletedPathSet.contains(PathCompareString(Path))) + { + const IoHash& RawHash = BaseContent.RawHashes[PathIndex]; + const uint64_t RawSize = BaseContent.RawSizes[PathIndex]; + Result.Paths.push_back(Path); + Result.RawSizes.push_back(RawSize); + Result.Attributes.push_back(BaseContent.Attributes[PathIndex]); + Result.RawHashes.push_back(RawHash); + if (RawSize > 0) + { + if (!RawHashToSequenceRawHashIndex.contains(RawHash)) + { + RawHashToSequenceRawHashIndex.insert( + {RawHash, gsl::narrow(Result.ChunkedContent.SequenceRawHashes.size())}); + const uint32_t SequenceRawHashIndex = BaseLookup.RawHashToSequenceRawHashIndex.at(RawHash); + const uint32_t OrderIndexOffset = BaseLookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + ChunkingStatistics Stats; + std::span OriginalChunkOrder = + std::span(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); + AddCunkSequence(Stats, + Result.ChunkedContent, + ChunkHashToChunkIndex, + RawHash, + OriginalChunkOrder, + BaseContent.ChunkedContent.ChunkHashes, + BaseContent.ChunkedContent.ChunkRawSizes); + Stats.UniqueSequencesFound++; + } + } + } + } + } + return Result; +} + +ChunkedFolderContent +ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag) +{ + Stopwatch Timer; + auto _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + ChunkedFolderContent Result = {.Platform = Content.Platform, + .Paths = Content.Paths, + .RawSizes = Content.RawSizes, + .Attributes = Content.Attributes}; + const size_t ItemCount = Result.Paths.size(); + Result.RawHashes.resize(ItemCount, IoHash::Zero); + Result.ChunkedContent.SequenceRawHashes.reserve(ItemCount); // Up to 1 per file, maybe less + Result.ChunkedContent.ChunkCounts.reserve(ItemCount); // Up to one per file + Result.ChunkedContent.ChunkOrders.reserve(ItemCount); // At least 1 per file, maybe more + Result.ChunkedContent.ChunkHashes.reserve(ItemCount); // At least 1 per file, maybe more + Result.ChunkedContent.ChunkRawSizes.reserve(ItemCount); // At least 1 per file, maybe more + tsl::robin_map ChunkHashToChunkIndex; + tsl::robin_map RawHashToChunkSequenceIndex; + RawHashToChunkSequenceIndex.reserve(ItemCount); + ChunkHashToChunkIndex.reserve(ItemCount); + { + std::vector Order; + Order.resize(ItemCount); + for (uint32_t I = 0; I < ItemCount; I++) + { + Order[I] = I; + } + + // Handle the biggest files first so we don't end up with one straggling large file at the end + // std::sort(Order.begin(), Order.end(), [&](uint32_t Lhs, uint32_t Rhs) { return Result.RawSizes[Lhs] > Result.RawSizes[Rhs]; + //}); + + tsl::robin_map RawHashToSequenceRawHashIndex; + RawHashToSequenceRawHashIndex.reserve(ItemCount); + + RwLock Lock; + + ParallellWork Work(AbortFlag); + + for (uint32_t PathIndex : Order) + { + if (Work.IsAborted()) + { + break; + } + Work.ScheduleWork( + WorkerPool, // GetSyncWorkerPool() + [&, PathIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + IoHash RawHash = HashOneFile(Stats, + InChunkingController, + Result, + ChunkHashToChunkIndex, + RawHashToSequenceRawHashIndex, + Lock, + RootPath, + PathIndex); + Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; }); + Stats.FilesProcessed++; + } + }, + [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { + ZEN_CONSOLE("Failed scanning file {}. Reason: {}", Result.Paths[PathIndex], Ex.what()); + AbortFlag = true; + }); + } + + Work.Wait(UpdateInteralMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted); + ZEN_UNUSED(PendingWork); + UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining()); + }); + } + return Result; +} + +ChunkedContentLookup +BuildChunkedContentLookup(const ChunkedFolderContent& Content) +{ + struct ChunkLocationReference + { + uint32_t ChunkIndex; + ChunkedContentLookup::ChunkLocation Location; + }; + + ChunkedContentLookup Result; + { + const uint32_t SequenceRawHashesCount = gsl::narrow(Content.ChunkedContent.SequenceRawHashes.size()); + Result.RawHashToSequenceRawHashIndex.reserve(SequenceRawHashesCount); + Result.SequenceRawHashIndexChunkOrderOffset.reserve(SequenceRawHashesCount); + uint32_t OrderOffset = 0; + for (uint32_t SequenceRawHashIndex = 0; SequenceRawHashIndex < Content.ChunkedContent.SequenceRawHashes.size(); + SequenceRawHashIndex++) + { + Result.RawHashToSequenceRawHashIndex.insert( + {Content.ChunkedContent.SequenceRawHashes[SequenceRawHashIndex], SequenceRawHashIndex}); + Result.SequenceRawHashIndexChunkOrderOffset.push_back(OrderOffset); + OrderOffset += Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + } + } + + std::vector Locations; + Locations.reserve(Content.ChunkedContent.ChunkOrders.size()); + for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++) + { + if (Content.RawSizes[PathIndex] > 0) + { + const IoHash& RawHash = Content.RawHashes[PathIndex]; + uint32_t SequenceRawHashIndex = Result.RawHashToSequenceRawHashIndex.at(RawHash); + const uint32_t OrderOffset = Result.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + uint64_t LocationOffset = 0; + for (size_t OrderIndex = OrderOffset; OrderIndex < OrderOffset + ChunkCount; OrderIndex++) + { + uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; + + Locations.push_back(ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkLocation{PathIndex, LocationOffset}}); + + LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + } + ZEN_ASSERT(LocationOffset == Content.RawSizes[PathIndex]); + } + } + + std::sort(Locations.begin(), Locations.end(), [](const ChunkLocationReference& Lhs, const ChunkLocationReference& Rhs) { + if (Lhs.ChunkIndex < Rhs.ChunkIndex) + { + return true; + } + if (Lhs.ChunkIndex > Rhs.ChunkIndex) + { + return false; + } + if (Lhs.Location.PathIndex < Rhs.Location.PathIndex) + { + return true; + } + if (Lhs.Location.PathIndex > Rhs.Location.PathIndex) + { + return false; + } + return Lhs.Location.Offset < Rhs.Location.Offset; + }); + + Result.ChunkLocations.reserve(Locations.size()); + const uint32_t ChunkCount = gsl::narrow(Content.ChunkedContent.ChunkHashes.size()); + Result.ChunkHashToChunkIndex.reserve(ChunkCount); + size_t RangeOffset = 0; + for (uint32_t ChunkIndex = 0; ChunkIndex < ChunkCount; ChunkIndex++) + { + Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex}); + uint32_t Count = 0; + while (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex) + { + Result.ChunkLocations.push_back(Locations[RangeOffset + Count].Location); + Count++; + } + Result.ChunkLocationOffset.push_back(RangeOffset); + Result.ChunkLocationCounts.push_back(Count); + RangeOffset += Count; + } + + return Result; +} + +} // namespace zen diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp new file mode 100644 index 000000000..bc0e57b14 --- /dev/null +++ b/src/zenutil/chunkingcontroller.cpp @@ -0,0 +1,265 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { +using namespace std::literals; + +namespace { + std::vector ReadStringArray(CbArrayView StringArray) + { + std::vector Result; + Result.reserve(StringArray.Num()); + for (CbFieldView FieldView : StringArray) + { + Result.emplace_back(FieldView.AsString()); + } + return Result; + } + + ChunkedParams ReadChunkParams(CbObjectView Params) + { + bool UseThreshold = Params["UseThreshold"sv].AsBool(true); + size_t MinSize = Params["MinSize"sv].AsUInt64(DefaultChunkedParams.MinSize); + size_t MaxSize = Params["MaxSize"sv].AsUInt64(DefaultChunkedParams.MaxSize); + size_t AvgSize = Params["AvgSize"sv].AsUInt64(DefaultChunkedParams.AvgSize); + + return ChunkedParams{.UseThreshold = UseThreshold, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize}; + } + +} // namespace + +class BasicChunkingController : public ChunkingController +{ +public: + BasicChunkingController(std::span ExcludeExtensions, + uint64_t ChunkFileSizeLimit, + const ChunkedParams& ChunkingParams) + : m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end()) + , m_ChunkFileSizeLimit(ChunkFileSizeLimit) + , m_ChunkingParams(ChunkingParams) + { + } + + BasicChunkingController(CbObjectView Parameters) + : m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView())) + , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) + , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) + { + } + + virtual bool ProcessFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + ChunkedInfoWithSource& OutChunked, + std::atomic& BytesProcessed) const override + { + const bool ExcludeFromChunking = + std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) != + m_ChunkExcludeExtensions.end(); + + if (ExcludeFromChunking || (RawSize < m_ChunkFileSizeLimit)) + { + return false; + } + + BasicFile Buffer(InputPath, BasicFile::Mode::kRead); + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed); + return true; + } + + virtual std::string_view GetName() const override { return Name; } + + virtual CbObject GetParameters() const override + { + CbObjectWriter Writer; + Writer.BeginArray("ChunkExcludeExtensions"sv); + { + for (const std::string& Extension : m_ChunkExcludeExtensions) + { + Writer.AddString(Extension); + } + } + Writer.EndArray(); // ChunkExcludeExtensions + Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); + Writer.BeginObject("ChunkingParams"sv); + { + Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold); + + Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize); + Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize); + Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize); + } + Writer.EndObject(); // ChunkingParams + return Writer.Save(); + } + static constexpr std::string_view Name = "BasicChunkingController"sv; + +protected: + const std::vector m_ChunkExcludeExtensions; + const uint64_t m_ChunkFileSizeLimit; + const ChunkedParams m_ChunkingParams; +}; + +class ChunkingControllerWithFixedChunking : public ChunkingController +{ +public: + ChunkingControllerWithFixedChunking(std::span FixedChunkingExtensions, + uint64_t ChunkFileSizeLimit, + const ChunkedParams& ChunkingParams, + uint32_t FixedChunkingChunkSize) + : m_FixedChunkingExtensions(FixedChunkingExtensions.begin(), FixedChunkingExtensions.end()) + , m_ChunkFileSizeLimit(ChunkFileSizeLimit) + , m_ChunkingParams(ChunkingParams) + , m_FixedChunkingChunkSize(FixedChunkingChunkSize) + { + } + + ChunkingControllerWithFixedChunking(CbObjectView Parameters) + : m_FixedChunkingExtensions(ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView())) + , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) + , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) + , m_FixedChunkingChunkSize(Parameters["FixedChunkingChunkSize"sv].AsUInt32(16u * 1024u * 1024u)) + { + } + + virtual bool ProcessFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + ChunkedInfoWithSource& OutChunked, + std::atomic& BytesProcessed) const override + { + if (RawSize < m_ChunkFileSizeLimit) + { + return false; + } + const bool FixedChunking = std::find(m_FixedChunkingExtensions.begin(), m_FixedChunkingExtensions.end(), InputPath.extension()) != + m_FixedChunkingExtensions.end(); + + if (FixedChunking) + { + IoHashStream FullHash; + IoBuffer Source = IoBufferBuilder::MakeFromFile(InputPath); + uint64_t Offset = 0; + tsl::robin_map ChunkHashToChunkIndex; + ChunkHashToChunkIndex.reserve(1 + (RawSize / m_FixedChunkingChunkSize)); + while (Offset < RawSize) + { + uint64_t ChunkSize = std::min(RawSize - Offset, m_FixedChunkingChunkSize); + IoBuffer Chunk(Source, Offset, ChunkSize); + MemoryView ChunkData = Chunk.GetView(); + FullHash.Append(ChunkData); + + IoHash ChunkHash = IoHash::HashBuffer(ChunkData); + if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end()) + { + OutChunked.Info.ChunkSequence.push_back(It->second); + } + else + { + uint32_t ChunkIndex = gsl::narrow(OutChunked.Info.ChunkHashes.size()); + OutChunked.Info.ChunkHashes.push_back(ChunkHash); + OutChunked.Info.ChunkSequence.push_back(ChunkIndex); + OutChunked.ChunkSources.push_back({.Offset = Offset, .Size = gsl::narrow(ChunkSize)}); + } + Offset += ChunkSize; + BytesProcessed.fetch_add(ChunkSize); + } + OutChunked.Info.RawSize = RawSize; + OutChunked.Info.RawHash = FullHash.GetHash(); + return true; + } + else + { + BasicFile Buffer(InputPath, BasicFile::Mode::kRead); + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed); + return true; + } + } + + virtual std::string_view GetName() const override { return Name; } + + virtual CbObject GetParameters() const override + { + CbObjectWriter Writer; + Writer.BeginArray("FixedChunkingExtensions"); + { + for (const std::string& Extension : m_FixedChunkingExtensions) + { + Writer.AddString(Extension); + } + } + Writer.EndArray(); // ChunkExcludeExtensions + Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); + Writer.BeginObject("ChunkingParams"sv); + { + Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold); + + Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize); + Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize); + Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize); + } + Writer.EndObject(); // ChunkingParams + Writer.AddInteger("FixedChunkingChunkSize"sv, m_FixedChunkingChunkSize); + return Writer.Save(); + } + + static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv; + +protected: + const std::vector m_FixedChunkingExtensions; + const uint64_t m_ChunkFileSizeLimit; + const ChunkedParams m_ChunkingParams; + const uint32_t m_FixedChunkingChunkSize; +}; + +std::unique_ptr +CreateBasicChunkingController(std::span ExcludeExtensions, + uint64_t ChunkFileSizeLimit, + const ChunkedParams& ChunkingParams) +{ + return std::make_unique(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams); +} +std::unique_ptr +CreateBasicChunkingController(CbObjectView Parameters) +{ + return std::make_unique(Parameters); +} + +std::unique_ptr +CreateChunkingControllerWithFixedChunking(std::span FixedChunkingExtensions, + uint64_t ChunkFileSizeLimit, + const ChunkedParams& ChunkingParams, + uint32_t FixedChunkingChunkSize) +{ + return std::make_unique(FixedChunkingExtensions, + ChunkFileSizeLimit, + ChunkingParams, + FixedChunkingChunkSize); +} +std::unique_ptr +CreateChunkingControllerWithFixedChunking(CbObjectView Parameters) +{ + return std::make_unique(Parameters); +} + +std::unique_ptr +CreateChunkingController(std::string_view Name, CbObjectView Parameters) +{ + if (Name == BasicChunkingController::Name) + { + return CreateBasicChunkingController(Parameters); + } + else if (Name == ChunkingControllerWithFixedChunking::Name) + { + return CreateChunkingControllerWithFixedChunking(Parameters); + } + return {}; +} + +} // namespace zen diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp new file mode 100644 index 000000000..78ebcdd55 --- /dev/null +++ b/src/zenutil/filebuildstorage.cpp @@ -0,0 +1,616 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +class FileBuildStorage : public BuildStorage +{ +public: + explicit FileBuildStorage(const std::filesystem::path& StoragePath, + BuildStorage::Statistics& Stats, + bool EnableJsonOutput, + double LatencySec, + double DelayPerKBSec) + : m_StoragePath(StoragePath) + , m_Stats(Stats) + , m_EnableJsonOutput(EnableJsonOutput) + , m_LatencySec(LatencySec) + , m_DelayPerKBSec(DelayPerKBSec) + { + CreateDirectories(GetBuildsFolder()); + CreateDirectories(GetBlobsFolder()); + CreateDirectories(GetBlobsMetadataFolder()); + } + + virtual ~FileBuildStorage() {} + + virtual CbObject ListBuilds(CbObject Query) override + { + ZEN_UNUSED(Query); + + SimulateLatency(Query.GetSize(), 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BuildFolder = GetBuildsFolder(); + DirectoryContent Content; + GetDirectoryContent(BuildFolder, DirectoryContentFlags::IncludeDirs, Content); + CbObjectWriter Writer; + Writer.BeginArray("results"); + { + for (const std::filesystem::path& BuildPath : Content.Directories) + { + Oid BuildId = Oid::TryFromHexString(BuildPath.stem().string()); + if (BuildId != Oid::Zero) + { + Writer.BeginObject(); + { + Writer.AddObjectId("buildId", BuildId); + Writer.AddObject("metadata", ReadBuild(BuildId)["metadata"sv].AsObjectView()); + } + Writer.EndObject(); + } + } + } + Writer.EndArray(); // builds + Writer.Save(); + SimulateLatency(Writer.GetSaveSize(), 0); + return Writer.Save(); + } + + virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override + { + SimulateLatency(MetaData.GetSize(), 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + CbObjectWriter BuildObject; + BuildObject.AddObject("metadata", MetaData); + BuildObject.AddInteger("chunkSize"sv, 32u * 1024u * 1024u); + WriteBuild(BuildId, BuildObject.Save()); + + CbObjectWriter BuildResponse; + BuildResponse.AddInteger("chunkSize"sv, 32u * 1024u * 1024u); + BuildResponse.Save(); + + SimulateLatency(0, BuildResponse.GetSaveSize()); + return BuildResponse.Save(); + } + + virtual CbObject GetBuild(const Oid& BuildId) override + { + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + CbObject Build = ReadBuild(BuildId); + SimulateLatency(0, Build.GetSize()); + return Build; + } + + virtual void FinalizeBuild(const Oid& BuildId) override + { + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + ZEN_UNUSED(BuildId); + SimulateLatency(0, 0); + } + + virtual std::pair> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) override + { + SimulateLatency(MetaData.GetSize(), 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); + CreateDirectories(BuildPartDataPath.parent_path()); + + TemporaryFile::SafeWriteFile(BuildPartDataPath, MetaData.GetView()); + m_WrittenBytes += MetaData.GetSize(); + WriteAsJson(BuildPartDataPath, MetaData); + + IoHash RawHash = IoHash::HashBuffer(MetaData.GetView()); + + CbObjectWriter Writer; + { + CbObject BuildObject = ReadBuild(BuildId); + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); + CbObjectView MetaDataView = BuildObject["metadata"sv].AsObjectView(); + + Writer.AddObject("metadata"sv, MetaDataView); + Writer.BeginObject("parts"sv); + { + for (CbFieldView PartView : PartsObject) + { + if (PartView.GetName() != PartName) + { + Writer.AddObjectId(PartView.GetName(), PartView.AsObjectId()); + } + } + Writer.AddObjectId(PartName, BuildPartId); + } + Writer.EndObject(); // parts + } + WriteBuild(BuildId, Writer.Save()); + + std::vector NeededAttachments = GetNeededAttachments(MetaData); + + SimulateLatency(0, sizeof(IoHash) * NeededAttachments.size()); + + return std::make_pair(RawHash, std::move(NeededAttachments)); + } + + virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override + { + SimulateLatency(0, 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); + + IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten(); + m_Stats.TotalBytesRead += Payload.GetSize(); + + ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None); + + CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); + + SimulateLatency(0, BuildPartObject.GetSize()); + + return BuildPartObject; + } + + virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override + { + SimulateLatency(0, 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BuildPartDataPath = GetBuildPartPath(BuildId, BuildPartId); + IoBuffer Payload = ReadFile(BuildPartDataPath).Flatten(); + m_Stats.TotalBytesRead += Payload.GetSize(); + IoHash RawHash = IoHash::HashBuffer(Payload.GetView()); + if (RawHash != PartHash) + { + throw std::runtime_error( + fmt::format("Failed finalizing build part {}: Expected hash {}, got {}", BuildPartId, PartHash, RawHash)); + } + + CbObject BuildPartObject = CbObject(SharedBuffer(Payload)); + std::vector NeededAttachments(GetNeededAttachments(BuildPartObject)); + + SimulateLatency(0, NeededAttachments.size() * sizeof(IoHash)); + + return NeededAttachments; + } + + virtual void PutBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + const CompositeBuffer& Payload) override + { + ZEN_UNUSED(BuildId); + ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); + SimulateLatency(Payload.GetSize(), 0); + + ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, Payload)); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); + if (!std::filesystem::is_regular_file(BlockPath)) + { + CreateDirectories(BlockPath.parent_path()); + TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); + } + m_Stats.TotalBytesWritten += Payload.GetSize(); + SimulateLatency(0, 0); + } + + virtual std::vector> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::function&& OnSentBytes) override + { + ZEN_UNUSED(BuildId); + ZEN_UNUSED(ContentType); + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); + if (!std::filesystem::is_regular_file(BlockPath)) + { + CreateDirectories(BlockPath.parent_path()); + + struct WorkloadData + { + std::function Transmitter; + std::function OnSentBytes; + TemporaryFile TempFile; + std::atomic PartsLeft; + }; + + std::shared_ptr Workload(std::make_shared()); + Workload->Transmitter = std::move(Transmitter); + Workload->OnSentBytes = std::move(OnSentBytes); + std::error_code Ec; + Workload->TempFile.CreateTemporary(BlockPath.parent_path(), Ec); + + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); + } + + std::vector> WorkItems; + uint64_t Offset = 0; + while (Offset < PayloadSize) + { + uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset); + + WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() { + IoBuffer PartPayload = Workload->Transmitter(Offset, Size); + SimulateLatency(PartPayload.GetSize(), 0); + + std::error_code Ec; + Workload->TempFile.Write(PartPayload, Offset, Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed writing to temporary file '{}': {} ({})", + Workload->TempFile.GetPath(), + Ec.message(), + Ec.value())); + } + uint64_t BytesWritten = PartPayload.GetSize(); + m_Stats.TotalBytesWritten += BytesWritten; + const bool IsLastPart = Workload->PartsLeft.fetch_sub(1) == 1; + if (IsLastPart) + { + Workload->TempFile.Flush(); + ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(Workload->TempFile.ReadAll()))); + Workload->TempFile.MoveTemporaryIntoPlace(BlockPath, Ec); + if (Ec) + { + throw std::runtime_error(fmt::format("Failed moving temporary file '{}' to '{}': {} ({})", + Workload->TempFile.GetPath(), + BlockPath, + Ec.message(), + Ec.value())); + } + } + Workload->OnSentBytes(BytesWritten, IsLastPart); + SimulateLatency(0, 0); + }); + + Offset += Size; + } + Workload->PartsLeft.store(WorkItems.size()); + + SimulateLatency(0, 0); + return WorkItems; + } + SimulateLatency(0, 0); + return {}; + } + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override + { + ZEN_UNUSED(BuildId); + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); + if (std::filesystem::is_regular_file(BlockPath)) + { + IoBuffer Payload = ReadFile(BlockPath).Flatten(); + ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload)))); + m_Stats.TotalBytesRead += Payload.GetSize(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + SimulateLatency(0, Payload.GetSize()); + return Payload; + } + SimulateLatency(0, 0); + return IoBuffer{}; + } + + virtual std::vector> GetLargeBuildBlob( + const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& Receiver) override + { + ZEN_UNUSED(BuildId); + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); + if (std::filesystem::is_regular_file(BlockPath)) + { + struct WorkloadData + { + std::atomic BytesRemaining; + IoBuffer BlobFile; + std::function Receiver; + }; + + std::shared_ptr Workload(std::make_shared()); + Workload->BlobFile = IoBufferBuilder::MakeFromFile(BlockPath); + const uint64_t BlobSize = Workload->BlobFile.GetSize(); + + Workload->Receiver = std::move(Receiver); + Workload->BytesRemaining = BlobSize; + + std::vector> WorkItems; + uint64_t Offset = 0; + while (Offset < BlobSize) + { + uint64_t Size = Min(ChunkSize, BlobSize - Offset); + WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() { + SimulateLatency(0, 0); + IoBuffer PartPayload(Workload->BlobFile, Offset, Size); + m_Stats.TotalBytesRead += PartPayload.GetSize(); + uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size); + Workload->Receiver(Offset, PartPayload, ByteRemaning); + SimulateLatency(Size, PartPayload.GetSize()); + }); + + Offset += Size; + } + SimulateLatency(0, 0); + return WorkItems; + } + return {}; + } + + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + { + ZEN_UNUSED(BuildId); + + SimulateLatency(MetaData.GetSize(), 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + const std::filesystem::path BlockMetaDataPath = GetBlobMetadataPath(BlockRawHash); + CreateDirectories(BlockMetaDataPath.parent_path()); + TemporaryFile::SafeWriteFile(BlockMetaDataPath, MetaData.GetView()); + m_Stats.TotalBytesWritten += MetaData.GetSize(); + WriteAsJson(BlockMetaDataPath, MetaData); + SimulateLatency(0, 0); + } + + virtual std::vector FindBlocks(const Oid& BuildId) override + { + ZEN_UNUSED(BuildId); + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + DirectoryContent Content; + GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); + std::vector Result; + for (const std::filesystem::path& MetaDataFile : Content.Files) + { + IoHash ChunkHash; + if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) + { + std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); + if (std::filesystem::is_regular_file(BlockPath)) + { + IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); + + m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); + + CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); + Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + } + } + } + SimulateLatency(0, sizeof(IoHash) * Result.size()); + return Result; + } + + virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + { + ZEN_UNUSED(BuildId); + SimulateLatency(0, 0); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + std::vector Result; + for (const IoHash& BlockHash : BlockHashes) + { + std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); + if (std::filesystem::is_regular_file(MetaDataFile)) + { + IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); + + m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); + + CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); + Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + } + } + SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size()); + return Result; + } + +protected: + std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; } + std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; } + std::filesystem::path GetBlobsMetadataFolder() const { return m_StoragePath / "blocks"; } + std::filesystem::path GetBuildFolder(const Oid& BuildId) const { return GetBuildsFolder() / BuildId.ToString(); } + + std::filesystem::path GetBuildPath(const Oid& BuildId) const { return GetBuildFolder(BuildId) / "metadata.cb"; } + + std::filesystem::path GetBuildPartFolder(const Oid& BuildId, const Oid& BuildPartId) const + { + return GetBuildFolder(BuildId) / "parts" / BuildPartId.ToString(); + } + + std::filesystem::path GetBuildPartPath(const Oid& BuildId, const Oid& BuildPartId) const + { + return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb"; + } + + std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); } + + std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const + { + return GetBlobsMetadataFolder() / fmt::format("{}.cb", RawHash); + } + + void SimulateLatency(uint64_t ReceiveSize, uint64_t SendSize) + { + double SleepSec = m_LatencySec; + if (m_DelayPerKBSec > 0.0) + { + SleepSec += m_DelayPerKBSec * (double(SendSize + ReceiveSize) / 1024u); + } + if (SleepSec > 0) + { + Sleep(int(SleepSec * 1000)); + } + } + + void WriteAsJson(const std::filesystem::path& OriginalPath, CbObjectView Data) const + { + if (m_EnableJsonOutput) + { + ExtendableStringBuilder<128> SB; + CompactBinaryToJson(Data, SB); + std::filesystem::path JsonPath = OriginalPath; + JsonPath.replace_extension(".json"); + std::string_view JsonMetaData = SB.ToView(); + TemporaryFile::SafeWriteFile(JsonPath, MemoryView(JsonMetaData.data(), JsonMetaData.length())); + } + } + + void WriteBuild(const Oid& BuildId, CbObjectView Data) + { + const std::filesystem::path BuildDataPath = GetBuildPath(BuildId); + CreateDirectories(BuildDataPath.parent_path()); + TemporaryFile::SafeWriteFile(BuildDataPath, Data.GetView()); + m_Stats.TotalBytesWritten += Data.GetSize(); + WriteAsJson(BuildDataPath, Data); + } + + CbObject ReadBuild(const Oid& BuildId) + { + const std::filesystem::path BuildDataPath = GetBuildPath(BuildId); + FileContents Content = ReadFile(BuildDataPath); + if (Content.ErrorCode) + { + throw std::runtime_error(fmt::format("Failed reading build '{}' from '{}': {} ({})", + BuildId, + BuildDataPath, + Content.ErrorCode.message(), + Content.ErrorCode.value())); + } + IoBuffer Payload = Content.Flatten(); + m_Stats.TotalBytesRead += Payload.GetSize(); + ZEN_ASSERT(ValidateCompactBinary(Payload.GetView(), CbValidateMode::Default) == CbValidateError::None); + CbObject BuildObject = CbObject(SharedBuffer(Payload)); + return BuildObject; + } + + std::vector GetNeededAttachments(CbObjectView BuildPartObject) + { + std::vector NeededAttachments; + BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); + const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); + if (!std::filesystem::is_regular_file(BlockPath)) + { + NeededAttachments.push_back(AttachmentHash); + } + }); + return NeededAttachments; + } + + bool ValidateCompressedBuffer(const IoHash& RawHash, const CompositeBuffer& Payload) + { + IoHash VerifyHash; + uint64_t VerifySize; + CompressedBuffer ValidateBuffer = CompressedBuffer::FromCompressed(Payload, VerifyHash, VerifySize); + if (!ValidateBuffer) + { + return false; + } + if (VerifyHash != RawHash) + { + return false; + } + CompositeBuffer Decompressed = ValidateBuffer.DecompressToComposite(); + if (!Decompressed) + { + return false; + } + IoHash Hash = IoHash::HashBuffer(Decompressed); + if (Hash != RawHash) + { + return false; + } + return true; + } + +private: + const std::filesystem::path m_StoragePath; + BuildStorage::Statistics& m_Stats; + const bool m_EnableJsonOutput = false; + std::atomic m_WrittenBytes; + + const double m_LatencySec = 0.0; + const double m_DelayPerKBSec = 0.0; +}; + +std::unique_ptr +CreateFileBuildStorage(const std::filesystem::path& StoragePath, + BuildStorage::Statistics& Stats, + bool EnableJsonOutput, + double LatencySec, + double DelayPerKBSec) +{ + return std::make_unique(StoragePath, Stats, EnableJsonOutput, LatencySec, DelayPerKBSec); +} + +} // namespace zen diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h new file mode 100644 index 000000000..9c236310f --- /dev/null +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class BuildStorage +{ +public: + struct Statistics + { + std::atomic TotalBytesRead = 0; + std::atomic TotalBytesWritten = 0; + std::atomic TotalRequestCount = 0; + std::atomic TotalRequestTimeUs = 0; + std::atomic TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorage() {} + + virtual CbObject ListBuilds(CbObject Query) = 0; + virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; + virtual CbObject GetBuild(const Oid& BuildId) = 0; + virtual void FinalizeBuild(const Oid& BuildId) = 0; + + virtual std::pair> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) = 0; + virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) = 0; + virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) = 0; + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual std::vector> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::function&& OnSentBytes) = 0; + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; + virtual std::vector> GetLargeBuildBlob( + const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& Receiver) = 0; + + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual std::vector FindBlocks(const Oid& BuildId) = 0; + virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h index 9b7414629..21107fb7c 100644 --- a/src/zenutil/include/zenutil/chunkblock.h +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -12,21 +12,28 @@ namespace zen { -struct ChunkBlockDescription +struct ThinChunkBlockDescription { - IoHash BlockHash; - std::vector ChunkHashes; + IoHash BlockHash; + std::vector ChunkRawHashes; +}; + +struct ChunkBlockDescription : public ThinChunkBlockDescription +{ + uint64_t HeaderSize; std::vector ChunkRawLengths; + std::vector ChunkCompressedLengths; }; std::vector ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); - +ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function(const IoHash& RawHash)> FetchChunkFunc; CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); bool IterateChunkBlock(const SharedBuffer& BlockPayload, - std::function Visitor); + std::function Visitor, + uint64_t& OutHeaderSize); } // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h new file mode 100644 index 000000000..15c687462 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -0,0 +1,256 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CbWriter; +class ChunkingController; +class WorkerThreadPool; + +enum class SourcePlatform +{ + Windows = 0, + Linux = 1, + MacOS = 2, + _Count +}; + +std::string_view ToString(SourcePlatform Platform); +SourcePlatform FromString(std::string_view Platform, SourcePlatform Default); +SourcePlatform GetSourceCurrentPlatform(); + +struct FolderContent +{ + SourcePlatform Platform = GetSourceCurrentPlatform(); + std::vector Paths; + std::vector RawSizes; + std::vector Attributes; + std::vector ModificationTicks; + + bool operator==(const FolderContent& Rhs) const; + + bool AreKnownFilesEqual(const FolderContent& Rhs) const; + void UpdateState(const FolderContent& Rhs, std::vector& PathIndexesOufOfDate); + static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs); +}; + +FolderContent GetUpdatedContent(const FolderContent& Old, + const FolderContent& New, + std::vector& OutDeletedPathIndexes); + +void SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output); +FolderContent LoadFolderContentToCompactBinary(CbObjectView Input); + +struct GetFolderContentStatistics +{ + std::atomic FoundFileCount = 0; + std::atomic FoundFileByteCount = 0; + std::atomic AcceptedFileCount = 0; + std::atomic AcceptedFileByteCount = 0; + uint64_t ElapsedWallTimeUS = 0; +}; + +FolderContent GetFolderContent(GetFolderContentStatistics& Stats, + const std::filesystem::path& RootPath, + std::function&& AcceptDirectory, + std::function&& AcceptFile, + WorkerThreadPool& WorkerPool, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag); + +struct ChunkedContentData +{ + // To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes + // ChunkCounts for that index will be the number of indexes in ChunkOrders that describe + // the sequence of chunks required to reconstruct the asset. + // Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses + std::vector SequenceRawHashes; // Raw hash for Chunk sequence + std::vector ChunkCounts; // Chunk count of ChunkOrder for SequenceRawHashes[n] + std::vector ChunkOrders; // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n] + std::vector ChunkHashes; // Unique chunk hashes + std::vector ChunkRawSizes; // Unique chunk raw size for ChunkHash[n] +}; + +struct ChunkedFolderContent +{ + SourcePlatform Platform = GetSourceCurrentPlatform(); + std::vector Paths; + std::vector RawSizes; + std::vector Attributes; + std::vector RawHashes; + ChunkedContentData ChunkedContent; +}; + +void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output); +ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input); + +ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays); +ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span DeletedPaths); + +struct ChunkingStatistics +{ + std::atomic FilesProcessed = 0; + std::atomic FilesChunked = 0; + std::atomic BytesHashed = 0; + std::atomic UniqueChunksFound = 0; + std::atomic UniqueSequencesFound = 0; + std::atomic UniqueBytesFound = 0; + uint64_t ElapsedWallTimeUS = 0; +}; + +ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag); + +struct ChunkedContentLookup +{ + struct ChunkLocation + { + uint32_t PathIndex; + uint64_t Offset; + }; + tsl::robin_map ChunkHashToChunkIndex; + tsl::robin_map RawHashToSequenceRawHashIndex; + std::vector SequenceRawHashIndexChunkOrderOffset; + std::vector ChunkLocations; + std::vector ChunkLocationOffset; // ChunkLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkLocationCounts; // ChunkLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex +}; + +ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); + +inline std::pair +GetChunkLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +{ + return std::make_pair(Lookup.ChunkLocationOffset[ChunkIndex], Lookup.ChunkLocationCounts[ChunkIndex]); +} + +inline std::span +GetChunkLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +{ + std::pair Range = GetChunkLocationRange(Lookup, ChunkIndex); + return std::span(Lookup.ChunkLocations).subspan(Range.first, Range.second); +} + +namespace compactbinary_helpers { + template + void WriteArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const Type Value : Values) + { + Output << Value; + } + Output.EndArray(); + } + + template + void WriteArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + template<> + inline void WriteArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const std::filesystem::path& Path : Values) + { + Output.AddString((const char*)Path.generic_u8string().c_str()); + } + Output.EndArray(); + } + + template<> + inline void WriteArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + inline void WriteBinaryAttachmentArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const IoHash& Hash : Values) + { + Output.AddBinaryAttachment(Hash); + } + Output.EndArray(); + } + + inline void WriteBinaryAttachmentArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsUInt32()); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsUInt64()); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + std::u8string_view U8Path = ItemView.AsU8String(); + Result.push_back(std::filesystem::path(U8Path)); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsHash()); + } + } + + inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsBinaryAttachment()); + } + } + +} // namespace compactbinary_helpers + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h new file mode 100644 index 000000000..fe4fc1bb5 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include + +namespace zen { + +const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; + +const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, + .MaxSize = 128u * 1024u, + .AvgSize = ((8u * 4u) * 1024u) + 128u}; + +const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize; + +const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u; + +struct ChunkedInfoWithSource; + +class ChunkingController +{ +public: + virtual ~ChunkingController() {} + + // Return true if the input file was processed. If true is returned OutChunked will contain the chunked info + virtual bool ProcessFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + ChunkedInfoWithSource& OutChunked, + std::atomic& BytesProcessed) const = 0; + virtual std::string_view GetName() const = 0; + virtual CbObject GetParameters() const = 0; +}; + +std::unique_ptr CreateBasicChunkingController( + std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, + const ChunkedParams& ChunkingParams = DefaultChunkedParams); +std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); + +std::unique_ptr CreateChunkingControllerWithFixedChunking( + std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, + const ChunkedParams& ChunkingParams = DefaultChunkedParams, + uint32_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize); +std::unique_ptr CreateChunkingControllerWithFixedChunking(CbObjectView Parameters); + +std::unique_ptr CreateChunkingController(std::string_view Name, CbObjectView Parameters); + +} // namespace zen diff --git a/src/zenutil/include/zenutil/filebuildstorage.h b/src/zenutil/include/zenutil/filebuildstorage.h new file mode 100644 index 000000000..c95fb32e6 --- /dev/null +++ b/src/zenutil/include/zenutil/filebuildstorage.h @@ -0,0 +1,16 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { +class HttpClient; + +std::unique_ptr CreateFileBuildStorage(const std::filesystem::path& StoragePath, + BuildStorage::Statistics& Stats, + bool EnableJsonOutput, + double LatencySec = 0.0, + double DelayPerKBSec = 0.0); +} // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h new file mode 100644 index 000000000..89fc70140 --- /dev/null +++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { +class HttpClient; + +std::unique_ptr CreateJupiterBuildStorage(LoggerRef InLog, + HttpClient& InHttpClient, + BuildStorage::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath); +} // namespace zen diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h new file mode 100644 index 000000000..7a8218c51 --- /dev/null +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -0,0 +1,69 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen { + +class ParallellWork +{ +public: + ParallellWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {} + + ~ParallellWork() + { + // Make sure to call Wait before destroying + ZEN_ASSERT(m_PendingWork.Remaining() == 0); + } + + void ScheduleWork(WorkerThreadPool& WorkerPool, + std::function& AbortFlag)>&& Work, + std::function& AbortFlag)>&& OnError) + { + m_PendingWork.AddCount(1); + try + { + WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] { + try + { + Work(m_AbortFlag); + } + catch (const std::exception& Ex) + { + OnError(Ex, m_AbortFlag); + } + m_PendingWork.CountDown(); + }); + } + catch (const std::exception&) + { + m_PendingWork.CountDown(); + throw; + } + } + + void Abort() { m_AbortFlag = true; } + + bool IsAborted() const { return m_AbortFlag.load(); } + + void Wait(int32_t UpdateInteralMS, std::function&& UpdateCallback) + { + ZEN_ASSERT(m_PendingWork.Remaining() > 0); + m_PendingWork.CountDown(); + while (!m_PendingWork.Wait(UpdateInteralMS)) + { + UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); + } + } + Latch& PendingWork() { return m_PendingWork; } + +private: + std::atomic& m_AbortFlag; + Latch m_PendingWork; +}; + +} // namespace zen diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp new file mode 100644 index 000000000..481e9146f --- /dev/null +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -0,0 +1,371 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +class JupiterBuildStorage : public BuildStorage +{ +public: + JupiterBuildStorage(LoggerRef InLog, + HttpClient& InHttpClient, + Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) + : m_Session(InLog, InHttpClient) + , m_Stats(Stats) + , m_Namespace(Namespace) + , m_Bucket(Bucket) + , m_TempFolderPath(TempFolderPath) + { + } + virtual ~JupiterBuildStorage() {} + + virtual CbObject ListBuilds(CbObject Query) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + IoBuffer Payload = Query.GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + JupiterResult ListResult = m_Session.ListBuilds(m_Namespace, m_Bucket, Payload); + AddStatistic(ListResult); + if (!ListResult.Success) + { + throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode)); + } + return PayloadToJson("Failed listing builds"sv, ListResult.Response); + } + + virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + JupiterResult PutResult = m_Session.PutBuild(m_Namespace, m_Bucket, BuildId, Payload); + AddStatistic(PutResult); + if (!PutResult.Success) + { + throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode)); + } + return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); + } + + virtual CbObject GetBuild(const Oid& BuildId) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId); + AddStatistic(GetBuildResult); + if (!GetBuildResult.Success) + { + throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode)); + } + return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); + } + + virtual void FinalizeBuild(const Oid& BuildId) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId); + AddStatistic(FinalizeBuildResult); + if (!FinalizeBuildResult.Success) + { + throw std::runtime_error( + fmt::format("Failed finalizing build part: {} ({})", FinalizeBuildResult.Reason, FinalizeBuildResult.ErrorCode)); + } + } + + virtual std::pair> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + PutBuildPartResult PutPartResult = m_Session.PutBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartName, Payload); + AddStatistic(PutPartResult); + if (!PutPartResult.Success) + { + throw std::runtime_error(fmt::format("Failed creating build part: {} ({})", PutPartResult.Reason, PutPartResult.ErrorCode)); + } + return std::make_pair(PutPartResult.RawHash, std::move(PutPartResult.Needs)); + } + + virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId); + AddStatistic(GetBuildPartResult); + if (!GetBuildPartResult.Success) + { + throw std::runtime_error(fmt::format("Failed fetching build part {}: {} ({})", + BuildPartId, + GetBuildPartResult.Reason, + GetBuildPartResult.ErrorCode)); + } + return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); + } + + virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash); + AddStatistic(FinalizePartResult); + if (!FinalizePartResult.Success) + { + throw std::runtime_error( + fmt::format("Failed finalizing build part: {} ({})", FinalizePartResult.Reason, FinalizePartResult.ErrorCode)); + } + return std::move(FinalizePartResult.Needs); + } + + virtual void PutBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + const CompositeBuffer& Payload) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload); + AddStatistic(PutBlobResult); + if (!PutBlobResult.Success) + { + throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PutBlobResult.Reason, PutBlobResult.ErrorCode)); + } + } + + virtual std::vector> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::function&& OnSentBytes) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + std::vector> WorkItems; + JupiterResult PutMultipartBlobResult = m_Session.PutMultipartBuildBlob(m_Namespace, + m_Bucket, + BuildId, + RawHash, + ContentType, + PayloadSize, + std::move(Transmitter), + WorkItems); + AddStatistic(PutMultipartBlobResult); + if (!PutMultipartBlobResult.Success) + { + throw std::runtime_error( + fmt::format("Failed putting build part: {} ({})", PutMultipartBlobResult.Reason, PutMultipartBlobResult.ErrorCode)); + } + OnSentBytes(PutMultipartBlobResult.SentBytes, WorkItems.empty()); + + std::vector> WorkList; + for (auto& WorkItem : WorkItems) + { + WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + bool IsComplete = false; + JupiterResult PartResult = WorkItem(IsComplete); + AddStatistic(PartResult); + if (!PartResult.Success) + { + throw std::runtime_error(fmt::format("Failed putting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode)); + } + OnSentBytes(PartResult.SentBytes, IsComplete); + }); + } + return WorkList; + } + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath); + AddStatistic(GetBuildBlobResult); + if (!GetBuildBlobResult.Success) + { + throw std::runtime_error( + fmt::format("Failed fetching build blob {}: {} ({})", RawHash, GetBuildBlobResult.Reason, GetBuildBlobResult.ErrorCode)); + } + return std::move(GetBuildBlobResult.Response); + } + + virtual std::vector> GetLargeBuildBlob( + const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& Receiver) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + std::vector> WorkItems; + JupiterResult GetMultipartBlobResult = + m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems); + + AddStatistic(GetMultipartBlobResult); + if (!GetMultipartBlobResult.Success) + { + throw std::runtime_error( + fmt::format("Failed getting build part: {} ({})", GetMultipartBlobResult.Reason, GetMultipartBlobResult.ErrorCode)); + } + std::vector> WorkList; + for (auto& WorkItem : WorkItems) + { + WorkList.emplace_back([this, WorkItem = std::move(WorkItem)]() { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult PartResult = WorkItem(); + AddStatistic(PartResult); + if (!PartResult.Success) + { + throw std::runtime_error(fmt::format("Failed getting build part: {} ({})", PartResult.Reason, PartResult.ErrorCode)); + } + }); + } + return WorkList; + } + + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + JupiterResult PutMetaResult = m_Session.PutBlockMetadata(m_Namespace, m_Bucket, BuildId, BlockRawHash, Payload); + AddStatistic(PutMetaResult); + if (!PutMetaResult.Success) + { + throw std::runtime_error( + fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode)); + } + } + + virtual std::vector FindBlocks(const Oid& BuildId) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId); + AddStatistic(FindResult); + if (!FindResult.Success) + { + throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode)); + } + return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response)); + } + + virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + { + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + CbObjectWriter Request; + + Request.BeginArray("blocks"sv); + for (const IoHash& BlockHash : BlockHashes) + { + Request.AddHash(BlockHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + JupiterResult GetBlockMetadataResult = m_Session.GetBlockMetadata(m_Namespace, m_Bucket, BuildId, Payload); + AddStatistic(GetBlockMetadataResult); + if (!GetBlockMetadataResult.Success) + { + throw std::runtime_error( + fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); + } + std::vector UnorderedList = + ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response)); + tsl::robin_map BlockDescriptionLookup; + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); + } + std::vector SortedBlockDescriptions; + SortedBlockDescriptions.reserve(BlockDescriptionLookup.size()); + for (const IoHash& BlockHash : BlockHashes) + { + if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) + { + SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second])); + } + } + return SortedBlockDescriptions; + } + +private: + static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload) + { + if (Payload.GetContentType() == ZenContentType::kJSON) + { + std::string_view Json(reinterpret_cast(Payload.GetData()), Payload.GetSize()); + return LoadCompactBinaryFromJson(Json).AsObject(); + } + else if (Payload.GetContentType() == ZenContentType::kCbObject) + { + return LoadCompactBinaryObject(Payload); + } + else if (Payload.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + return LoadCompactBinaryObject(CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize)); + } + else + { + throw std::runtime_error( + fmt::format("{}: {} ({})", "Unsupported response format", Context, ToString(Payload.GetContentType()))); + } + } + + void AddStatistic(const JupiterResult& Result) + { + m_Stats.TotalBytesWritten += Result.SentBytes; + m_Stats.TotalBytesRead += Result.ReceivedBytes; + m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); + m_Stats.TotalRequestCount++; + } + + JupiterSession m_Session; + Statistics& m_Stats; + const std::string m_Namespace; + const std::string m_Bucket; + const std::filesystem::path m_TempFolderPath; +}; + +std::unique_ptr +CreateJupiterBuildStorage(LoggerRef InLog, + HttpClient& InHttpClient, + BuildStorage::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) +{ + return std::make_unique(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath); +} + +} // namespace zen -- cgit v1.2.3 From 40bf8e292618e21a07223994ce438e580d6fdd20 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 26 Feb 2025 18:08:37 +0100 Subject: added include to zentest-appstub.cpp --- src/zentest-appstub/zentest-appstub.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/zentest-appstub/zentest-appstub.cpp b/src/zentest-appstub/zentest-appstub.cpp index 66e6e03fd..e6b1c4bd1 100644 --- a/src/zentest-appstub/zentest-appstub.cpp +++ b/src/zentest-appstub/zentest-appstub.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace std::chrono_literals; -- cgit v1.2.3 From 1ee30cf395b3a57865ed21c7be22c894a8fdae38 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 26 Feb 2025 18:18:11 +0100 Subject: clang-format fix --- src/zentest-appstub/zentest-appstub.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zentest-appstub/zentest-appstub.cpp b/src/zentest-appstub/zentest-appstub.cpp index e6b1c4bd1..24cf21e97 100644 --- a/src/zentest-appstub/zentest-appstub.cpp +++ b/src/zentest-appstub/zentest-appstub.cpp @@ -1,10 +1,10 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include +#include #include #include #include -#include using namespace std::chrono_literals; -- cgit v1.2.3 From aef396302e8f9177aa9185bd37d5a8657056f73c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Feb 2025 21:27:13 +0100 Subject: Fix bug causing crash if large file was exact multiple of 256KB when using zen builds command (#286) --- src/zen/cmds/builds_cmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ececab29e..ed1eb2d19 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -855,7 +855,7 @@ namespace { ZEN_ASSERT((Offset + Size) <= SourceSize); const uint64_t BlockIndexStart = Offset / BlockSize; - const uint64_t BlockIndexEnd = (Offset + Size) / BlockSize; + const uint64_t BlockIndexEnd = (Offset + Size - 1) / BlockSize; std::vector BufferRanges; BufferRanges.reserve(BlockIndexEnd - BlockIndexStart + 1); -- cgit v1.2.3 From 38a58059214bacc18c8a6406acf7f46c57f51e86 Mon Sep 17 00:00:00 2001 From: Zousar Shaker Date: Thu, 27 Feb 2025 01:27:34 -0700 Subject: Zs/auth bad function fix (#287) * Describe fix in changelog * remove JupiterClient::m_TokenProvider --------- Co-authored-by: zousar <2936246+zousar@users.noreply.github.com> Co-authored-by: Dan Engelbrecht --- src/zenutil/include/zenutil/jupiter/jupiterclient.h | 11 +++++------ src/zenutil/jupiter/jupiterclient.cpp | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zenutil/include/zenutil/jupiter/jupiterclient.h b/src/zenutil/include/zenutil/jupiter/jupiterclient.h index defe50edc..8a51bd60a 100644 --- a/src/zenutil/include/zenutil/jupiter/jupiterclient.h +++ b/src/zenutil/include/zenutil/jupiter/jupiterclient.h @@ -44,12 +44,11 @@ public: HttpClient& Client() { return m_HttpClient; } private: - LoggerRef m_Log; - const std::string m_DefaultDdcNamespace; - const std::string m_DefaultBlobStoreNamespace; - const std::string m_ComputeCluster; - std::function m_TokenProvider; - HttpClient m_HttpClient; + LoggerRef m_Log; + const std::string m_DefaultDdcNamespace; + const std::string m_DefaultBlobStoreNamespace; + const std::string m_ComputeCluster; + HttpClient m_HttpClient; friend class JupiterSession; }; diff --git a/src/zenutil/jupiter/jupiterclient.cpp b/src/zenutil/jupiter/jupiterclient.cpp index 5e5da3750..dbac218a4 100644 --- a/src/zenutil/jupiter/jupiterclient.cpp +++ b/src/zenutil/jupiter/jupiterclient.cpp @@ -11,7 +11,6 @@ JupiterClient::JupiterClient(const JupiterClientOptions& Options, std::function< , m_DefaultDdcNamespace(Options.DdcNamespace) , m_DefaultBlobStoreNamespace(Options.BlobStoreNamespace) , m_ComputeCluster(Options.ComputeCluster) -, m_TokenProvider(std::move(TokenProvider)) , m_HttpClient(Options.ServiceUrl, HttpClientSettings{.ConnectTimeout = Options.ConnectTimeout, .Timeout = Options.Timeout, -- cgit v1.2.3 From 5791f51cccea1d4e5365456c8da89dbac0dd3ec0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 28 Feb 2025 12:39:48 +0100 Subject: improve error handling (#289) * clearer errors * quicker abort * handle deleted local files * simplify parallellwork error handling * don't finish progress on destructor - gives wrong impression * graceful ctrl-c handling --- src/zen/cmds/builds_cmd.cpp | 1922 +++++++++++----------- src/zen/zen.cpp | 2 +- src/zenutil/chunkedcontent.cpp | 13 +- src/zenutil/chunkedfile.cpp | 11 +- src/zenutil/chunkingcontroller.cpp | 12 +- src/zenutil/include/zenutil/chunkedfile.h | 3 +- src/zenutil/include/zenutil/chunkingcontroller.h | 7 +- src/zenutil/include/zenutil/parallellwork.h | 48 + 8 files changed, 1040 insertions(+), 978 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ed1eb2d19..18cc7cf9e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -28,6 +28,7 @@ #include #include +#include #include ZEN_THIRD_PARTY_INCLUDES_START @@ -49,6 +50,21 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace { + static std::atomic AbortFlag = false; + static void SignalCallbackHandler(int SigNum) + { + if (SigNum == SIGINT) + { + AbortFlag = true; + } +#if ZEN_PLATFORM_WINDOWS + if (SigNum == SIGBREAK) + { + AbortFlag = true; + } +#endif // ZEN_PLATFORM_WINDOWS + } + using namespace std::literals; static const size_t DefaultMaxBlockSize = 64u * 1024u * 1024u; @@ -273,8 +289,7 @@ namespace { const std::filesystem::path& Path, std::function&& IsAcceptedFolder, std::function&& IsAcceptedFile, - ChunkingController& ChunkController, - std::atomic& AbortFlag) + ChunkingController& ChunkController) { FolderContent Content = GetFolderContent( GetFolderContentStats, @@ -316,6 +331,10 @@ namespace { false); }, AbortFlag); + if (AbortFlag) + { + return {}; + } FilteredBytesHashed.Stop(); ProgressBar.Finish(); @@ -1214,7 +1233,6 @@ namespace { const ChunkedContentLookup& Lookup, BuildStorage& Storage, const Oid& BuildId, - std::atomic& AbortFlag, const std::vector>& NewBlockChunks, GeneratedBlocks& OutBlocks, DiskStatistics& DiskStats, @@ -1254,7 +1272,7 @@ namespace { const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; Work.ScheduleWork( GenerateBlobsPool, - [&, BlockIndex](std::atomic& AbortFlag) { + [&, BlockIndex](std::atomic&) { if (!AbortFlag) { FilteredGeneratedBytesPerSecond.Start(); @@ -1295,7 +1313,7 @@ namespace { PendingUploadCount++; Work.ScheduleWork( UploadBlocksPool, - [&, BlockIndex, Payload = std::move(Payload)](std::atomic& AbortFlag) { + [&, BlockIndex, Payload = std::move(Payload)](std::atomic&) { if (!AbortFlag) { if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) @@ -1340,17 +1358,11 @@ namespace { } } }, - [&](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed uploading block. Reason: {}", Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } }, - [&](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed generating block. Reason: {}", Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -1394,7 +1406,6 @@ namespace { GeneratedBlocks& NewBlocks, std::span LooseChunkIndexes, const std::uint64_t LargeAttachmentSize, - std::atomic& AbortFlag, DiskStatistics& DiskStats, UploadStatistics& UploadStats, GenerateBlocksStatistics& GenerateBlocksStats, @@ -1424,9 +1435,6 @@ namespace { ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex); } - std::vector FoundChunkHashes; - FoundChunkHashes.reserve(RawHashes.size()); - std::vector BlockIndexes; std::vector LooseChunkOrderIndexes; @@ -1458,7 +1466,7 @@ namespace { auto AsyncUploadBlock = [&](const size_t BlockIndex, const IoHash BlockHash, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, BlockIndex, BlockHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic& AbortFlag) { + [&, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) { if (!AbortFlag) { FilteredUploadedBytesPerSecond.Start(); @@ -1490,16 +1498,13 @@ namespace { } } }, - [&](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed uploading block. Reason: {}", Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); }; auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, RawHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic& AbortFlag) { + [&, RawHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) { if (!AbortFlag) { const uint64_t PayloadSize = Payload.GetSize(); @@ -1536,16 +1541,13 @@ namespace { { Work.ScheduleWork( UploadChunkPool, - [Work = std::move(WorkPart)](std::atomic& AbortFlag) { + [Work = std::move(WorkPart)](std::atomic&) { if (!AbortFlag) { Work(); } }, - [&, RawHash](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed uploading multipart blob {}. Reason: {}", RawHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } ZEN_CONSOLE_VERBOSE("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize)); } @@ -1564,10 +1566,7 @@ namespace { } } }, - [&](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed uploading chunk. Reason: {}", Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); }; std::vector GenerateBlockIndexes; @@ -1581,7 +1580,6 @@ namespace { if (CompositeBuffer BlockPayload = std::move(NewBlocks.BlockBuffers[BlockIndex]); BlockPayload) { const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - FoundChunkHashes.push_back(BlockHash); if (!AbortFlag) { AsyncUploadBlock(BlockIndex, BlockHash, std::move(BlockPayload)); @@ -1606,12 +1604,11 @@ namespace { for (const size_t BlockIndex : GenerateBlockIndexes) { const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - FoundChunkHashes.push_back(BlockHash); if (!AbortFlag) { Work.ScheduleWork( ReadChunkPool, - [&, BlockIndex](std::atomic& AbortFlag) { + [&, BlockIndex](std::atomic&) { if (!AbortFlag) { FilteredGenerateBlockBytesPerSecond.Start(); @@ -1646,10 +1643,7 @@ namespace { NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); } }, - [&](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed generating block. Reason: {}", Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } @@ -1662,7 +1656,7 @@ namespace { const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; Work.ScheduleWork( ReadChunkPool, - [&, ChunkIndex](std::atomic& AbortFlag) { + [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { FilteredCompressedBytesPerSecond.Start(); @@ -1686,12 +1680,7 @@ namespace { } } }, - [&, ChunkIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed compressing part blob {}. Reason: {}", - Content.ChunkedContent.ChunkHashes[ChunkIndex], - Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -1884,7 +1873,7 @@ namespace { return FilteredReuseBlockIndexes; }; - bool UploadFolder(BuildStorage& Storage, + void UploadFolder(BuildStorage& Storage, const Oid& BuildId, const Oid& BuildPartId, const std::string_view BuildPartName, @@ -1898,8 +1887,6 @@ namespace { { Stopwatch ProcessTimer; - std::atomic AbortFlag = false; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; CreateDirectories(ZenTempFolder); CleanDirectory(ZenTempFolder, {}); @@ -2101,15 +2088,14 @@ namespace { false); }, AbortFlag); + if (AbortFlag) + { + return; + } FilteredBytesHashed.Stop(); ProgressBar.Finish(); } - if (AbortFlag) - { - return true; - } - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", LocalContent.Paths.size(), NiceBytes(TotalRawSize), @@ -2289,7 +2275,6 @@ namespace { LocalLookup, Storage, BuildId, - AbortFlag, NewBlockChunks, NewBlocks, DiskStats, @@ -2297,11 +2282,6 @@ namespace { GenerateBlocksStats); } - if (AbortFlag) - { - return true; - } - CbObject PartManifest; { CbObjectWriter PartManifestWriter; @@ -2508,7 +2488,6 @@ namespace { NewBlocks, LooseChunkIndexes, LargeAttachmentSize, - AbortFlag, DiskStats, TempUploadStats, TempGenerateBlocksStats, @@ -2558,21 +2537,8 @@ namespace { { break; } - if (AbortFlag) - { - return true; - } ZEN_CONSOLE_VERBOSE("FinalizeBuildPart needs attachments: {}", FormatArray(Needs, "\n "sv)); UploadAttachments(Needs); - if (AbortFlag) - { - return true; - } - } - - if (AbortFlag) - { - return true; } if (CreateBuild) @@ -2612,9 +2578,8 @@ namespace { std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockHashes); if (VerifyBlockDescriptions.size() != BlockHashes.size()) { - ZEN_CONSOLE("Uploaded blocks could not all be found, {} blocks are missing", - BlockHashes.size() - VerifyBlockDescriptions.size()); - return true; + throw std::runtime_error(fmt::format("Uploaded blocks could not all be found, {} blocks are missing", + BlockHashes.size() - VerifyBlockDescriptions.size())); } } @@ -2774,10 +2739,9 @@ namespace { BuildPartName, BuildPartId, NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs())); - return false; } - void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, std::atomic& AbortFlag) + void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path) { ProgressBar ProgressBar(UsePlainProgress); std::atomic FilesVerified(0); @@ -2822,7 +2786,7 @@ namespace { Work.ScheduleWork( VerifyPool, - [&, PathIndex](std::atomic& AbortFlag) { + [&, PathIndex](std::atomic&) { if (!AbortFlag) { // TODO: Convert ScheduleWork body to function @@ -2912,9 +2876,7 @@ namespace { FilesVerified++; } }, - [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_UNUSED(AbortFlag); - + [&, PathIndex](const std::exception& Ex, std::atomic&) { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", (Path / Content.Paths[PathIndex]).make_preferred(), @@ -3085,36 +3047,39 @@ namespace { { if (!WriteOps.empty()) { - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOpData& Lhs, const WriteOpData& Rhs) { - if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) - { - return true; - } - if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + if (!AbortFlag) + { + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOpData& Lhs, const WriteOpData& Rhs) { + if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + { + return true; + } + if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + + WriteFileCache OpenFileCache; + for (const WriteOpData& WriteOp : WriteOps) { - return false; + const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t PathIndex = WriteOp.Target->PathIndex; + const uint64_t ChunkSize = Chunk.GetSize(); + const uint64_t FileOffset = WriteOp.Target->Offset; + ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); + + OpenFileCache.WriteToFile( + PathIndex, + [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, + Chunk, + FileOffset, + Content.RawSizes[PathIndex]); + OutBytesWritten += ChunkSize; } - return Lhs.Target->Offset < Rhs.Target->Offset; - }); - - WriteFileCache OpenFileCache; - for (const WriteOpData& WriteOp : WriteOps) - { - const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; - const uint32_t PathIndex = WriteOp.Target->PathIndex; - const uint64_t ChunkSize = Chunk.GetSize(); - const uint64_t FileOffset = WriteOp.Target->Offset; - ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); - - OpenFileCache.WriteToFile( - PathIndex, - [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, - Chunk, - FileOffset, - Content.RawSizes[PathIndex]); - OutBytesWritten += ChunkSize; + OutChunksComplete += gsl::narrow(ChunkBuffers.size()); } - OutChunksComplete += gsl::narrow(ChunkBuffers.size()); } return true; } @@ -3186,7 +3151,6 @@ namespace { ParallellWork& Work, WorkerThreadPool& WritePool, WorkerThreadPool& NetworkPool, - std::atomic& AbortFlag, std::atomic& BytesWritten, std::atomic& WriteToDiskBytes, std::atomic& BytesDownloaded, @@ -3225,8 +3189,8 @@ namespace { &WriteToDiskBytes, &DownloadedChunks, &ChunksComplete, - ChunkTargetPtrs = std::vector(ChunkTargetPtrs), - &AbortFlag](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { + ChunkTargetPtrs = std::vector( + ChunkTargetPtrs)](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { BytesDownloaded += Chunk.GetSize(); LooseChunksBytes += Chunk.GetSize(); @@ -3249,7 +3213,7 @@ namespace { &ChunksComplete, &BytesWritten, &WriteToDiskBytes, - ChunkTargetPtrs](std::atomic& AbortFlag) { + ChunkTargetPtrs](std::atomic&) { if (!AbortFlag) { uint64_t CompressedSize = Workload->TempFile.FileSize(); @@ -3276,6 +3240,7 @@ namespace { // ZEN_ASSERT_SLOW(ChunkHash == // IoHash::HashBuffer(Chunk.AsIoBuffer())); + if (!AbortFlag) { WriteFileCache OpenFileCache; @@ -3291,10 +3256,7 @@ namespace { } } }, - [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed writing chunk {}. Reason: {}", ChunkHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } }); @@ -3306,20 +3268,17 @@ namespace { { Work.ScheduleWork( NetworkPool, - [WorkItem = std::move(WorkItem)](std::atomic& AbortFlag) { + [WorkItem = std::move(WorkItem)](std::atomic&) { if (!AbortFlag) { WorkItem(); } }, - [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed uploading multipart blob {}. Reason: {}", ChunkHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } - bool UpdateFolder(BuildStorage& Storage, + void UpdateFolder(BuildStorage& Storage, const Oid& BuildId, const std::filesystem::path& Path, const std::uint64_t LargeAttachmentSize, @@ -3329,7 +3288,6 @@ namespace { const std::vector& BlockDescriptions, const std::vector& LooseChunkHashes, bool WipeTargetFolder, - std::atomic& AbortFlag, FolderContent& OutLocalFolderState) { std::atomic DownloadedBlocks = 0; @@ -3433,7 +3391,7 @@ namespace { if (AbortFlag) { - return true; + return; } CleanDirectory(Path, DefaultExcludeFolders); @@ -3646,7 +3604,7 @@ namespace { Work.ScheduleWork( WritePool, // GetSyncWorkerPool(),// - [&, CopyDataIndex](std::atomic& AbortFlag) { + [&, CopyDataIndex](std::atomic&) { if (!AbortFlag) { const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; @@ -3703,6 +3661,10 @@ namespace { WriteFileCache OpenFileCache; for (const WriteOp& Op : WriteOps) { + if (AbortFlag) + { + break; + } const uint32_t RemotePathIndex = Op.Target->PathIndex; const uint64_t ChunkSize = Op.ChunkSize; CompositeBuffer ChunkSource = SourceFile.GetRange(Op.LocalFileOffset, ChunkSize); @@ -3721,15 +3683,18 @@ namespace { WriteToDiskBytes += ChunkSize; CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? } - ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), CopyData.OriginalSourceFileName); + if (!AbortFlag) + { + ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); + ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), CopyData.OriginalSourceFileName); + } } if (CopyData.RemotePathIndexes.empty()) { std::filesystem::remove(CacheFilePath); } - else + else if (!AbortFlag) { uint64_t LocalBytesWritten = 0; CloneFullFileFromCache(Path, @@ -3753,12 +3718,7 @@ namespace { } } }, - [&, CopyDataIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed reading cached file {}. Reason: {}", - CacheCopyDatas[CopyDataIndex].OriginalSourceFileName, - Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } for (const IoHash ChunkHash : LooseChunkHashes) @@ -3788,7 +3748,7 @@ namespace { { Work.ScheduleWork( NetworkPool, - [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic& AbortFlag) { + [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { if (!AbortFlag) { if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) @@ -3804,7 +3764,6 @@ namespace { Work, WritePool, NetworkPool, - AbortFlag, BytesWritten, WriteToDiskBytes, BytesDownloaded, @@ -3829,7 +3788,7 @@ namespace { Work.ScheduleWork( WritePool, [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(CompressedPart)]( - std::atomic& AbortFlag) { + std::atomic&) { if (!AbortFlag) { uint64_t TotalBytesWritten = 0; @@ -3850,18 +3809,12 @@ namespace { WriteToDiskBytes += TotalBytesWritten; } }, - [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed writing chunk {}. Reason: {}", ChunkHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } } }, - [&, ChunkHash](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed downloading chunk {}. Reason: {}", ChunkHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } } @@ -3893,98 +3846,97 @@ namespace { } Work.ScheduleWork( WritePool, - [&, BlockIndex](std::atomic& AbortFlag) { + [&, BlockIndex](std::atomic&) { if (!AbortFlag) { if (IsBlockNeeded(BlockDescriptions[BlockIndex])) { Work.ScheduleWork( NetworkPool, - [&, BlockIndex](std::atomic& AbortFlag) { - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); - if (!BlockBuffer) - { - throw std::runtime_error( - fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); - } - BytesDownloaded += BlockBuffer.GetSize(); - BlockBytes += BlockBuffer.GetSize(); - DownloadedBlocks++; - + [&, BlockIndex](std::atomic&) { if (!AbortFlag) { - Work.ScheduleWork( - WritePool, - [&, BlockIndex, BlockBuffer = std::move(BlockBuffer)](std::atomic& AbortFlag) { - if (!AbortFlag) - { - IoHash BlockRawHash; - uint64_t BlockRawSize; - CompressedBuffer CompressedBlockBuffer = - CompressedBuffer::FromCompressed(SharedBuffer(std::move(BlockBuffer)), - BlockRawHash, - BlockRawSize); - if (!CompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", - BlockDescriptions[BlockIndex].BlockHash)); - } - - if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) - { - throw std::runtime_error( - fmt::format("Block {} header has a mismatching raw hash {}", - BlockDescriptions[BlockIndex].BlockHash, - BlockRawHash)); - } - - CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); - if (!DecompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} failed to decompress", - BlockDescriptions[BlockIndex].BlockHash)); - } + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); + } + BytesDownloaded += BlockBuffer.GetSize(); + BlockBytes += BlockBuffer.GetSize(); + DownloadedBlocks++; - ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == - IoHash::HashBuffer(DecompressedBlockBuffer)); - - uint64_t BytesWrittenToDisk = 0; - uint32_t ChunksReadFromBlock = 0; - if (WriteBlockToDisk(Path, - RemoteContent, - RemotePathIndexWantsCopyFromCacheFlags, - DecompressedBlockBuffer, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags.data(), - ChunksReadFromBlock, - BytesWrittenToDisk)) - { - BytesWritten += BytesWrittenToDisk; - WriteToDiskBytes += BytesWrittenToDisk; - ChunkCountWritten += ChunksReadFromBlock; - } - else + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockBuffer = std::move(BlockBuffer)](std::atomic&) { + if (!AbortFlag) { - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescriptions[BlockIndex].BlockHash)); + IoHash BlockRawHash; + uint64_t BlockRawSize; + CompressedBuffer CompressedBlockBuffer = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(BlockBuffer)), + BlockRawHash, + BlockRawSize); + if (!CompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", + BlockDescriptions[BlockIndex].BlockHash)); + } + + if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) + { + throw std::runtime_error( + fmt::format("Block {} header has a mismatching raw hash {}", + BlockDescriptions[BlockIndex].BlockHash, + BlockRawHash)); + } + + CompositeBuffer DecompressedBlockBuffer = + CompressedBlockBuffer.DecompressToComposite(); + if (!DecompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} failed to decompress", + BlockDescriptions[BlockIndex].BlockHash)); + } + + ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == + IoHash::HashBuffer(DecompressedBlockBuffer)); + + uint64_t BytesWrittenToDisk = 0; + uint32_t ChunksReadFromBlock = 0; + if (WriteBlockToDisk(Path, + RemoteContent, + RemotePathIndexWantsCopyFromCacheFlags, + DecompressedBlockBuffer, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags.data(), + ChunksReadFromBlock, + BytesWrittenToDisk)) + { + BytesWritten += BytesWrittenToDisk; + WriteToDiskBytes += BytesWrittenToDisk; + ChunkCountWritten += ChunksReadFromBlock; + } + else + { + throw std::runtime_error(fmt::format("Block {} is malformed", + BlockDescriptions[BlockIndex].BlockHash)); + } + BlocksComplete++; } - BlocksComplete++; - } - }, - [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed writing block {}. Reason: {}", - BlockDescriptions[BlockIndex].BlockHash, - Ex.what()); - AbortFlag = true; - }); + }, + [&, BlockIndex](const std::exception& Ex, std::atomic&) { + ZEN_ERROR("Failed writing block {}. Reason: {}", + BlockDescriptions[BlockIndex].BlockHash, + Ex.what()); + AbortFlag = true; + }); + } } }, - [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed downloading block {}. Reason: {}", - BlockDescriptions[BlockIndex].BlockHash, - Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } else { @@ -3993,10 +3945,7 @@ namespace { } } }, - [&, BlockIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed determning if block {} is needed. Reason: {}", BlockDescriptions[BlockIndex].BlockHash, Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } for (uint32_t PathIndex = 0; PathIndex < RemoteContent.Paths.size(); PathIndex++) { @@ -4008,7 +3957,7 @@ namespace { { Work.ScheduleWork( WritePool, - [&, PathIndex](std::atomic& AbortFlag) { + [&, PathIndex](std::atomic&) { if (!AbortFlag) { const std::filesystem::path TargetPath = (Path / RemoteContent.Paths[PathIndex]).make_preferred(); @@ -4017,10 +3966,7 @@ namespace { OutputFile.Open(TargetPath, BasicFile::Mode::kTruncate); } }, - [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed creating file at {}. Reason: {}", RemoteContent.Paths[PathIndex], Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } } @@ -4040,6 +3986,12 @@ namespace { .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, false); }); + + if (AbortFlag) + { + return; + } + WriteProgressBar.Finish(); { @@ -4109,8 +4061,6 @@ namespace { } PremissionsProgressBar.Finish(); } - - return false; } std::vector> ResolveBuildPartNames(BuildStorage& Storage, @@ -4421,8 +4371,7 @@ namespace { ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, - ChunkingController& ChunkController, - std::atomic& AbortFlag) + ChunkingController& ChunkController) { ChunkedFolderContent LocalContent; @@ -4564,6 +4513,26 @@ namespace { } else { + // Remove files from LocalContent no longer in LocalFolderState + tsl::robin_set LocalFolderPaths; + LocalFolderPaths.reserve(LocalFolderState.Paths.size()); + for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) + { + LocalFolderPaths.insert(LocalFolderPath.generic_string()); + } + std::vector DeletedPaths; + for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) + { + if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) + { + DeletedPaths.push_back(LocalContentPath); + } + } + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } + ZEN_CONSOLE("Using cached local state"); } ZEN_CONSOLE("Read local state in {}", NiceLatencyNs(ReadStateTimer.GetElapsedTimeUs() * 1000)); @@ -4611,18 +4580,19 @@ namespace { false); }, AbortFlag); - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); if (AbortFlag) { return {}; } + + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); } return LocalContent; } - bool DownloadFolder(BuildStorage& Storage, + void DownloadFolder(BuildStorage& Storage, const Oid& BuildId, const std::vector& BuildPartIds, std::span BuildPartNames, @@ -4630,8 +4600,7 @@ namespace { bool AllowMultiparts, bool WipeTargetFolder) { - Stopwatch DownloadTimer; - std::atomic AbortFlag(false); + Stopwatch DownloadTimer; const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; CreateDirectories(ZenTempFolder); @@ -4672,19 +4641,19 @@ namespace { { if (!WipeTargetFolder) { - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController, AbortFlag); + LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); } } else { CreateDirectories(Path); } - if (AbortFlag.load()) + if (AbortFlag) { - return true; + return; } - auto CompareContent = [](const ChunkedFolderContent& Lsh, const ChunkedFolderContent& Rhs) { + auto CompareContent = [](const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs) { tsl::robin_map RhsPathToIndex; const size_t RhsPathCount = Rhs.Paths.size(); RhsPathToIndex.reserve(RhsPathCount); @@ -4692,14 +4661,14 @@ namespace { { RhsPathToIndex.insert({Rhs.Paths[RhsPathIndex].generic_string(), RhsPathIndex}); } - const size_t LhsPathCount = Lsh.Paths.size(); + const size_t LhsPathCount = Lhs.Paths.size(); for (size_t LhsPathIndex = 0; LhsPathIndex < LhsPathCount; LhsPathIndex++) { - if (auto It = RhsPathToIndex.find(Lsh.Paths[LhsPathIndex].generic_string()); It != RhsPathToIndex.end()) + if (auto It = RhsPathToIndex.find(Lhs.Paths[LhsPathIndex].generic_string()); It != RhsPathToIndex.end()) { const size_t RhsPathIndex = It->second; - if ((Lsh.RawHashes[LhsPathIndex] != Rhs.RawHashes[RhsPathIndex]) || - (!FolderContent::AreFileAttributesEqual(Lsh.Attributes[LhsPathIndex], Rhs.Attributes[RhsPathIndex]))) + if ((Lhs.RawHashes[LhsPathIndex] != Rhs.RawHashes[RhsPathIndex]) || + (!FolderContent::AreFileAttributesEqual(Lhs.Attributes[LhsPathIndex], Rhs.Attributes[RhsPathIndex]))) { return false; } @@ -4709,6 +4678,20 @@ namespace { return false; } } + tsl::robin_set LhsPathExists; + LhsPathExists.reserve(LhsPathCount); + for (size_t LhsPathIndex = 0; LhsPathIndex < LhsPathCount; LhsPathIndex++) + { + LhsPathExists.insert({Lhs.Paths[LhsPathIndex].generic_string()}); + } + for (size_t RhsPathIndex = 0; RhsPathIndex < RhsPathCount; RhsPathIndex++) + { + if (!LhsPathExists.contains(Rhs.Paths[RhsPathIndex].generic_string())) + { + return false; + } + } + return true; }; @@ -4726,49 +4709,42 @@ namespace { } ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, SB.ToView()); FolderContent LocalFolderState; - if (UpdateFolder(Storage, - BuildId, - Path, - LargeAttachmentSize, - PreferredMultipartChunkSize, - LocalContent, - RemoteContent, - BlockDescriptions, - LooseChunkHashes, - WipeTargetFolder, - AbortFlag, - LocalFolderState)) - { - AbortFlag = true; - } + UpdateFolder(Storage, + BuildId, + Path, + LargeAttachmentSize, + PreferredMultipartChunkSize, + LocalContent, + RemoteContent, + BlockDescriptions, + LooseChunkHashes, + WipeTargetFolder, + LocalFolderState); + if (!AbortFlag) { - VerifyFolder(RemoteContent, Path, AbortFlag); - } + VerifyFolder(RemoteContent, Path); - Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceLatencyNs(WriteStateTimer.GetElapsedTimeUs() * 1000)); + CreateDirectories((Path / ZenStateFilePath).parent_path()); + TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceLatencyNs(WriteStateTimer.GetElapsedTimeUs() * 1000)); #if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 - ZEN_CONSOLE("Downloaded build in {}.", NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("Downloaded build in {}.", NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + } } - - return AbortFlag.load(); } - bool DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) + void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) { - std::atomic AbortFlag(false); - ChunkedFolderContent BaseFolderContent; ChunkedFolderContent CompareFolderContent; @@ -4818,8 +4794,11 @@ namespace { BasePath, IsAcceptedFolder, IsAcceptedFile, - *ChunkController, - AbortFlag); + *ChunkController); + if (AbortFlag) + { + return; + } GetFolderContentStatistics CompareGetFolderContentStats; ChunkingStatistics CompareChunkingStats; @@ -4828,8 +4807,12 @@ namespace { ComparePath, IsAcceptedFolder, IsAcceptedFile, - *ChunkController, - AbortFlag); + *ChunkController); + + if (AbortFlag) + { + return; + } } std::vector AddedHashes; @@ -4918,8 +4901,6 @@ namespace { NewChunkCount, NiceBytes(NewChunkSize), NewPercent); - - return false; } } // namespace @@ -5190,6 +5171,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_UNUSED(GlobalOptions); + signal(SIGINT, SignalCallbackHandler); +#if ZEN_PLATFORM_WINDOWS + signal(SIGBREAK, SignalCallbackHandler); +#endif // ZEN_PLATFORM_WINDOWS + using namespace std::literals; std::vector SubCommandArguments; @@ -5320,786 +5306,798 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseOutputOptions(); - if (SubOption == &m_ListOptions) + try { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - - CbObjectWriter QueryWriter; - QueryWriter.BeginObject("query"); - { - // QueryWriter.BeginObject("platform"); - // { - // QueryWriter.AddString("$eq", "Windows"); - // } - // QueryWriter.EndObject(); // changelist - } - QueryWriter.EndObject(); // query - - BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Querying builds in folder '{}'.", m_StoragePath); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 - } - else + if (SubOption == &m_ListOptions) { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } - - CbObject Response = Storage->ListBuilds(QueryWriter.Save()); - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); - return 0; - } + ParseStorageOptions(); + ParseAuthOptions(); - if (SubOption == &m_UploadOptions) - { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); - } + HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_CreateBuild) - { - if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) + CbObjectWriter QueryWriter; + QueryWriter.BeginObject("query"); { - throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + // QueryWriter.BeginObject("platform"); + // { + // QueryWriter.AddString("$eq", "Windows"); + // } + // QueryWriter.EndObject(); // changelist } - if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) + QueryWriter.EndObject(); // query + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + if (!m_BuildsUrl.empty()) { - throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); + ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); } - } - else - { - if (!m_BuildMetadataPath.empty()) + else if (!m_StoragePath.empty()) { - throw zen::OptionParseException( - fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + ZEN_CONSOLE("Querying builds in folder '{}'.", m_StoragePath); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 } - if (!m_BuildMetadata.empty()) + else { - throw zen::OptionParseException( - fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } - } - if (m_BuildPartName.empty()) - { - m_BuildPartName = m_Path.filename().string(); + CbObject Response = Storage->ListBuilds(QueryWriter.Save()); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + return 0; } - const bool GeneratedBuildId = m_BuildId.empty(); - if (GeneratedBuildId) - { - m_BuildId = Oid::NewOid().ToString(); - } - else if (m_BuildId.length() != Oid::StringLength) + if (SubOption == &m_UploadOptions) { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } - else if (Oid::FromHexString(m_BuildId) == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } + ParseStorageOptions(); + ParseAuthOptions(); - const bool GeneratedBuildPartId = m_BuildPartId.empty(); - if (GeneratedBuildPartId) - { - m_BuildPartId = Oid::NewOid().ToString(); - } - else if (m_BuildPartId.length() != Oid::StringLength) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } - else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); - } + HttpClient Http(m_BuildsUrl, ClientSettings); - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", - m_BuildPartName, - m_Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - GeneratedBuildId ? "Generated " : "", - BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", - m_BuildPartName, - m_Path, - m_StoragePath, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson); // , .0015, 0.00004 - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); + } - CbObject MetaData; - if (m_CreateBuild) - { - if (!m_BuildMetadataPath.empty()) + if (m_CreateBuild) { - std::filesystem::path MetadataPath(m_BuildMetadataPath); - IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); - std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - if (!JsonError.empty()) + if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); } } - if (!m_BuildMetadata.empty()) + else { - CbObjectWriter MetaDataWriter(1024); - ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { - size_t SplitPos = Pair.find('='); - if (SplitPos == std::string::npos || SplitPos == 0) - { - throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); - } - MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); - return true; - }); - MetaData = MetaDataWriter.Save(); + if (!m_BuildMetadataPath.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadata.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + } } - } - - bool Aborted = UploadFolder(*Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - m_ManifestPath, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData, - m_CreateBuild, - m_Clean); - if (Aborted) - { - ZEN_CONSOLE("Upload failed."); - } - - if (false) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } - return Aborted ? 11 : 0; - } - - if (SubOption == &m_DownloadOptions) - { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } + if (m_BuildPartName.empty()) + { + m_BuildPartName = m_Path.filename().string(); + } - std::vector BuildPartIds; - for (const std::string& BuildPartId : m_BuildPartIds) - { - BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); - if (BuildPartIds.back() == Oid::Zero) + const bool GeneratedBuildId = m_BuildId.empty(); + if (GeneratedBuildId) { - throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + m_BuildId = Oid::NewOid().ToString(); + } + else if (m_BuildId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid::FromHexString(m_BuildId) == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); } - } - BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - BuildId, - m_Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, m_Path, m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + const bool GeneratedBuildPartId = m_BuildPartId.empty(); + if (GeneratedBuildPartId) + { + m_BuildPartId = Oid::NewOid().ToString(); + } + else if (m_BuildPartId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } - bool Aborted = DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean); - if (Aborted) - { - ZEN_CONSOLE("Download failed."); - } - if (false) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + std::unique_ptr Storage; + std::string StorageName; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", + m_BuildPartName, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + GeneratedBuildId ? "Generated " : "", + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", + m_BuildPartName, + m_Path, + m_StoragePath, + GeneratedBuildId ? "Generated " : "", + BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } - return Aborted ? 11 : 0; - } - if (SubOption == &m_DiffOptions) - { - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + CbObject MetaData; + if (m_CreateBuild) + { + if (!m_BuildMetadataPath.empty()) + { + std::filesystem::path MetadataPath(m_BuildMetadataPath); + IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); + std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + } + } + if (!m_BuildMetadata.empty()) + { + CbObjectWriter MetaDataWriter(1024); + ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { + size_t SplitPos = Pair.find('='); + if (SplitPos == std::string::npos || SplitPos == 0) + { + throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); + } + MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); + return true; + }); + MetaData = MetaDataWriter.Save(); + } + } + + UploadFolder(*Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + m_ManifestPath, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + m_CreateBuild, + m_Clean); + + if (false) + { + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); + } + return AbortFlag ? 11 : 0; } - if (m_DiffPath.empty()) + + if (SubOption == &m_DownloadOptions) { - throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); - } - bool Aborted = DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); - return Aborted ? 11 : 0; - } + ParseStorageOptions(); + ParseAuthOptions(); - if (SubOption == &m_TestOptions) - { - ParseStorageOptions(); - ParseAuthOptions(); + HttpClient Http(m_BuildsUrl, ClientSettings); - HttpClient Http(m_BuildsUrl, ClientSettings); + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } + if (m_BuildId.empty()) + { + throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); + } + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); + } - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + } + + std::vector BuildPartIds; + for (const std::string& BuildPartId : m_BuildPartIds) + { + BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); + if (BuildPartIds.back() == Oid::Zero) + { + throw zen::OptionParseException( + fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + } + } - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = m_Path.filename().string(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + std::string StorageName; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + BuildId, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, m_Path, m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; + DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean); - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - m_StoragePath = GetRunningExecutablePath().parent_path() / ".tmpstore"; - CreateDirectories(m_StoragePath); - CleanDirectory(m_StoragePath); - } - auto _ = MakeGuard([&]() { - if (m_BuildsUrl.empty() && m_StoragePath.empty()) + if (false) { - DeleteDirectories(m_StoragePath); + ZEN_CONSOLE( + "{}:\n" + "Read: {}\n" + "Write: {}\n" + "Requests: {}\n" + "Avg Request Time: {}\n" + "Avg I/O Time: {}", + StorageName, + NiceBytes(StorageStats.TotalBytesRead.load()), + NiceBytes(StorageStats.TotalBytesWritten.load()), + StorageStats.TotalRequestCount.load(), + StorageStats.TotalExecutionTimeUs.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0, + StorageStats.TotalRequestCount.load() > 0 + ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) + : 0); } - }); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - m_Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - m_Path, - m_StoragePath, - BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + return AbortFlag ? 11 : 0; } - else + if (SubOption == &m_DiffOptions) { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } - - auto MakeMetaData = [](const Oid& BuildId) -> CbObject { - CbObjectWriter BuildMetaDataWriter; + if (m_Path.empty()) { - const uint32_t CL = BuildId.OidBits[2]; - BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); - BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); - BuildMetaDataWriter.AddString("platform", "Windows"); - BuildMetaDataWriter.AddString("project", "Test"); - BuildMetaDataWriter.AddInteger("changelist", CL); - BuildMetaDataWriter.AddString("buildType", "test-folder"); + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - return BuildMetaDataWriter.Save(); - }; - CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); - { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); + if (m_DiffPath.empty()) + { + throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); + } + DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); + return AbortFlag ? 11 : 0; } - bool Aborted = UploadFolder(*Storage, - BuildId, - BuildPartId, - m_BuildPartName, - m_Path, - {}, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData, - m_CreateBuild, - m_Clean); - if (Aborted) + if (SubOption == &m_TestOptions) { - ZEN_CONSOLE("Upload failed."); - return 11; - } + ParseStorageOptions(); + ParseAuthOptions(); - const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true); - if (Aborted) - { - ZEN_CONSOLE("Download failed."); - return 11; - } + HttpClient Http(m_BuildsUrl, ClientSettings); - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); - if (Aborted) - { - ZEN_CONSOLE("Re-download failed. (identical target)"); - return 11; - } + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } - auto ScrambleDir = [](const std::filesystem::path& Path) { - ZEN_CONSOLE("\nScrambling '{}'", Path); - Stopwatch Timer; - DirectoryContent DownloadContent; - GetDirectoryContent( - Path, - DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, - DownloadContent); - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { - std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); - for (const std::string_view& ExcludeFolder : ExcludeFolders) + m_BuildId = Oid::NewOid().ToString(); + m_BuildPartName = m_Path.filename().string(); + m_BuildPartId = Oid::NewOid().ToString(); + m_CreateBuild = true; + + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + std::unique_ptr Storage; + std::string StorageName; + + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + m_StoragePath = GetRunningExecutablePath().parent_path() / ".tmpstore"; + CreateDirectories(m_StoragePath); + CleanDirectory(m_StoragePath); + } + auto _ = MakeGuard([&]() { + if (m_BuildsUrl.empty() && m_StoragePath.empty()) { - if (RelativePath.starts_with(ExcludeFolder)) + DeleteDirectories(m_StoragePath); + } + }); + + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", + m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, + m_Path, + m_StoragePath, + BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + auto MakeMetaData = [](const Oid& BuildId) -> CbObject { + CbObjectWriter BuildMetaDataWriter; + { + const uint32_t CL = BuildId.OidBits[2]; + BuildMetaDataWriter.AddString("name", fmt::format("++Test+Main-CL-{}", CL)); + BuildMetaDataWriter.AddString("branch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("baselineBranch", "ZenTestBuild"); + BuildMetaDataWriter.AddString("platform", "Windows"); + BuildMetaDataWriter.AddString("project", "Test"); + BuildMetaDataWriter.AddInteger("changelist", CL); + BuildMetaDataWriter.AddString("buildType", "test-folder"); + } + return BuildMetaDataWriter.Save(); + }; + CbObject MetaData = MakeMetaData(Oid::TryFromHexString(m_BuildId)); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); + } + + UploadFolder(*Storage, + BuildId, + BuildPartId, + m_BuildPartName, + m_Path, + {}, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData, + m_CreateBuild, + m_Clean); + if (AbortFlag) + { + ZEN_CONSOLE("Upload failed."); + return 11; + } + + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true); + if (AbortFlag) + { + ZEN_CONSOLE("Download failed."); + return 11; + } + + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (identical target)", + BuildId, + BuildPartId, + m_BuildPartName, + DownloadPath); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed. (identical target)"); + return 11; + } + + auto ScrambleDir = [](const std::filesystem::path& Path) { + ZEN_CONSOLE("\nScrambling '{}'", Path); + Stopwatch Timer; + DirectoryContent DownloadContent; + GetDirectoryContent( + Path, + DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + DownloadContent); + auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders, Path](const std::filesystem::path& AbsolutePath) -> bool { + std::string RelativePath = std::filesystem::relative(AbsolutePath, Path).generic_string(); + for (const std::string_view& ExcludeFolder : ExcludeFolders) { - if (RelativePath.length() == ExcludeFolder.length()) + if (RelativePath.starts_with(ExcludeFolder)) { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; + if (RelativePath.length() == ExcludeFolder.length()) + { + return false; + } + else if (RelativePath[ExcludeFolder.length()] == '/') + { + return false; + } } } - } - return true; - }; + return true; + }; - std::atomic AbortFlag = false; - ParallellWork Work(AbortFlag); + ParallellWork Work(AbortFlag); - uint32_t Randomizer = 0; - auto FileSizeIt = DownloadContent.FileSizes.begin(); - for (const std::filesystem::path& FilePath : DownloadContent.Files) - { - if (IsAcceptedFolder(FilePath)) + uint32_t Randomizer = 0; + auto FileSizeIt = DownloadContent.FileSizes.begin(); + for (const std::filesystem::path& FilePath : DownloadContent.Files) { - uint32_t Case = (Randomizer++) % 7; - switch (Case) + if (IsAcceptedFolder(FilePath)) { - case 0: - { - uint64_t SourceSize = *FileSizeIt; - if (SourceSize > 0) + uint32_t Case = (Randomizer++) % 7; + switch (Case) + { + case 0: { - Work.ScheduleWork( - GetMediumWorkerPool(EWorkloadType::Burst), - [SourceSize, FilePath](std::atomic& AbortFlag) { - if (!AbortFlag) - { - IoBuffer Scrambled(SourceSize); - { - IoBuffer Source = IoBufferBuilder::MakeFromFile(FilePath); - Scrambled.GetMutableView().CopyFrom( - Source.GetView().Mid(SourceSize / 3, SourceSize / 3)); - Scrambled.GetMutableView() - .Mid(SourceSize / 3) - .CopyFrom(Source.GetView().Mid(0, SourceSize / 3)); - Scrambled.GetMutableView() - .Mid((SourceSize / 3) * 2) - .CopyFrom(Source.GetView().Mid(SourceSize / 2, SourceSize / 3)); - } - bool IsReadOnly = SetFileReadOnly(FilePath, false); - WriteFile(FilePath, Scrambled); - if (IsReadOnly) + uint64_t SourceSize = *FileSizeIt; + if (SourceSize > 0) + { + Work.ScheduleWork( + GetMediumWorkerPool(EWorkloadType::Burst), + [SourceSize, FilePath](std::atomic&) { + if (!AbortFlag) { - SetFileReadOnly(FilePath, true); + IoBuffer Scrambled(SourceSize); + { + IoBuffer Source = IoBufferBuilder::MakeFromFile(FilePath); + Scrambled.GetMutableView().CopyFrom( + Source.GetView().Mid(SourceSize / 3, SourceSize / 3)); + Scrambled.GetMutableView() + .Mid(SourceSize / 3) + .CopyFrom(Source.GetView().Mid(0, SourceSize / 3)); + Scrambled.GetMutableView() + .Mid((SourceSize / 3) * 2) + .CopyFrom(Source.GetView().Mid(SourceSize / 2, SourceSize / 3)); + } + bool IsReadOnly = SetFileReadOnly(FilePath, false); + WriteFile(FilePath, Scrambled); + if (IsReadOnly) + { + SetFileReadOnly(FilePath, true); + } } - } - }, - [FilePath](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed scrambling file {}. Reason: {}", FilePath, Ex.what()); - AbortFlag = true; - }); + }, + Work.DefaultErrorFunction()); + } } - } - break; - case 1: - std::filesystem::remove(FilePath); - break; - default: - break; + break; + case 1: + std::filesystem::remove(FilePath); + break; + default: + break; + } } + FileSizeIt++; } - FileSizeIt++; - } - Work.Wait(5000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted); - ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); - }); - ZEN_ASSERT(!AbortFlag.load()); - ZEN_CONSOLE("Scrambled files in {}", NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - }; - - ScrambleDir(DownloadPath); - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); - if (Aborted) - { - ZEN_CONSOLE("Re-download failed. (scrambled target)"); - return 11; - } - - ScrambleDir(DownloadPath); - - Oid BuildId2 = Oid::NewOid(); - Oid BuildPartId2 = Oid::NewOid(); + Work.Wait(5000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted); + ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); + }); + ZEN_ASSERT(!AbortFlag.load()); + ZEN_CONSOLE("Scrambled files in {}", NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + }; - CbObject MetaData2 = MakeMetaData(BuildId2); - { - ExtendableStringBuilder<256> SB; - CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); - } + ScrambleDir(DownloadPath); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled target)", + BuildId, + BuildPartId, + m_BuildPartName, + DownloadPath); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed. (scrambled target)"); + return 11; + } - Aborted = UploadFolder(*Storage, - BuildId2, - BuildPartId2, - m_BuildPartName, - DownloadPath, - {}, - m_BlockReuseMinPercentLimit, - m_AllowMultiparts, - MetaData2, - true, - false); - if (Aborted) - { - ZEN_CONSOLE("Upload of scrambled failed."); - return 11; - } + ScrambleDir(DownloadPath); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); - if (Aborted) - { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + Oid BuildId2 = Oid::NewOid(); + Oid BuildPartId2 = Oid::NewOid(); - ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); - if (Aborted) - { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + CbObject MetaData2 = MakeMetaData(BuildId2); + { + ExtendableStringBuilder<256> SB; + CompactBinaryToJson(MetaData, SB); + ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); + } - ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - Aborted = DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); - if (Aborted) - { - ZEN_CONSOLE("Re-download failed."); - return 11; - } + UploadFolder(*Storage, + BuildId2, + BuildPartId2, + m_BuildPartName, + DownloadPath, + {}, + m_BlockReuseMinPercentLimit, + m_AllowMultiparts, + MetaData2, + true, + false); + if (AbortFlag) + { + ZEN_CONSOLE("Upload of scrambled failed."); + return 11; + } - return 0; - } + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } - if (SubOption == &m_FetchBlobOptions) - { - ParseStorageOptions(); - ParseAuthOptions(); + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } - HttpClient Http(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); + DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } - if (m_BlobHash.empty()) - { - throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + return 0; } - IoHash BlobHash; - if (!IoHash::TryParse(m_BlobHash, BlobHash)) + if (SubOption == &m_FetchBlobOptions) { - throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); - } + ParseStorageOptions(); + ParseAuthOptions(); - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } + HttpClient Http(m_BuildsUrl, ClientSettings); - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - std::unique_ptr Storage; - std::string StorageName; + if (m_BlobHash.empty()) + { + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + } - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + IoHash BlobHash; + if (!IoHash::TryParse(m_BlobHash, BlobHash)) + { + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + } - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", BlobHash, CompressedSize, DecompressedSize); - return 0; - } + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } - if (SubOption == &m_ValidateBuildPartOptions) - { - ParseStorageOptions(); - ParseAuthOptions(); + BuildStorage::Statistics StorageStats; + const Oid BuildId = Oid::FromHexString(m_BuildId); + std::unique_ptr Storage; + std::string StorageName; - HttpClient Http(m_BuildsUrl, ClientSettings); + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag) + { + return 11; + } + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", + BlobHash, + CompressedSize, + DecompressedSize); + return 0; } - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) + if (SubOption == &m_ValidateBuildPartOptions) { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } + ParseStorageOptions(); + ParseAuthOptions(); - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } + HttpClient Http(m_BuildsUrl, ClientSettings); - BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; + if (m_BuildsUrl.empty() && m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } - Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); - CbObject Build = Storage->GetBuild(BuildId); - if (!m_BuildPartName.empty()) - { - BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); - if (BuildPartId == Oid::Zero) + if (m_BuildId.empty()) { - throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); + throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); + } + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); } - } - CbObject BuildPart = Storage->GetBuildPart(BuildId, BuildPartId); - ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); - std::vector ChunkAttachments; - for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) - { - ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); - } - std::vector BlockAttachments; - for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) - { - BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); - } - for (const IoHash& ChunkAttachment : ChunkAttachments) - { - uint64_t CompressedSize; - uint64_t DecompressedSize; - try + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { - ValidateBlob(*Storage, BuildId, ChunkAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE("Chunk attachment {} ({} -> {}) is valid", - ChunkAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - catch (const std::exception& Ex) + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + std::string StorageName; + + if (!m_BuildsUrl.empty()) { - ZEN_CONSOLE("Failed validating chunk attachment {}: {}", ChunkAttachment, Ex.what()); + ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket, + BuildId); + CreateDirectories(m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); + CbObject Build = Storage->GetBuild(BuildId); + if (!m_BuildPartName.empty()) + { + BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); + if (BuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); + } + } + CbObject BuildPart = Storage->GetBuildPart(BuildId, BuildPartId); + ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + std::vector ChunkAttachments; + for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) + { + ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); + } + std::vector BlockAttachments; + for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) + { + BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); } - } - for (const IoHash& BlockAttachment : BlockAttachments) - { - uint64_t CompressedSize; - uint64_t DecompressedSize; - try + for (const IoHash& ChunkAttachment : ChunkAttachments) { - ValidateChunkBlock(*Storage, BuildId, BlockAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE("Block attachment {} ({} -> {}) is valid", - BlockAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); + uint64_t CompressedSize; + uint64_t DecompressedSize; + try + { + ValidateBlob(*Storage, BuildId, ChunkAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE("Chunk attachment {} ({} -> {}) is valid", + ChunkAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed validating chunk attachment {}: {}", ChunkAttachment, Ex.what()); + } } - catch (const std::exception& Ex) + + for (const IoHash& BlockAttachment : BlockAttachments) { - ZEN_CONSOLE("Failed validating block attachment {}: {}", BlockAttachment, Ex.what()); + uint64_t CompressedSize; + uint64_t DecompressedSize; + try + { + ValidateChunkBlock(*Storage, BuildId, BlockAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE("Block attachment {} ({} -> {}) is valid", + BlockAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed validating block attachment {}: {}", BlockAttachment, Ex.what()); + } } - } - return 0; + return AbortFlag ? 13 : 0; + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("{}", Ex.what()); + return 3; } - ZEN_ASSERT(false); } diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 2e230ed53..bee1fd676 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -264,7 +264,7 @@ ProgressBar::~ProgressBar() { try { - Finish(); + ForceLinebreak(); } catch (const std::exception& Ex) { diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index a41b71972..6dc2a20d8 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -92,7 +92,8 @@ namespace { tsl::robin_map& RawHashToSequenceRawHashIndex, RwLock& Lock, const std::filesystem::path& FolderPath, - uint32_t PathIndex) + uint32_t PathIndex, + std::atomic& AbortFlag) { const uint64_t RawSize = OutChunkedContent.RawSizes[PathIndex]; const std::filesystem::path& Path = OutChunkedContent.Paths[PathIndex]; @@ -105,7 +106,7 @@ namespace { { ChunkedInfoWithSource Chunked; const bool DidChunking = - InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed); + InChunkingController.ProcessFile((FolderPath / Path).make_preferred(), RawSize, Chunked, Stats.BytesHashed, AbortFlag); if (DidChunking) { Lock.WithExclusiveLock([&]() { @@ -753,15 +754,13 @@ ChunkFolderContent(ChunkingStatistics& Stats, RawHashToSequenceRawHashIndex, Lock, RootPath, - PathIndex); + PathIndex, + AbortFlag); Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; }); Stats.FilesProcessed++; } }, - [&, PathIndex](const std::exception& Ex, std::atomic& AbortFlag) { - ZEN_CONSOLE("Failed scanning file {}. Reason: {}", Result.Paths[PathIndex], Ex.what()); - AbortFlag = true; - }); + Work.DefaultErrorFunction()); } Work.Wait(UpdateInteralMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) { diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp index 3f3a6661c..4f9344039 100644 --- a/src/zenutil/chunkedfile.cpp +++ b/src/zenutil/chunkedfile.cpp @@ -112,7 +112,12 @@ Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, st } ChunkedInfoWithSource -ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params, std::atomic* BytesProcessed) +ChunkData(BasicFile& RawData, + uint64_t Offset, + uint64_t Size, + ChunkedParams Params, + std::atomic* BytesProcessed, + std::atomic* AbortFlag) { ChunkedInfoWithSource Result; tsl::robin_map FoundChunks; @@ -129,6 +134,10 @@ ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Para IoHashStream RawHashStream; while (Offset < End) { + if (AbortFlag != nullptr && AbortFlag->load()) + { + return {}; + } size_t ScanLength = Chunker.ScanChunk(SliceView.GetData(), SliceSize); if (ScanLength == ZenChunkHelper::kNoBoundaryFound) { diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index bc0e57b14..017d12433 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -58,7 +58,8 @@ public: virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, ChunkedInfoWithSource& OutChunked, - std::atomic& BytesProcessed) const override + std::atomic& BytesProcessed, + std::atomic& AbortFlag) const override { const bool ExcludeFromChunking = std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) != @@ -70,7 +71,7 @@ public: } BasicFile Buffer(InputPath, BasicFile::Mode::kRead); - OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed); + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag); return true; } @@ -132,7 +133,8 @@ public: virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, ChunkedInfoWithSource& OutChunked, - std::atomic& BytesProcessed) const override + std::atomic& BytesProcessed, + std::atomic& AbortFlag) const override { if (RawSize < m_ChunkFileSizeLimit) { @@ -150,6 +152,10 @@ public: ChunkHashToChunkIndex.reserve(1 + (RawSize / m_FixedChunkingChunkSize)); while (Offset < RawSize) { + if (AbortFlag) + { + return false; + } uint64_t ChunkSize = std::min(RawSize - Offset, m_FixedChunkingChunkSize); IoBuffer Chunk(Source, Offset, ChunkSize); MemoryView ChunkData = Chunk.GetView(); diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h index 7110ad317..4cec80fdb 100644 --- a/src/zenutil/include/zenutil/chunkedfile.h +++ b/src/zenutil/include/zenutil/chunkedfile.h @@ -47,7 +47,8 @@ ChunkedInfoWithSource ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params = {}, - std::atomic* BytesProcessed = nullptr); + std::atomic* BytesProcessed = nullptr, + std::atomic* AbortFlag = nullptr); void Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function GetChunk); diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index fe4fc1bb5..ebc80e207 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -32,9 +32,10 @@ public: virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, ChunkedInfoWithSource& OutChunked, - std::atomic& BytesProcessed) const = 0; - virtual std::string_view GetName() const = 0; - virtual CbObject GetParameters() const = 0; + std::atomic& BytesProcessed, + std::atomic& AbortFlag) const = 0; + virtual std::string_view GetName() const = 0; + virtual CbObject GetParameters() const = 0; }; std::unique_ptr CreateBasicChunkingController( diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h index 7a8218c51..79798fc8d 100644 --- a/src/zenutil/include/zenutil/parallellwork.h +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -2,6 +2,8 @@ #pragma once +#include +#include #include #include @@ -20,6 +22,14 @@ public: ZEN_ASSERT(m_PendingWork.Remaining() == 0); } + std::function& AbortFlag)> DefaultErrorFunction() + { + return [&](const std::exception& Ex, std::atomic& AbortFlag) { + m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); }); + AbortFlag = true; + }; + } + void ScheduleWork(WorkerThreadPool& WorkerPool, std::function& AbortFlag)>&& Work, std::function& AbortFlag)>&& OnError) @@ -32,6 +42,27 @@ public: { Work(m_AbortFlag); } + catch (const AssertException& AssertEx) + { + OnError( + std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())), + m_AbortFlag); + } + catch (const std::system_error& SystemError) + { + if (IsOOM(SystemError.code())) + { + OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag); + } + else if (IsOOD(SystemError.code())) + { + OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag); + } + else + { + OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag); + } + } catch (const std::exception& Ex) { OnError(Ex, m_AbortFlag); @@ -58,12 +89,29 @@ public: { UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); } + if (m_Errors.size() == 1) + { + throw std::runtime_error(m_Errors.front()); + } + else if (m_Errors.size() > 1) + { + ExtendableStringBuilder<128> SB; + SB.Append("Multiple errors:"); + for (const std::string& Error : m_Errors) + { + SB.Append(fmt::format("\n {}", Error)); + } + throw std::runtime_error(SB.ToString()); + } } Latch& PendingWork() { return m_PendingWork; } private: std::atomic& m_AbortFlag; Latch m_PendingWork; + + RwLock m_ErrorLock; + std::vector m_Errors; }; } // namespace zen -- cgit v1.2.3 From 19b3c492dcc0fc3f8879ecb60124ca64dea9b7ef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 1 Mar 2025 10:10:53 +0100 Subject: builds download incremental (#290) * incremental download * merge rebuild state and output state building * fix writing when > 1 zero size file --- src/zen/cmds/builds_cmd.cpp | 1235 +++++++++++++++--------------- src/zencore/filesystem.cpp | 7 +- src/zencore/include/zencore/filesystem.h | 10 +- src/zenutil/filebuildstorage.cpp | 12 +- 4 files changed, 635 insertions(+), 629 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 18cc7cf9e..28c794559 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -82,7 +82,7 @@ namespace { const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); - const std::string ZenTempReuseFolderName = fmt::format("{}/reuse", ZenTempFolderName); + const std::string ZenTempCacheFolderName = fmt::format("{}/cache", ZenTempFolderName); const std::string ZenTempStorageFolderName = fmt::format("{}/storage", ZenTempFolderName); const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); const std::string ZenTempChunkFolderName = fmt::format("{}/chunks", ZenTempFolderName); @@ -115,6 +115,54 @@ namespace { ); + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) + { +#if ZEN_PLATFORM_WINDOWS + if (SourcePlatform == SourcePlatform::Windows) + { + SetFileAttributes(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentAttributes = GetFileAttributes(FilePath); + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, IsFileModeReadOnly(Attributes)); + if (CurrentAttributes != NewAttributes) + { + SetFileAttributes(FilePath, NewAttributes); + } + return NewAttributes; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + if (SourcePlatform != SourcePlatform::Windows) + { + SetFileMode(FilePath, Attributes); + return Attributes; + } + else + { + uint32_t CurrentMode = GetFileMode(FilePath); + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, IsFileAttributeReadOnly(Attributes)); + if (CurrentMode != NewMode) + { + SetFileMode(FilePath, NewMode); + } + return NewMode; + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + }; + + uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath) + { +#if ZEN_PLATFORM_WINDOWS + return GetFileAttributes(FilePath); +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + return GetFileMode(FilePath); +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + } + template std::string FormatArray(std::span Items, std::string_view Prefix) { @@ -181,9 +229,8 @@ namespace { // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory std::span Segments = Buffer.GetSegments(); ZEN_ASSERT(Buffer.GetSegments().size() > 0); - size_t SegmentIndexToCheck = Segments.size() > 1 ? 1 : 0; IoBufferFileReference FileRef; - if (Segments[SegmentIndexToCheck].GetFileReference(FileRef)) + if (Segments.back().GetFileReference(FileRef)) { return Buffer; } @@ -2114,7 +2161,7 @@ namespace { Stopwatch PutBuildTimer; CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); ZEN_CONSOLE("PutBuild took {}. Payload size: {}", - NiceLatencyNs(PutBuildTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(PutBuildTimer.GetElapsedTimeMs()), NiceBytes(MetaData.GetSize())); PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(PreferredMultipartChunkSize); } @@ -2122,9 +2169,7 @@ namespace { { Stopwatch GetBuildTimer; CbObject Build = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Payload size: {}", - NiceLatencyNs(GetBuildTimer.GetElapsedTimeUs() * 1000), - NiceBytes(Build.GetSize())); + ZEN_CONSOLE("GetBuild took {}. Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), NiceBytes(Build.GetSize())); if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) { PreferredMultipartChunkSize = ChunkSize; @@ -2439,7 +2484,7 @@ namespace { Stopwatch PutBuildPartResultTimer; std::pair> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are missing.", - NiceLatencyNs(PutBuildPartResultTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), PutBuildPartResult.second.size()); IoHash PartHash = PutBuildPartResult.first; @@ -2531,7 +2576,7 @@ namespace { Stopwatch FinalizeBuildPartTimer; std::vector Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", - NiceLatencyNs(FinalizeBuildPartTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), Needs.size()); if (Needs.empty()) { @@ -2545,7 +2590,7 @@ namespace { { Stopwatch FinalizeBuildTimer; Storage.FinalizeBuild(BuildId); - ZEN_CONSOLE("FinalizeBuild took {}", NiceLatencyNs(FinalizeBuildTimer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); } if (!NewBlocks.BlockDescriptions.empty()) @@ -2949,7 +2994,7 @@ namespace { const bool CacheWriter = TargetFinalSize > Buffer.GetSize(); if (CacheWriter) { - ZEN_ASSERT(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); + ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); OutputFile = std::move(NewOutputFile); OpenFileWriter = std::make_unique(*OutputFile, Min(TargetFinalSize, 256u * 1024u)); @@ -2994,7 +3039,7 @@ namespace { return ChunkTargetPtrs; }; - bool WriteBlockToDisk(const std::filesystem::path& Path, + bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, const ChunkedFolderContent& Content, const std::vector& RemotePathIndexWantsCopyFromCacheFlags, const CompositeBuffer& DecompressedBlockBuffer, @@ -3061,22 +3106,30 @@ namespace { return Lhs.Target->Offset < Rhs.Target->Offset; }); - WriteFileCache OpenFileCache; - for (const WriteOpData& WriteOp : WriteOps) { - const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; - const uint32_t PathIndex = WriteOp.Target->PathIndex; - const uint64_t ChunkSize = Chunk.GetSize(); - const uint64_t FileOffset = WriteOp.Target->Offset; - ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); - - OpenFileCache.WriteToFile( - PathIndex, - [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, - Chunk, - FileOffset, - Content.RawSizes[PathIndex]); - OutBytesWritten += ChunkSize; + WriteFileCache OpenFileCache; + for (const WriteOpData& WriteOp : WriteOps) + { + if (AbortFlag) + { + break; + } + const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t PathIndex = WriteOp.Target->PathIndex; + const uint64_t ChunkSize = Chunk.GetSize(); + const uint64_t FileOffset = WriteOp.Target->Offset; + ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); + + OpenFileCache.WriteToFile( + PathIndex, + [&CacheFolderPath, &Content](uint32_t TargetIndex) { + return (CacheFolderPath / Content.RawHashes[TargetIndex].ToHexString()).make_preferred(); + }, + Chunk, + FileOffset, + Content.RawSizes[PathIndex]); + OutBytesWritten += ChunkSize; + } } OutChunksComplete += gsl::narrow(ChunkBuffers.size()); } @@ -3086,11 +3139,11 @@ namespace { return false; } - SharedBuffer Decompress(const IoBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) + SharedBuffer Decompress(const CompositeBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) { IoHash RawHash; uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(CompressedChunk), RawHash, RawSize); + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedChunk, RawHash, RawSize); if (!Compressed) { throw std::runtime_error(fmt::format("Invalid build blob format for chunk {}", ChunkHash)); @@ -3118,7 +3171,7 @@ namespace { return Decompressed; } - void WriteChunkToDisk(const std::filesystem::path& Path, + void WriteChunkToDisk(const std::filesystem::path& CacheFolderPath, const ChunkedFolderContent& Content, std::span ChunkTargets, const CompositeBuffer& ChunkData, @@ -3132,7 +3185,10 @@ namespace { OpenFileCache.WriteToFile( Target.PathIndex, - [&Path, &Content](uint32_t TargetIndex) { return (Path / Content.Paths[TargetIndex]).make_preferred(); }, + [&CacheFolderPath, &Content](uint32_t TargetIndex) { + return (CacheFolderPath / Content.RawHashes[TargetIndex].ToHexString()).make_preferred(); + // return (Path / Content.Paths[TargetIndex]).make_preferred(); + }, ChunkData, FileOffset, Content.RawSizes[Target.PathIndex]); @@ -3141,7 +3197,8 @@ namespace { } void DownloadLargeBlob(BuildStorage& Storage, - const std::filesystem::path& Path, + const std::filesystem::path& TempFolderPath, + const std::filesystem::path& CacheFolderPath, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, const Oid& BuildId, @@ -3166,7 +3223,7 @@ namespace { std::shared_ptr Workload(std::make_shared()); std::error_code Ec; - Workload->TempFile.CreateTemporary(Path / ZenTempChunkFolderName, Ec); + Workload->TempFile.CreateTemporary(TempFolderPath, Ec); if (Ec) { throw std::runtime_error( @@ -3176,7 +3233,7 @@ namespace { BuildId, ChunkHash, PreferredMultipartChunkSize, - [&Path, + [&CacheFolderPath, &RemoteContent, &RemoteLookup, &Work, @@ -3203,7 +3260,7 @@ namespace { Work.ScheduleWork( WritePool, - [&Path, + [&CacheFolderPath, &RemoteContent, &RemoteLookup, ChunkHash, @@ -3234,8 +3291,9 @@ namespace { uint32_t ChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); - SharedBuffer Chunk = - Decompress(CompressedPart, ChunkHash, RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), + ChunkHash, + RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); // ZEN_ASSERT_SLOW(ChunkHash == // IoHash::HashBuffer(Chunk.AsIoBuffer())); @@ -3244,7 +3302,7 @@ namespace { { WriteFileCache OpenFileCache; - WriteChunkToDisk(Path, + WriteChunkToDisk(CacheFolderPath, RemoteContent, ChunkTargetPtrs, CompositeBuffer(Chunk), @@ -3290,6 +3348,7 @@ namespace { bool WipeTargetFolder, FolderContent& OutLocalFolderState) { + ZEN_UNUSED(WipeTargetFolder); std::atomic DownloadedBlocks = 0; std::atomic BlockBytes = 0; std::atomic DownloadedChunks = 0; @@ -3307,16 +3366,17 @@ namespace { ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - const std::filesystem::path CacheFolderPath = Path / ZenTempReuseFolderName; + const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName; - tsl::robin_map LocalRawHashToPathIndex; + tsl::robin_map RawHashToLocalPathIndex; - if (!WipeTargetFolder) { Stopwatch CacheTimer; for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { + ZEN_ASSERT_SLOW(std::filesystem::exists(Path / LocalContent.Paths[LocalPathIndex])); + if (LocalContent.RawSizes[LocalPathIndex] > 0) { const uint32_t SequenceRawHashIndex = @@ -3325,99 +3385,33 @@ namespace { if (ChunkCount > 0) { const IoHash LocalRawHash = LocalContent.RawHashes[LocalPathIndex]; - if (!LocalRawHashToPathIndex.contains(LocalRawHash)) + if (!RawHashToLocalPathIndex.contains(LocalRawHash)) { - LocalRawHashToPathIndex.insert_or_assign(LocalRawHash, LocalPathIndex); + RawHashToLocalPathIndex.insert_or_assign(LocalRawHash, LocalPathIndex); } } } } - - { - std::vector IncludeLocalFiles(LocalContent.Paths.size(), false); - - for (const IoHash& ChunkHash : RemoteContent.ChunkedContent.ChunkHashes) - { - if (auto It = LocalLookup.ChunkHashToChunkIndex.find(ChunkHash); It != LocalLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t LocalChunkIndex = It->second; - std::span LocalChunkTargetRange = - GetChunkLocations(LocalLookup, LocalChunkIndex); - if (!LocalChunkTargetRange.empty()) - { - std::uint32_t LocalPathIndex = LocalChunkTargetRange[0].PathIndex; - IncludeLocalFiles[LocalPathIndex] = true; - } - } - } - for (const IoHash& RawHash : RemoteContent.RawHashes) - { - if (auto It = LocalRawHashToPathIndex.find(RawHash); It != LocalRawHashToPathIndex.end()) - { - uint32_t LocalPathIndex = It->second; - IncludeLocalFiles[LocalPathIndex] = true; - } - } - - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) - { - if (!IncludeLocalFiles[LocalPathIndex]) - { - LocalRawHashToPathIndex.erase(LocalContent.RawHashes[LocalPathIndex]); - } - } - } - - uint64_t CachedBytes = 0; - CreateDirectories(CacheFolderPath); - for (auto& CachedLocalFile : LocalRawHashToPathIndex) - { - const IoHash& LocalRawHash = CachedLocalFile.first; - const uint32_t LocalPathIndex = CachedLocalFile.second; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - const std::filesystem::path CacheFilePath = (CacheFolderPath / LocalRawHash.ToHexString()).make_preferred(); - - SetFileReadOnly(LocalFilePath, false); - - std::filesystem::rename(LocalFilePath, CacheFilePath); - CachedBytes += std::filesystem::file_size(CacheFilePath); - } - - ZEN_CONSOLE("Cached {} ({}) local files in {}", - LocalRawHashToPathIndex.size(), - NiceBytes(CachedBytes), - NiceTimeSpanMs(CacheTimer.GetElapsedTimeMs())); - } - - if (AbortFlag) - { - return; } - - CleanDirectory(Path, DefaultExcludeFolders); - Stopwatch CacheMappingTimer; std::atomic BytesWritten = 0; uint64_t CacheMappedBytesForReuse = 0; - std::vector RemotePathIndexWantsCopyFromCacheFlags(RemoteContent.Paths.size(), false); - std::vector> RemoteChunkIndexWantsCopyFromCacheFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + std::vector RemotePathIndexWantsCopyFromCacheFlags(RemoteContent.Paths.size(), false); + std::vector RemoteChunkIndexWantsCopyFromCacheFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); struct CacheCopyData { - std::filesystem::path OriginalSourceFileName; - IoHash LocalFileRawHash; - uint64_t LocalFileRawSize = 0; - std::vector RemotePathIndexes; - std::vector ChunkSourcePtrs; + uint32_t LocalPathIndex; + std::vector TargetChunkLocationPtrs; struct ChunkTarget { - uint32_t ChunkSourceCount; + uint32_t TargetChunkLocationCount; uint64_t ChunkRawSize; - uint64_t LocalFileOffset; + uint64_t CacheFileOffset; }; std::vector ChunkTargets; }; @@ -3426,42 +3420,24 @@ namespace { std::vector CacheCopyDatas; uint32_t ChunkCountToWrite = 0; - // Pick up all whole files to copy and/or move + // Pick up all whole files we can use from current local state for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { const IoHash& RemoteRawHash = RemoteContent.RawHashes[RemotePathIndex]; - if (auto It = LocalRawHashToPathIndex.find(RemoteRawHash); It != LocalRawHashToPathIndex.end()) + if (auto It = RawHashToLocalPathIndex.find(RemoteRawHash); It != RawHashToLocalPathIndex.end()) { - if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(RemoteRawHash); CopySourceIt != RawHashToCacheCopyDataIndex.end()) - { - CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; - Data.RemotePathIndexes.push_back(RemotePathIndex); - } - else - { - const uint32_t LocalPathIndex = It->second; - ZEN_ASSERT(LocalContent.RawSizes[LocalPathIndex] == RemoteContent.RawSizes[RemotePathIndex]); - ZEN_ASSERT(LocalContent.RawHashes[LocalPathIndex] == RemoteContent.RawHashes[RemotePathIndex]); - RawHashToCacheCopyDataIndex.insert_or_assign(RemoteRawHash, CacheCopyDatas.size()); - CacheCopyDatas.push_back(CacheCopyData{.OriginalSourceFileName = LocalContent.Paths[LocalPathIndex], - .LocalFileRawHash = RemoteRawHash, - .LocalFileRawSize = LocalContent.RawSizes[LocalPathIndex], - .RemotePathIndexes = {RemotePathIndex}}); - CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; - ChunkCountToWrite++; - } RemotePathIndexWantsCopyFromCacheFlags[RemotePathIndex] = true; + CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; } } - // Pick up all chunks in cached files and make sure we block moving of cache files if we need part of them - for (auto& CachedLocalFile : LocalRawHashToPathIndex) + // Pick up all chunks in current local state + for (auto& CachedLocalFile : RawHashToLocalPathIndex) { const IoHash& LocalFileRawHash = CachedLocalFile.first; const uint32_t LocalPathIndex = CachedLocalFile.second; const uint32_t LocalSequenceRawHashIndex = LocalLookup.RawHashToSequenceRawHashIndex.at(LocalFileRawHash); - const uint32_t LocalOrderOffset = - LocalLookup.SequenceRawHashIndexChunkOrderOffset[LocalSequenceRawHashIndex]; // CachedLocalFile.second.ChunkOrderOffset; + const uint32_t LocalOrderOffset = LocalLookup.SequenceRawHashIndexChunkOrderOffset[LocalSequenceRawHashIndex]; { uint64_t SourceOffset = 0; @@ -3482,30 +3458,30 @@ namespace { if (!ChunkTargetPtrs.empty()) { - CacheCopyData::ChunkTarget Target = {.ChunkSourceCount = gsl::narrow(ChunkTargetPtrs.size()), - .ChunkRawSize = LocalChunkRawSize, - .LocalFileOffset = SourceOffset}; + CacheCopyData::ChunkTarget Target = { + .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), + .ChunkRawSize = LocalChunkRawSize, + .CacheFileOffset = SourceOffset}; if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalFileRawHash); CopySourceIt != RawHashToCacheCopyDataIndex.end()) { CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; - Data.ChunkSourcePtrs.insert(Data.ChunkSourcePtrs.end(), ChunkTargetPtrs.begin(), ChunkTargetPtrs.end()); + Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), + ChunkTargetPtrs.begin(), + ChunkTargetPtrs.end()); Data.ChunkTargets.push_back(Target); } else { RawHashToCacheCopyDataIndex.insert_or_assign(LocalFileRawHash, CacheCopyDatas.size()); CacheCopyDatas.push_back( - CacheCopyData{.OriginalSourceFileName = LocalContent.Paths[LocalPathIndex], - .LocalFileRawHash = LocalFileRawHash, - .LocalFileRawSize = LocalContent.RawSizes[LocalPathIndex], - .RemotePathIndexes = {}, - .ChunkSourcePtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector{Target}}); + CacheCopyData{.LocalPathIndex = LocalPathIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); } CacheMappedBytesForReuse += LocalChunkRawSize; + RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex] = true; } - RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex] = true; } } SourceOffset += LocalChunkRawSize; @@ -3535,531 +3511,554 @@ namespace { ZEN_CONSOLE("Mapped {} cached data for reuse in {}", NiceBytes(CacheMappedBytesForReuse), NiceTimeSpanMs(CacheMappingTimer.GetElapsedTimeMs())); + { + WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - auto CopyChunksFromCacheFile = [](const std::filesystem::path& Path, - BufferedOpenFile& SourceFile, - WriteFileCache& OpenFileCache, - const ChunkedFolderContent& RemoteContent, - const uint64_t LocalFileSourceOffset, - const uint64_t LocalChunkRawSize, - std::span ChunkTargetPtrs, - uint64_t& OutBytesWritten) { - CompositeBuffer Chunk = SourceFile.GetRange(LocalFileSourceOffset, LocalChunkRawSize); - uint64_t TotalBytesWritten = 0; - - WriteChunkToDisk(Path, RemoteContent, ChunkTargetPtrs, Chunk, OpenFileCache, TotalBytesWritten); - OutBytesWritten += TotalBytesWritten; - }; - - auto CloneFullFileFromCache = [](const std::filesystem::path& Path, - const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& RemoteContent, - const IoHash& FileRawHash, - const uint64_t FileRawSize, - std::span FullCloneRemotePathIndexes, - bool CanMove, - uint64_t& OutBytesWritten) { - const std::filesystem::path CacheFilePath = (CacheFolderPath / FileRawHash.ToHexString()).make_preferred(); + ProgressBar WriteProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); - size_t CopyCount = FullCloneRemotePathIndexes.size(); - if (CanMove) - { - // If every reference to this chunk has are full files we can move the cache file to the last target - CopyCount--; - } + std::atomic BytesDownloaded = 0; - for (uint32_t RemotePathIndex : FullCloneRemotePathIndexes) + for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { - const std::filesystem::path TargetPath = (Path / RemoteContent.Paths[RemotePathIndex]).make_preferred(); - CreateDirectories(TargetPath.parent_path()); - - if (CopyCount == 0) - { - std::filesystem::rename(CacheFilePath, TargetPath); - } - else + if (AbortFlag) { - CopyFile(CacheFilePath, TargetPath, {.EnableClone = false}); - ZEN_ASSERT(CopyCount > 0); - CopyCount--; + break; } - OutBytesWritten += FileRawSize; - } - }; - - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - - ProgressBar WriteProgressBar(UsePlainProgress); - ParallellWork Work(AbortFlag); - - std::atomic BytesDownloaded = 0; - - for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) - { - if (AbortFlag) - { - break; - } - - Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// - [&, CopyDataIndex](std::atomic&) { - if (!AbortFlag) - { - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - const std::filesystem::path CacheFilePath = - (CacheFolderPath / CopyData.LocalFileRawHash.ToHexString()).make_preferred(); - if (!CopyData.ChunkSourcePtrs.empty()) + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(),// + [&, CopyDataIndex](std::atomic&) { + if (!AbortFlag) { - uint64_t CacheLocalFileBytesRead = 0; - - size_t TargetStart = 0; - const std::span AllTargets(CopyData.ChunkSourcePtrs); - - struct WriteOp + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + const std::filesystem::path LocalFilePath = + (Path / LocalContent.Paths[CopyData.LocalPathIndex]).make_preferred(); + if (!CopyData.TargetChunkLocationPtrs.empty()) { - const ChunkedContentLookup::ChunkLocation* Target; - uint64_t LocalFileOffset; - uint64_t ChunkSize; - }; + uint64_t CacheLocalFileBytesRead = 0; - std::vector WriteOps; - WriteOps.reserve(CopyData.ChunkSourcePtrs.size()); + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) - { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.ChunkSourceCount); - for (const ChunkedContentLookup::ChunkLocation* Target : TargetRange) + struct WriteOp { - WriteOps.push_back(WriteOp{.Target = Target, - .LocalFileOffset = ChunkTarget.LocalFileOffset, - .ChunkSize = ChunkTarget.ChunkRawSize}); - } - TargetStart += ChunkTarget.ChunkSourceCount; - } + const ChunkedContentLookup::ChunkLocation* Target; + uint64_t CacheFileOffset; + uint64_t ChunkSize; + }; - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + std::vector WriteOps; + WriteOps.reserve(AllTargets.size()); + + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) { - return true; + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkLocation* Target : TargetRange) + { + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkSize = ChunkTarget.ChunkRawSize}); + } + TargetStart += ChunkTarget.TargetChunkLocationCount; } - else if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) - { + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + { + return true; + } + else if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } return false; + }); + + { + BufferedOpenFile SourceFile(LocalFilePath); + WriteFileCache OpenFileCache; + for (const WriteOp& Op : WriteOps) + { + if (AbortFlag) + { + break; + } + const uint32_t RemotePathIndex = Op.Target->PathIndex; + const uint64_t ChunkSize = Op.ChunkSize; + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); + + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + + OpenFileCache.WriteToFile( + RemotePathIndex, + [&CacheFolderPath, &RemoteContent](uint32_t TargetIndex) { + return (CacheFolderPath / RemoteContent.RawHashes[TargetIndex].ToHexString()) + .make_preferred(); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); + BytesWritten += ChunkSize; + WriteToDiskBytes += ChunkSize; + CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? + } } - if (Lhs.Target->Offset < Rhs.Target->Offset) + if (!AbortFlag) { - return true; + ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); + ZEN_DEBUG("Copied {} from {}", + NiceBytes(CacheLocalFileBytesRead), + LocalContent.Paths[CopyData.LocalPathIndex]); } - return false; - }); + } + } + }, + Work.DefaultErrorFunction()); + } - BufferedOpenFile SourceFile(CacheFilePath); - WriteFileCache OpenFileCache; - for (const WriteOp& Op : WriteOps) - { - if (AbortFlag) + for (const IoHash ChunkHash : LooseChunkHashes) + { + if (AbortFlag) + { + break; + } + + uint32_t RemoteChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); + if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + { + ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + continue; + } + bool NeedsCopy = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + + if (ChunkTargetPtrs.empty()) + { + ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { + if (!AbortFlag) { - break; - } - const uint32_t RemotePathIndex = Op.Target->PathIndex; - const uint64_t ChunkSize = Op.ChunkSize; - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.LocalFileOffset, ChunkSize); + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + { + DownloadLargeBlob(Storage, + Path / ZenTempChunkFolderName, + CacheFolderPath, + RemoteContent, + RemoteLookup, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + ChunkTargetPtrs, + Work, + WritePool, + NetworkPool, + BytesWritten, + WriteToDiskBytes, + BytesDownloaded, + LooseChunksBytes, + DownloadedChunks, + ChunkCountWritten, + MultipartAttachmentCount); + } + else + { + IoBuffer CompressedPart = Storage.GetBuildBlob(BuildId, ChunkHash); + if (!CompressedPart) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + BytesDownloaded += CompressedPart.GetSize(); + LooseChunksBytes += CompressedPart.GetSize(); + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(CompressedPart)), + Path / ZenTempChunkFolderName, + ChunkHash); + DownloadedChunks++; - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(Payload)]( + std::atomic&) { + if (!AbortFlag) + { + uint64_t TotalBytesWritten = 0; + SharedBuffer Chunk = + Decompress(CompressedPart, + ChunkHash, + RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); - OpenFileCache.WriteToFile( - RemotePathIndex, - [&Path, &RemoteContent](uint32_t TargetIndex) { - return (Path / RemoteContent.Paths[TargetIndex]).make_preferred(); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); - BytesWritten += ChunkSize; - WriteToDiskBytes += ChunkSize; - CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? - } - if (!AbortFlag) - { - ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), CopyData.OriginalSourceFileName); - } - } + { + WriteFileCache OpenFileCache; + WriteChunkToDisk(CacheFolderPath, + RemoteContent, + ChunkTargetPtrs, + CompositeBuffer(Chunk), + OpenFileCache, + TotalBytesWritten); + } + ChunkCountWritten++; + BytesWritten += TotalBytesWritten; + WriteToDiskBytes += TotalBytesWritten; + } + }, + Work.DefaultErrorFunction()); + } + } + } + }, + Work.DefaultErrorFunction()); + } + } + } - if (CopyData.RemotePathIndexes.empty()) - { - std::filesystem::remove(CacheFilePath); - } - else if (!AbortFlag) + size_t BlockCount = BlockDescriptions.size(); + std::atomic BlocksComplete = 0; + + auto IsBlockNeeded = [&RemoteContent, &RemoteLookup, &RemoteChunkIndexNeedsCopyFromSourceFlags]( + const ChunkBlockDescription& BlockDescription) -> bool { + for (const IoHash& ChunkHash : BlockDescription.ChunkRawHashes) + { + if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) { - uint64_t LocalBytesWritten = 0; - CloneFullFileFromCache(Path, - CacheFolderPath, - RemoteContent, - CopyData.LocalFileRawHash, - CopyData.LocalFileRawSize, - CopyData.RemotePathIndexes, - true, - LocalBytesWritten); - // CacheLocalFileBytesRead += CopyData.LocalFileRawSize; - BytesWritten += LocalBytesWritten; - WriteToDiskBytes += LocalBytesWritten; - ChunkCountWritten++; - - ZEN_DEBUG("Used full cached file {} ({}) for {} ({}) targets", - CopyData.OriginalSourceFileName, - NiceBytes(CopyData.LocalFileRawSize), - CopyData.RemotePathIndexes.size(), - NiceBytes(LocalBytesWritten)); + return true; } } - }, - Work.DefaultErrorFunction()); - } - - for (const IoHash ChunkHash : LooseChunkHashes) - { - if (AbortFlag) - { - break; - } + } + return false; + }; - uint32_t RemoteChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); - if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + size_t BlocksNeededCount = 0; + for (size_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); - continue; - } - bool NeedsCopy = true; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) - { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); - - if (ChunkTargetPtrs.empty()) + if (Work.IsAborted()) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + break; } - else + if (IsBlockNeeded(BlockDescriptions[BlockIndex])) { + BlocksNeededCount++; Work.ScheduleWork( NetworkPool, - [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { + [&, BlockIndex](std::atomic&) { if (!AbortFlag) { - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); + if (!BlockBuffer) { - DownloadLargeBlob(Storage, - Path, - RemoteContent, - RemoteLookup, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - ChunkTargetPtrs, - Work, - WritePool, - NetworkPool, - BytesWritten, - WriteToDiskBytes, - BytesDownloaded, - LooseChunksBytes, - DownloadedChunks, - ChunkCountWritten, - MultipartAttachmentCount); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); } - else + BytesDownloaded += BlockBuffer.GetSize(); + BlockBytes += BlockBuffer.GetSize(); + DownloadedBlocks++; + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), + Path / ZenTempBlockFolderName, + BlockDescriptions[BlockIndex].BlockHash); + + if (!AbortFlag) { - IoBuffer CompressedPart = Storage.GetBuildBlob(BuildId, ChunkHash); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - BytesDownloaded += CompressedPart.GetSize(); - LooseChunksBytes += CompressedPart.GetSize(); - DownloadedChunks++; + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { + if (!AbortFlag) + { + IoHash BlockRawHash; + uint64_t BlockRawSize; + CompressedBuffer CompressedBlockBuffer = + CompressedBuffer::FromCompressed(std::move(BlockBuffer), BlockRawHash, BlockRawSize); + if (!CompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", + BlockDescriptions[BlockIndex].BlockHash)); + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(CompressedPart)]( - std::atomic&) { - if (!AbortFlag) + if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) + { + throw std::runtime_error(fmt::format("Block {} header has a mismatching raw hash {}", + BlockDescriptions[BlockIndex].BlockHash, + BlockRawHash)); + } + + CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); + if (!DecompressedBlockBuffer) { - uint64_t TotalBytesWritten = 0; - SharedBuffer Chunk = - Decompress(CompressedPart, - ChunkHash, - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); - WriteFileCache OpenFileCache; - - WriteChunkToDisk(Path, + throw std::runtime_error(fmt::format("Block {} failed to decompress", + BlockDescriptions[BlockIndex].BlockHash)); + } + + ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == + IoHash::HashBuffer(DecompressedBlockBuffer)); + + uint64_t BytesWrittenToDisk = 0; + uint32_t ChunksReadFromBlock = 0; + if (WriteBlockToDisk(CacheFolderPath, RemoteContent, - ChunkTargetPtrs, - CompositeBuffer(Chunk), - OpenFileCache, - TotalBytesWritten); - ChunkCountWritten++; - BytesWritten += TotalBytesWritten; - WriteToDiskBytes += TotalBytesWritten; + RemotePathIndexWantsCopyFromCacheFlags, + DecompressedBlockBuffer, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags.data(), + ChunksReadFromBlock, + BytesWrittenToDisk)) + { + BytesWritten += BytesWrittenToDisk; + WriteToDiskBytes += BytesWrittenToDisk; + ChunkCountWritten += ChunksReadFromBlock; } - }, - Work.DefaultErrorFunction()); - } + else + { + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescriptions[BlockIndex].BlockHash)); + } + BlocksComplete++; + } + }, + [&, BlockIndex](const std::exception& Ex, std::atomic&) { + ZEN_ERROR("Failed writing block {}. Reason: {}", + BlockDescriptions[BlockIndex].BlockHash, + Ex.what()); + AbortFlag = true; + }); } } }, Work.DefaultErrorFunction()); } + else + { + ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + } } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); + WriteProgressBar.UpdateState( + {.Task = "Writing chunks ", + .Details = fmt::format("Written {} chunks out of {}. {} ouf of {} blocks complete. Downloaded: {}. Written: {}", + ChunkCountWritten.load(), + ChunkCountToWrite, + BlocksComplete.load(), + BlocksNeededCount, + NiceBytes(BytesDownloaded.load()), + NiceBytes(BytesWritten.load())), + .TotalCount = gsl::narrow(ChunkCountToWrite), + .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, + false); + }); + + if (AbortFlag) + { + return; + } + + WriteProgressBar.Finish(); } - size_t BlockCount = BlockDescriptions.size(); - std::atomic BlocksComplete = 0; + std::vector> Targets; + Targets.reserve(RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + { + Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + } + std::sort(Targets.begin(), Targets.end(), [](const std::pair& Lhs, const std::pair& Rhs) { + return Lhs.first < Rhs.first; + }); + + // Move all files we will reuse to cache folder + for (auto It : RawHashToLocalPathIndex) + { + const IoHash& RawHash = It.first; + if (RemoteLookup.RawHashToSequenceRawHashIndex.contains(RawHash)) + { + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[It.second]).make_preferred(); + const std::filesystem::path CacheFilePath = (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); + SetFileReadOnly(LocalFilePath, false); + std::filesystem::rename(LocalFilePath, CacheFilePath); + } + } - auto IsBlockNeeded = [&RemoteContent, &RemoteLookup, &RemoteChunkIndexNeedsCopyFromSourceFlags]( - const ChunkBlockDescription& BlockDescription) -> bool { - for (const IoHash& ChunkHash : BlockDescription.ChunkRawHashes) + if (WipeTargetFolder) + { + // Clean target folder + ZEN_CONSOLE("Wiping {}", Path); + CleanDirectory(Path, DefaultExcludeFolders); + } + else + { + // Remove unused tracked files + tsl::robin_map RemotePathToRemoteIndex; + RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + { + RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); + } + std::vector LocalFilesToRemove; + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { - if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) + if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string())) { - const uint32_t RemoteChunkIndex = It->second; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + if (std::filesystem::exists(LocalFilePath)) { - return true; + LocalFilesToRemove.emplace_back(std::move(LocalFilePath)); } } } - return false; - }; - - for (size_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) - { - if (Work.IsAborted()) + if (!LocalFilesToRemove.empty()) { - break; + ZEN_CONSOLE("Cleaning {} removed files from {}", LocalFilesToRemove.size(), Path); + for (const std::filesystem::path& LocalFilePath : LocalFilesToRemove) + { + SetFileReadOnly(LocalFilePath, false); + std::filesystem::remove(LocalFilePath); + } } - Work.ScheduleWork( - WritePool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - if (IsBlockNeeded(BlockDescriptions[BlockIndex])) - { - Work.ScheduleWork( - NetworkPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); - if (!BlockBuffer) - { - throw std::runtime_error( - fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); - } - BytesDownloaded += BlockBuffer.GetSize(); - BlockBytes += BlockBuffer.GetSize(); - DownloadedBlocks++; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&, BlockIndex, BlockBuffer = std::move(BlockBuffer)](std::atomic&) { - if (!AbortFlag) - { - IoHash BlockRawHash; - uint64_t BlockRawSize; - CompressedBuffer CompressedBlockBuffer = - CompressedBuffer::FromCompressed(SharedBuffer(std::move(BlockBuffer)), - BlockRawHash, - BlockRawSize); - if (!CompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", - BlockDescriptions[BlockIndex].BlockHash)); - } + { + WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) - { - throw std::runtime_error( - fmt::format("Block {} header has a mismatching raw hash {}", - BlockDescriptions[BlockIndex].BlockHash, - BlockRawHash)); - } + ProgressBar RebuildProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); - CompositeBuffer DecompressedBlockBuffer = - CompressedBlockBuffer.DecompressToComposite(); - if (!DecompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} failed to decompress", - BlockDescriptions[BlockIndex].BlockHash)); - } + OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); + OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size()); + OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size()); + OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size()); - ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == - IoHash::HashBuffer(DecompressedBlockBuffer)); + std::atomic TargetsComplete = 0; - uint64_t BytesWrittenToDisk = 0; - uint32_t ChunksReadFromBlock = 0; - if (WriteBlockToDisk(Path, - RemoteContent, - RemotePathIndexWantsCopyFromCacheFlags, - DecompressedBlockBuffer, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags.data(), - ChunksReadFromBlock, - BytesWrittenToDisk)) - { - BytesWritten += BytesWrittenToDisk; - WriteToDiskBytes += BytesWrittenToDisk; - ChunkCountWritten += ChunksReadFromBlock; - } - else - { - throw std::runtime_error(fmt::format("Block {} is malformed", - BlockDescriptions[BlockIndex].BlockHash)); - } - BlocksComplete++; - } - }, - [&, BlockIndex](const std::exception& Ex, std::atomic&) { - ZEN_ERROR("Failed writing block {}. Reason: {}", - BlockDescriptions[BlockIndex].BlockHash, - Ex.what()); - AbortFlag = true; - }); - } - } - }, - Work.DefaultErrorFunction()); - } - else - { - ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); - BlocksComplete++; - } - } - }, - Work.DefaultErrorFunction()); - } - for (uint32_t PathIndex = 0; PathIndex < RemoteContent.Paths.size(); PathIndex++) - { - if (Work.IsAborted()) - { - break; - } - if (RemoteContent.RawSizes[PathIndex] == 0) + size_t TargetOffset = 0; + while (TargetOffset < Targets.size()) { + if (AbortFlag) + { + break; + } + + size_t TargetCount = 1; + const IoHash& RawHash = Targets[TargetOffset].first; + while (Targets[TargetOffset + TargetCount].first == RawHash) + { + TargetCount++; + } + Work.ScheduleWork( - WritePool, - [&, PathIndex](std::atomic&) { + WritePool, // GetSyncWorkerPool(),// + [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { if (!AbortFlag) { - const std::filesystem::path TargetPath = (Path / RemoteContent.Paths[PathIndex]).make_preferred(); - CreateDirectories(TargetPath.parent_path()); - BasicFile OutputFile; - OutputFile.Open(TargetPath, BasicFile::Mode::kTruncate); + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].first; + const uint32_t FirstTargetPathIndex = Targets[TargetOffset].second; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstTargetPathIndex]; + OutLocalFolderState.Paths[FirstTargetPathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstTargetPathIndex] = RemoteContent.RawSizes[FirstTargetPathIndex]; + const std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + if (RawHash == IoHash::Zero) + { + if (std::filesystem::exists(FirstTargetFilePath)) + { + SetFileReadOnly(FirstTargetFilePath, false); + } + CreateDirectories(FirstTargetFilePath.parent_path()); + { + BasicFile OutputFile; + OutputFile.Open(FirstTargetFilePath, BasicFile::Mode::kTruncate); + } + } + else + { + const std::filesystem::path CacheFilePath = (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + CreateDirectories(FirstTargetFilePath.parent_path()); + if (std::filesystem::exists(FirstTargetFilePath)) + { + SetFileReadOnly(FirstTargetFilePath, false); + } + std::filesystem::rename(CacheFilePath, FirstTargetFilePath); + } + + OutLocalFolderState.Attributes[FirstTargetPathIndex] = + RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[FirstTargetPathIndex]); + OutLocalFolderState.ModificationTicks[FirstTargetPathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + + TargetOffset++; + TargetsComplete++; + while (TargetOffset < (BaseTargetOffset + TargetCount)) + { + ZEN_ASSERT(Targets[TargetOffset].first == RawHash); + ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + const uint32_t ExtraTargetPathIndex = Targets[TargetOffset].second; + const std::filesystem::path& ExtraTargetPath = RemoteContent.Paths[ExtraTargetPathIndex]; + const std::filesystem::path ExtraTargetFilePath = (Path / ExtraTargetPath).make_preferred(); + OutLocalFolderState.Paths[ExtraTargetPathIndex] = ExtraTargetPath; + OutLocalFolderState.RawSizes[ExtraTargetPathIndex] = RemoteContent.RawSizes[ExtraTargetPathIndex]; + CreateDirectories(ExtraTargetFilePath.parent_path()); + if (std::filesystem::exists(ExtraTargetFilePath)) + { + SetFileReadOnly(ExtraTargetFilePath, false); + } + CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false}); + + OutLocalFolderState.Attributes[ExtraTargetPathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(ExtraTargetFilePath) + : SetNativeFileAttributes(ExtraTargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[ExtraTargetPathIndex]); + OutLocalFolderState.ModificationTicks[ExtraTargetPathIndex] = + GetModificationTickFromPath(ExtraTargetFilePath); + + TargetOffset++; + TargetsComplete++; + } } }, Work.DefaultErrorFunction()); - } - } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); - WriteProgressBar.UpdateState( - {.Task = "Writing chunks ", - .Details = fmt::format("Written {} chunks out of {}. {} ouf of {} blocks complete. Downloaded: {}. Written: {}", - ChunkCountWritten.load(), - ChunkCountToWrite, - BlocksComplete.load(), - BlockCount, - NiceBytes(BytesDownloaded.load()), - NiceBytes(BytesWritten.load())), - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, - false); - }); - - if (AbortFlag) - { - return; - } + TargetOffset += TargetCount; + } - WriteProgressBar.Finish(); + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + RebuildProgressBar.UpdateState( + {.Task = "Rebuilding state ", + .Details = fmt::format("Written {} files out of {}", TargetsComplete.load(), Targets.size()), + .TotalCount = gsl::narrow(Targets.size()), + .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, + false); + }); - { - ProgressBar PremissionsProgressBar(false); - if (!RemoteContent.Attributes.empty()) + if (AbortFlag) { - auto SetNativeFileAttributes = - [](const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) -> uint32_t { -#if ZEN_PLATFORM_WINDOWS - if (SourcePlatform == SourcePlatform::Windows) - { - SetFileAttributes(FilePath, Attributes); - return Attributes; - } - else - { - uint32_t CurrentAttributes = GetFileAttributes(FilePath); - uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, IsFileModeReadOnly(Attributes)); - if (CurrentAttributes != NewAttributes) - { - SetFileAttributes(FilePath, NewAttributes); - } - return NewAttributes; - } -#endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - if (SourcePlatform != SourcePlatform::Windows) - { - SetFileMode(FilePath, Attributes); - return Attributes; - } - else - { - uint32_t CurrentMode = GetFileMode(FilePath); - uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, IsFileAttributeReadOnly(Attributes)); - if (CurrentMode != NewMode) - { - SetFileMode(FilePath, NewMode); - } - return NewMode; - } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - }; - - OutLocalFolderState.Paths.reserve(RemoteContent.Paths.size()); - OutLocalFolderState.RawSizes.reserve(RemoteContent.Paths.size()); - OutLocalFolderState.Attributes.reserve(RemoteContent.Paths.size()); - OutLocalFolderState.ModificationTicks.reserve(RemoteContent.Paths.size()); - for (uint32_t PathIndex = 0; PathIndex < RemoteContent.Paths.size(); PathIndex++) - { - const std::filesystem::path LocalFilePath = (Path / RemoteContent.Paths[PathIndex]); - const uint32_t CurrentPlatformAttributes = - SetNativeFileAttributes(LocalFilePath, RemoteContent.Platform, RemoteContent.Attributes[PathIndex]); - - OutLocalFolderState.Paths.push_back(RemoteContent.Paths[PathIndex]); - OutLocalFolderState.RawSizes.push_back(RemoteContent.RawSizes[PathIndex]); - OutLocalFolderState.Attributes.push_back(CurrentPlatformAttributes); - OutLocalFolderState.ModificationTicks.push_back(GetModificationTickFromPath(LocalFilePath)); - - PremissionsProgressBar.UpdateState( - {.Task = "Set permissions ", - .Details = fmt::format("Updated {} files out of {}", PathIndex, RemoteContent.Paths.size()), - .TotalCount = RemoteContent.Paths.size(), - .RemainingCount = (RemoteContent.Paths.size() - PathIndex)}, - false); - } + return; } - PremissionsProgressBar.Finish(); + + RebuildProgressBar.Finish(); } } @@ -4077,7 +4076,7 @@ namespace { CbObject BuildObject = Storage.GetBuild(BuildId); ZEN_CONSOLE("GetBuild took {}. Payload size: {}", - NiceLatencyNs(GetBuildTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), NiceBytes(BuildObject.GetSize())); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); @@ -4162,7 +4161,7 @@ namespace { ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", BuildParts[0].first, BuildParts[0].second, - NiceLatencyNs(GetBuildPartTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), NiceBytes(BuildPartManifest.GetSize())); { @@ -4202,7 +4201,7 @@ namespace { OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes); ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", BuildPartId, - NiceLatencyNs(GetBlockMetadataTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), OutBlockDescriptions.size()); if (OutBlockDescriptions.size() != BlockRawHashes.size()) @@ -4309,7 +4308,7 @@ namespace { ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", OverlayBuildPartId, OverlayBuildPartName, - NiceLatencyNs(GetOverlayBuildPartTimer.GetElapsedTimeUs() * 1000), + NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()), NiceBytes(OverlayBuildPartManifest.GetSize())); ChunkedFolderContent OverlayPartContent; @@ -4457,6 +4456,7 @@ namespace { if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) { + const size_t LocaStatePathCount = LocalFolderState.Paths.size(); std::vector DeletedPaths; FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); if (!DeletedPaths.empty()) @@ -4464,9 +4464,10 @@ namespace { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated", + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", DeletedPaths.size(), - UpdatedContent.Paths.size()); + UpdatedContent.Paths.size(), + LocaStatePathCount); if (UpdatedContent.Paths.size() > 0) { uint64_t ByteCountToScan = 0; @@ -4535,7 +4536,7 @@ namespace { ZEN_CONSOLE("Using cached local state"); } - ZEN_CONSOLE("Read local state in {}", NiceLatencyNs(ReadStateTimer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); ScanContent = false; } } @@ -4610,7 +4611,7 @@ namespace { }); CreateDirectories(Path / ZenTempBlockFolderName); CreateDirectories(Path / ZenTempChunkFolderName); - CreateDirectories(Path / ZenTempReuseFolderName); + CreateDirectories(Path / ZenTempCacheFolderName); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -4698,7 +4699,7 @@ namespace { if (CompareContent(RemoteContent, LocalContent)) { ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", - NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); } else { @@ -4730,7 +4731,7 @@ namespace { CreateDirectories((Path / ZenStateFilePath).parent_path()); TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceLatencyNs(WriteStateTimer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 ExtendableStringBuilder<1024> SB; @@ -4738,7 +4739,7 @@ namespace { WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 - ZEN_CONSOLE("Downloaded build in {}.", NiceLatencyNs(DownloadTimer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("Downloaded build in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); } } } @@ -5095,13 +5096,13 @@ BuildsCommand::BuildsCommand() "Build part Ids list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", cxxopts::value(m_BuildPartIds), ""); - m_DownloadOptions.add_option( - "", - "", - "build-part-name", - "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given all parts will be downloaded", - cxxopts::value(m_BuildPartNames), - ""); + m_DownloadOptions.add_option("", + "", + "build-part-name", + "Name of the build parts list separated by ',', if no build-part-ids or build-part-names are given " + "all parts will be downloaded", + cxxopts::value(m_BuildPartNames), + ""); m_DownloadOptions .add_option("", "", "clean", "Delete all data in target folder before downloading", cxxopts::value(m_Clean), ""); m_DownloadOptions.add_option("", @@ -5841,7 +5842,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); ZEN_ASSERT(!AbortFlag.load()); - ZEN_CONSOLE("Scrambled files in {}", NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + ZEN_CONSOLE("Scrambled files in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }; ScrambleDir(DownloadPath); diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8279fb952..85feab2f7 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -531,7 +531,10 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) } void -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options, std::error_code& OutErrorCode) +CopyFile(const std::filesystem::path& FromPath, + const std::filesystem::path& ToPath, + const CopyFileOptions& Options, + std::error_code& OutErrorCode) { OutErrorCode.clear(); @@ -544,7 +547,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop } bool -CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) +CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options) { bool Success = false; diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 20f6dc56c..e020668fc 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -97,11 +97,11 @@ struct CopyFileOptions bool MustClone = false; }; -ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); -ZENCORE_API void CopyFile(std::filesystem::path FromPath, - std::filesystem::path ToPath, - const CopyFileOptions& Options, - std::error_code& OutError); +ZENCORE_API bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options); +ZENCORE_API void CopyFile(const std::filesystem::path& FromPath, + const std::filesystem::path& ToPath, + const CopyFileOptions& Options, + std::error_code& OutError); ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 78ebcdd55..a4bb759e7 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -336,7 +336,8 @@ public: const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); if (std::filesystem::is_regular_file(BlockPath)) { - IoBuffer Payload = ReadFile(BlockPath).Flatten(); + BasicFile File(BlockPath, BasicFile::Mode::kRead); + IoBuffer Payload = File.ReadAll(); ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload)))); m_Stats.TotalBytesRead += Payload.GetSize(); Payload.SetContentType(ZenContentType::kCompressedBinary); @@ -365,13 +366,13 @@ public: struct WorkloadData { std::atomic BytesRemaining; - IoBuffer BlobFile; + BasicFile BlobFile; std::function Receiver; }; std::shared_ptr Workload(std::make_shared()); - Workload->BlobFile = IoBufferBuilder::MakeFromFile(BlockPath); - const uint64_t BlobSize = Workload->BlobFile.GetSize(); + Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead); + const uint64_t BlobSize = Workload->BlobFile.FileSize(); Workload->Receiver = std::move(Receiver); Workload->BytesRemaining = BlobSize; @@ -383,7 +384,8 @@ public: uint64_t Size = Min(ChunkSize, BlobSize - Offset); WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() { SimulateLatency(0, 0); - IoBuffer PartPayload(Workload->BlobFile, Offset, Size); + IoBuffer PartPayload(Size); + Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset); m_Stats.TotalBytesRead += PartPayload.GetSize(); uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size); Workload->Receiver(Offset, PartPayload, ByteRemaning); -- cgit v1.2.3 From 1270bfeffbc81b1e4940c5c454ee6acde43e696a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 3 Mar 2025 17:53:11 +0100 Subject: refactor use chunk sequence download (#291) * work on chunk sequences on download, not paths * write chunksequences to .tmp file and move when complete * cleanup * Added on the fly validation `zen builds download` of files built from smaller chunks as each file is completed Added `--verify` option to `zen builds upload` to verify all uploaded data once entire upload is complete Added `--verify` option to `zen builds download` to verify all files in target folder once entire download is complete Fixed/improved progress updated Multithreaded part validation * added rates to Write Chunks task * b/s -> bits/s * dont validate partial content as complete payload * handle legacy c# builds --- src/zen/cmds/builds_cmd.cpp | 1324 ++++++++++++++++---------- src/zen/cmds/builds_cmd.h | 2 + src/zenhttp/httpclient.cpp | 5 + src/zenutil/chunkedcontent.cpp | 78 +- src/zenutil/include/zenutil/chunkedcontent.h | 53 +- 5 files changed, 910 insertions(+), 552 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 28c794559..219d01240 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -286,7 +286,7 @@ namespace { } } - uint64_t GetCurrent() const + uint64_t GetCurrent() const // If Stopped - return total count / total time { if (LastTimeUS == (uint64_t)-1) { @@ -330,6 +330,16 @@ namespace { return Count * 1000000 / ElapsedWallTimeUS; } + std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) + { + return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred(); + } + + std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) + { + return (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + } + ChunkedFolderContent ScanAndChunkFolder( GetFolderContentStatistics& GetFolderContentStats, ChunkingStatistics& ChunkingStats, @@ -364,15 +374,16 @@ namespace { UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + GetFolderContentStats.AcceptedFileCount.load(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(GetFolderContentStats.FoundFileByteCount), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - GetFolderContentStats.AcceptedFileCount.load(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(GetFolderContentStats.FoundFileByteCount), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .Details = Details, .TotalCount = GetFolderContentStats.AcceptedFileByteCount, .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()}, false); @@ -766,9 +777,19 @@ namespace { } } - if (FilesObject["chunkcounts"sv]) + if (CbObjectView ChunkContentView = BuildPartManifest["chunkedContent"sv].AsObjectView(); ChunkContentView) + { + compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkContentView, OutSequenceRawHashes); + compactbinary_helpers::ReadArray("chunkcounts"sv, ChunkContentView, OutChunkCounts); + if (OutChunkCounts.size() != OutSequenceRawHashes.size()) + { + throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths")); + } + compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders); + } + else if (FilesObject["chunkcounts"sv]) { - // Legacy style + // Legacy zen style std::vector LegacyChunkCounts; compactbinary_helpers::ReadArray("chunkcounts"sv, FilesObject, LegacyChunkCounts); @@ -805,15 +826,29 @@ namespace { OrderIndexOffset += LegacyChunkCounts[PathIndex]; } } - if (CbObjectView ChunkContentView = BuildPartManifest["chunkedContent"sv].AsObjectView(); ChunkContentView) + else { - compactbinary_helpers::ReadArray("sequenceRawHashes"sv, ChunkContentView, OutSequenceRawHashes); - compactbinary_helpers::ReadArray("chunkcounts"sv, ChunkContentView, OutChunkCounts); - if (OutChunkCounts.size() != OutSequenceRawHashes.size()) + // Legacy C# style + + tsl::robin_set FoundRawHashes; + FoundRawHashes.reserve(PathCount); + uint32_t OrderIndexOffset = 0; + for (uint32_t PathIndex = 0; PathIndex < OutPaths.size(); PathIndex++) { - throw std::runtime_error(fmt::format("Number of chunk count entries does not match number of paths")); + if (OutRawSizes[PathIndex] > 0) + { + const IoHash& PathRawHash = OutRawHashes[PathIndex]; + if (FoundRawHashes.insert(PathRawHash).second) + { + OutSequenceRawHashes.push_back(PathRawHash); + OutChunkCounts.push_back(1); + OutAbsoluteChunkOrders.push_back(OrderIndexOffset); + OutLooseChunkHashes.push_back(PathRawHash); + OutLooseChunkRawSizes.push_back(OutRawSizes[PathIndex]); + OrderIndexOffset += 1; + } + } } - compactbinary_helpers::ReadArray("chunkorders"sv, ChunkContentView, OutAbsoluteChunkOrders); } CbObjectView ChunkAttachmentsView = BuildPartManifest["chunkAttachments"sv].AsObjectView(); @@ -963,14 +998,14 @@ namespace { { public: // A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten - ReadFileCache(DiskStatistics& DiskStats, - const std::filesystem::path& Path, - const std::vector& Paths, - const std::vector& RawSizes, - size_t MaxOpenFileCount) + ReadFileCache(DiskStatistics& DiskStats, + const std::filesystem::path& Path, + const ChunkedFolderContent& LocalContent, + const ChunkedContentLookup& LocalLookup, + size_t MaxOpenFileCount) : m_Path(Path) - , m_Paths(Paths) - , m_RawSizes(RawSizes) + , m_LocalContent(LocalContent) + , m_LocalLookup(LocalLookup) , m_DiskStats(DiskStats) { m_OpenFiles.reserve(MaxOpenFileCount); @@ -981,26 +1016,28 @@ namespace { m_OpenFiles.clear(); } - CompositeBuffer GetRange(uint32_t PathIndex, uint64_t Offset, uint64_t Size) + CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) { - auto CacheIt = - std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [PathIndex](const auto& Lhs) { return Lhs.first == PathIndex; }); + auto CacheIt = std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { + return Lhs.first == SequenceIndex; + }); if (CacheIt != m_OpenFiles.end()) { if (CacheIt != m_OpenFiles.begin()) { auto CachedFile(std::move(CacheIt->second)); m_OpenFiles.erase(CacheIt); - m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(PathIndex, std::move(CachedFile))); + m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile))); } CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); m_DiskStats.ReadByteCount += Result.GetSize(); return Result; } - const std::filesystem::path AttachmentPath = (m_Path / m_Paths[PathIndex]).make_preferred(); - if (Size == m_RawSizes[PathIndex]) + const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred(); + if (Size == m_LocalContent.RawSizes[LocalPathIndex]) { - IoBuffer Result = IoBufferBuilder::MakeFromFile(AttachmentPath); + IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath); m_DiskStats.OpenReadCount++; m_DiskStats.ReadByteCount += Result.GetSize(); return CompositeBuffer(SharedBuffer(Result)); @@ -1010,7 +1047,7 @@ namespace { m_OpenFiles.pop_back(); m_DiskStats.CurrentOpenFileCount--; } - m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(PathIndex, std::make_unique(AttachmentPath))); + m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::make_unique(LocalFilePath))); CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); m_DiskStats.ReadByteCount += Result.GetSize(); m_DiskStats.OpenReadCount++; @@ -1020,23 +1057,14 @@ namespace { private: const std::filesystem::path m_Path; - const std::vector& m_Paths; - const std::vector& m_RawSizes; + const ChunkedFolderContent& m_LocalContent; + const ChunkedContentLookup& m_LocalLookup; std::vector>> m_OpenFiles; DiskStatistics& m_DiskStats; }; - CompositeBuffer ValidateBlob(BuildStorage& Storage, - const Oid& BuildId, - const IoHash& BlobHash, - uint64_t& OutCompressedSize, - uint64_t& OutDecompressedSize) + CompositeBuffer ValidateBlob(IoBuffer&& Payload, const IoHash& BlobHash, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize) { - IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); - if (!Payload) - { - throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash)); - } if (Payload.GetContentType() != ZenContentType::kCompressedBinary) { throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'", @@ -1080,13 +1108,26 @@ namespace { return DecompressedComposite; } - ChunkBlockDescription ValidateChunkBlock(BuildStorage& Storage, - const Oid& BuildId, + CompositeBuffer ValidateBlob(BuildStorage& Storage, + const Oid& BuildId, + const IoHash& BlobHash, + uint64_t& OutCompressedSize, + uint64_t& OutDecompressedSize) + { + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); + if (!Payload) + { + throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash)); + } + return ValidateBlob(std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); + } + + ChunkBlockDescription ValidateChunkBlock(IoBuffer&& Payload, const IoHash& BlobHash, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize) { - CompositeBuffer BlockBuffer = ValidateBlob(Storage, BuildId, BlobHash, OutCompressedSize, OutDecompressedSize); + CompositeBuffer BlockBuffer = ValidateBlob(std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); } @@ -1097,11 +1138,12 @@ namespace { { auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end()); - uint32_t ChunkIndex = It->second; - std::span ChunkLocations = GetChunkLocations(Lookup, ChunkIndex); + uint32_t ChunkIndex = It->second; + std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = - OpenFileCache.GetRange(ChunkLocations[0].PathIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + ChunkLocations[0].Offset, + Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); return Chunk; }; @@ -1113,7 +1155,7 @@ namespace { ChunkBlockDescription& OutBlockDescription, DiskStatistics& DiskStats) { - ReadFileCache OpenFileCache(DiskStats, Path, Content.Paths, Content.RawSizes, 4); + ReadFileCache OpenFileCache(DiskStats, Path, Content, Lookup, 4); std::vector> BlockContent; BlockContent.reserve(ChunksInBlock.size()); @@ -1135,6 +1177,187 @@ namespace { return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription); }; + void ValidateBuildPart(BuildStorage& Storage, const Oid& BuildId, Oid BuildPartId, const std::string_view BuildPartName) + { + Stopwatch Timer; + auto _ = MakeGuard([&]() { + ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", + BuildId, + BuildPartId, + BuildPartName, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + CbObject Build = Storage.GetBuild(BuildId); + if (!BuildPartName.empty()) + { + BuildPartId = Build["parts"sv].AsObjectView()[BuildPartName].AsObjectId(); + if (BuildPartId == Oid::Zero) + { + throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", BuildId, BuildPartName)); + } + } + CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); + ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); + std::vector ChunkAttachments; + for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) + { + ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); + } + std::vector BlockAttachments; + for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) + { + BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); + } + + std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); + if (VerifyBlockDescriptions.size() != BlockAttachments.size()) + { + throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", + BlockAttachments.size() - VerifyBlockDescriptions.size())); + } + + WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + ParallellWork Work(AbortFlag); + + ProgressBar ProgressBar(UsePlainProgress); + + uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); + std::atomic DownloadedAttachmentCount = 0; + std::atomic VerifiedAttachmentCount = 0; + std::atomic DownloadedByteCount = 0; + std::atomic VerifiedByteCount = 0; + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredVerifiedBytesPerSecond; + + for (const IoHash& ChunkAttachment : ChunkAttachments) + { + Work.ScheduleWork( + NetworkPool, + [&, ChunkAttachment](std::atomic&) { + if (!AbortFlag) + { + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer Payload = Storage.GetBuildBlob(BuildId, ChunkAttachment); + DownloadedAttachmentCount++; + DownloadedByteCount += Payload.GetSize(); + if (DownloadedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!Payload) + { + throw std::runtime_error(fmt::format("Chunk attachment {} could not be found", ChunkAttachment)); + } + if (!AbortFlag) + { + Work.ScheduleWork( + VerifyPool, + [&, Payload = std::move(Payload), ChunkAttachment](std::atomic&) { + if (!AbortFlag) + { + FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(IoBuffer(Payload), ChunkAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid", + ChunkAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + VerifiedAttachmentCount++; + VerifiedByteCount += DecompressedSize; + if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredVerifiedBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + } + }, + Work.DefaultErrorFunction()); + } + + for (const IoHash& BlockAttachment : BlockAttachments) + { + Work.ScheduleWork( + NetworkPool, + [&, BlockAttachment](std::atomic&) { + if (!AbortFlag) + { + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); + DownloadedAttachmentCount++; + DownloadedByteCount += Payload.GetSize(); + if (DownloadedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!Payload) + { + throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); + } + if (!AbortFlag) + { + Work.ScheduleWork( + VerifyPool, + [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) { + if (!AbortFlag) + { + FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateChunkBlock(IoBuffer(Payload), BlockAttachment, CompressedSize, DecompressedSize); + ZEN_CONSOLE_VERBOSE("Chunk block {} ({} -> {}) is valid", + BlockAttachment, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + VerifiedAttachmentCount++; + VerifiedByteCount += DecompressedSize; + if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredVerifiedBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + } + }, + Work.DefaultErrorFunction()); + } + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + + FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount); + FilteredVerifiedBytesPerSecond.Update(VerifiedByteCount); + + std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)", + DownloadedAttachmentCount.load(), + AttachmentsToVerifyCount, + NiceBytes(DownloadedByteCount.load()), + NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), + VerifiedAttachmentCount.load(), + AttachmentsToVerifyCount, + NiceBytes(VerifiedByteCount.load()), + NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent())); + + ProgressBar.UpdateState( + {.Task = "Validating blobs ", + .Details = Details, + .TotalCount = gsl::narrow(AttachmentsToVerifyCount * 2), + .RemainingCount = gsl::narrow(AttachmentsToVerifyCount * 2 - + (DownloadedAttachmentCount.load() + VerifiedAttachmentCount.load()))}, + false); + }); + + ProgressBar.Finish(); + } + void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, uint64_t MaxBlockSize, @@ -1142,13 +1365,13 @@ namespace { std::vector>& OutBlocks) { std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { - const ChunkedContentLookup::ChunkLocation& LhsLocation = GetChunkLocations(Lookup, Lhs)[0]; - const ChunkedContentLookup::ChunkLocation& RhsLocation = GetChunkLocations(Lookup, Rhs)[0]; - if (LhsLocation.PathIndex < RhsLocation.PathIndex) + const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0]; + const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0]; + if (LhsLocation.SequenceIndex < RhsLocation.SequenceIndex) { return true; } - else if (LhsLocation.PathIndex > RhsLocation.PathIndex) + else if (LhsLocation.SequenceIndex > RhsLocation.SequenceIndex) { return false; } @@ -1171,7 +1394,8 @@ namespace { // between source paths for chunks. Break the block at the last such break if any. ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart); - const uint32_t ChunkPathIndex = Lookup.ChunkLocations[Lookup.ChunkLocationOffset[ChunkIndex]].PathIndex; + const uint32_t ChunkSequenceIndex = + Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[ChunkIndex]].SequenceIndex; uint64_t ScanBlockSize = BlockSize; @@ -1185,8 +1409,9 @@ namespace { break; } - const uint32_t TestPathIndex = Lookup.ChunkLocations[Lookup.ChunkLocationOffset[TestChunkIndex]].PathIndex; - if (ChunkPathIndex != TestPathIndex) + const uint32_t TestSequenceIndex = + Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[TestChunkIndex]].SequenceIndex; + if (ChunkSequenceIndex != TestSequenceIndex) { ChunkIndexOffset = ScanChunkIndexOffset + 1; break; @@ -1235,10 +1460,9 @@ namespace { const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - const ChunkedContentLookup::ChunkLocation& Source = GetChunkLocations(Lookup, ChunkIndex)[0]; - - IoBuffer RawSource = - IoBufferBuilder::MakeFromFile((Path / Content.Paths[Source.PathIndex]).make_preferred(), Source.Offset, ChunkSize); + const ChunkedContentLookup::ChunkSequenceLocation& Source = GetChunkSequenceLocations(Lookup, ChunkIndex)[0]; + const std::uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[Source.SequenceIndex]; + IoBuffer RawSource = IoBufferBuilder::MakeFromFile((Path / Content.Paths[PathIndex]).make_preferred(), Source.Offset, ChunkSize); if (!RawSource) { throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash)); @@ -1418,7 +1642,7 @@ namespace { FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); - std::string Details = fmt::format("Generated {}/{} ({}, {}B/s) and uploaded {}/{} ({}, {}bits/s) blocks", + std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)", GenerateBlocksStats.GeneratedBlockCount.load(), NewBlockCount, NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), @@ -1470,10 +1694,11 @@ namespace { ParallellWork Work(AbortFlag); - std::atomic UploadedBlockSize = 0; - std::atomic UploadedBlockCount = 0; - std::atomic UploadedChunkSize = 0; - std::atomic UploadedChunkCount = 0; + std::atomic UploadedBlockSize = 0; + std::atomic UploadedBlockCount = 0; + std::atomic UploadedRawChunkSize = 0; + std::atomic UploadedCompressedChunkSize = 0; + std::atomic UploadedChunkCount = 0; tsl::robin_map ChunkIndexToLooseChunkOrderIndex; ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size()); @@ -1485,8 +1710,8 @@ namespace { std::vector BlockIndexes; std::vector LooseChunkOrderIndexes; - uint64_t TotalChunksSize = 0; - uint64_t TotalBlocksSize = 0; + uint64_t TotalLooseChunksSize = 0; + uint64_t TotalBlocksSize = 0; for (const IoHash& RawHash : RawHashes) { if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end()) @@ -1501,11 +1726,11 @@ namespace { LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end()) { LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second); - TotalChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; } } } - uint64_t TotalSize = TotalChunksSize + TotalBlocksSize; + uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize; const size_t UploadBlockCount = BlockIndexes.size(); const uint32_t UploadChunkCount = gsl::narrow(LooseChunkOrderIndexes.size()); @@ -1548,10 +1773,10 @@ namespace { Work.DefaultErrorFunction()); }; - auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, CompositeBuffer&& Payload) { + auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, RawHash, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) { + [&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) { if (!AbortFlag) { const uint64_t PayloadSize = Payload.GetSize(); @@ -1571,9 +1796,9 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&](uint64_t SentBytes, bool IsComplete) { + [&, RawSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; - UploadedChunkSize += SentBytes; + UploadedCompressedChunkSize += SentBytes; if (IsComplete) { UploadStats.ChunkCount++; @@ -1582,6 +1807,7 @@ namespace { { FilteredUploadedBytesPerSecond.Stop(); } + UploadedRawChunkSize += RawSize; } }); for (auto& WorkPart : MultipartWork) @@ -1604,7 +1830,8 @@ namespace { ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); UploadStats.ChunksBytes += Payload.GetSize(); UploadStats.ChunkCount++; - UploadedChunkSize += Payload.GetSize(); + UploadedCompressedChunkSize += Payload.GetSize(); + UploadedRawChunkSize += RawSize; UploadedChunkCount++; if (UploadedChunkCount == UploadChunkCount) { @@ -1696,6 +1923,7 @@ namespace { std::atomic CompressedLooseChunkCount = 0; std::atomic CompressedLooseChunkByteCount = 0; + std::atomic RawLooseChunkByteCount = 0; // Start compression of any non-precompressed loose chunks and schedule upload for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) @@ -1712,18 +1940,20 @@ namespace { Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), NiceBytes(Payload.GetSize())); - UploadStats.ReadFromDiskBytes += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + UploadStats.ReadFromDiskBytes += ChunkRawSize; LooseChunksStats.CompressedChunkBytes += Payload.GetSize(); LooseChunksStats.CompressedChunkCount++; CompressedLooseChunkByteCount += Payload.GetSize(); CompressedLooseChunkCount++; + RawLooseChunkByteCount += ChunkRawSize; if (CompressedLooseChunkCount == CompressLooseChunkOrderIndexes.size()) { FilteredCompressedBytesPerSecond.Stop(); } if (!AbortFlag) { - AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], std::move(Payload)); + AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload)); } } }, @@ -1734,22 +1964,31 @@ namespace { ZEN_UNUSED(IsAborted, PendingWork); FilteredCompressedBytesPerSecond.Update(CompressedLooseChunkByteCount.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); - FilteredUploadedBytesPerSecond.Update(UploadedChunkSize.load() + UploadedBlockSize.load()); - uint64_t UploadedSize = UploadedChunkSize.load() + UploadedBlockSize.load(); + FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); + uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); + uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load(); + + std::string Details = fmt::format( + "Compressed {}/{} ({}/{}) chunks. " + "Uploaded {}/{} ({}/{}) blobs " + "({} {}bits/s)", + CompressedLooseChunkCount.load(), + CompressLooseChunkOrderIndexes.size(), + NiceBytes(RawLooseChunkByteCount), + NiceBytes(TotalLooseChunksSize), + + UploadedBlockCount.load() + UploadedChunkCount.load(), + UploadBlockCount + UploadChunkCount, + NiceBytes(UploadedRawSize), + NiceBytes(TotalRawSize), + + NiceBytes(UploadedCompressedSize), + NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())); + ProgressBar.UpdateState({.Task = "Uploading blobs ", - .Details = fmt::format("Compressed {}/{} chunks. " - "Uploaded {}/{} blobs ({}/{} {}bits/s)", - CompressedLooseChunkCount.load(), - CompressLooseChunkOrderIndexes.size(), - - UploadedBlockCount.load() + UploadedChunkCount.load(), - UploadBlockCount + UploadChunkCount, - - NiceBytes(UploadedChunkSize.load() + UploadedBlockSize.load()), - NiceBytes(TotalSize), - NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())), - .TotalCount = gsl::narrow(TotalSize), - .RemainingCount = gsl::narrow(TotalSize - UploadedSize)}, + .Details = Details, + .TotalCount = gsl::narrow(TotalRawSize), + .RemainingCount = gsl::narrow(TotalRawSize - UploadedRawSize)}, false); }); @@ -1930,7 +2169,8 @@ namespace { bool AllowMultiparts, const CbObject& MetaData, bool CreateBuild, - bool IgnoreExistingBlocks) + bool IgnoreExistingBlocks, + bool PostUploadVerify) { Stopwatch ProcessTimer; @@ -2121,15 +2361,16 @@ namespace { UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + Content.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(TotalRawSize), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - Content.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(TotalRawSize), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .Details = Details, .TotalCount = TotalRawSize, .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load()}, false); @@ -2422,19 +2663,18 @@ namespace { std::vector OutLooseChunkHashes; std::vector OutLooseChunkRawSizes; std::vector OutBlockRawHashes; - - ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), - VerifyFolderContent.Platform, - VerifyFolderContent.Paths, - VerifyFolderContent.RawHashes, - VerifyFolderContent.RawSizes, - VerifyFolderContent.Attributes, - VerifyFolderContent.ChunkedContent.SequenceRawHashes, - VerifyFolderContent.ChunkedContent.ChunkCounts, - OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - OutBlockRawHashes); + 4 ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), + VerifyFolderContent.Platform, + VerifyFolderContent.Paths, + VerifyFolderContent.RawHashes, + VerifyFolderContent.RawSizes, + VerifyFolderContent.Attributes, + VerifyFolderContent.ChunkedContent.SequenceRawHashes, + VerifyFolderContent.ChunkedContent.ChunkCounts, + OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + OutBlockRawHashes); ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes); for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) @@ -2483,7 +2723,7 @@ namespace { Stopwatch PutBuildPartResultTimer; std::pair> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); - ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are missing.", + ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), PutBuildPartResult.second.size()); @@ -2504,7 +2744,7 @@ namespace { ZEN_CONSOLE( "Generated {} ({} {}B/s) and uploaded {} ({}) blocks. " "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " - "Transferred {} ({}B/s) in {}", + "Transferred {} ({}bits/s) in {}", TempGenerateBlocksStats.GeneratedBlockCount.load(), NiceBytes(TempGenerateBlocksStats.GeneratedBlockByteCount.load()), NiceNum(GetBytesPerSecond(TempGenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, @@ -2571,7 +2811,7 @@ namespace { UploadAttachments(PutBuildPartResult.second); } - while (true) + while (!AbortFlag) { Stopwatch FinalizeBuildPartTimer; std::vector Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); @@ -2586,14 +2826,14 @@ namespace { UploadAttachments(Needs); } - if (CreateBuild) + if (CreateBuild && !AbortFlag) { Stopwatch FinalizeBuildTimer; Storage.FinalizeBuild(BuildId); ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); } - if (!NewBlocks.BlockDescriptions.empty()) + if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) { uint64_t UploadBlockMetadataCount = 0; std::vector BlockHashes; @@ -2619,13 +2859,11 @@ namespace { UploadStats.ElapsedWallTimeUS += ElapsedUS; ZEN_CONSOLE("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000)); } + } - std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockHashes); - if (VerifyBlockDescriptions.size() != BlockHashes.size()) - { - throw std::runtime_error(fmt::format("Uploaded blocks could not all be found, {} blocks are missing", - BlockHashes.size() - VerifyBlockDescriptions.size())); - } + if (PostUploadVerify && !AbortFlag) + { + ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName); } const double DeltaByteCountPercent = @@ -2786,7 +3024,7 @@ namespace { NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs())); } - void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path) + void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, bool VerifyFileHash) { ProgressBar ProgressBar(UsePlainProgress); std::atomic FilesVerified(0); @@ -2879,18 +3117,18 @@ namespace { }); FilesFailed++; } - else if (SizeOnDisk > 0) + else if (SizeOnDisk > 0 && VerifyFileHash) { const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex]; IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath); IoHash RawHash = IoHash::HashBuffer(Buffer); if (RawHash != ExpectedRawHash) { - uint64_t FileOffset = 0; - const uint32_t SequenceRawHashesIndex = Lookup.RawHashToSequenceRawHashIndex.at(ExpectedRawHash); - const uint32_t OrderOffset = Lookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashesIndex]; + uint64_t FileOffset = 0; + const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash); + const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex]; for (uint32_t OrderIndex = OrderOffset; - OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceRawHashesIndex]; + OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex]; OrderIndex++) { uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; @@ -2933,12 +3171,13 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); + std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", + FilesVerified.load(), + PathCount, + NiceBytes(ReadBytes.load()), + FilesFailed.load()); ProgressBar.UpdateState({.Task = "Verifying files ", - .Details = fmt::format("Verified {} files out of {}. Verfied: {}. Failed files: {}", - FilesVerified.load(), - PathCount, - NiceBytes(ReadBytes.load()), - FilesFailed.load()), + .Details = Details, .TotalCount = gsl::narrow(PathCount), .RemainingCount = gsl::narrow(PathCount - FilesVerified.load())}, false); @@ -3018,19 +3257,19 @@ namespace { std::unique_ptr OpenFileWriter; }; - std::vector GetRemainingChunkTargets( - const std::vector& RemotePathIndexWantsCopyFromCacheFlags, - const ChunkedContentLookup& Lookup, - uint32_t ChunkIndex) + std::vector GetRemainingChunkTargets( + std::span> SequenceIndexChunksLeftToWriteCounters, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) { - std::span ChunkSources = GetChunkLocations(Lookup, ChunkIndex); - std::vector ChunkTargetPtrs; + std::span ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); + std::vector ChunkTargetPtrs; if (!ChunkSources.empty()) { ChunkTargetPtrs.reserve(ChunkSources.size()); - for (const ChunkedContentLookup::ChunkLocation& Source : ChunkSources) + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) { - if (!RemotePathIndexWantsCopyFromCacheFlags[Source.PathIndex]) + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) { ChunkTargetPtrs.push_back(&Source); } @@ -3039,20 +3278,20 @@ namespace { return ChunkTargetPtrs; }; - bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& Content, - const std::vector& RemotePathIndexWantsCopyFromCacheFlags, - const CompositeBuffer& DecompressedBlockBuffer, - const ChunkedContentLookup& Lookup, - std::atomic* RemoteChunkIndexNeedsCopyFromSourceFlags, - uint32_t& OutChunksComplete, - uint64_t& OutBytesWritten) + bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + std::span> SequenceIndexChunksLeftToWriteCounters, + const CompositeBuffer& DecompressedBlockBuffer, + const ChunkedContentLookup& Lookup, + std::atomic* RemoteChunkIndexNeedsCopyFromSourceFlags, + uint32_t& OutChunksComplete, + uint64_t& OutBytesWritten) { std::vector ChunkBuffers; struct WriteOpData { - const ChunkedContentLookup::ChunkLocation* Target; - size_t ChunkBufferIndex; + const ChunkedContentLookup::ChunkSequenceLocation* Target; + size_t ChunkBufferIndex; }; std::vector WriteOps; @@ -3063,9 +3302,9 @@ namespace { [&](CompressedBuffer&& Chunk, const IoHash& ChunkHash) { if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) { - const uint32_t ChunkIndex = It->second; - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, Lookup, ChunkIndex); + const uint32_t ChunkIndex = It->second; + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, Lookup, ChunkIndex); if (!ChunkTargetPtrs.empty()) { @@ -3078,8 +3317,8 @@ namespace { throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); } ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); - ZEN_ASSERT(Decompressed.GetSize() == Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - for (const ChunkedContentLookup::ChunkLocation* Target : ChunkTargetPtrs) + ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { WriteOps.push_back(WriteOpData{.Target = Target, .ChunkBufferIndex = ChunkBuffers.size()}); } @@ -3095,11 +3334,11 @@ namespace { if (!AbortFlag) { std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOpData& Lhs, const WriteOpData& Rhs) { - if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) { return true; } - if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) + if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) { return false; } @@ -3114,23 +3353,50 @@ namespace { { break; } - const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; - const uint32_t PathIndex = WriteOp.Target->PathIndex; - const uint64_t ChunkSize = Chunk.GetSize(); - const uint64_t FileOffset = WriteOp.Target->Offset; - ZEN_ASSERT(FileOffset + ChunkSize <= Content.RawSizes[PathIndex]); + const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); + const uint64_t ChunkSize = Chunk.GetSize(); + const uint64_t FileOffset = WriteOp.Target->Offset; + const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; + ZEN_ASSERT(FileOffset + ChunkSize <= RemoteContent.RawSizes[PathIndex]); OpenFileCache.WriteToFile( - PathIndex, - [&CacheFolderPath, &Content](uint32_t TargetIndex) { - return (CacheFolderPath / Content.RawHashes[TargetIndex].ToHexString()).make_preferred(); + SequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); }, Chunk, FileOffset, - Content.RawSizes[PathIndex]); + RemoteContent.RawSizes[PathIndex]); OutBytesWritten += ChunkSize; } } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const WriteOpData& WriteOp : WriteOps) + { + const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = IoHash::HashBuffer( + IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format("Written hunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + } OutChunksComplete += gsl::narrow(ChunkBuffers.size()); } } @@ -3171,50 +3437,52 @@ namespace { return Decompressed; } - void WriteChunkToDisk(const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& Content, - std::span ChunkTargets, - const CompositeBuffer& ChunkData, - WriteFileCache& OpenFileCache, - uint64_t& OutBytesWritten) + void WriteChunkToDisk(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + std::span ChunkTargets, + const CompositeBuffer& ChunkData, + WriteFileCache& OpenFileCache, + uint64_t& OutBytesWritten) { - for (const ChunkedContentLookup::ChunkLocation* TargetPtr : ChunkTargets) + for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets) { - const auto& Target = *TargetPtr; - const uint64_t FileOffset = Target.Offset; + const auto& Target = *TargetPtr; + const uint64_t FileOffset = Target.Offset; + const uint32_t SequenceIndex = Target.SequenceIndex; + const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; OpenFileCache.WriteToFile( - Target.PathIndex, - [&CacheFolderPath, &Content](uint32_t TargetIndex) { - return (CacheFolderPath / Content.RawHashes[TargetIndex].ToHexString()).make_preferred(); - // return (Path / Content.Paths[TargetIndex]).make_preferred(); + SequenceIndex, + [&CacheFolderPath, &Content](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName(CacheFolderPath, Content.ChunkedContent.SequenceRawHashes[SequenceIndex]); }, ChunkData, FileOffset, - Content.RawSizes[Target.PathIndex]); + Content.RawSizes[PathIndex]); OutBytesWritten += ChunkData.GetSize(); } } - void DownloadLargeBlob(BuildStorage& Storage, - const std::filesystem::path& TempFolderPath, - const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& RemoteLookup, - const Oid& BuildId, - const IoHash& ChunkHash, - const std::uint64_t PreferredMultipartChunkSize, - const std::vector& ChunkTargetPtrs, - ParallellWork& Work, - WorkerThreadPool& WritePool, - WorkerThreadPool& NetworkPool, - std::atomic& BytesWritten, - std::atomic& WriteToDiskBytes, - std::atomic& BytesDownloaded, - std::atomic& LooseChunksBytes, - std::atomic& DownloadedChunks, - std::atomic& ChunksComplete, - std::atomic& MultipartAttachmentCount) + void DownloadLargeBlob(BuildStorage& Storage, + const std::filesystem::path& TempFolderPath, + const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + const std::vector& ChunkTargetPtrs, + std::span> SequenceIndexChunksLeftToWriteCounters, + ParallellWork& Work, + WorkerThreadPool& WritePool, + WorkerThreadPool& NetworkPool, + std::atomic& WriteToDiskBytes, + std::atomic& BytesDownloaded, + std::atomic& LooseChunksBytes, + std::atomic& DownloadedChunks, + std::atomic& ChunksComplete, + std::atomic& MultipartAttachmentCount) { struct WorkloadData { @@ -3242,11 +3510,11 @@ namespace { ChunkHash, &BytesDownloaded, &LooseChunksBytes, - &BytesWritten, &WriteToDiskBytes, &DownloadedChunks, &ChunksComplete, - ChunkTargetPtrs = std::vector( + SequenceIndexChunksLeftToWriteCounters, + ChunkTargetPtrs = std::vector( ChunkTargetPtrs)](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { BytesDownloaded += Chunk.GetSize(); LooseChunksBytes += Chunk.GetSize(); @@ -3268,8 +3536,8 @@ namespace { Offset, BytesRemaining, &ChunksComplete, - &BytesWritten, &WriteToDiskBytes, + SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs](std::atomic&) { if (!AbortFlag) { @@ -3289,7 +3557,9 @@ namespace { uint64_t TotalBytesWritten = 0; - uint32_t ChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); + auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, @@ -3304,14 +3574,38 @@ namespace { WriteChunkToDisk(CacheFolderPath, RemoteContent, + RemoteLookup, ChunkTargetPtrs, CompositeBuffer(Chunk), OpenFileCache, TotalBytesWritten); ChunksComplete++; - BytesWritten += TotalBytesWritten; WriteToDiskBytes += TotalBytesWritten; } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) + { + const uint32_t RemoteSequenceIndex = Location->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + } } }, Work.DefaultErrorFunction()); @@ -3368,45 +3662,37 @@ namespace { const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName; - tsl::robin_map RawHashToLocalPathIndex; + Stopwatch CacheMappingTimer; - { - Stopwatch CacheTimer; + uint64_t CacheMappedBytesForReuse = 0; + std::vector> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); + // std::vector RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false); + std::vector RemoteChunkIndexIsCachedFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) + std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + // Pick up all whole files we can use from current local state + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(Path / LocalContent.Paths[LocalPathIndex])); - - if (LocalContent.RawSizes[LocalPathIndex] > 0) - { - const uint32_t SequenceRawHashIndex = - LocalLookup.RawHashToSequenceRawHashIndex.at(LocalContent.RawHashes[LocalPathIndex]); - uint32_t ChunkCount = LocalContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; - if (ChunkCount > 0) - { - const IoHash LocalRawHash = LocalContent.RawHashes[LocalPathIndex]; - if (!RawHashToLocalPathIndex.contains(LocalRawHash)) - { - RawHashToLocalPathIndex.insert_or_assign(LocalRawHash, LocalPathIndex); - } - } - } + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0; + const uint32_t RemotePathIndex = GetFirstPathIndexForRawHash(RemoteLookup, RemoteSequenceRawHash); + CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; + } + else + { + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; } } - Stopwatch CacheMappingTimer; - - std::atomic BytesWritten = 0; - uint64_t CacheMappedBytesForReuse = 0; - - std::vector RemotePathIndexWantsCopyFromCacheFlags(RemoteContent.Paths.size(), false); - std::vector RemoteChunkIndexWantsCopyFromCacheFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); - // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) - std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + // Pick up all chunks in current local state struct CacheCopyData { - uint32_t LocalPathIndex; - std::vector TargetChunkLocationPtrs; + uint32_t LocalSequenceIndex; + std::vector TargetChunkLocationPtrs; struct ChunkTarget { uint32_t TargetChunkLocationCount; @@ -3418,43 +3704,30 @@ namespace { tsl::robin_map RawHashToCacheCopyDataIndex; std::vector CacheCopyDatas; - uint32_t ChunkCountToWrite = 0; - // Pick up all whole files we can use from current local state - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size(); + LocalSequenceIndex++) { - const IoHash& RemoteRawHash = RemoteContent.RawHashes[RemotePathIndex]; - if (auto It = RawHashToLocalPathIndex.find(RemoteRawHash); It != RawHashToLocalPathIndex.end()) - { - RemotePathIndexWantsCopyFromCacheFlags[RemotePathIndex] = true; - CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; - } - } - - // Pick up all chunks in current local state - for (auto& CachedLocalFile : RawHashToLocalPathIndex) - { - const IoHash& LocalFileRawHash = CachedLocalFile.first; - const uint32_t LocalPathIndex = CachedLocalFile.second; - const uint32_t LocalSequenceRawHashIndex = LocalLookup.RawHashToSequenceRawHashIndex.at(LocalFileRawHash); - const uint32_t LocalOrderOffset = LocalLookup.SequenceRawHashIndexChunkOrderOffset[LocalSequenceRawHashIndex]; + const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; + const uint32_t LocalOrderOffset = LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; { uint64_t SourceOffset = 0; - const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceRawHashIndex]; + const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) { const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) { const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + if (!RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) { @@ -3462,7 +3735,7 @@ namespace { .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), .ChunkRawSize = LocalChunkRawSize, .CacheFileOffset = SourceOffset}; - if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalFileRawHash); + if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalSequenceRawHash); CopySourceIt != RawHashToCacheCopyDataIndex.end()) { CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; @@ -3473,14 +3746,14 @@ namespace { } else { - RawHashToCacheCopyDataIndex.insert_or_assign(LocalFileRawHash, CacheCopyDatas.size()); + RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); CacheCopyDatas.push_back( - CacheCopyData{.LocalPathIndex = LocalPathIndex, + CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector{Target}}); } CacheMappedBytesForReuse += LocalChunkRawSize; - RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex] = true; + RemoteChunkIndexIsCachedFlags[RemoteChunkIndex] = true; } } } @@ -3489,16 +3762,24 @@ namespace { } } + if (CacheMappedBytesForReuse > 0) + { + ZEN_CONSOLE("Mapped {} cached data for reuse in {}", + NiceBytes(CacheMappedBytesForReuse), + NiceTimeSpanMs(CacheMappingTimer.GetElapsedTimeMs())); + } + + uint32_t ChunkCountToWrite = 0; for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) { - if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + if (RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) { ChunkCountToWrite++; } else { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (!ChunkTargetPtrs.empty()) { RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; @@ -3506,12 +3787,13 @@ namespace { } } } + std::atomic ChunkCountWritten = 0; - ZEN_CONSOLE("Mapped {} cached data for reuse in {}", - NiceBytes(CacheMappedBytesForReuse), - NiceTimeSpanMs(CacheMappingTimer.GetElapsedTimeMs())); { + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredWrittenBytesPerSecond; + WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // @@ -3520,110 +3802,6 @@ namespace { std::atomic BytesDownloaded = 0; - for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) - { - if (AbortFlag) - { - break; - } - - Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// - [&, CopyDataIndex](std::atomic&) { - if (!AbortFlag) - { - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - const std::filesystem::path LocalFilePath = - (Path / LocalContent.Paths[CopyData.LocalPathIndex]).make_preferred(); - if (!CopyData.TargetChunkLocationPtrs.empty()) - { - uint64_t CacheLocalFileBytesRead = 0; - - size_t TargetStart = 0; - const std::span AllTargets( - CopyData.TargetChunkLocationPtrs); - - struct WriteOp - { - const ChunkedContentLookup::ChunkLocation* Target; - uint64_t CacheFileOffset; - uint64_t ChunkSize; - }; - - std::vector WriteOps; - WriteOps.reserve(AllTargets.size()); - - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) - { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkLocation* Target : TargetRange) - { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkSize = ChunkTarget.ChunkRawSize}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; - } - - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->PathIndex < Rhs.Target->PathIndex) - { - return true; - } - else if (Lhs.Target->PathIndex > Rhs.Target->PathIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } - return false; - }); - - { - BufferedOpenFile SourceFile(LocalFilePath); - WriteFileCache OpenFileCache; - for (const WriteOp& Op : WriteOps) - { - if (AbortFlag) - { - break; - } - const uint32_t RemotePathIndex = Op.Target->PathIndex; - const uint64_t ChunkSize = Op.ChunkSize; - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); - - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - - OpenFileCache.WriteToFile( - RemotePathIndex, - [&CacheFolderPath, &RemoteContent](uint32_t TargetIndex) { - return (CacheFolderPath / RemoteContent.RawHashes[TargetIndex].ToHexString()) - .make_preferred(); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); - BytesWritten += ChunkSize; - WriteToDiskBytes += ChunkSize; - CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? - } - } - if (!AbortFlag) - { - ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); - ZEN_DEBUG("Copied {} from {}", - NiceBytes(CacheLocalFileBytesRead), - LocalContent.Paths[CopyData.LocalPathIndex]); - } - } - } - }, - Work.DefaultErrorFunction()); - } - for (const IoHash ChunkHash : LooseChunkHashes) { if (AbortFlag) @@ -3631,8 +3809,10 @@ namespace { break; } - uint32_t RemoteChunkIndex = RemoteLookup.ChunkHashToChunkIndex.at(ChunkHash); - if (RemoteChunkIndexWantsCopyFromCacheFlags[RemoteChunkIndex]) + auto RemoteChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; + if (RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) { ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); continue; @@ -3640,8 +3820,8 @@ namespace { bool NeedsCopy = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(RemotePathIndexWantsCopyFromCacheFlags, RemoteLookup, RemoteChunkIndex); + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); if (ChunkTargetPtrs.empty()) { @@ -3654,6 +3834,7 @@ namespace { [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { if (!AbortFlag) { + FilteredDownloadedBytesPerSecond.Start(); if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { DownloadLargeBlob(Storage, @@ -3665,10 +3846,10 @@ namespace { ChunkHash, PreferredMultipartChunkSize, ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, Work, WritePool, NetworkPool, - BytesWritten, WriteToDiskBytes, BytesDownloaded, LooseChunksBytes, @@ -3698,6 +3879,7 @@ namespace { std::atomic&) { if (!AbortFlag) { + FilteredWrittenBytesPerSecond.Start(); uint64_t TotalBytesWritten = 0; SharedBuffer Chunk = Decompress(CompressedPart, @@ -3708,14 +3890,45 @@ namespace { WriteFileCache OpenFileCache; WriteChunkToDisk(CacheFolderPath, RemoteContent, + RemoteLookup, ChunkTargetPtrs, CompositeBuffer(Chunk), OpenFileCache, TotalBytesWritten); } - ChunkCountWritten++; - BytesWritten += TotalBytesWritten; - WriteToDiskBytes += TotalBytesWritten; + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open + // (WriteFileCache) + for (const ChunkedContentLookup::ChunkSequenceLocation* Location : + ChunkTargetPtrs) + { + const uint32_t RemoteSequenceIndex = Location->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub( + 1) == 1) + { + const IoHash& SequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = + IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, + SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format( + "Written hunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + std::filesystem::rename( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + + ChunkCountWritten++; + WriteToDiskBytes += TotalBytesWritten; + } } }, Work.DefaultErrorFunction()); @@ -3728,6 +3941,139 @@ namespace { } } + for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) + { + if (AbortFlag) + { + break; + } + + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(),// + [&, CopyDataIndex](std::atomic&) { + if (!AbortFlag) + { + FilteredWrittenBytesPerSecond.Start(); + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + if (!CopyData.TargetChunkLocationPtrs.empty()) + { + uint64_t CacheLocalFileBytesRead = 0; + + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target; + uint64_t CacheFileOffset; + uint64_t ChunkSize; + }; + + std::vector WriteOps; + WriteOps.reserve(AllTargets.size()); + + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + { + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) + { + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkSize = ChunkTarget.ChunkRawSize}); + } + TargetStart += ChunkTarget.TargetChunkLocationCount; + } + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); + + if (!AbortFlag) + { + BufferedOpenFile SourceFile(LocalFilePath); + WriteFileCache OpenFileCache; + for (const WriteOp& Op : WriteOps) + { + if (AbortFlag) + { + break; + } + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = Op.ChunkSize; + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); + + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + + OpenFileCache.WriteToFile( + RemoteSequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName( + CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); + WriteToDiskBytes += ChunkSize; + CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? + } + } + if (!AbortFlag) + { + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const WriteOp& Op : WriteOps) + { + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written hunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + } + + ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); + ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + } + } + } + }, + Work.DefaultErrorFunction()); + } + size_t BlockCount = BlockDescriptions.size(); std::atomic BlocksComplete = 0; @@ -3762,6 +4108,7 @@ namespace { [&, BlockIndex](std::atomic&) { if (!AbortFlag) { + FilteredDownloadedBytesPerSecond.Start(); IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); if (!BlockBuffer) { @@ -3781,6 +4128,7 @@ namespace { [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { if (!AbortFlag) { + FilteredWrittenBytesPerSecond.Start(); IoHash BlockRawHash; uint64_t BlockRawSize; CompressedBuffer CompressedBlockBuffer = @@ -3812,14 +4160,13 @@ namespace { uint32_t ChunksReadFromBlock = 0; if (WriteBlockToDisk(CacheFolderPath, RemoteContent, - RemotePathIndexWantsCopyFromCacheFlags, + SequenceIndexChunksLeftToWriteCounters, DecompressedBlockBuffer, RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags.data(), ChunksReadFromBlock, BytesWrittenToDisk)) { - BytesWritten += BytesWrittenToDisk; WriteToDiskBytes += BytesWrittenToDisk; ChunkCountWritten += ChunksReadFromBlock; } @@ -3851,20 +4198,27 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); - WriteProgressBar.UpdateState( - {.Task = "Writing chunks ", - .Details = fmt::format("Written {} chunks out of {}. {} ouf of {} blocks complete. Downloaded: {}. Written: {}", - ChunkCountWritten.load(), - ChunkCountToWrite, - BlocksComplete.load(), - BlocksNeededCount, - NiceBytes(BytesDownloaded.load()), - NiceBytes(BytesWritten.load())), - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, - false); + FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load()); + FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load()); + std::string Details = fmt::format("{}/{} chunks. {}/{} blocks. {} {}bits/s downloaded. {} {}B/s written", + ChunkCountWritten.load(), + ChunkCountToWrite, + BlocksComplete.load(), + BlocksNeededCount, + NiceBytes(BytesDownloaded.load()), + NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), + NiceBytes(WriteToDiskBytes.load()), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + WriteProgressBar.UpdateState({.Task = "Writing chunks ", + .Details = Details, + .TotalCount = gsl::narrow(ChunkCountToWrite), + .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, + false); }); + FilteredWrittenBytesPerSecond.Stop(); + FilteredDownloadedBytesPerSecond.Stop(); + if (AbortFlag) { return; @@ -3873,6 +4227,11 @@ namespace { WriteProgressBar.Finish(); } + for (const auto& SequenceIndexChunksLeftToWriteCounter : SequenceIndexChunksLeftToWriteCounters) + { + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() == 0); + } + std::vector> Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) @@ -3884,13 +4243,13 @@ namespace { }); // Move all files we will reuse to cache folder - for (auto It : RawHashToLocalPathIndex) + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { - const IoHash& RawHash = It.first; - if (RemoteLookup.RawHashToSequenceRawHashIndex.contains(RawHash)) + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[It.second]).make_preferred(); - const std::filesystem::path CacheFilePath = (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); SetFileReadOnly(LocalFilePath, false); std::filesystem::rename(LocalFilePath, CacheFilePath); @@ -3989,7 +4348,7 @@ namespace { } else { - const std::filesystem::path CacheFilePath = (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); CreateDirectories(FirstTargetFilePath.parent_path()); if (std::filesystem::exists(FirstTargetFilePath)) @@ -4045,12 +4404,12 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - RebuildProgressBar.UpdateState( - {.Task = "Rebuilding state ", - .Details = fmt::format("Written {} files out of {}", TargetsComplete.load(), Targets.size()), - .TotalCount = gsl::narrow(Targets.size()), - .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, - false); + std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); + RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", + .Details = Details, + .TotalCount = gsl::narrow(Targets.size()), + .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, + false); }); if (AbortFlag) @@ -4487,20 +4846,19 @@ namespace { UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - - ProgressBar.UpdateState( - {.Task = "Scanning files ", - .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())), - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, - false); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + false); }, AbortFlag); if (AbortFlag) @@ -4567,15 +4925,16 @@ namespace { UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + CurrentLocalFolderContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + ByteCountToScan, + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = fmt::format("{}/{} ({}/{}, {}B/s) files, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - CurrentLocalFolderContent.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - ByteCountToScan, - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())), + .Details = Details, .TotalCount = ByteCountToScan, .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load())}, false); @@ -4599,7 +4958,8 @@ namespace { std::span BuildPartNames, const std::filesystem::path& Path, bool AllowMultiparts, - bool WipeTargetFolder) + bool WipeTargetFolder, + bool PostDownloadVerify) { Stopwatch DownloadTimer; @@ -4610,8 +4970,10 @@ namespace { std::filesystem::remove(ZenTempFolder); }); CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempChunkFolderName); - CreateDirectories(Path / ZenTempCacheFolderName); + CreateDirectories(Path / ZenTempChunkFolderName); // TODO: Don't clear this - pick up files -> chunks to use + CreateDirectories(Path / + ZenTempCacheFolderName); // TODO: Don't clear this - pick up files and use as sequences (non .tmp extension) and + // delete .tmp (maybe?) - chunk them? How do we know the file is worth chunking? std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -4724,7 +5086,7 @@ namespace { if (!AbortFlag) { - VerifyFolder(RemoteContent, Path); + VerifyFolder(RemoteContent, Path, PostDownloadVerify); Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); @@ -5078,6 +5440,8 @@ BuildsCommand::BuildsCommand() "Path to a text file with one line of [TAB] per file to include.", cxxopts::value(m_ManifestPath), ""); + m_UploadOptions + .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), ""); m_UploadOptions.parse_positional({"local-path", "build-id"}); m_UploadOptions.positional_help("local-path build-id"); @@ -5111,6 +5475,8 @@ BuildsCommand::BuildsCommand() "Allow large attachments to be transfered using multipart protocol. Defaults to true.", cxxopts::value(m_AllowMultiparts), ""); + m_DownloadOptions + .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), ""); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); m_DownloadOptions.positional_help("local-path build-id build-part-name"); @@ -5503,7 +5869,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, MetaData, m_CreateBuild, - m_Clean); + m_Clean, + m_PostUploadVerify); if (false) { @@ -5593,7 +5960,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } - DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean); + DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean, m_PostDownloadVerify); if (false) { @@ -5727,8 +6094,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, - m_CreateBuild, - m_Clean); + true, + false, + true); if (AbortFlag) { ZEN_CONSOLE("Upload failed."); @@ -5737,7 +6105,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true, true); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -5749,7 +6117,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -5851,7 +6219,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -5880,7 +6248,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, MetaData2, true, - false); + false, + true); if (AbortFlag) { ZEN_CONSOLE("Upload of scrambled failed."); @@ -5888,7 +6257,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -5896,7 +6265,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -5904,7 +6273,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false); + DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -6032,64 +6401,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } - Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); - CbObject Build = Storage->GetBuild(BuildId); - if (!m_BuildPartName.empty()) - { - BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId(); - if (BuildPartId == Oid::Zero) - { - throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName)); - } - } - CbObject BuildPart = Storage->GetBuildPart(BuildId, BuildPartId); - ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); - std::vector ChunkAttachments; - for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) - { - ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); - } - std::vector BlockAttachments; - for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) - { - BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); - } + Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); - for (const IoHash& ChunkAttachment : ChunkAttachments) - { - uint64_t CompressedSize; - uint64_t DecompressedSize; - try - { - ValidateBlob(*Storage, BuildId, ChunkAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE("Chunk attachment {} ({} -> {}) is valid", - ChunkAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); - } - catch (const std::exception& Ex) - { - ZEN_CONSOLE("Failed validating chunk attachment {}: {}", ChunkAttachment, Ex.what()); - } - } - - for (const IoHash& BlockAttachment : BlockAttachments) - { - uint64_t CompressedSize; - uint64_t DecompressedSize; - try - { - ValidateChunkBlock(*Storage, BuildId, BlockAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE("Block attachment {} ({} -> {}) is valid", - BlockAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); - } - catch (const std::exception& Ex) - { - ZEN_CONSOLE("Failed validating block attachment {}: {}", BlockAttachment, Ex.what()); - } - } + ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName); return AbortFlag ? 13 : 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index fa223943b..c54fb4db1 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -78,10 +78,12 @@ private: std::filesystem::path m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; + bool m_PostUploadVerify = false; cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; std::vector m_BuildPartNames; std::vector m_BuildPartIds; + bool m_PostDownloadVerify = false; cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; std::filesystem::path m_DiffPath; diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index bb15a6fce..7f7e70fef 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -414,6 +414,11 @@ ValidatePayload(cpr::Response& Response, std::unique_ptrsecond == "application/x-ue-comp") diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 6dc2a20d8..1552ea823 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -599,10 +599,10 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span(Result.ChunkedContent.SequenceRawHashes.size())}); - const uint32_t SequenceRawHashIndex = OverlayLookup.RawHashToSequenceRawHashIndex.at(RawHash); - const uint32_t OrderIndexOffset = OverlayLookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; - const uint32_t ChunkCount = OverlayContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; - ChunkingStatistics Stats; + const uint32_t SequenceRawHashIndex = OverlayLookup.RawHashToSequenceIndex.at(RawHash); + const uint32_t OrderIndexOffset = OverlayLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = OverlayContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + ChunkingStatistics Stats; std::span OriginalChunkOrder = std::span(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); AddCunkSequence(Stats, @@ -667,9 +667,9 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span { RawHashToSequenceRawHashIndex.insert( {RawHash, gsl::narrow(Result.ChunkedContent.SequenceRawHashes.size())}); - const uint32_t SequenceRawHashIndex = BaseLookup.RawHashToSequenceRawHashIndex.at(RawHash); - const uint32_t OrderIndexOffset = BaseLookup.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; - const uint32_t ChunkCount = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + const uint32_t SequenceRawHashIndex = BaseLookup.RawHashToSequenceIndex.at(RawHash); + const uint32_t OrderIndexOffset = BaseLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; ChunkingStatistics Stats; std::span OriginalChunkOrder = std::span(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); @@ -777,46 +777,40 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) { struct ChunkLocationReference { - uint32_t ChunkIndex; - ChunkedContentLookup::ChunkLocation Location; + uint32_t ChunkIndex; + ChunkedContentLookup::ChunkSequenceLocation Location; }; ChunkedContentLookup Result; { const uint32_t SequenceRawHashesCount = gsl::narrow(Content.ChunkedContent.SequenceRawHashes.size()); - Result.RawHashToSequenceRawHashIndex.reserve(SequenceRawHashesCount); - Result.SequenceRawHashIndexChunkOrderOffset.reserve(SequenceRawHashesCount); + Result.RawHashToSequenceIndex.reserve(SequenceRawHashesCount); + Result.SequenceIndexChunkOrderOffset.reserve(SequenceRawHashesCount); uint32_t OrderOffset = 0; for (uint32_t SequenceRawHashIndex = 0; SequenceRawHashIndex < Content.ChunkedContent.SequenceRawHashes.size(); SequenceRawHashIndex++) { - Result.RawHashToSequenceRawHashIndex.insert( - {Content.ChunkedContent.SequenceRawHashes[SequenceRawHashIndex], SequenceRawHashIndex}); - Result.SequenceRawHashIndexChunkOrderOffset.push_back(OrderOffset); + Result.RawHashToSequenceIndex.insert({Content.ChunkedContent.SequenceRawHashes[SequenceRawHashIndex], SequenceRawHashIndex}); + Result.SequenceIndexChunkOrderOffset.push_back(OrderOffset); OrderOffset += Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; } } std::vector Locations; Locations.reserve(Content.ChunkedContent.ChunkOrders.size()); - for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++) + for (uint32_t SequenceIndex = 0; SequenceIndex < Content.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++) { - if (Content.RawSizes[PathIndex] > 0) + const uint32_t OrderOffset = Result.SequenceIndexChunkOrderOffset[SequenceIndex]; + const uint32_t ChunkCount = Content.ChunkedContent.ChunkCounts[SequenceIndex]; + uint64_t LocationOffset = 0; + for (size_t OrderIndex = OrderOffset; OrderIndex < OrderOffset + ChunkCount; OrderIndex++) { - const IoHash& RawHash = Content.RawHashes[PathIndex]; - uint32_t SequenceRawHashIndex = Result.RawHashToSequenceRawHashIndex.at(RawHash); - const uint32_t OrderOffset = Result.SequenceRawHashIndexChunkOrderOffset[SequenceRawHashIndex]; - const uint32_t ChunkCount = Content.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; - uint64_t LocationOffset = 0; - for (size_t OrderIndex = OrderOffset; OrderIndex < OrderOffset + ChunkCount; OrderIndex++) - { - uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; + uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - Locations.push_back(ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkLocation{PathIndex, LocationOffset}}); + Locations.push_back( + ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkSequenceLocation{SequenceIndex, LocationOffset}}); - LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - } - ZEN_ASSERT(LocationOffset == Content.RawSizes[PathIndex]); + LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; } } @@ -829,18 +823,18 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) { return false; } - if (Lhs.Location.PathIndex < Rhs.Location.PathIndex) + if (Lhs.Location.SequenceIndex < Rhs.Location.SequenceIndex) { return true; } - if (Lhs.Location.PathIndex > Rhs.Location.PathIndex) + if (Lhs.Location.SequenceIndex > Rhs.Location.SequenceIndex) { return false; } return Lhs.Location.Offset < Rhs.Location.Offset; }); - Result.ChunkLocations.reserve(Locations.size()); + Result.ChunkSequenceLocations.reserve(Locations.size()); const uint32_t ChunkCount = gsl::narrow(Content.ChunkedContent.ChunkHashes.size()); Result.ChunkHashToChunkIndex.reserve(ChunkCount); size_t RangeOffset = 0; @@ -850,14 +844,30 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) uint32_t Count = 0; while (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex) { - Result.ChunkLocations.push_back(Locations[RangeOffset + Count].Location); + Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location); Count++; } - Result.ChunkLocationOffset.push_back(RangeOffset); - Result.ChunkLocationCounts.push_back(Count); + Result.ChunkSequenceLocationOffset.push_back(RangeOffset); + Result.ChunkSequenceLocationCounts.push_back(Count); RangeOffset += Count; } + Result.SequenceIndexFirstPathIndex.resize(Content.ChunkedContent.SequenceRawHashes.size(), (uint32_t)-1); + for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++) + { + if (Content.RawSizes[PathIndex] > 0) + { + const IoHash& RawHash = Content.RawHashes[PathIndex]; + auto SequenceIndexIt = Result.RawHashToSequenceIndex.find(RawHash); + ZEN_ASSERT(SequenceIndexIt != Result.RawHashToSequenceIndex.end()); + const uint32_t SequenceIndex = SequenceIndexIt->second; + if (Result.SequenceIndexFirstPathIndex[SequenceIndex] == (uint32_t)-1) + { + Result.SequenceIndexFirstPathIndex[SequenceIndex] = PathIndex; + } + } + } + return Result; } diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 15c687462..309341550 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -122,32 +122,59 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, struct ChunkedContentLookup { - struct ChunkLocation + struct ChunkSequenceLocation { - uint32_t PathIndex; + uint32_t SequenceIndex; uint64_t Offset; }; tsl::robin_map ChunkHashToChunkIndex; - tsl::robin_map RawHashToSequenceRawHashIndex; - std::vector SequenceRawHashIndexChunkOrderOffset; - std::vector ChunkLocations; - std::vector ChunkLocationOffset; // ChunkLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex - std::vector ChunkLocationCounts; // ChunkLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + tsl::robin_map RawHashToSequenceIndex; + std::vector SequenceIndexChunkOrderOffset; + std::vector ChunkSequenceLocations; + std::vector + ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash }; ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); inline std::pair -GetChunkLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) { - return std::make_pair(Lookup.ChunkLocationOffset[ChunkIndex], Lookup.ChunkLocationCounts[ChunkIndex]); + return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]); } -inline std::span -GetChunkLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +inline std::span +GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) { - std::pair Range = GetChunkLocationRange(Lookup, ChunkIndex); - return std::span(Lookup.ChunkLocations).subspan(Range.first, Range.second); + std::pair Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex); + return std::span(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second); +} + +inline uint32_t +GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + return Lookup.RawHashToSequenceIndex.at(RawHash); +} + +inline uint32_t +GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + return Lookup.RawHashToSequenceIndex.at(RawHash); +} + +inline uint32_t +GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex) +{ + return Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; +} + +inline uint32_t +GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash); + return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex); } namespace compactbinary_helpers { -- cgit v1.2.3 From 2232eb28256ec54beaf3dbe06f5176698c7245a0 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 4 Mar 2025 09:38:13 +0100 Subject: limit and validate responses before logging the text (#292) Improvement: When logging HTTP responses, the body is now sanity checked to ensure it is human readable, and the length of the output is capped to prevent inadvertent log bloat --- src/zencore/include/zencore/string.h | 3 ++ src/zencore/string.cpp | 14 ++++++++++ src/zenhttp/httpclient.cpp | 46 ++++++++++++++++++++++++++++++ src/zenhttp/include/zenhttp/formatters.h | 48 +++++++++++++++++++++++++++++++- 4 files changed, 110 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index e2ef1c1a0..68129b691 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -522,6 +522,9 @@ public: ////////////////////////////////////////////////////////////////////////// +bool IsValidUtf8(const std::string_view& str); +std::string_view::const_iterator FindFirstInvalidUtf8Byte(const std::string_view& str); + void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 242d41abe..a0d8c927f 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -99,6 +99,20 @@ FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch ////////////////////////////////////////////////////////////////////////// +bool +IsValidUtf8(const std::string_view& str) +{ + return utf8::is_valid(begin(str), end(str)); +} + +std::string_view::const_iterator +FindFirstInvalidUtf8Byte(const std::string_view& str) +{ + return utf8::find_invalid(begin(str), end(str)); +} + +////////////////////////////////////////////////////////////////////////// + void Utf8ToWide(const char8_t* Str8, WideStringBuilderBase& OutString) { diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 7f7e70fef..e4c6d243d 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -1422,6 +1422,52 @@ HttpClient::Response::ThrowError(std::string_view ErrorPrefix) #if ZEN_WITH_TESTS +TEST_CASE("responseformat") +{ + using namespace std::literals; + + SUBCASE("identity") + { + BodyLogFormatter _{"abcd"}; + CHECK_EQ(_.GetText(), "abcd"sv); + } + + SUBCASE("very long") + { + std::string_view LongView = + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz" + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; + + BodyLogFormatter _{LongView}; + + CHECK(_.GetText().size() < LongView.size()); + CHECK(_.GetText().starts_with("[truncated"sv)); + } + + SUBCASE("invalid text") + { + std::string_view BadText = "totobaba\xff\xfe"; + + BodyLogFormatter _{BadText}; + + CHECK_EQ(_.GetText(), "totobaba"); + } +} + TEST_CASE("httpclient") { using namespace std::literals; diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index 538136238..0fa5dc6da 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -13,6 +13,50 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +namespace zen { + +struct BodyLogFormatter +{ +private: + std::string_view ResponseText; + zen::ExtendableStringBuilder<128> ModifiedResponse; + +public: + explicit BodyLogFormatter(std::string_view InResponseText) : ResponseText(InResponseText) + { + using namespace std::literals; + + const int TextSizeLimit = 1024; + + // Trim invalid UTF8 + + auto InvalidIt = zen::FindFirstInvalidUtf8Byte(ResponseText); + + if (InvalidIt != end(ResponseText)) + { + ResponseText = ResponseText.substr(0, InvalidIt - begin(ResponseText)); + } + + if (ResponseText.empty()) + { + ResponseText = ""sv; + } + + if (ResponseText.size() > TextSizeLimit) + { + const auto TruncatedString = "[truncated response] "sv; + ModifiedResponse.Append(TruncatedString); + ModifiedResponse.Append(ResponseText.data(), TextSizeLimit - TruncatedString.size()); + + ResponseText = ModifiedResponse; + } + } + + inline std::string_view GetText() const { return ResponseText; } +}; + +} // namespace zen + template<> struct fmt::formatter { @@ -57,6 +101,8 @@ struct fmt::formatter } else { + zen::BodyLogFormatter Body(Response.text); + return fmt::format_to(Ctx.out(), "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s, Reponse: '{}', Reason: '{}'", Response.url.str(), @@ -64,7 +110,7 @@ struct fmt::formatter Response.uploaded_bytes, Response.downloaded_bytes, Response.elapsed, - Response.text, + Body.GetText(), Response.reason); } } -- cgit v1.2.3 From 7cb1de5a966aa23e0c098d2bbf9009d0f82a6477 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 4 Mar 2025 10:22:53 +0100 Subject: stream decompress (#293) * clean up latency parameters and slow down rate updates * add DecompressToStream --- src/zen/cmds/builds_cmd.cpp | 198 +++++++++++++++++++++++---------- src/zencore/compress.cpp | 186 +++++++++++++++++++++++++++++++ src/zencore/include/zencore/compress.h | 10 ++ 3 files changed, 337 insertions(+), 57 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 219d01240..a9852151f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -78,15 +78,19 @@ namespace { const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; - const std::string ZenFolderName = ".zen"; - const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); - const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); - const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); - const std::string ZenTempCacheFolderName = fmt::format("{}/cache", ZenTempFolderName); - const std::string ZenTempStorageFolderName = fmt::format("{}/storage", ZenTempFolderName); - const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); - const std::string ZenTempChunkFolderName = fmt::format("{}/chunks", ZenTempFolderName); - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; + + const double DefaultLatency = 0; // .0010; + const double DefaultDelayPerKBSec = 0; // 0.00005; + + const std::string ZenFolderName = ".zen"; + const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); + const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); + const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + const std::string ZenTempCacheFolderName = fmt::format("{}/cache", ZenTempFolderName); + const std::string ZenTempStorageFolderName = fmt::format("{}/storage", ZenTempFolderName); + const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); + const std::string ZenTempChunkFolderName = fmt::format("{}/chunks", ZenTempFolderName); + const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; const std::string UnsyncFolderName = ".unsync"; @@ -271,7 +275,7 @@ namespace { } uint64_t TimeUS = Timer.GetElapsedTimeUs(); uint64_t TimeDeltaUS = TimeUS - LastTimeUS; - if (TimeDeltaUS >= 1000000) + if (TimeDeltaUS >= 2000000) { uint64_t Delta = Count - LastCount; uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS; @@ -3464,6 +3468,61 @@ namespace { } } + bool CanStreamDecompress(const ChunkedFolderContent& RemoteContent, + const std::vector Locations) + { + if (Locations.size() == 1) + { + const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex; + if (Locations[0]->Offset == 0 && RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1) + { + return true; + } + } + return false; + } + + void StreamDecompress(const std::filesystem::path& CacheFolderPath, + const IoHash& SequenceRawHash, + CompositeBuffer&& CompressedPart, + std::atomic& WriteToDiskBytes) + { + const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); + TemporaryFile DecompressedTemp; + std::error_code Ec; + DecompressedTemp.CreateTemporary(CacheFolderPath, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed creating temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); + } + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedPart, RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", SequenceRawHash)); + } + if (RawHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); + } + bool CouldDecompress = Compressed.DecompressToStream(0, (uint64_t)-1, [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { + DecompressedTemp.Write(RangeBuffer, Offset); + WriteToDiskBytes += RangeBuffer.GetSize(); + }); + if (!CouldDecompress) + { + throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash)); + } + DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); + } + } + void DownloadLargeBlob(BuildStorage& Storage, const std::filesystem::path& TempFolderPath, const std::filesystem::path& CacheFolderPath, @@ -3527,7 +3586,7 @@ namespace { DownloadedChunks++; Work.ScheduleWork( - WritePool, + WritePool, // GetSyncWorkerPool(),// [&CacheFolderPath, &RemoteContent, &RemoteLookup, @@ -3555,33 +3614,44 @@ namespace { } CompressedPart.SetDeleteOnClose(true); - uint64_t TotalBytesWritten = 0; - auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); - uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; - - SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), - ChunkHash, - RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) + { + const IoHash& SequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; + StreamDecompress(CacheFolderPath, + SequenceRawHash, + CompositeBuffer(std::move(CompressedPart)), + WriteToDiskBytes); + } + else + { + const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; + SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), + ChunkHash, + RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); - // ZEN_ASSERT_SLOW(ChunkHash == - // IoHash::HashBuffer(Chunk.AsIoBuffer())); + // ZEN_ASSERT_SLOW(ChunkHash == + // IoHash::HashBuffer(Chunk.AsIoBuffer())); - if (!AbortFlag) - { - WriteFileCache OpenFileCache; - - WriteChunkToDisk(CacheFolderPath, - RemoteContent, - RemoteLookup, - ChunkTargetPtrs, - CompositeBuffer(Chunk), - OpenFileCache, - TotalBytesWritten); - ChunksComplete++; - WriteToDiskBytes += TotalBytesWritten; + if (!AbortFlag) + { + WriteFileCache OpenFileCache; + + uint64_t TotalBytesWritten = 0; + WriteChunkToDisk(CacheFolderPath, + RemoteContent, + RemoteLookup, + ChunkTargetPtrs, + CompositeBuffer(Chunk), + OpenFileCache, + TotalBytesWritten); + ChunksComplete++; + WriteToDiskBytes += TotalBytesWritten; + } } + if (!AbortFlag) { // Write tracking, updating this must be done without any files open (WriteFileCache) @@ -3619,7 +3689,7 @@ namespace { for (auto& WorkItem : WorkItems) { Work.ScheduleWork( - NetworkPool, + NetworkPool, // GetSyncWorkerPool(),// [WorkItem = std::move(WorkItem)](std::atomic&) { if (!AbortFlag) { @@ -3830,7 +3900,7 @@ namespace { else { Work.ScheduleWork( - NetworkPool, + NetworkPool, // GetSyncWorkerPool(),// [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { if (!AbortFlag) { @@ -3880,21 +3950,38 @@ namespace { if (!AbortFlag) { FilteredWrittenBytesPerSecond.Start(); - uint64_t TotalBytesWritten = 0; - SharedBuffer Chunk = - Decompress(CompressedPart, - ChunkHash, - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); + if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) { - WriteFileCache OpenFileCache; - WriteChunkToDisk(CacheFolderPath, - RemoteContent, - RemoteLookup, - ChunkTargetPtrs, - CompositeBuffer(Chunk), - OpenFileCache, - TotalBytesWritten); + const IoHash& SequenceRawHash = + RemoteContent.ChunkedContent + .SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; + StreamDecompress(CacheFolderPath, + SequenceRawHash, + CompositeBuffer(CompressedPart), + WriteToDiskBytes); + ChunkCountWritten++; + } + else + { + uint64_t TotalBytesWritten = 0; + SharedBuffer Chunk = + Decompress(CompressedPart, + ChunkHash, + RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); + + { + WriteFileCache OpenFileCache; + WriteChunkToDisk(CacheFolderPath, + RemoteContent, + RemoteLookup, + ChunkTargetPtrs, + CompositeBuffer(Chunk), + OpenFileCache, + TotalBytesWritten); + ChunkCountWritten++; + WriteToDiskBytes += TotalBytesWritten; + } } if (!AbortFlag) { @@ -3925,9 +4012,6 @@ namespace { GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } } - - ChunkCountWritten++; - WriteToDiskBytes += TotalBytesWritten; } } }, @@ -5707,7 +5791,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { ZEN_CONSOLE("Querying builds in folder '{}'.", m_StoragePath); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); } else { @@ -5819,7 +5903,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_StoragePath, GeneratedBuildId ? "Generated " : "", BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else @@ -5952,7 +6036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, m_Path, m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else @@ -6056,7 +6140,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Path, m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else @@ -6326,7 +6410,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else @@ -6394,7 +6478,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false); // , .0015, 0.00004 + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 0e2ce2b54..f13f8b9ca 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -185,6 +185,12 @@ public: const MemoryView HeaderView, uint64_t RawOffset, uint64_t RawSize) const = 0; + + virtual bool DecompressToStream(const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const = 0; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -263,6 +269,22 @@ public: } [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader&) const final { return sizeof(BufferHeader); } + + virtual bool DecompressToStream(const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const final + { + if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() && + Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize && + (RawOffset + RawSize) <= Header.TotalRawSize) + { + Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)); + return true; + } + return false; + } }; ////////////////////////////////////////////////////////////////////////// @@ -447,6 +469,12 @@ public: MutableMemoryView RawView, uint64_t RawOffset) const final; + virtual bool DecompressToStream(const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const final; + protected: virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; }; @@ -568,6 +596,143 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB return CompositeBuffer(std::move(Segments)); } +bool +BlockDecoder::DecompressToStream(const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const +{ + if (Header.TotalCompressedSize != CompressedData.GetSize()) + { + return false; + } + + const uint64_t BlockSize = uint64_t(1) << Header.BlockSizeExponent; + + UniqueBuffer BlockSizeBuffer; + MemoryView BlockSizeView = CompressedData.ViewOrCopyRange(sizeof(BufferHeader), Header.BlockCount * sizeof(uint32_t), BlockSizeBuffer); + std::span CompressedBlockSizes(reinterpret_cast(BlockSizeView.GetData()), Header.BlockCount); + + UniqueBuffer CompressedBlockCopy; + + const size_t FirstBlockIndex = uint64_t(RawOffset / BlockSize); + const size_t LastBlockIndex = uint64_t((RawOffset + RawSize - 1) / BlockSize); + const uint64_t LastBlockSize = BlockSize - ((Header.BlockCount * BlockSize) - Header.TotalRawSize); + uint64_t OffsetInFirstBlock = RawOffset % BlockSize; + uint64_t CompressedOffset = sizeof(BufferHeader) + uint64_t(Header.BlockCount) * sizeof(uint32_t); + uint64_t RemainingRawSize = RawSize; + + for (size_t BlockIndex = 0; BlockIndex < FirstBlockIndex; BlockIndex++) + { + const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); + CompressedOffset += CompressedBlockSize; + } + + UniqueBuffer RawDataBuffer; + + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + { + ZEN_ASSERT(FileRef.FileHandle != nullptr); + BasicFile Source; + Source.Attach(FileRef.FileHandle); + + for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) + { + const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; + const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); + const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; + + const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawSize, UncompressedBlockSize - OffsetInFirstBlock) + : zen::Min(RemainingRawSize, BlockSize); + + if (CompressedBlockCopy.GetSize() < CompressedBlockSize) + { + CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize); + } + Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset); + + MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize); + + if (IsCompressed) + { + if (RawDataBuffer.IsNull()) + { + RawDataBuffer = UniqueBuffer::Alloc(zen::Min(RawSize, UncompressedBlockSize)); + } + else + { + ZEN_ASSERT(RawDataBuffer.GetSize() >= UncompressedBlockSize); + } + MutableMemoryView UncompressedBlock = RawDataBuffer.GetMutableView().Left(UncompressedBlockSize); + if (!DecompressBlock(UncompressedBlock, CompressedBlock)) + { + Source.Detach(); + return false; + } + Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))); + } + else + { + Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer( + IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))); + } + + OffsetInFirstBlock = 0; + RemainingRawSize -= BytesToUncompress; + CompressedOffset += CompressedBlockSize; + } + Source.Detach(); + } + else + { + for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++) + { + const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize; + const uint32_t CompressedBlockSize = ByteSwap(CompressedBlockSizes[BlockIndex]); + const bool IsCompressed = CompressedBlockSize < UncompressedBlockSize; + + const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawSize, UncompressedBlockSize - OffsetInFirstBlock) + : zen::Min(RemainingRawSize, BlockSize); + + MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy); + + if (IsCompressed) + { + if (RawDataBuffer.IsNull()) + { + RawDataBuffer = UniqueBuffer::Alloc(zen::Min(RawSize, UncompressedBlockSize)); + } + else + { + ZEN_ASSERT(RawDataBuffer.GetSize() >= UncompressedBlockSize); + } + MutableMemoryView UncompressedBlock = RawDataBuffer.GetMutableView().Left(UncompressedBlockSize); + if (!DecompressBlock(UncompressedBlock, CompressedBlock)) + { + return false; + } + Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))); + } + else + { + Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer( + IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))); + } + + OffsetInFirstBlock = 0; + RemainingRawSize -= BytesToUncompress; + CompressedOffset += CompressedBlockSize; + } + } + return true; +} + bool BlockDecoder::TryDecompressTo(const BufferHeader& Header, const CompositeBuffer& CompressedData, @@ -1643,6 +1808,27 @@ CompressedBuffer::DecompressToComposite() const return CompositeBuffer(); } +bool +CompressedBuffer::DecompressToStream(uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const +{ + using namespace detail; + if (CompressedData) + { + const BufferHeader Header = BufferHeader::Read(CompressedData); + if (Header.Magic == BufferHeader::ExpectedMagic) + { + if (const BaseDecoder* const Decoder = GetDecoder(Header.Method)) + { + const uint64_t TotalRawSize = RawSize < ~uint64_t(0) ? RawSize : Header.TotalRawSize - RawOffset; + return Decoder->DecompressToStream(Header, CompressedData, RawOffset, TotalRawSize, std::move(Callback)); + } + } + } + return false; +} + bool CompressedBuffer::TryGetCompressParameters(OodleCompressor& OutCompressor, OodleCompressionLevel& OutCompressionLevel, diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 5e761ceef..c056d7561 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -196,6 +196,16 @@ public: */ [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const; + /** + * Decompress into and call callback for ranges of decompressed data. + * The buffer in the callback will be overwritten when the callback returns. + * + * @return True if the buffer is valid and can be decompressed. + */ + [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const; + /** A null compressed buffer. */ static const CompressedBuffer Null; -- cgit v1.2.3 From 5867d04117645cfd8e558ad6d7888e929ca6e816 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 4 Mar 2025 12:43:05 +0100 Subject: do direct update of stats numbers (#294) --- src/zen/cmds/builds_cmd.cpp | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index a9852151f..132f5db86 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3288,8 +3288,8 @@ namespace { const CompositeBuffer& DecompressedBlockBuffer, const ChunkedContentLookup& Lookup, std::atomic* RemoteChunkIndexNeedsCopyFromSourceFlags, - uint32_t& OutChunksComplete, - uint64_t& OutBytesWritten) + std::atomic& OutChunksComplete, + std::atomic& OutBytesWritten) { std::vector ChunkBuffers; struct WriteOpData @@ -3447,7 +3447,7 @@ namespace { std::span ChunkTargets, const CompositeBuffer& ChunkData, WriteFileCache& OpenFileCache, - uint64_t& OutBytesWritten) + std::atomic& OutBytesWritten) { for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets) { @@ -3639,16 +3639,14 @@ namespace { { WriteFileCache OpenFileCache; - uint64_t TotalBytesWritten = 0; WriteChunkToDisk(CacheFolderPath, RemoteContent, RemoteLookup, ChunkTargetPtrs, CompositeBuffer(Chunk), OpenFileCache, - TotalBytesWritten); + WriteToDiskBytes); ChunksComplete++; - WriteToDiskBytes += TotalBytesWritten; } } @@ -3964,7 +3962,6 @@ namespace { } else { - uint64_t TotalBytesWritten = 0; SharedBuffer Chunk = Decompress(CompressedPart, ChunkHash, @@ -3978,9 +3975,8 @@ namespace { ChunkTargetPtrs, CompositeBuffer(Chunk), OpenFileCache, - TotalBytesWritten); + WriteToDiskBytes); ChunkCountWritten++; - WriteToDiskBytes += TotalBytesWritten; } } if (!AbortFlag) @@ -4240,21 +4236,14 @@ namespace { ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == IoHash::HashBuffer(DecompressedBlockBuffer)); - uint64_t BytesWrittenToDisk = 0; - uint32_t ChunksReadFromBlock = 0; - if (WriteBlockToDisk(CacheFolderPath, - RemoteContent, - SequenceIndexChunksLeftToWriteCounters, - DecompressedBlockBuffer, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags.data(), - ChunksReadFromBlock, - BytesWrittenToDisk)) - { - WriteToDiskBytes += BytesWrittenToDisk; - ChunkCountWritten += ChunksReadFromBlock; - } - else + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + SequenceIndexChunksLeftToWriteCounters, + DecompressedBlockBuffer, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags.data(), + ChunkCountWritten, + WriteToDiskBytes)) { throw std::runtime_error( fmt::format("Block {} is malformed", BlockDescriptions[BlockIndex].BlockHash)); -- cgit v1.2.3 From 4b981aed0281ac995c326e3d03dc482353f66405 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Mar 2025 11:27:54 +0100 Subject: streaming compress (#295) - Improvement: Validate hash of decompressed data inline with streaming decompression - Improvement: Do streaming compression of large blobs to improve memory and I/O performance --- src/zen/cmds/builds_cmd.cpp | 150 ++++++++++++++++++++++------- src/zencore/compress.cpp | 171 ++++++++++++++++++++++++++++++++- src/zencore/include/zencore/compress.h | 5 + 3 files changed, 289 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 132f5db86..fb9da021d 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1088,13 +1088,21 @@ namespace { throw std::runtime_error( fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash)); } - SharedBuffer Decompressed = Compressed.Decompress(); - if (!Decompressed) + + IoHashStream Hash; + bool CouldDecompress = Compressed.DecompressToStream(0, RawSize, [&Hash](uint64_t, const CompositeBuffer& RangeBuffer) { + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + }); + + if (!CouldDecompress) { throw std::runtime_error( fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize())); } - IoHash ValidateRawHash = IoHash::HashBuffer(Decompressed); + IoHash ValidateRawHash = Hash.GetHash(); if (ValidateRawHash != BlobHash) { throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information", @@ -1102,14 +1110,26 @@ namespace { Payload.GetSize(), ValidateRawHash)); } - CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite(); - if (!DecompressedComposite) + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize; + if (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize)) { - throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize())); + throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to get compression details", BlobHash, Payload.GetSize())); } OutCompressedSize = Payload.GetSize(); OutDecompressedSize = RawSize; - return DecompressedComposite; + if (CompressionLevel == OodleCompressionLevel::None) + { + // Only decompress to composite if we need it for block verification + CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite(); + if (!DecompressedComposite) + { + throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize())); + } + return DecompressedComposite; + } + return CompositeBuffer{}; } CompositeBuffer ValidateBlob(BuildStorage& Storage, @@ -1132,6 +1152,10 @@ namespace { uint64_t& OutDecompressedSize) { CompositeBuffer BlockBuffer = ValidateBlob(std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Chunk block blob {} is not compressed using 'None' compression level", BlobHash)); + } return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash); } @@ -1461,6 +1485,7 @@ namespace { uint32_t ChunkIndex, const std::filesystem::path& TempFolderPath) { + ZEN_ASSERT(!TempFolderPath.empty()); const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; @@ -1476,21 +1501,53 @@ namespace { throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash)); } ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); + { + std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred(); + BasicFile CompressedFile; + std::error_code Ec; + CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed creating temporary file for compressing blob {}. Reason: {}", ChunkHash, Ec.message())); + } + + bool CouldCompress = CompressedBuffer::CompressToStream( + CompositeBuffer(SharedBuffer(RawSource)), + [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { CompressedFile.Write(RangeBuffer, Offset); }); + if (CouldCompress) + { + uint64_t CompressedSize = CompressedFile.FileSize(); + void* FileHandle = CompressedFile.Detach(); + IoBuffer TempPayload = IoBuffer(IoBuffer::File, + FileHandle, + 0, + CompressedSize, + /*IsWholeFile*/ true); + ZEN_ASSERT(TempPayload); + TempPayload.SetDeleteOnClose(true); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(TempPayload), RawHash, RawSize); + ZEN_ASSERT(Compressed); + ZEN_ASSERT(RawHash == ChunkHash); + ZEN_ASSERT(RawSize == ChunkSize); + return Compressed.GetCompressed(); + } + CompressedFile.Close(); + std::filesystem::remove(TempFilePath, Ec); + ZEN_UNUSED(Ec); + } + + // Try regular compress - decompress may fail if compressed data is larger than non-compressed CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(RawSource))); if (!CompressedBlob) { - throw std::runtime_error(fmt::format("Failed decompressing chunk {}", ChunkHash)); - } - if (TempFolderPath.empty()) - { - return CompressedBlob.GetCompressed().MakeOwned(); - } - else - { - CompositeBuffer TempPayload = WriteToTempFileIfNeeded(CompressedBlob.GetCompressed(), TempFolderPath, ChunkHash); - return CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)).GetCompressed(); + throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash)); } + CompositeBuffer TempPayload = WriteToTempFileIfNeeded(CompressedBlob.GetCompressed(), TempFolderPath, ChunkHash); + return CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)).GetCompressed(); } struct GeneratedBlocks @@ -1934,7 +1991,7 @@ namespace { { const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; Work.ScheduleWork( - ReadChunkPool, + ReadChunkPool, // GetSyncWorkerPool(),// ReadChunkPool, [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { @@ -3490,7 +3547,7 @@ namespace { const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); TemporaryFile DecompressedTemp; std::error_code Ec; - DecompressedTemp.CreateTemporary(CacheFolderPath, Ec); + DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec); if (Ec) { throw std::runtime_error( @@ -3507,14 +3564,25 @@ namespace { { throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); } + IoHashStream Hash; bool CouldDecompress = Compressed.DecompressToStream(0, (uint64_t)-1, [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { DecompressedTemp.Write(RangeBuffer, Offset); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } WriteToDiskBytes += RangeBuffer.GetSize(); }); if (!CouldDecompress) { throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash)); } + const IoHash VerifyHash = Hash.GetHash(); + if (VerifyHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Decompressed blob payload hash {} does not match expected hash {}", VerifyHash, SequenceRawHash)); + } DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec); if (Ec) { @@ -3616,6 +3684,7 @@ namespace { auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + bool NeedHashVerify = true; if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) { const IoHash& SequenceRawHash = @@ -3624,6 +3693,8 @@ namespace { SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), WriteToDiskBytes); + NeedHashVerify = false; + ChunksComplete++; } else { @@ -3660,14 +3731,17 @@ namespace { { const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) + if (NeedHashVerify) { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != ChunkHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + ChunkHash)); + } } std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); @@ -3949,6 +4023,7 @@ namespace { { FilteredWrittenBytesPerSecond.Start(); + bool NeedHashVerify = true; if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) { const IoHash& SequenceRawHash = @@ -3959,6 +4034,7 @@ namespace { CompositeBuffer(CompressedPart), WriteToDiskBytes); ChunkCountWritten++; + NeedHashVerify = false; } else { @@ -3992,16 +4068,20 @@ namespace { { const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, - SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) + if (NeedHashVerify) { - throw std::runtime_error(fmt::format( - "Written hunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); + const IoHash VerifyChunkHash = + IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, + SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written hunk sequence {} hash does not match " + "expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } } std::filesystem::rename( GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index f13f8b9ca..1844f6a63 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -158,6 +158,9 @@ class BaseEncoder { public: [[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0; + [[nodiscard]] virtual bool CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize = DefaultBlockSize) const = 0; }; class BaseDecoder @@ -198,11 +201,21 @@ public: class NoneEncoder final : public BaseEncoder { public: - [[nodiscard]] CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final + [[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t /* BlockSize */) const final { UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned()); } + + [[nodiscard]] virtual bool CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t /* BlockSize */) const final + { + UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); + Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); + Callback(HeaderData.GetSize(), RawData); + return true; + } }; class NoneDecoder final : public BaseDecoder @@ -292,7 +305,10 @@ public: class BlockEncoder : public BaseEncoder { public: - CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const final; + virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize) const final; + virtual bool CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize) const final; protected: virtual CompressionMethod GetMethod() const = 0; @@ -440,6 +456,133 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) return CompositeBuffer(SharedBuffer::MakeView(CompositeView, CompressedData.MoveToShared())); } +bool +BlockEncoder::CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize = DefaultBlockSize) const +{ + ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31))); + + const uint64_t RawSize = RawData.GetSize(); + BLAKE3Stream RawHash; + + const uint64_t BlockCount = RoundUp(RawSize, BlockSize) / BlockSize; + ZEN_ASSERT(BlockCount <= ~uint32_t(0)); + + const uint64_t MetaSize = BlockCount * sizeof(uint32_t); + const uint64_t FullHeaderSize = sizeof(BufferHeader) + MetaSize; + + std::vector CompressedBlockSizes; + CompressedBlockSizes.reserve(BlockCount); + uint64_t CompressedSize = 0; + { + UniqueBuffer CompressedBlockBuffer = UniqueBuffer::Alloc(GetCompressedBlocksBound(1, BlockSize, Min(RawSize, BlockSize))); + + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + { + ZEN_ASSERT(FileRef.FileHandle != nullptr); + UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize); + BasicFile Source; + Source.Attach(FileRef.FileHandle); + for (uint64_t RawOffset = 0; RawOffset < RawSize;) + { + const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); + Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset); + const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize); + RawHash.Append(RawBlock); + MutableMemoryView CompressedBlock = CompressedBlockBuffer.GetMutableView(); + if (!CompressBlock(CompressedBlock, RawBlock)) + { + Source.Detach(); + return false; + } + + uint64_t CompressedBlockSize = CompressedBlock.GetSize(); + if (RawBlockSize <= CompressedBlockSize) + { + Callback(FullHeaderSize + CompressedSize, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlockCopy.GetView().GetData(), RawBlockSize))); + CompressedBlockSize = RawBlockSize; + } + else + { + Callback(FullHeaderSize + CompressedSize, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize))); + } + + CompressedBlockSizes.push_back(static_cast(CompressedBlockSize)); + CompressedSize += CompressedBlockSize; + RawOffset += RawBlockSize; + } + Source.Detach(); + } + else + { + UniqueBuffer RawBlockCopy; + CompositeBuffer::Iterator It = RawData.GetIterator(0); + + for (uint64_t RawOffset = 0; RawOffset < RawSize;) + { + const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize); + const MemoryView RawBlock = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy); + RawHash.Append(RawBlock); + + MutableMemoryView CompressedBlock = CompressedBlockBuffer.GetMutableView(); + if (!CompressBlock(CompressedBlock, RawBlock)) + { + return false; + } + + uint64_t CompressedBlockSize = CompressedBlock.GetSize(); + if (RawBlockSize <= CompressedBlockSize) + { + Callback(FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize))); + CompressedBlockSize = RawBlockSize; + } + else + { + Callback(FullHeaderSize + CompressedSize, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize))); + } + + CompressedBlockSizes.push_back(static_cast(CompressedBlockSize)); + CompressedSize += CompressedBlockSize; + RawOffset += RawBlockSize; + } + } + } + + // Return failure if the compressed data is larger than the raw data. + if (RawSize <= MetaSize + CompressedSize) + { + return false; + } + + // Write the header and calculate the CRC-32. + for (uint32_t& Size : CompressedBlockSizes) + { + Size = ByteSwap(Size); + } + UniqueBuffer HeaderBuffer = UniqueBuffer::Alloc(sizeof(BufferHeader) + MetaSize); + + BufferHeader Header; + Header.Method = GetMethod(); + Header.Compressor = GetCompressor(); + Header.CompressionLevel = GetCompressionLevel(); + Header.BlockSizeExponent = static_cast(zen::FloorLog2_64(BlockSize)); + Header.BlockCount = static_cast(BlockCount); + Header.TotalRawSize = RawSize; + Header.TotalCompressedSize = sizeof(BufferHeader) + MetaSize + CompressedSize; + Header.RawHash = RawHash.GetHash(); + + HeaderBuffer.GetMutableView().Mid(sizeof(BufferHeader), MetaSize).CopyFrom(MakeMemoryView(CompressedBlockSizes)); + Header.Write(HeaderBuffer.GetMutableView()); + + Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize()))); + return true; +} + class BlockDecoder : public BaseDecoder { public: @@ -1615,6 +1758,30 @@ CompressedBuffer::Compress(const SharedBuffer& RawData, return Compress(CompositeBuffer(RawData), Compressor, CompressionLevel, BlockSize); } +bool +CompressedBuffer::CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor, + OodleCompressionLevel CompressionLevel, + uint64_t BlockSize) +{ + using namespace detail; + + if (BlockSize == 0) + { + BlockSize = DefaultBlockSize; + } + + if (CompressionLevel == OodleCompressionLevel::None) + { + return NoneEncoder().CompressToStream(RawData, std::move(Callback), BlockSize); + } + else + { + return OodleEncoder(Compressor, CompressionLevel).CompressToStream(RawData, std::move(Callback), BlockSize); + } +} + CompressedBuffer CompressedBuffer::FromCompressed(const CompositeBuffer& InCompressedData, IoHash& OutRawHash, uint64_t& OutRawSize) { diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index c056d7561..3969e9dbd 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -74,6 +74,11 @@ public: OodleCompressor Compressor = OodleCompressor::Mermaid, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, uint64_t BlockSize = 0); + [[nodiscard]] ZENCORE_API static bool CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); /** * Construct from a compressed buffer previously created by Compress(). -- cgit v1.2.3 From 7b1c99f53da3a08b844cc7d7ce99530758e34be2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 5 Mar 2025 12:25:51 +0100 Subject: Add trace support for zen CLI command (#296) - This change adds support for `--trace`, `--tracehost` and `--tracefile` command arguments to enable and control tracing to Insights - It also adds profiling scopes primarily to build download command related code --- src/zen/cmds/builds_cmd.cpp | 83 +++++++++++++++++++++++++++-- src/zen/zen.cpp | 54 +++++++++++++++++++ src/zenutil/chunkedcontent.cpp | 79 +++++++++++++++------------ src/zenutil/jupiter/jupiterbuildstorage.cpp | 31 +++++++++++ 4 files changed, 210 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index fb9da021d..eb0650c4d 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -180,6 +181,8 @@ namespace { void CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) { + ZEN_TRACE_CPU("CleanDirectory"); + DirectoryContent LocalDirectoryContent; GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) @@ -352,6 +355,8 @@ namespace { std::function&& IsAcceptedFile, ChunkingController& ChunkController) { + ZEN_TRACE_CPU("ScanAndChunkFolder"); + FolderContent Content = GetFolderContent( GetFolderContentStats, Path, @@ -506,6 +511,8 @@ namespace { const std::span& LooseChunkIndexes, const std::span& BlockDescriptions) { + ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders"); + #if EXTRA_VERIFY std::vector TmpAbsoluteChunkHashes; TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size()); @@ -572,6 +579,8 @@ namespace { std::vector& OutLocalChunkRawSizes, std::vector& OutLocalChunkOrders) { + ZEN_TRACE_CPU("CalculateLocalChunkOrders"); + std::vector AbsoluteChunkHashes; std::vector AbsoluteChunkRawSizes; AbsoluteChunkHashes.insert(AbsoluteChunkHashes.end(), LooseChunkHashes.begin(), LooseChunkHashes.end()); @@ -955,6 +964,8 @@ namespace { const uint64_t BlockSize = 256u * 1024u; CompositeBuffer GetRange(uint64_t Offset, uint64_t Size) { + ZEN_TRACE_CPU("BufferedOpenFile::GetRange"); + ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); auto _ = MakeGuard([&]() { ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); }); @@ -1022,6 +1033,8 @@ namespace { CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) { + ZEN_TRACE_CPU("ReadFileCache::GetRange"); + auto CacheIt = std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; }); @@ -1069,6 +1082,8 @@ namespace { CompositeBuffer ValidateBlob(IoBuffer&& Payload, const IoHash& BlobHash, uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize) { + ZEN_TRACE_CPU("ValidateBlob"); + if (Payload.GetContentType() != ZenContentType::kCompressedBinary) { throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'", @@ -3087,6 +3102,8 @@ namespace { void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, bool VerifyFileHash) { + ZEN_TRACE_CPU("VerifyFolder"); + ProgressBar ProgressBar(UsePlainProgress); std::atomic FilesVerified(0); std::atomic FilesFailed(0); @@ -3133,6 +3150,8 @@ namespace { [&, PathIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("VerifyFile_work"); + // TODO: Convert ScheduleWork body to function const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred(); @@ -3348,6 +3367,8 @@ namespace { std::atomic& OutChunksComplete, std::atomic& OutBytesWritten) { + ZEN_TRACE_CPU("WriteBlockToDisk"); + std::vector ChunkBuffers; struct WriteOpData { @@ -3438,6 +3459,8 @@ namespace { } if (!AbortFlag) { + ZEN_TRACE_CPU("WriteBlockToDisk_VerifyHash"); + // Write tracking, updating this must be done without any files open (WriteFileCache) for (const WriteOpData& WriteOp : WriteOps) { @@ -3449,7 +3472,7 @@ namespace { IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); if (VerifyChunkHash != SequenceRawHash) { - throw std::runtime_error(fmt::format("Written hunk sequence {} hash does not match expected hash {}", + throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); } @@ -3468,6 +3491,8 @@ namespace { SharedBuffer Decompress(const CompositeBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) { + ZEN_TRACE_CPU("Decompress"); + IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedChunk, RawHash, RawSize); @@ -3506,6 +3531,8 @@ namespace { WriteFileCache& OpenFileCache, std::atomic& OutBytesWritten) { + ZEN_TRACE_CPU("WriteChunkToDisk"); + for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets) { const auto& Target = *TargetPtr; @@ -3611,6 +3638,8 @@ namespace { std::atomic& ChunksComplete, std::atomic& MultipartAttachmentCount) { + ZEN_TRACE_CPU("DownloadLargeBlob"); + struct WorkloadData { TemporaryFile TempFile; @@ -3666,6 +3695,8 @@ namespace { &WriteToDiskBytes, SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs](std::atomic&) { + ZEN_TRACE_CPU("DownloadLargeBlob_Work"); + if (!AbortFlag) { uint64_t CompressedSize = Workload->TempFile.FileSize(); @@ -3729,6 +3760,8 @@ namespace { const uint32_t RemoteSequenceIndex = Location->SequenceIndex; if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { + ZEN_TRACE_CPU("VerifyChunkHash"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; if (NeedHashVerify) @@ -3743,6 +3776,8 @@ namespace { ChunkHash)); } } + + ZEN_TRACE_CPU("VerifyChunkHashes_rename"); std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } @@ -3784,6 +3819,8 @@ namespace { bool WipeTargetFolder, FolderContent& OutLocalFolderState) { + ZEN_TRACE_CPU("UpdateFolder"); + ZEN_UNUSED(WipeTargetFolder); std::atomic DownloadedBlocks = 0; std::atomic BlockBytes = 0; @@ -3933,6 +3970,8 @@ namespace { std::atomic ChunkCountWritten = 0; { + ZEN_TRACE_CPU("HandleChunks"); + FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; @@ -3974,6 +4013,8 @@ namespace { Work.ScheduleWork( NetworkPool, // GetSyncWorkerPool(),// [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_LooseChunk"); + if (!AbortFlag) { FilteredDownloadedBytesPerSecond.Start(); @@ -4019,6 +4060,8 @@ namespace { WritePool, [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(Payload)]( std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_WriteBlob"); + if (!AbortFlag) { FilteredWrittenBytesPerSecond.Start(); @@ -4066,6 +4109,8 @@ namespace { if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub( 1) == 1) { + ZEN_TRACE_CPU("UpdateFolder_VerifyHash"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; if (NeedHashVerify) @@ -4077,12 +4122,14 @@ namespace { if (VerifyChunkHash != SequenceRawHash) { throw std::runtime_error( - fmt::format("Written hunk sequence {} hash does not match " + fmt::format("Written chunk sequence {} hash does not match " "expected hash {}", VerifyChunkHash, SequenceRawHash)); } } + + ZEN_TRACE_CPU("UpdateFolder_rename"); std::filesystem::rename( GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); @@ -4113,6 +4160,8 @@ namespace { [&, CopyDataIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("UpdateFolder_Copy"); + FilteredWrittenBytesPerSecond.Start(); const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; @@ -4205,6 +4254,8 @@ namespace { // Write tracking, updating this must be done without any files open (WriteFileCache) for (const WriteOp& Op : WriteOps) { + ZEN_TRACE_CPU("UpdateFolder_Copy_VerifyHash"); + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { @@ -4215,10 +4266,12 @@ namespace { if (VerifyChunkHash != SequenceRawHash) { throw std::runtime_error( - fmt::format("Written hunk sequence {} hash does not match expected hash {}", + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); } + + ZEN_TRACE_CPU("UpdateFolder_Copy_rename"); std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } @@ -4268,6 +4321,8 @@ namespace { [&, BlockIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Read"); + FilteredDownloadedBytesPerSecond.Start(); IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); if (!BlockBuffer) @@ -4288,6 +4343,8 @@ namespace { [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Write"); + FilteredWrittenBytesPerSecond.Start(); IoHash BlockRawHash; uint64_t BlockRawSize; @@ -4348,6 +4405,8 @@ namespace { } } + ZEN_TRACE_CPU("HandleChunks_Wait"); + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); @@ -4411,12 +4470,16 @@ namespace { if (WipeTargetFolder) { + ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); + // Clean target folder ZEN_CONSOLE("Wiping {}", Path); CleanDirectory(Path, DefaultExcludeFolders); } else { + ZEN_TRACE_CPU("UpdateFolder_RemoveUnused"); + // Remove unused tracked files tsl::robin_map RemotePathToRemoteIndex; RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); @@ -4468,6 +4531,8 @@ namespace { break; } + ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); + size_t TargetCount = 1; const IoHash& RawHash = Targets[TargetOffset].first; while (Targets[TargetOffset + TargetCount].first == RawHash) @@ -4480,6 +4545,8 @@ namespace { [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("FinalizeTree_Work"); + size_t TargetOffset = BaseTargetOffset; const IoHash& RawHash = Targets[TargetOffset].first; const uint32_t FirstTargetPathIndex = Targets[TargetOffset].second; @@ -4501,6 +4568,8 @@ namespace { } else { + ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace"); + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); CreateDirectories(FirstTargetFilePath.parent_path()); @@ -4522,6 +4591,8 @@ namespace { TargetsComplete++; while (TargetOffset < (BaseTargetOffset + TargetCount)) { + ZEN_TRACE_CPU("FinalizeTree_Copy"); + ZEN_ASSERT(Targets[TargetOffset].first == RawHash); ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); const uint32_t ExtraTargetPathIndex = Targets[TargetOffset].second; @@ -4555,6 +4626,8 @@ namespace { TargetOffset += TargetCount; } + ZEN_TRACE_CPU("FinalizeTree_Wait"); + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); @@ -4668,6 +4741,8 @@ namespace { std::vector& OutBlockDescriptions, std::vector& OutLooseChunkHashes) { + ZEN_TRACE_CPU("GetRemoteContent"); + Stopwatch GetBuildPartTimer; CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildParts[0].first); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", @@ -5114,6 +5189,8 @@ namespace { bool WipeTargetFolder, bool PostDownloadVerify) { + ZEN_TRACE_CPU("DownloadFolder"); + Stopwatch DownloadTimer; const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index bee1fd676..713a0d66d 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -31,9 +31,16 @@ #include #include #include +#include #include #include +#include +#include +#include +#include +#include + #if ZEN_WITH_TESTS # define ZEN_TEST_WITH_RUNNER 1 # include @@ -603,6 +610,29 @@ main(int argc, char** argv) Options.add_options()("help", "Show command line help"); Options.add_options()("c, command", "Sub command", cxxopts::value(SubCommand)); +#if ZEN_WITH_TRACE + std::string TraceChannels; + std::string TraceHost; + std::string TraceFile; + + Options.add_option("ue-trace", + "", + "trace", + "Specify which trace channels should be enabled", + cxxopts::value(TraceChannels)->default_value(""), + ""); + + Options.add_option("ue-trace", + "", + "tracehost", + "Hostname to send the trace to", + cxxopts::value(TraceHost)->default_value(""), + ""); + + Options + .add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value(TraceFile)->default_value(""), ""); +#endif // ZEN_WITH_TRACE + Options.parse_positional({"command"}); const bool IsNullInvoke = (argc == 1); // If no arguments are passed we want to print usage information @@ -653,6 +683,30 @@ main(int argc, char** argv) logging::SetLogLevel(logging::level::Trace); } +#if ZEN_WITH_TRACE + if (TraceHost.size()) + { + TraceStart("zen", TraceHost.c_str(), TraceType::Network); + } + else if (TraceFile.size()) + { + TraceStart("zen", TraceFile.c_str(), TraceType::File); + } + else + { + TraceInit("zen"); + } +#endif // ZEN_WITH_TRACE + +#if ZEN_WITH_MEMTRACK + FMalloc* TraceMalloc = MemoryTrace_Create(GMalloc); + if (TraceMalloc != GMalloc) + { + GMalloc = TraceMalloc; + MemoryTrace_Initialize(); + } +#endif + for (const CommandInfo& CmdInfo : Commands) { if (StrCaseCompare(SubCommand.c_str(), CmdInfo.CmdName) == 0) diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 1552ea823..4ca89d996 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -23,13 +24,13 @@ namespace zen { using namespace std::literals; namespace { - void AddCunkSequence(ChunkingStatistics& Stats, - ChunkedContentData& InOutChunkedContent, - tsl::robin_map& ChunkHashToChunkIndex, - const IoHash& RawHash, - std::span ChunkSequence, - std::span ChunkHashes, - std::span ChunkRawSizes) + void AddChunkSequence(ChunkingStatistics& Stats, + ChunkedContentData& InOutChunkedContent, + tsl::robin_map& ChunkHashToChunkIndex, + const IoHash& RawHash, + std::span ChunkSequence, + std::span ChunkHashes, + std::span ChunkRawSizes) { ZEN_ASSERT(ChunkHashes.size() == ChunkRawSizes.size()); InOutChunkedContent.ChunkCounts.push_back(gsl::narrow(ChunkSequence.size())); @@ -58,11 +59,11 @@ namespace { Stats.UniqueSequencesFound++; } - void AddCunkSequence(ChunkingStatistics& Stats, - ChunkedContentData& InOutChunkedContent, - tsl::robin_map& ChunkHashToChunkIndex, - const IoHash& RawHash, - const uint64_t RawSize) + void AddChunkSequence(ChunkingStatistics& Stats, + ChunkedContentData& InOutChunkedContent, + tsl::robin_map& ChunkHashToChunkIndex, + const IoHash& RawHash, + const uint64_t RawSize) { InOutChunkedContent.ChunkCounts.push_back(1); @@ -120,13 +121,13 @@ namespace { { ChunkSizes.push_back(Source.Size); } - AddCunkSequence(Stats, - OutChunkedContent.ChunkedContent, - ChunkHashToChunkIndex, - Chunked.Info.RawHash, - Chunked.Info.ChunkSequence, - Chunked.Info.ChunkHashes, - ChunkSizes); + AddChunkSequence(Stats, + OutChunkedContent.ChunkedContent, + ChunkHashToChunkIndex, + Chunked.Info.RawHash, + Chunked.Info.ChunkSequence, + Chunked.Info.ChunkHashes, + ChunkSizes); Stats.UniqueSequencesFound++; } }); @@ -143,7 +144,7 @@ namespace { { RawHashToSequenceRawHashIndex.insert( {Hash, gsl::narrow(OutChunkedContent.ChunkedContent.SequenceRawHashes.size())}); - AddCunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize); + AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Hash, RawSize); Stats.UniqueSequencesFound++; } }); @@ -360,6 +361,8 @@ GetFolderContent(GetFolderContentStatistics& Stats, std::function&& UpdateCallback, std::atomic& AbortFlag) { + ZEN_TRACE_CPU("GetFolderContent"); + Stopwatch Timer; auto _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); @@ -528,6 +531,8 @@ LoadChunkedFolderContentToCompactBinary(CbObjectView Input) ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays) { + ZEN_TRACE_CPU("MergeChunkedFolderContents"); + ZEN_ASSERT(!Overlays.empty()); ChunkedFolderContent Result; @@ -605,13 +610,13 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span OriginalChunkOrder = std::span(OverlayContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); - AddCunkSequence(Stats, - Result.ChunkedContent, - ChunkHashToChunkIndex, - RawHash, - OriginalChunkOrder, - OverlayContent.ChunkedContent.ChunkHashes, - OverlayContent.ChunkedContent.ChunkRawSizes); + AddChunkSequence(Stats, + Result.ChunkedContent, + ChunkHashToChunkIndex, + RawHash, + OriginalChunkOrder, + OverlayContent.ChunkedContent.ChunkHashes, + OverlayContent.ChunkedContent.ChunkRawSizes); Stats.UniqueSequencesFound++; } } @@ -636,6 +641,8 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span DeletedPaths) { + ZEN_TRACE_CPU("DeletePathsFromChunkedContent"); + ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size()); ChunkedFolderContent Result = {.Platform = BaseContent.Platform}; if (DeletedPaths.size() < BaseContent.Paths.size()) @@ -673,13 +680,13 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span ChunkingStatistics Stats; std::span OriginalChunkOrder = std::span(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); - AddCunkSequence(Stats, - Result.ChunkedContent, - ChunkHashToChunkIndex, - RawHash, - OriginalChunkOrder, - BaseContent.ChunkedContent.ChunkHashes, - BaseContent.ChunkedContent.ChunkRawSizes); + AddChunkSequence(Stats, + Result.ChunkedContent, + ChunkHashToChunkIndex, + RawHash, + OriginalChunkOrder, + BaseContent.ChunkedContent.ChunkHashes, + BaseContent.ChunkedContent.ChunkRawSizes); Stats.UniqueSequencesFound++; } } @@ -699,6 +706,8 @@ ChunkFolderContent(ChunkingStatistics& Stats, std::function&& UpdateCallback, std::atomic& AbortFlag) { + ZEN_TRACE_CPU("ChunkFolderContent"); + Stopwatch Timer; auto _ = MakeGuard([&Stats, &Timer]() { Stats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); @@ -775,6 +784,8 @@ ChunkFolderContent(ChunkingStatistics& Stats, ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content) { + ZEN_TRACE_CPU("BuildChunkedContentLookup"); + struct ChunkLocationReference { uint32_t ChunkIndex; diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 481e9146f..309885b05 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include ZEN_THIRD_PARTY_INCLUDES_START @@ -36,6 +37,8 @@ public: virtual CbObject ListBuilds(CbObject Query) override { + ZEN_TRACE_CPU("Jupiter::ListBuilds"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); IoBuffer Payload = Query.GetBuffer().AsIoBuffer(); @@ -51,6 +54,8 @@ public: virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override { + ZEN_TRACE_CPU("Jupiter::PutBuild"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); @@ -66,6 +71,8 @@ public: virtual CbObject GetBuild(const Oid& BuildId) override { + ZEN_TRACE_CPU("Jupiter::GetBuild"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult GetBuildResult = m_Session.GetBuild(m_Namespace, m_Bucket, BuildId); @@ -79,6 +86,8 @@ public: virtual void FinalizeBuild(const Oid& BuildId) override { + ZEN_TRACE_CPU("Jupiter::FinalizeBuild"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult FinalizeBuildResult = m_Session.FinalizeBuild(m_Namespace, m_Bucket, BuildId); @@ -95,6 +104,8 @@ public: std::string_view PartName, const CbObject& MetaData) override { + ZEN_TRACE_CPU("Jupiter::PutBuildPart"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); @@ -110,6 +121,8 @@ public: virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override { + ZEN_TRACE_CPU("Jupiter::GetBuildPart"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult GetBuildPartResult = m_Session.GetBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId); @@ -126,6 +139,8 @@ public: virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override { + ZEN_TRACE_CPU("Jupiter::FinalizeBuildPart"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); FinalizeBuildPartResult FinalizePartResult = m_Session.FinalizeBuildPart(m_Namespace, m_Bucket, BuildId, BuildPartId, PartHash); @@ -143,6 +158,8 @@ public: ZenContentType ContentType, const CompositeBuffer& Payload) override { + ZEN_TRACE_CPU("Jupiter::PutBuildBlob"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult PutBlobResult = m_Session.PutBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ContentType, Payload); @@ -160,6 +177,8 @@ public: std::function&& Transmitter, std::function&& OnSentBytes) override { + ZEN_TRACE_CPU("Jupiter::PutLargeBuildBlob"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); std::vector> WorkItems; @@ -200,6 +219,8 @@ public: virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override { + ZEN_TRACE_CPU("Jupiter::GetBuildBlob"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath); @@ -218,6 +239,8 @@ public: uint64_t ChunkSize, std::function&& Receiver) override { + ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); std::vector> WorkItems; @@ -249,6 +272,8 @@ public: virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { + ZEN_TRACE_CPU("Jupiter::PutBlockMetadata"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); IoBuffer Payload = MetaData.GetBuffer().AsIoBuffer(); @@ -264,6 +289,8 @@ public: virtual std::vector FindBlocks(const Oid& BuildId) override { + ZEN_TRACE_CPU("Jupiter::FindBlocks"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId); @@ -277,6 +304,8 @@ public: virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override { + ZEN_TRACE_CPU("Jupiter::GetBlockMetadata"); + Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); CbObjectWriter Request; @@ -365,6 +394,8 @@ CreateJupiterBuildStorage(LoggerRef InLog, std::string_view Bucket, const std::filesystem::path& TempFolderPath) { + ZEN_TRACE_CPU("CreateJupiterBuildStorage"); + return std::make_unique(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath); } -- cgit v1.2.3 From 920120bbcec9f91df3336f62970b3e010a4fa6c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 6 Mar 2025 16:18:32 +0100 Subject: reduced memory churn using fixed_xxx containers (#236) * Added EASTL to help with eliminating memory allocations * Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F) --- src/zencore/compactbinary.cpp | 10 +- src/zencore/compactbinarybuilder.cpp | 11 +- src/zencore/compactbinarypackage.cpp | 14 ++- src/zencore/include/zencore/compactbinarybuilder.h | 8 +- src/zencore/include/zencore/compactbinarypackage.h | 15 ++- src/zencore/include/zencore/compositebuffer.h | 38 ++++++- src/zencore/include/zencore/eastlutil.h | 20 ++++ src/zencore/include/zencore/memory/newdelete.h | 26 +++++ src/zencore/xmake.lua | 1 + src/zenhttp/httpserver.cpp | 6 +- src/zenhttp/include/zenhttp/httpserver.h | 8 +- src/zenhttp/packageformat.cpp | 24 +++- src/zenhttp/servers/httpsys.cpp | 18 +-- src/zenserver/objectstore/objectstore.cpp | 27 +++-- src/zenserver/objectstore/objectstore.h | 4 +- src/zenserver/projectstore/httpprojectstore.cpp | 32 +++--- src/zenserver/projectstore/projectstore.cpp | 45 +++++--- src/zenserver/workspaces/httpworkspaces.cpp | 12 +- src/zenstore/cache/cachedisklayer.cpp | 121 ++++++++++++--------- src/zenstore/cache/cacherpc.cpp | 73 +++++++------ src/zenstore/cache/structuredcachestore.cpp | 8 +- .../include/zenstore/cache/cachedisklayer.h | 38 ++++--- src/zenstore/include/zenstore/cache/cacheshared.h | 4 + .../include/zenstore/cache/structuredcachestore.h | 6 +- src/zenstore/xmake.lua | 1 + src/zenutil/include/zenutil/cache/cachekey.h | 6 + 26 files changed, 377 insertions(+), 199 deletions(-) create mode 100644 src/zencore/include/zencore/eastlutil.h (limited to 'src') diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index adccaba70..b43cc18f1 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include @@ -1376,9 +1378,9 @@ TryMeasureCompactBinary(MemoryView View, CbFieldType& OutType, uint64_t& OutSize CbField LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) { - std::vector HeaderBytes; - CbFieldType FieldType; - uint64_t FieldSize = 1; + eastl::fixed_vector HeaderBytes; + CbFieldType FieldType; + uint64_t FieldSize = 1; for (const int64_t StartPos = Ar.CurrentOffset(); FieldSize > 0;) { @@ -1393,7 +1395,7 @@ LoadCompactBinary(BinaryReader& Ar, BufferAllocator Allocator) HeaderBytes.resize(ReadOffset + ReadSize); Ar.Read(HeaderBytes.data() + ReadOffset, ReadSize); - if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes), FieldType, FieldSize)) + if (TryMeasureCompactBinary(MakeMemoryView(HeaderBytes.data(), HeaderBytes.size()), FieldType, FieldSize)) { if (FieldSize <= uint64_t(Ar.Size() - StartPos)) { diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp index a60de023d..63c0b9c5c 100644 --- a/src/zencore/compactbinarybuilder.cpp +++ b/src/zencore/compactbinarybuilder.cpp @@ -15,23 +15,21 @@ namespace zen { -template uint64_t -AddUninitialized(std::vector& Vector, uint64_t Count) +AddUninitialized(CbWriter::CbWriterData_t& Vector, uint64_t Count) { const uint64_t Offset = Vector.size(); Vector.resize(Offset + Count); return Offset; } -template uint64_t -Append(std::vector& Vector, const T* Data, uint64_t Count) +Append(CbWriter::CbWriterData_t& Vector, const uint8_t* Data, uint64_t Count) { const uint64_t Offset = Vector.size(); Vector.resize(Offset + Count); - memcpy(Vector.data() + Offset, Data, sizeof(T) * Count); + memcpy(Vector.data() + Offset, Data, sizeof(uint8_t) * Count); return Offset; } @@ -76,7 +74,7 @@ IsUniformType(const CbFieldType Type) /** Append the payload from the compact binary value to the array and return its type. */ static inline CbFieldType -AppendCompactBinary(const CbFieldView& Value, std::vector& OutData) +AppendCompactBinary(const CbFieldView& Value, CbWriter::CbWriterData_t& OutData) { struct FCopy : public CbFieldView { @@ -93,7 +91,6 @@ AppendCompactBinary(const CbFieldView& Value, std::vector& OutData) CbWriter::CbWriter() { - States.reserve(4); States.emplace_back(); } diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp index 7de161845..ffe64f2e9 100644 --- a/src/zencore/compactbinarypackage.cpp +++ b/src/zencore/compactbinarypackage.cpp @@ -3,10 +3,13 @@ #include "zencore/compactbinarypackage.h" #include #include +#include #include #include #include +#include + namespace zen { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -340,6 +343,12 @@ CbPackage::SetObject(CbObject InObject, const IoHash* InObjectHash, AttachmentRe } } +void +CbPackage::ReserveAttachments(size_t Count) +{ + Attachments.reserve(Count); +} + void CbPackage::AddAttachment(const CbAttachment& Attachment, AttachmentResolver* Resolver) { @@ -374,17 +383,18 @@ CbPackage::AddAttachments(std::span InAttachments) { ZEN_ASSERT(!Attachment.IsNull()); } + // Assume we have no duplicates! Attachments.insert(Attachments.end(), InAttachments.begin(), InAttachments.end()); std::sort(Attachments.begin(), Attachments.end()); - ZEN_ASSERT_SLOW(std::unique(Attachments.begin(), Attachments.end()) == Attachments.end()); + ZEN_ASSERT_SLOW(eastl::unique(Attachments.begin(), Attachments.end()) == Attachments.end()); } int32_t CbPackage::RemoveAttachment(const IoHash& Hash) { return gsl::narrow_cast( - std::erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); + erase_if(Attachments, [&Hash](const CbAttachment& Attachment) -> bool { return Attachment.GetHash() == Hash; })); } bool diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h index 1c625cacc..f11717453 100644 --- a/src/zencore/include/zencore/compactbinarybuilder.h +++ b/src/zencore/include/zencore/compactbinarybuilder.h @@ -18,6 +18,8 @@ #include #include +#include + #include namespace zen { @@ -367,6 +369,8 @@ public: /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ enum class StateFlags : uint8_t; + typedef eastl::fixed_vector CbWriterData_t; + protected: /** Reserve the specified size up front until the format is optimized. */ ZENCORE_API explicit CbWriter(int64_t InitialSize); @@ -409,8 +413,8 @@ private: // provided externally, such as on the stack. That format will store the offsets that require // object or array sizes to be inserted and field types to be removed, and will perform those // operations only when saving to a buffer. - std::vector Data; - std::vector States; + eastl::fixed_vector States; + CbWriterData_t Data; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h index 12fcc41b7..9ec12cb0f 100644 --- a/src/zencore/include/zencore/compactbinarypackage.h +++ b/src/zencore/include/zencore/compactbinarypackage.h @@ -12,6 +12,8 @@ #include #include +#include + #ifdef GetObject # error "windows.h pollution" # undef GetObject @@ -265,7 +267,10 @@ public: } /** Returns the attachments in this package. */ - inline std::span GetAttachments() const { return Attachments; } + inline std::span GetAttachments() const + { + return std::span(begin(Attachments), end(Attachments)); + } /** * Find an attachment by its hash. @@ -286,6 +291,8 @@ public: void AddAttachments(std::span Attachments); + void ReserveAttachments(size_t Count); + /** * Remove an attachment by hash. * @@ -324,9 +331,9 @@ private: void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver); /** Attachments ordered by their hash. */ - std::vector Attachments; - CbObject Object; - IoHash ObjectHash; + eastl::fixed_vector Attachments; + CbObject Object; + IoHash ObjectHash; }; namespace legacy { diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h index b435c5e74..1e1611de9 100644 --- a/src/zencore/include/zencore/compositebuffer.h +++ b/src/zencore/include/zencore/compositebuffer.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include @@ -9,6 +10,8 @@ #include #include +#include + namespace zen { /** @@ -35,7 +38,7 @@ public: { m_Segments.reserve((GetBufferCount(std::forward(Buffers)) + ...)); (AppendBuffers(std::forward(Buffers)), ...); - std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); + erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); } } @@ -46,7 +49,10 @@ public: [[nodiscard]] ZENCORE_API uint64_t GetSize() const; /** Returns the segments that the buffer is composed from. */ - [[nodiscard]] inline std::span GetSegments() const { return std::span{m_Segments}; } + [[nodiscard]] inline std::span GetSegments() const + { + return std::span{begin(m_Segments), end(m_Segments)}; + } /** Returns true if the composite buffer is not null. */ [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } @@ -120,6 +126,8 @@ public: static const CompositeBuffer Null; private: + typedef eastl::fixed_vector SharedBufferVector_t; + static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); } inline void AppendBuffers(const CompositeBuffer& Buffer) { @@ -134,12 +142,25 @@ private: inline void AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); } inline void AppendBuffers(IoBuffer&& Buffer) { m_Segments.push_back(SharedBuffer(std::move(Buffer))); } + static inline size_t GetBufferCount(std::span&& Container) { return Container.size(); } + inline void AppendBuffers(std::span&& Container) + { + m_Segments.reserve(m_Segments.size() + Container.size()); + for (IoBuffer& Buffer : Container) + { + m_Segments.emplace_back(SharedBuffer(std::move(Buffer))); + } + } + static inline size_t GetBufferCount(std::vector&& Container) { return Container.size(); } static inline size_t GetBufferCount(std::vector&& Container) { return Container.size(); } inline void AppendBuffers(std::vector&& Container) { m_Segments.reserve(m_Segments.size() + Container.size()); - m_Segments.insert(m_Segments.end(), std::make_move_iterator(Container.begin()), std::make_move_iterator(Container.end())); + for (SharedBuffer& Buffer : Container) + { + m_Segments.emplace_back(std::move(Buffer)); + } } inline void AppendBuffers(std::vector&& Container) { @@ -150,8 +171,17 @@ private: } } + inline void AppendBuffers(SharedBufferVector_t&& Container) + { + m_Segments.reserve(m_Segments.size() + Container.size()); + for (SharedBuffer& Buffer : Container) + { + m_Segments.emplace_back(std::move(Buffer)); + } + } + private: - std::vector m_Segments; + SharedBufferVector_t m_Segments; }; void compositebuffer_forcelink(); // internal diff --git a/src/zencore/include/zencore/eastlutil.h b/src/zencore/include/zencore/eastlutil.h new file mode 100644 index 000000000..642321dae --- /dev/null +++ b/src/zencore/include/zencore/eastlutil.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +size_t +erase_if(auto& _Cont, auto Predicate) +{ + auto _First = _Cont.begin(); + const auto _Last = _Cont.end(); + const auto _Old_size = _Cont.size(); + _First = std::remove_if(_First, _Last, Predicate); + _Cont.erase(_First, _Last); + return _Old_size - _Cont.size(); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h index d22c8604f..059f1d5ea 100644 --- a/src/zencore/include/zencore/memory/newdelete.h +++ b/src/zencore/include/zencore/memory/newdelete.h @@ -153,3 +153,29 @@ operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexce return zen_new_aligned_nothrow(n, static_cast(al)); } #endif + +// EASTL operator new + +void* +operator new[](size_t size, const char* pName, int flags, unsigned debugFlags, const char* file, int line) +{ + ZEN_UNUSED(pName, flags, debugFlags, file, line); + return zen_new(size); +} + +void* +operator new[](size_t size, + size_t alignment, + size_t alignmentOffset, + const char* pName, + int flags, + unsigned debugFlags, + const char* file, + int line) +{ + ZEN_UNUSED(alignmentOffset, pName, flags, debugFlags, file, line); + + ZEN_ASSERT_SLOW(alignmentOffset == 0); // currently not supported + + return zen_new_aligned(size, alignment); +} diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 2efa3fdb8..b8b14084c 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -55,6 +55,7 @@ target('zencore') add_packages( "vcpkg::doctest", + "vcpkg::eastl", "vcpkg::fmt", "vcpkg::gsl-lite", "vcpkg::lz4", diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 1fbe22628..27a09f339 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -31,6 +31,8 @@ #include #include +#include + namespace zen { using namespace std::literals; @@ -529,7 +531,7 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType { std::span Segments = Payload.GetSegments(); - std::vector Buffers; + eastl::fixed_vector Buffers; Buffers.reserve(Segments.size()); for (auto& Segment : Segments) @@ -537,7 +539,7 @@ HttpServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentType Buffers.push_back(Segment.AsIoBuffer()); } - WriteResponse(ResponseCode, ContentType, Buffers); + WriteResponse(ResponseCode, ContentType, std::span(begin(Buffers), end(Buffers))); } std::string diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 7b87cb84b..217455dba 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -208,7 +208,7 @@ class HttpRouterRequest public: HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} - ZENCORE_API std::string GetCapture(uint32_t Index) const; + std::string_view GetCapture(uint32_t Index) const; inline HttpServerRequest& ServerRequest() { return m_HttpRequest; } private: @@ -220,12 +220,14 @@ private: friend class HttpRequestRouter; }; -inline std::string +inline std::string_view HttpRouterRequest::GetCapture(uint32_t Index) const { ZEN_ASSERT(Index < m_Match.size()); - return m_Match[Index]; + const auto& Match = m_Match[Index]; + + return std::string_view(&*Match.first, Match.second - Match.first); } /** HTTP request router helper diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index 676fc73fd..ae80851e4 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -19,6 +19,8 @@ #include #include +#include + #if ZEN_PLATFORM_WINDOWS # include #endif @@ -31,6 +33,10 @@ namespace zen { const std::string_view HandlePrefix(":?#:"); +typedef eastl::fixed_vector IoBufferVec_t; + +IoBufferVec_t FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle); + std::vector FormatPackageMessage(const CbPackage& Data, void* TargetProcessHandle) { @@ -42,10 +48,18 @@ FormatPackageMessageBuffer(const CbPackage& Data, void* TargetProcessHandle) return FormatPackageMessageBuffer(Data, FormatFlags::kDefault, TargetProcessHandle); } +std::vector +FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) +{ + auto Vec = FormatPackageMessageInternal(Data, Flags, TargetProcessHandle); + return std::vector(begin(Vec), end(Vec)); +} + CompositeBuffer FormatPackageMessageBuffer(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) { - return CompositeBuffer(FormatPackageMessage(Data, Flags, TargetProcessHandle)); + auto Vec = FormatPackageMessageInternal(Data, Flags, TargetProcessHandle); + return CompositeBuffer(std::span{begin(Vec), end(Vec)}); } static void @@ -54,7 +68,7 @@ MarshalLocal(CbAttachmentEntry*& AttachmentInfo, CbAttachmentReferenceHeader& LocalRef, const IoHash& AttachmentHash, bool IsCompressed, - std::vector& ResponseBuffers) + IoBufferVec_t& ResponseBuffers) { IoBuffer RefBuffer(sizeof(CbAttachmentReferenceHeader) + Path8.size()); @@ -146,8 +160,8 @@ IsLocalRef(tsl::robin_map& FileNameMap, return true; }; -std::vector -FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) +IoBufferVec_t +FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* TargetProcessHandle) { ZEN_TRACE_CPU("FormatPackageMessage"); @@ -177,7 +191,7 @@ FormatPackageMessage(const CbPackage& Data, FormatFlags Flags, void* TargetProce #endif // ZEN_PLATFORM_WINDOWS const std::span& Attachments = Data.GetAttachments(); - std::vector ResponseBuffers; + IoBufferVec_t ResponseBuffers; ResponseBuffers.reserve(2 + Attachments.size()); // TODO: may want to use an additional fudge factor here to avoid growing since each // attachment is likely to consist of several buffers diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 87128c0c9..3bdcdf098 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -16,6 +16,8 @@ #include #include +#include + #if ZEN_WITH_HTTPSYS # define _WINSOCKAPI_ # include @@ -381,14 +383,14 @@ public: void SuppressResponseBody(); // typically used for HEAD requests private: - std::vector m_HttpDataChunks; - uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes - uint16_t m_ResponseCode = 0; - uint32_t m_NextDataChunkOffset = 0; // Cursor used for very large chunk lists - uint32_t m_RemainingChunkCount = 0; // Backlog for multi-call sends - bool m_IsInitialResponse = true; - HttpContentType m_ContentType = HttpContentType::kBinary; - std::vector m_DataBuffers; + eastl::fixed_vector m_HttpDataChunks; + uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes + uint16_t m_ResponseCode = 0; + uint32_t m_NextDataChunkOffset = 0; // Cursor used for very large chunk lists + uint32_t m_RemainingChunkCount = 0; // Backlog for multi-call sends + bool m_IsInitialResponse = true; + HttpContentType m_ContentType = HttpContentType::kBinary; + eastl::fixed_vector m_DataBuffers; void InitializeForPayload(uint16_t ResponseCode, std::span Blobs); }; diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index 5d96de225..e757ef84e 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -269,9 +269,9 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "bucket/{path}", [this](zen::HttpRouterRequest& Request) { - const std::string Path = Request.GetCapture(1); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; if (IsObject) { @@ -337,18 +337,18 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string& Path) +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; - const auto Sep = Path.find_first_of('/'); - const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; if (BucketName.empty()) { return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } - std::string BucketPrefix = Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; if (BucketPrefix.empty()) { const auto QueryParms = Request.ServerRequest().GetQueryParams(); @@ -450,14 +450,13 @@ HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) } void -HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string& Path) +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) { namespace fs = std::filesystem; - const auto Sep = Path.find_first_of('/'); - const std::string BucketName = Sep == std::string::npos ? Path : Path.substr(0, Sep); - const std::string BucketPrefix = - Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1); + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; const fs::path BucketDir = GetBucketDirectory(BucketName); @@ -554,8 +553,8 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { namespace fs = std::filesystem; - const std::string& BucketName = Request.GetCapture(1); - const fs::path BucketDir = GetBucketDirectory(BucketName); + const std::string_view BucketName = Request.GetCapture(1); + const fs::path BucketDir = GetBucketDirectory(BucketName); if (BucketDir.empty()) { diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index c905ceab3..dae979c4c 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -36,9 +36,9 @@ private: void Inititalize(); std::filesystem::path GetBucketDirectory(std::string_view BucketName); void CreateBucket(zen::HttpRouterRequest& Request); - void ListBucket(zen::HttpRouterRequest& Request, const std::string& Path); + void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); void DeleteBucket(zen::HttpRouterRequest& Request); - void GetObject(zen::HttpRouterRequest& Request, const std::string& Path); + void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); void PutObject(zen::HttpRouterRequest& Request); ObjectStoreConfig m_Cfg; diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 0b8e5f13b..47748dd90 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -983,15 +983,19 @@ HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) IoBuffer Payload = HttpReq.ReadPayload(); CbObject RequestObject = LoadCompactBinaryObject(Payload); - std::vector ChunkList; - CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); - ChunkList.reserve(HaveList.Num()); - for (auto& Entry : HaveList) + std::vector NeedList; + { - ChunkList.push_back(Entry.AsHash()); - } + eastl::fixed_vector ChunkList; + CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); + ChunkList.reserve(HaveList.Num()); + for (auto& Entry : HaveList) + { + ChunkList.push_back(Entry.AsHash()); + } - std::vector NeedList = FoundLog->CheckPendingChunkReferences(ChunkList, std::chrono::minutes(2)); + NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); + } CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); Cbo.BeginArray("need"); @@ -1151,7 +1155,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); } - std::vector ReferencedChunks; + eastl::fixed_vector ReferencedChunks; Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); }); // Write core to oplog @@ -1169,7 +1173,7 @@ HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) // Once we stored the op, we no longer need to retain any chunks this op references if (!ReferencedChunks.empty()) { - FoundLog->RemovePendingChunkReferences(ReferencedChunks); + FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks))); } m_ProjectStats.OpWriteCount++; @@ -1301,9 +1305,9 @@ HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) HttpServerRequest& HttpReq = Req.ServerRequest(); - const std::string& ProjectId = Req.GetCapture(1); - const std::string& OplogId = Req.GetCapture(2); - const std::string& OpIdString = Req.GetCapture(3); + const std::string_view ProjectId = Req.GetCapture(1); + const std::string_view OplogId = Req.GetCapture(2); + const std::string_view OpIdString = Req.GetCapture(3); Ref Project = m_ProjectStore->OpenProject(ProjectId); if (!Project) @@ -1690,8 +1694,8 @@ HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) using namespace std::literals; - HttpServerRequest& HttpReq = Req.ServerRequest(); - const std::string ProjectId = Req.GetCapture(1); + HttpServerRequest& HttpReq = Req.ServerRequest(); + const std::string_view ProjectId = Req.GetCapture(1); switch (HttpReq.RequestVerb()) { diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 53df12b14..86791e29a 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -423,9 +423,13 @@ ComputeOpKey(const CbObjectView& Op) { using namespace std::literals; - BinaryWriter KeyStream; + eastl::fixed_vector KeyData; - Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyStream.Write(Data, Size); }); + Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { + auto Begin = reinterpret_cast(Data); + auto End = Begin + Size; + KeyData.insert(KeyData.end(), Begin, End); + }); XXH3_128 KeyHash128; @@ -434,15 +438,15 @@ ComputeOpKey(const CbObjectView& Op) // path but longer paths are evaluated properly. In the future all key lengths // should be evaluated using the proper path, this is a temporary workaround to // maintain compatibility with existing disk state. - if (KeyStream.GetSize() < 240) + if (KeyData.size() < 240) { XXH3_128Stream_deprecated KeyHasher; - KeyHasher.Append(KeyStream.Data(), KeyStream.Size()); + KeyHasher.Append(KeyData.data(), KeyData.size()); KeyHash128 = KeyHasher.GetHash(); } else { - KeyHash128 = XXH3_128::HashMemory(KeyStream.GetView()); + KeyHash128 = XXH3_128::HashMemory(KeyData.data(), KeyData.size()); } Oid KeyHash; @@ -2735,7 +2739,7 @@ ProjectStore::Oplog::CheckPendingChunkReferences(std::span ChunkHa MissingChunks.reserve(ChunkHashes.size()); for (const IoHash& FileHash : ChunkHashes) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(FileHash); !Payload) + if (!m_CidStore.ContainsChunk(FileHash)) { MissingChunks.push_back(FileHash); } @@ -3359,7 +3363,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Store::OpenOplog"); - std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); { RwLock::SharedLockScope ProjectLock(m_ProjectLock); @@ -3367,21 +3370,35 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo if (OplogIt != m_Oplogs.end()) { - if (!VerifyPathOnDisk || Oplog::ExistsAt(OplogBasePath)) + bool ReOpen = false; + + if (VerifyPathOnDisk) { - return OplogIt->second.get(); + std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); + + if (!Oplog::ExistsAt(OplogBasePath)) + { + // Somebody deleted the oplog on disk behind our back + ProjectLock.ReleaseNow(); + std::filesystem::path DeletePath; + if (!RemoveOplog(OplogId, DeletePath)) + { + ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath); + } + + ReOpen = true; + } } - // Somebody deleted the oplog on disk behind our back - ProjectLock.ReleaseNow(); - std::filesystem::path DeletePath; - if (!RemoveOplog(OplogId, DeletePath)) + if (!ReOpen) { - ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath); + return OplogIt->second.get(); } } } + std::filesystem::path OplogBasePath = BasePathForOplog(OplogId); + RwLock::ExclusiveLockScope Lock(m_ProjectLock); if (auto It = m_Oplogs.find(std::string{OplogId}); It != m_Oplogs.end()) { diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 905ba5ab2..8a4b977ad 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -589,7 +589,7 @@ void HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, @@ -608,7 +608,7 @@ void HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, @@ -635,7 +635,7 @@ void HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, @@ -654,7 +654,7 @@ void HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, @@ -673,7 +673,7 @@ void HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, @@ -700,7 +700,7 @@ void HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req) { HttpServerRequest& ServerRequest = Req.ServerRequest(); - std::string Alias = Req.GetCapture(1); + std::string_view Alias = Req.GetCapture(1); if (Alias.empty()) { return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 25f68330a..61552fafc 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -708,11 +708,11 @@ namespace zen { ZenCacheDiskLayer::CacheBucket::CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, - std::string BucketName, + std::string_view BucketName, const BucketConfiguration& Config) : m_Gc(Gc) , m_OuterCacheMemoryUsage(OuterCacheMemoryUsage) -, m_BucketName(std::move(BucketName)) +, m_BucketName(BucketName) , m_Configuration(Config) , m_BucketId(Oid::Zero) { @@ -1329,7 +1329,7 @@ ZenCacheDiskLayer::CacheBucket::EndPutBatch(PutBatchHandle* Batch) noexcept struct ZenCacheDiskLayer::CacheBucket::GetBatchHandle { - GetBatchHandle(std::vector& OutResults) : OutResults(OutResults) + GetBatchHandle(ZenCacheValueVec_t& OutResults) : OutResults(OutResults) { Keys.reserve(OutResults.capacity()); ResultIndexes.reserve(OutResults.capacity()); @@ -1340,11 +1340,11 @@ struct ZenCacheDiskLayer::CacheBucket::GetBatchHandle std::vector Keys; std::vector ResultIndexes; - std::vector& OutResults; + ZenCacheValueVec_t& OutResults; }; ZenCacheDiskLayer::CacheBucket::GetBatchHandle* -ZenCacheDiskLayer::CacheBucket::BeginGetBatch(std::vector& OutResult) +ZenCacheDiskLayer::CacheBucket::BeginGetBatch(ZenCacheValueVec_t& OutResult) { ZEN_TRACE_CPU("Z$::Bucket::BeginGetBatch"); return new GetBatchHandle(OutResult); @@ -1364,13 +1364,13 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept if (!Batch->ResultIndexes.empty()) { - std::vector StandaloneDiskLocations; - std::vector StandaloneKeyIndexes; - std::vector MemCachedKeyIndexes; - std::vector InlineDiskLocations; - std::vector InlineBlockLocations; - std::vector InlineKeyIndexes; - std::vector FillRawHashAndRawSize(Batch->Keys.size(), false); + eastl::fixed_vector StandaloneDiskLocations; + eastl::fixed_vector StandaloneKeyIndexes; + eastl::fixed_vector MemCachedKeyIndexes; + eastl::fixed_vector InlineDiskLocations; + eastl::fixed_vector InlineBlockLocations; + eastl::fixed_vector InlineKeyIndexes; + eastl::fixed_vector FillRawHashAndRawSize(Batch->Keys.size(), false); { RwLock::SharedLockScope IndexLock(m_IndexLock); for (size_t KeyIndex = 0; KeyIndex < Batch->Keys.size(); KeyIndex++) @@ -1526,33 +1526,35 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept if (!InlineDiskLocations.empty()) { ZEN_TRACE_CPU("Z$::Bucket::EndGetBatch::ReadInline"); - m_BlockStore.IterateChunks(InlineBlockLocations, [&](uint32_t, std::span ChunkIndexes) -> bool { - // Only read into memory the IoBuffers we could potentially add to memcache - const uint64_t LargeChunkSizeLimit = Max(m_Configuration.MemCacheSizeThreshold, 1u * 1024u); - m_BlockStore.IterateBlock( - InlineBlockLocations, - ChunkIndexes, - [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, - const void* Data, - uint64_t Size) -> bool { - if (Data != nullptr) - { - FillOne(InlineDiskLocations[ChunkIndex], - InlineKeyIndexes[ChunkIndex], - IoBufferBuilder::MakeCloneFromMemory(Data, Size)); - } - return true; - }, - [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, - BlockStoreFile& File, - uint64_t Offset, - uint64_t Size) -> bool { - FillOne(InlineDiskLocations[ChunkIndex], InlineKeyIndexes[ChunkIndex], File.GetChunk(Offset, Size)); - return true; - }, - LargeChunkSizeLimit); - return true; - }); + m_BlockStore.IterateChunks( + std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, + [&](uint32_t, std::span ChunkIndexes) -> bool { + // Only read into memory the IoBuffers we could potentially add to memcache + const uint64_t LargeChunkSizeLimit = Max(m_Configuration.MemCacheSizeThreshold, 1u * 1024u); + m_BlockStore.IterateBlock( + std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, + ChunkIndexes, + [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, + const void* Data, + uint64_t Size) -> bool { + if (Data != nullptr) + { + FillOne(InlineDiskLocations[ChunkIndex], + InlineKeyIndexes[ChunkIndex], + IoBufferBuilder::MakeCloneFromMemory(Data, Size)); + } + return true; + }, + [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, + BlockStoreFile& File, + uint64_t Offset, + uint64_t Size) -> bool { + FillOne(InlineDiskLocations[ChunkIndex], InlineKeyIndexes[ChunkIndex], File.GetChunk(Offset, Size)); + return true; + }, + LargeChunkSizeLimit); + return true; + }); } if (!StandaloneDiskLocations.empty()) @@ -3581,15 +3583,29 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer() } } +template +struct equal_to_2 : public eastl::binary_function +{ + constexpr bool operator()(const T& a, const U& b) const { return a == b; } + + template, eastl::remove_const_t>>> + constexpr bool operator()(const U& b, const T& a) const + { + return b == a; + } +}; + ZenCacheDiskLayer::CacheBucket* ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) { ZEN_TRACE_CPU("Z$::GetOrCreateBucket"); - const auto BucketName = std::string(InBucket); { RwLock::SharedLockScope SharedLock(m_Lock); - if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) + if (auto It = m_Buckets.find_as(InBucket, std::hash(), equal_to_2()); + It != m_Buckets.end()) { return It->second.get(); } @@ -3597,31 +3613,32 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) // We create the bucket without holding a lock since contructor calls GcManager::AddGcReferencer which takes an exclusive lock. // This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock - std::unique_ptr Bucket( - std::make_unique(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig)); + std::unique_ptr Bucket(std::make_unique(m_Gc, m_TotalMemCachedSize, InBucket, m_Configuration.BucketConfig)); RwLock::ExclusiveLockScope Lock(m_Lock); - if (auto It = m_Buckets.find(BucketName); It != m_Buckets.end()) + if (auto It = m_Buckets.find_as(InBucket, std::hash(), equal_to_2()); + It != m_Buckets.end()) { return It->second.get(); } std::filesystem::path BucketPath = m_RootDir; - BucketPath /= BucketName; + BucketPath /= InBucket; try { if (!Bucket->OpenOrCreate(BucketPath)) { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", InBucket, m_RootDir); return nullptr; } } catch (const std::exception& Err) { - ZEN_WARN("Creating bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what()); + ZEN_WARN("Creating bucket '{}' in '{}' FAILED, reason: '{}'", InBucket, BucketPath, Err.what()); throw; } + std::string BucketName{InBucket}; CacheBucket* Result = Bucket.get(); m_Buckets.emplace(BucketName, std::move(Bucket)); if (m_CapturedBuckets) @@ -3720,7 +3737,7 @@ ZenCacheDiskLayer::EndPutBatch(PutBatchHandle* Batch) noexcept struct ZenCacheDiskLayer::GetBatchHandle { - GetBatchHandle(std::vector& OutResults) : OutResults(OutResults) {} + GetBatchHandle(ZenCacheValueVec_t& OutResults) : OutResults(OutResults) {} struct BucketHandle { CacheBucket* Bucket; @@ -3780,13 +3797,13 @@ struct ZenCacheDiskLayer::GetBatchHandle return NewBucketHandle; } - RwLock Lock; - std::vector BucketHandles; - std::vector& OutResults; + RwLock Lock; + eastl::fixed_vector BucketHandles; + ZenCacheValueVec_t& OutResults; }; ZenCacheDiskLayer::GetBatchHandle* -ZenCacheDiskLayer::BeginGetBatch(std::vector& OutResults) +ZenCacheDiskLayer::BeginGetBatch(ZenCacheValueVec_t& OutResults) { return new GetBatchHandle(OutResults); } diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index cca51e63e..97e26a38d 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -20,6 +20,8 @@ #include +#include + ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -89,7 +91,7 @@ GetRpcRequestCacheKey(const CbObjectView& KeyView, CacheKey& Key) return false; } IoHash Hash = HashField.AsHash(); - Key = CacheKey::Create(*Bucket, Hash); + Key = CacheKey::CreateValidated(std::move(*Bucket), Hash); return true; } @@ -305,7 +307,7 @@ CacheRpcHandler::HandleRpcPutCacheRecords(const CacheRequestContext& Context, co } DefaultPolicy = !PolicyText.empty() ? ParseCachePolicy(PolicyText) : CachePolicy::Default; - std::vector Results; + eastl::fixed_vector Results; CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); for (CbFieldView RequestField : RequestsArray) @@ -481,16 +483,15 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb bool Exists = false; bool ReadFromUpstream = false; }; - struct RecordRequestData + struct RecordRequestData : public CacheKeyRequest { - CacheKeyRequest Upstream; - CbObjectView RecordObject; - IoBuffer RecordCacheValue; - CacheRecordPolicy DownstreamPolicy; - std::vector Values; - bool Complete = false; - const UpstreamEndpointInfo* Source = nullptr; - uint64_t ElapsedTimeUs; + CbObjectView RecordObject; + IoBuffer RecordCacheValue; + CacheRecordPolicy DownstreamPolicy; + eastl::fixed_vector Values; + bool Complete = false; + const UpstreamEndpointInfo* Source = nullptr; + uint64_t ElapsedTimeUs; }; std::string_view PolicyText = Params["DefaultPolicy"sv].AsString(); @@ -503,8 +504,8 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb const bool HasUpstream = m_UpstreamCache.IsActive(); - std::vector Requests; - std::vector UpstreamIndexes; + eastl::fixed_vector Requests; + eastl::fixed_vector UpstreamIndexes; auto ParseValues = [](RecordRequestData& Request) { CbArrayView ValuesArray = Request.RecordObject["Values"sv].AsArrayView(); @@ -535,7 +536,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb CbObjectView RequestObject = RequestField.AsObjectView(); CbObjectView KeyObject = RequestObject["Key"sv].AsObjectView(); - CacheKey& Key = Request.Upstream.Key; + CacheKey& Key = Request.Key; if (!GetRpcRequestCacheKey(KeyObject, Key)) { return CbPackage{}; @@ -707,7 +708,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb for (size_t Index : UpstreamIndexes) { RecordRequestData& Request = Requests[Index]; - UpstreamRequests.push_back(&Request.Upstream); + UpstreamRequests.push_back(&Request); if (Request.Values.size()) { @@ -721,13 +722,13 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb UpstreamPolicy |= !Value.ReadFromUpstream ? CachePolicy::SkipData : CachePolicy::None; Builder.AddValuePolicy(Value.ValueId, UpstreamPolicy); } - Request.Upstream.Policy = Builder.Build(); + Request.Policy = Builder.Build(); } else { // We don't know which Values exist in the Record; ask the upstrem for all values that the client wants, // and convert the CacheRecordPolicy to an upstream policy - Request.Upstream.Policy = Request.DownstreamPolicy.ConvertToUpstream(); + Request.Policy = Request.DownstreamPolicy.ConvertToUpstream(); } } @@ -737,10 +738,9 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb return; } - RecordRequestData& Request = - *reinterpret_cast(reinterpret_cast(&Params.Request) - offsetof(RecordRequestData, Upstream)); + RecordRequestData& Request = *static_cast(&Params.Request); Request.ElapsedTimeUs += static_cast(Params.ElapsedSeconds * 1000000.0); - const CacheKey& Key = Request.Upstream.Key; + const CacheKey& Key = Request.Key; Stopwatch Timer; auto TimeGuard = MakeGuard([&Timer, &Request]() { Request.ElapsedTimeUs += Timer.GetElapsedTimeUs(); }); if (!Request.RecordObject) @@ -832,10 +832,12 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb CbPackage ResponsePackage; CbObjectWriter ResponseObject{2048}; + ResponsePackage.ReserveAttachments(Requests.size()); + ResponseObject.BeginArray("Result"sv); for (RecordRequestData& Request : Requests) { - const CacheKey& Key = Request.Upstream.Key; + const CacheKey& Key = Request.Key; if (Request.Complete || (Request.RecordObject && EnumHasAllFlags(Request.DownstreamPolicy.GetRecordPolicy(), CachePolicy::PartialRecord))) { @@ -910,11 +912,12 @@ CacheRpcHandler::HandleRpcPutCacheValues(const CacheRequestContext& Context, con const bool HasUpstream = m_UpstreamCache.IsActive(); CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - std::vector BatchResults; - std::vector BatchResultIndexes; - std::vector Results; - std::vector UpstreamCacheKeys; - uint64_t RequestCount = RequestsArray.Num(); + std::vector BatchResults; + eastl::fixed_vector BatchResultIndexes; + eastl::fixed_vector Results; + eastl::fixed_vector UpstreamCacheKeys; + + uint64_t RequestCount = RequestsArray.Num(); { Results.reserve(RequestCount); std::unique_ptr Batch; @@ -1099,15 +1102,15 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO uint64_t RawSize = 0; CompressedBuffer Result; }; - std::vector Requests; + eastl::fixed_vector Requests; - std::vector RemoteRequestIndexes; + eastl::fixed_vector RemoteRequestIndexes; const bool HasUpstream = m_UpstreamCache.IsActive(); - CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); - std::vector CacheValues; - const uint64_t RequestCount = RequestsArray.Num(); + CbArrayView RequestsArray = Params["Requests"sv].AsArrayView(); + ZenCacheValueVec_t CacheValues; + const uint64_t RequestCount = RequestsArray.Num(); CacheValues.reserve(RequestCount); { std::unique_ptr Batch; @@ -1136,7 +1139,6 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO CacheKey& Key = Request.Key; CachePolicy Policy = Request.Policy; - ZenCacheValue CacheValue; if (EnumHasAllFlags(Policy, CachePolicy::QueryLocal)) { if (Batch) @@ -1276,6 +1278,9 @@ CacheRpcHandler::HandleRpcGetCacheValues(const CacheRequestContext& Context, CbO ZEN_TRACE_CPU("Z$::RpcGetCacheValues::Response"); CbPackage RpcResponse; CbObjectWriter ResponseObject{1024}; + + RpcResponse.ReserveAttachments(Requests.size()); + ResponseObject.BeginArray("Result"sv); for (const RequestData& Request : Requests) { @@ -1642,7 +1647,7 @@ CacheRpcHandler::GetLocalCacheValues(const CacheRequestContext& Context, using namespace cache::detail; const bool HasUpstream = m_UpstreamCache.IsActive(); - std::vector Chunks; + ZenCacheValueVec_t Chunks; Chunks.reserve(ValueRequests.size()); { std::unique_ptr Batch; @@ -1796,6 +1801,8 @@ CacheRpcHandler::WriteGetCacheChunksResponse([[maybe_unused]] const CacheRequest CbPackage RpcResponse; CbObjectWriter Writer{1024}; + RpcResponse.ReserveAttachments(Requests.size()); + Writer.BeginArray("Result"sv); for (ChunkRequest& Request : Requests) { diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 133cb42d7..7d277329e 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -178,13 +178,13 @@ ZenCacheNamespace::EndPutBatch(PutBatchHandle* Batch) noexcept struct ZenCacheNamespace::GetBatchHandle { - GetBatchHandle(std::vector& OutResult) : Results(OutResult) {} - std::vector& Results; + GetBatchHandle(ZenCacheValueVec_t& OutResult) : Results(OutResult) {} + ZenCacheValueVec_t& Results; ZenCacheDiskLayer::GetBatchHandle* DiskLayerHandle = nullptr; }; ZenCacheNamespace::GetBatchHandle* -ZenCacheNamespace::BeginGetBatch(std::vector& OutResult) +ZenCacheNamespace::BeginGetBatch(ZenCacheValueVec_t& OutResult) { ZenCacheNamespace::GetBatchHandle* Handle = new ZenCacheNamespace::GetBatchHandle(OutResult); Handle->DiskLayerHandle = m_DiskLayer.BeginGetBatch(OutResult); @@ -580,7 +580,7 @@ ZenCacheStore::PutBatch::~PutBatch() } } -ZenCacheStore::GetBatch::GetBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, std::vector& OutResult) +ZenCacheStore::GetBatch::GetBatch(ZenCacheStore& CacheStore, std::string_view InNamespace, ZenCacheValueVec_t& OutResult) : m_CacheStore(CacheStore) , Results(OutResult) { diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index b0b4f22cb..05400c784 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -12,8 +12,9 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +#include +#include #include -#include namespace zen { @@ -169,7 +170,7 @@ public: ~ZenCacheDiskLayer(); struct GetBatchHandle; - GetBatchHandle* BeginGetBatch(std::vector& OutResult); + GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResult); void EndGetBatch(GetBatchHandle* Batch) noexcept; bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); void Get(std::string_view Bucket, const IoHash& HashKey, GetBatchHandle& BatchHandle); @@ -216,13 +217,16 @@ public: */ struct CacheBucket : public GcReferencer { - CacheBucket(GcManager& Gc, std::atomic_uint64_t& OuterCacheMemoryUsage, std::string BucketName, const BucketConfiguration& Config); + CacheBucket(GcManager& Gc, + std::atomic_uint64_t& OuterCacheMemoryUsage, + std::string_view BucketName, + const BucketConfiguration& Config); ~CacheBucket(); bool OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true); struct GetBatchHandle; - GetBatchHandle* BeginGetBatch(std::vector& OutResult); + GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResult); void EndGetBatch(GetBatchHandle* Batch) noexcept; bool Get(const IoHash& HashKey, ZenCacheValue& OutValue); void Get(const IoHash& HashKey, GetBatchHandle& BatchHandle); @@ -486,18 +490,20 @@ private: bool StartAsyncMemCacheTrim(); void MemCacheTrim(); - GcManager& m_Gc; - JobQueue& m_JobQueue; - std::filesystem::path m_RootDir; - Configuration m_Configuration; - std::atomic_uint64_t m_TotalMemCachedSize{}; - std::atomic_bool m_IsMemCacheTrimming = false; - std::atomic m_NextAllowedTrimTick; - mutable RwLock m_Lock; - std::unordered_map> m_Buckets; - std::vector> m_DroppedBuckets; - uint32_t m_UpdateCaptureRefCounter = 0; - std::unique_ptr> m_CapturedBuckets; + typedef eastl::unordered_map, std::hash, std::equal_to> BucketMap_t; + + GcManager& m_Gc; + JobQueue& m_JobQueue; + std::filesystem::path m_RootDir; + Configuration m_Configuration; + std::atomic_uint64_t m_TotalMemCachedSize{}; + std::atomic_bool m_IsMemCacheTrimming = false; + std::atomic m_NextAllowedTrimTick; + mutable RwLock m_Lock; + BucketMap_t m_Buckets; + std::vector> m_DroppedBuckets; + uint32_t m_UpdateCaptureRefCounter = 0; + std::unique_ptr> m_CapturedBuckets; ZenCacheDiskLayer(const ZenCacheDiskLayer&) = delete; ZenCacheDiskLayer& operator=(const ZenCacheDiskLayer&) = delete; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 9b45c7b21..521c78bb1 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -6,6 +6,8 @@ #include #include +#include + #include #include @@ -32,6 +34,8 @@ struct ZenCacheValue IoHash RawHash = IoHash::Zero; }; +typedef eastl::fixed_vector ZenCacheValueVec_t; + struct CacheValueDetails { struct ValueDetails diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 82fec9b0e..5e056cf2d 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -86,7 +86,7 @@ public: void EndPutBatch(PutBatchHandle* Batch) noexcept; struct GetBatchHandle; - GetBatchHandle* BeginGetBatch(std::vector& OutResults); + GetBatchHandle* BeginGetBatch(ZenCacheValueVec_t& OutResults); void EndGetBatch(GetBatchHandle* Batch) noexcept; bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue); @@ -220,14 +220,14 @@ public: class GetBatch { public: - GetBatch(ZenCacheStore& CacheStore, std::string_view Namespace, std::vector& OutResult); + GetBatch(ZenCacheStore& CacheStore, std::string_view Namespace, ZenCacheValueVec_t& OutResult); ~GetBatch(); private: ZenCacheStore& m_CacheStore; ZenCacheNamespace* m_Store = nullptr; ZenCacheNamespace::GetBatchHandle* m_NamespaceBatchHandle = nullptr; - std::vector& Results; + ZenCacheValueVec_t& Results; friend class ZenCacheStore; }; diff --git a/src/zenstore/xmake.lua b/src/zenstore/xmake.lua index f0bd64d2e..031a66829 100644 --- a/src/zenstore/xmake.lua +++ b/src/zenstore/xmake.lua @@ -8,3 +8,4 @@ target('zenstore') add_includedirs("include", {public=true}) add_deps("zencore", "zenutil") add_packages("vcpkg::robin-map") + add_packages("vcpkg::eastl", {public=true}); diff --git a/src/zenutil/include/zenutil/cache/cachekey.h b/src/zenutil/include/zenutil/cache/cachekey.h index 741375946..0ab05f4f1 100644 --- a/src/zenutil/include/zenutil/cache/cachekey.h +++ b/src/zenutil/include/zenutil/cache/cachekey.h @@ -17,6 +17,12 @@ struct CacheKey static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; } + // This should be used whenever the bucket name has already been validated to avoid redundant ToLower calls + static CacheKey CreateValidated(std::string&& BucketValidated, const IoHash& Hash) + { + return {.Bucket = std::move(BucketValidated), .Hash = Hash}; + } + auto operator<=>(const CacheKey& that) const { if (auto b = caseSensitiveCompareStrings(Bucket, that.Bucket); b != std::strong_ordering::equal) -- cgit v1.2.3 From 9b24647facccc9c7848a52f1f4c5e32055bf2f01 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 7 Mar 2025 09:58:20 +0100 Subject: partial block fetch (#298) - Improvement: Do partial requests of blocks if not all of the block is needed - Improvement: Better progress/statistics on download - Bugfix: Ensure that temporary folder for Jupiter downloads exists during verify phase --- src/zen/cmds/builds_cmd.cpp | 1099 ++++++++++++++------ src/zen/cmds/builds_cmd.h | 1 + src/zenhttp/httpclient.cpp | 161 +-- src/zenutil/filebuildstorage.cpp | 17 +- src/zenutil/include/zenutil/buildstorage.h | 5 +- .../include/zenutil/jupiter/jupitersession.h | 4 +- src/zenutil/jupiter/jupiterbuildstorage.cpp | 10 +- src/zenutil/jupiter/jupitersession.cpp | 12 +- 8 files changed, 907 insertions(+), 402 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index eb0650c4d..1c9476b96 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -231,7 +231,10 @@ namespace { return AuthToken; } - CompositeBuffer WriteToTempFileIfNeeded(const CompositeBuffer& Buffer, const std::filesystem::path& TempFolderPath, const IoHash& Hash) + CompositeBuffer WriteToTempFileIfNeeded(const CompositeBuffer& Buffer, + const std::filesystem::path& TempFolderPath, + const IoHash& Hash, + const std::string& Suffix = {}) { // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory std::span Segments = Buffer.GetSegments(); @@ -241,7 +244,7 @@ namespace { { return Buffer; } - std::filesystem::path TempFilePath = (TempFolderPath / Hash.ToHexString()).make_preferred(); + std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); return CompositeBuffer(WriteToTempFile(Buffer, TempFilePath)); } @@ -302,7 +305,7 @@ namespace { return FilteredPerSecond; } - uint64_t GetElapsedTime() const + uint64_t GetElapsedTimeUS() const { if (StartTimeUS == (uint64_t)-1) { @@ -1738,8 +1741,8 @@ namespace { ProgressBar.Finish(); - GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTime(); - UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTime(); + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); } } @@ -2069,9 +2072,9 @@ namespace { }); ProgressBar.Finish(); - UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTime(); - GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTime(); - LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTime(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); + GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTimeUS(); + LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); } } @@ -3358,17 +3361,8 @@ namespace { return ChunkTargetPtrs; }; - bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& RemoteContent, - std::span> SequenceIndexChunksLeftToWriteCounters, - const CompositeBuffer& DecompressedBlockBuffer, - const ChunkedContentLookup& Lookup, - std::atomic* RemoteChunkIndexNeedsCopyFromSourceFlags, - std::atomic& OutChunksComplete, - std::atomic& OutBytesWritten) + struct BlockWriteOps { - ZEN_TRACE_CPU("WriteBlockToDisk"); - std::vector ChunkBuffers; struct WriteOpData { @@ -3376,7 +3370,79 @@ namespace { size_t ChunkBufferIndex; }; std::vector WriteOps; + }; + void WriteBlockChunkOps(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + std::span> SequenceIndexChunksLeftToWriteCounters, + const BlockWriteOps Ops, + std::atomic& OutChunksComplete, + std::atomic& OutBytesWritten) + { + ZEN_TRACE_CPU("WriteBlockChunkOps"); + { + WriteFileCache OpenFileCache; + for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) + { + if (AbortFlag) + { + break; + } + const CompositeBuffer& Chunk = Ops.ChunkBuffers[WriteOp.ChunkBufferIndex]; + const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); + const uint64_t ChunkSize = Chunk.GetSize(); + const uint64_t FileOffset = WriteOp.Target->Offset; + const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; + ZEN_ASSERT(FileOffset + ChunkSize <= RemoteContent.RawSizes[PathIndex]); + + OpenFileCache.WriteToFile( + SequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + Chunk, + FileOffset, + RemoteContent.RawSizes[PathIndex]); + OutBytesWritten += ChunkSize; + } + } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) + { + const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = + IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written hunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + OutChunksComplete += gsl::narrow(Ops.ChunkBuffers.size()); + } + } + + bool GetBlockWriteOps(const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + const CompositeBuffer& DecompressedBlockBuffer, + BlockWriteOps& OutOps) + { + ZEN_TRACE_CPU("GetBlockWriteOps"); SharedBuffer BlockBuffer = DecompressedBlockBuffer.Flatten(); uint64_t HeaderSize = 0; if (IterateChunkBlock( @@ -3402,91 +3468,207 @@ namespace { ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { - WriteOps.push_back(WriteOpData{.Target = Target, .ChunkBufferIndex = ChunkBuffers.size()}); + OutOps.WriteOps.push_back( + BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()}); } - ChunkBuffers.emplace_back(std::move(Decompressed)); + OutOps.ChunkBuffers.emplace_back(std::move(Decompressed)); } } } }, HeaderSize)) { - if (!WriteOps.empty()) + std::sort(OutOps.WriteOps.begin(), + OutOps.WriteOps.end(), + [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + + return true; + } + else + { + return false; + } + } + + bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + const CompositeBuffer& BlockBuffer, + const ChunkedContentLookup& Lookup, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + std::atomic& OutChunksComplete, + std::atomic& OutBytesWritten) + { + ZEN_TRACE_CPU("WriteBlockToDisk"); + + IoHash BlockRawHash; + uint64_t BlockRawSize; + CompressedBuffer CompressedBlockBuffer = CompressedBuffer::FromCompressed(BlockBuffer, BlockRawHash, BlockRawSize); + if (!CompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", BlockDescription.BlockHash)); + } + + if (BlockRawHash != BlockDescription.BlockHash) + { + throw std::runtime_error( + fmt::format("Block {} header has a mismatching raw hash {}", BlockDescription.BlockHash, BlockRawHash)); + } + + CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); + if (!DecompressedBlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} failed to decompress", BlockDescription.BlockHash)); + } + + ZEN_ASSERT_SLOW(BlockDescription.BlockHash == IoHash::HashBuffer(DecompressedBlockBuffer)); + + BlockWriteOps Ops; + if (GetBlockWriteOps(RemoteContent, + Lookup, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DecompressedBlockBuffer, + Ops)) + { + WriteBlockChunkOps(CacheFolderPath, + RemoteContent, + Lookup, + SequenceIndexChunksLeftToWriteCounters, + Ops, + OutChunksComplete, + OutBytesWritten); + return true; + } + return false; + } + + bool GetPartialBlockWriteOps(const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + const CompositeBuffer& PartialBlockBuffer, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + BlockWriteOps& OutOps) + { + ZEN_TRACE_CPU("GetPartialBlockWriteOps"); + uint32_t OffsetInBlock = 0; + for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++) + { + const uint32_t ChunkCompressedSize = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) { - if (!AbortFlag) + const uint32_t ChunkIndex = It->second; + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, Lookup, ChunkIndex); + + if (!ChunkTargetPtrs.empty()) { - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOpData& Lhs, const WriteOpData& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + bool NeedsWrite = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) + { + CompositeBuffer Chunk = PartialBlockBuffer.Mid(OffsetInBlock, ChunkCompressedSize); + IoHash VerifyChunkHash; + uint64_t VerifyRawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Chunk, VerifyChunkHash, VerifyRawSize); + if (!Compressed) { - return true; + ZEN_ASSERT(false); } - if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + if (VerifyChunkHash != ChunkHash) { - return false; + ZEN_ASSERT(false); } - return Lhs.Target->Offset < Rhs.Target->Offset; - }); - - { - WriteFileCache OpenFileCache; - for (const WriteOpData& WriteOp : WriteOps) + if (VerifyRawSize != BlockDescription.ChunkRawLengths[ChunkBlockIndex]) { - if (AbortFlag) - { - break; - } - const CompositeBuffer& Chunk = ChunkBuffers[WriteOp.ChunkBufferIndex]; - const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); - const uint64_t ChunkSize = Chunk.GetSize(); - const uint64_t FileOffset = WriteOp.Target->Offset; - const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; - ZEN_ASSERT(FileOffset + ChunkSize <= RemoteContent.RawSizes[PathIndex]); - - OpenFileCache.WriteToFile( - SequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName(CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - Chunk, - FileOffset, - RemoteContent.RawSizes[PathIndex]); - OutBytesWritten += ChunkSize; + ZEN_ASSERT(false); } - } - if (!AbortFlag) - { - ZEN_TRACE_CPU("WriteBlockToDisk_VerifyHash"); - - // Write tracking, updating this must be done without any files open (WriteFileCache) - for (const WriteOpData& WriteOp : WriteOps) + CompositeBuffer Decompressed = Compressed.DecompressToComposite(); + if (!Decompressed) { - const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) - { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = IoHash::HashBuffer( - IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); - } + throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); } + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) + { + OutOps.WriteOps.push_back( + BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()}); + } + OutOps.ChunkBuffers.emplace_back(std::move(Decompressed)); } - OutChunksComplete += gsl::narrow(ChunkBuffers.size()); } } + + OffsetInBlock += ChunkCompressedSize; + } + std::sort(OutOps.WriteOps.begin(), + OutOps.WriteOps.end(), + [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + return true; + } + + bool WritePartialBlockToDisk(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + const CompositeBuffer& PartialBlockBuffer, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, + const ChunkedContentLookup& Lookup, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + std::atomic& OutChunksComplete, + std::atomic& OutBytesWritten) + { + ZEN_TRACE_CPU("WritePartialBlockToDisk"); + BlockWriteOps Ops; + if (GetPartialBlockWriteOps(RemoteContent, + Lookup, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + PartialBlockBuffer, + FirstIncludedBlockChunkIndex, + LastIncludedBlockChunkIndex, + Ops)) + { + WriteBlockChunkOps(CacheFolderPath, + RemoteContent, + Lookup, + SequenceIndexChunksLeftToWriteCounters, + Ops, + OutChunksComplete, + OutBytesWritten); return true; } - return false; + else + { + return false; + } } SharedBuffer Decompress(const CompositeBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) @@ -3571,6 +3753,7 @@ namespace { CompositeBuffer&& CompressedPart, std::atomic& WriteToDiskBytes) { + ZEN_TRACE_CPU("StreamDecompress"); const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); TemporaryFile DecompressedTemp; std::error_code Ec; @@ -3633,10 +3816,9 @@ namespace { WorkerThreadPool& NetworkPool, std::atomic& WriteToDiskBytes, std::atomic& BytesDownloaded, - std::atomic& LooseChunksBytes, - std::atomic& DownloadedChunks, - std::atomic& ChunksComplete, - std::atomic& MultipartAttachmentCount) + std::atomic& MultipartAttachmentCount, + std::function&& OnDownloadComplete, + std::function&& OnWriteComplete) { ZEN_TRACE_CPU("DownloadLargeBlob"); @@ -3665,22 +3847,20 @@ namespace { Workload, ChunkHash, &BytesDownloaded, - &LooseChunksBytes, + OnDownloadComplete = std::move(OnDownloadComplete), + OnWriteComplete = std::move(OnWriteComplete), &WriteToDiskBytes, - &DownloadedChunks, - &ChunksComplete, SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs = std::vector( ChunkTargetPtrs)](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { BytesDownloaded += Chunk.GetSize(); - LooseChunksBytes += Chunk.GetSize(); if (!AbortFlag.load()) { Workload->TempFile.Write(Chunk.GetView(), Offset); if (Chunk.GetSize() == BytesRemaining) { - DownloadedChunks++; + OnDownloadComplete(Workload->TempFile.FileSize()); Work.ScheduleWork( WritePool, // GetSyncWorkerPool(),// @@ -3690,8 +3870,7 @@ namespace { ChunkHash, Workload, Offset, - BytesRemaining, - &ChunksComplete, + OnWriteComplete = std::move(OnWriteComplete), &WriteToDiskBytes, SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs](std::atomic&) { @@ -3725,7 +3904,7 @@ namespace { CompositeBuffer(std::move(CompressedPart)), WriteToDiskBytes); NeedHashVerify = false; - ChunksComplete++; + OnWriteComplete(); } else { @@ -3748,7 +3927,7 @@ namespace { CompositeBuffer(Chunk), OpenFileCache, WriteToDiskBytes); - ChunksComplete++; + OnWriteComplete(); } } @@ -3760,12 +3939,12 @@ namespace { const uint32_t RemoteSequenceIndex = Location->SequenceIndex; if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { - ZEN_TRACE_CPU("VerifyChunkHash"); - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; if (NeedHashVerify) { + ZEN_TRACE_CPU("VerifyChunkHash"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); if (VerifyChunkHash != ChunkHash) @@ -3816,6 +3995,7 @@ namespace { const ChunkedFolderContent& RemoteContent, const std::vector& BlockDescriptions, const std::vector& LooseChunkHashes, + bool AllowPartialBlockRequests, bool WipeTargetFolder, FolderContent& OutLocalFolderState) { @@ -3967,11 +4147,17 @@ namespace { } } - std::atomic ChunkCountWritten = 0; + uint64_t TotalRequestCount = 0; + std::atomic RequestsComplete = 0; + std::atomic ChunkCountWritten = 0; + std::atomic TotalPartWriteCount = 0; + std::atomic WritePartsComplete = 0; { ZEN_TRACE_CPU("HandleChunks"); + Stopwatch WriteTimer; + FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; @@ -4010,6 +4196,8 @@ namespace { } else { + TotalRequestCount++; + TotalPartWriteCount++; Work.ScheduleWork( NetworkPool, // GetSyncWorkerPool(),// [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { @@ -4020,25 +4208,39 @@ namespace { FilteredDownloadedBytesPerSecond.Start(); if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - DownloadLargeBlob(Storage, - Path / ZenTempChunkFolderName, - CacheFolderPath, - RemoteContent, - RemoteLookup, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - NetworkPool, - WriteToDiskBytes, - BytesDownloaded, - LooseChunksBytes, - DownloadedChunks, - ChunkCountWritten, - MultipartAttachmentCount); + DownloadLargeBlob( + Storage, + Path / ZenTempChunkFolderName, + CacheFolderPath, + RemoteContent, + RemoteLookup, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + NetworkPool, + WriteToDiskBytes, + BytesDownloaded, + MultipartAttachmentCount, + [&](uint64_t BytesDownloaded) { + LooseChunksBytes += BytesDownloaded; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + }, + [&]() { + ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + }); } else { @@ -4049,6 +4251,11 @@ namespace { } BytesDownloaded += CompressedPart.GetSize(); LooseChunksBytes += CompressedPart.GetSize(); + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(CompressedPart)), Path / ZenTempChunkFolderName, ChunkHash); @@ -4077,6 +4284,11 @@ namespace { CompositeBuffer(CompressedPart), WriteToDiskBytes); ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } NeedHashVerify = false; } else @@ -4096,10 +4308,17 @@ namespace { OpenFileCache, WriteToDiskBytes); ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } } if (!AbortFlag) { + WritePartsComplete++; + // Write tracking, updating this must be done without any files open // (WriteFileCache) for (const ChunkedContentLookup::ChunkSequenceLocation* Location : @@ -4109,12 +4328,12 @@ namespace { if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub( 1) == 1) { - ZEN_TRACE_CPU("UpdateFolder_VerifyHash"); - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; if (NeedHashVerify) { + ZEN_TRACE_CPU("UpdateFolder_VerifyHash"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( GetTempChunkedSequenceFileName(CacheFolderPath, @@ -4155,6 +4374,8 @@ namespace { break; } + TotalPartWriteCount++; + Work.ScheduleWork( WritePool, // GetSyncWorkerPool(),// [&, CopyDataIndex](std::atomic&) { @@ -4166,245 +4387,428 @@ namespace { const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - if (!CopyData.TargetChunkLocationPtrs.empty()) - { - uint64_t CacheLocalFileBytesRead = 0; + ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - size_t TargetStart = 0; - const std::span AllTargets( - CopyData.TargetChunkLocationPtrs); + uint64_t CacheLocalFileBytesRead = 0; - struct WriteOp - { - const ChunkedContentLookup::ChunkSequenceLocation* Target; - uint64_t CacheFileOffset; - uint64_t ChunkSize; - }; + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target; + uint64_t CacheFileOffset; + uint64_t ChunkSize; + }; - std::vector WriteOps; - WriteOps.reserve(AllTargets.size()); + std::vector WriteOps; + WriteOps.reserve(AllTargets.size()); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + { + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) - { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkSize = ChunkTarget.ChunkRawSize}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkSize = ChunkTarget.ChunkRawSize}); } + TargetStart += ChunkTarget.TargetChunkLocationCount; + } - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { return false; - }); + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); - if (!AbortFlag) + if (!AbortFlag) + { + BufferedOpenFile SourceFile(LocalFilePath); + WriteFileCache OpenFileCache; + for (const WriteOp& Op : WriteOps) { - BufferedOpenFile SourceFile(LocalFilePath); - WriteFileCache OpenFileCache; - for (const WriteOp& Op : WriteOps) + if (AbortFlag) { - if (AbortFlag) - { - break; - } - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); - const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = Op.ChunkSize; - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); - - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - - OpenFileCache.WriteToFile( - RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); - WriteToDiskBytes += ChunkSize; - CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? + break; } + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = Op.ChunkSize; + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); + + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + + OpenFileCache.WriteToFile( + RemoteSequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName( + CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); + WriteToDiskBytes += ChunkSize; + CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? } - if (!AbortFlag) + } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const WriteOp& Op : WriteOps) { - if (!AbortFlag) + ZEN_TRACE_CPU("UpdateFolder_Copy_VerifyHash"); + + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { - // Write tracking, updating this must be done without any files open (WriteFileCache) - for (const WriteOp& Op : WriteOps) + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) { - ZEN_TRACE_CPU("UpdateFolder_Copy_VerifyHash"); - - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) - { - const IoHash& SequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - - ZEN_TRACE_CPU("UpdateFolder_Copy_rename"); - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); - } + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); } - } - ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + ZEN_TRACE_CPU("UpdateFolder_Copy_rename"); + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } } + + ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); + ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); } } }, Work.DefaultErrorFunction()); } - size_t BlockCount = BlockDescriptions.size(); - std::atomic BlocksComplete = 0; - - auto IsBlockNeeded = [&RemoteContent, &RemoteLookup, &RemoteChunkIndexNeedsCopyFromSourceFlags]( - const ChunkBlockDescription& BlockDescription) -> bool { - for (const IoHash& ChunkHash : BlockDescription.ChunkRawHashes) - { - if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = It->second; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) - { - return true; - } - } - } - return false; + size_t BlockCount = BlockDescriptions.size(); + + std::vector ChunkIsPickedUpByBlock(RemoteContent.ChunkedContent.ChunkHashes.size(), false); + auto GetNeededChunkBlockIndexes = [&RemoteContent, + &RemoteLookup, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) { + std::vector NeededBlockChunkIndexes; + for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) + { + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) + { + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + { + ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; + NeededBlockChunkIndexes.push_back(ChunkBlockIndex); + } + } + } + } + return NeededBlockChunkIndexes; }; - size_t BlocksNeededCount = 0; + size_t BlocksNeededCount = 0; + uint64_t AllBlocksSize = 0; + uint64_t AllBlocksFetch = 0; + uint64_t AllBlocksSlack = 0; + uint64_t AllBlockRequests = 0; + uint64_t AllBlockChunksSize = 0; for (size_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { if (Work.IsAborted()) { break; } - if (IsBlockNeeded(BlockDescriptions[BlockIndex])) + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); + if (!BlockChunkIndexNeeded.empty()) { - BlocksNeededCount++; - Work.ScheduleWork( - NetworkPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) + bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = (BlockDescription.HeaderSize > 0) && (BlockDescription.ChunkCompressedLengths.size() == + BlockDescription.ChunkRawHashes.size()); + if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + { + struct BlockRangeDescriptor + { + uint64_t RangeStart = 0; + uint64_t RangeLength = 0; + uint32_t ChunkBlockIndexStart = 0; + uint32_t ChunkBlockIndexCount = 0; + }; + std::vector BlockRanges; + + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalisys"); + + uint32_t NeedBlockChunkIndexOffset = 0; + uint32_t ChunkBlockIndex = 0; + uint32_t CurrentOffset = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + BlockRangeDescriptor NextRange; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && + ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) + { + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Read"); - - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescriptions[BlockIndex].BlockHash); - if (!BlockBuffer) + if (NextRange.RangeLength > 0) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescriptions[BlockIndex].BlockHash)); + BlockRanges.push_back(NextRange); + NextRange = {}; } - BytesDownloaded += BlockBuffer.GetSize(); - BlockBytes += BlockBuffer.GetSize(); - DownloadedBlocks++; - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), - Path / ZenTempBlockFolderName, - BlockDescriptions[BlockIndex].BlockHash); - - if (!AbortFlag) + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + AllBlockChunksSize += ChunkCompressedLength; + if (NextRange.RangeLength == 0) { - Work.ScheduleWork( - WritePool, - [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Write"); - - FilteredWrittenBytesPerSecond.Start(); - IoHash BlockRawHash; - uint64_t BlockRawSize; - CompressedBuffer CompressedBlockBuffer = - CompressedBuffer::FromCompressed(std::move(BlockBuffer), BlockRawHash, BlockRawSize); - if (!CompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", - BlockDescriptions[BlockIndex].BlockHash)); - } + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; + } + else + { + ZEN_ASSERT(false); + } + } + AllBlocksSize += CurrentOffset; + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + NextRange = {}; + } - if (BlockRawHash != BlockDescriptions[BlockIndex].BlockHash) - { - throw std::runtime_error(fmt::format("Block {} header has a mismatching raw hash {}", - BlockDescriptions[BlockIndex].BlockHash, - BlockRawHash)); - } + ZEN_ASSERT(!BlockRanges.empty()); + std::vector CollapsedBlockRanges; + auto It = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*It++); + uint64_t TotalSlack = 0; + while (It != BlockRanges.end()) + { + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; + if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + { + LastRange.ChunkBlockIndexCount = + (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + TotalSlack += Slack; + } + else + { + CollapsedBlockRanges.push_back(*It); + } + ++It; + } + + uint64_t TotalFetch = 0; + for (const BlockRangeDescriptor& Range : CollapsedBlockRanges) + { + TotalFetch += Range.RangeLength; + } + + AllBlocksFetch += TotalFetch; + AllBlocksSlack += TotalSlack; + BlocksNeededCount++; + AllBlockRequests += CollapsedBlockRanges.size(); + + for (size_t BlockRangeIndex = 0; BlockRangeIndex < CollapsedBlockRanges.size(); BlockRangeIndex++) + { + TotalRequestCount++; + TotalPartWriteCount++; + const BlockRangeDescriptor BlockRange = CollapsedBlockRanges[BlockRangeIndex]; + // Partial block schedule + Work.ScheduleWork( + NetworkPool, // NetworkPool, // GetSyncWorkerPool() + [&, BlockIndex, BlockRange](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialGet"); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + BytesDownloaded += BlockBuffer.GetSize(); + BlockBytes += BlockBuffer.GetSize(); + DownloadedBlocks++; + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), + Path / ZenTempBlockFolderName, + BlockDescription.BlockHash, + fmt::format("_{}", BlockRange.RangeStart)); + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); - if (!DecompressedBlockBuffer) + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, // WritePool, // GetSyncWorkerPool(), + [&, BlockIndex, BlockRange, BlockPartialBuffer = std::move(Payload)](std::atomic&) { + if (!AbortFlag) { - throw std::runtime_error(fmt::format("Block {} failed to decompress", - BlockDescriptions[BlockIndex].BlockHash)); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + BlockPartialBuffer, + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + ChunkCountWritten, + WriteToDiskBytes)) + { + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } + }, + [&, BlockIndex](const std::exception& Ex, std::atomic&) { + ZEN_ERROR("Failed writing block {}. Reason: {}", + BlockDescriptions[BlockIndex].BlockHash, + Ex.what()); + AbortFlag = true; + }); + } + }, + Work.DefaultErrorFunction()); + } + } + else + { + BlocksNeededCount++; + TotalRequestCount++; + TotalPartWriteCount++; + + Work.ScheduleWork( + NetworkPool, // GetSyncWorkerPool(), // NetworkPool, + [&, BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Get"); + + // Full block schedule - ZEN_ASSERT_SLOW(BlockDescriptions[BlockIndex].BlockHash == - IoHash::HashBuffer(DecompressedBlockBuffer)); - - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - SequenceIndexChunksLeftToWriteCounters, - DecompressedBlockBuffer, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags.data(), - ChunkCountWritten, - WriteToDiskBytes)) + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + BytesDownloaded += BlockBuffer.GetSize(); + BlockBytes += BlockBuffer.GetSize(); + DownloadedBlocks++; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), + Path / ZenTempBlockFolderName, + BlockDescription.BlockHash); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { + if (!AbortFlag) { - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescriptions[BlockIndex].BlockHash)); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + BlockBuffer, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + ChunkCountWritten, + WriteToDiskBytes)) + { + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - BlocksComplete++; - } - }, - [&, BlockIndex](const std::exception& Ex, std::atomic&) { - ZEN_ERROR("Failed writing block {}. Reason: {}", - BlockDescriptions[BlockIndex].BlockHash, - Ex.what()); - AbortFlag = true; - }); + }, + Work.DefaultErrorFunction()); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } else { ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); } } - + ZEN_DEBUG("Fetching {} with {} slack (ideal {}) out of {} using {} requests for {} blocks", + NiceBytes(AllBlocksFetch), + NiceBytes(AllBlocksSlack), + NiceBytes(AllBlockChunksSize), + NiceBytes(AllBlocksSize), + AllBlockRequests, + BlocksNeededCount); ZEN_TRACE_CPU("HandleChunks_Wait"); Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -4412,13 +4816,13 @@ namespace { ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load()); FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load()); - std::string Details = fmt::format("{}/{} chunks. {}/{} blocks. {} {}bits/s downloaded. {} {}B/s written", - ChunkCountWritten.load(), - ChunkCountToWrite, - BlocksComplete.load(), - BlocksNeededCount, + std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", + RequestsComplete.load(), + TotalRequestCount, NiceBytes(BytesDownloaded.load()), NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), + ChunkCountWritten.load(), + ChunkCountToWrite, NiceBytes(WriteToDiskBytes.load()), NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); WriteProgressBar.UpdateState({.Task = "Writing chunks ", @@ -4437,6 +4841,13 @@ namespace { } WriteProgressBar.Finish(); + + ZEN_CONSOLE("Downloaded {} ({}bits/s). Wrote {} ({}B/s). Completed in {}", + NiceBytes(BytesDownloaded.load()), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), BytesDownloaded * 8)), + NiceBytes(WriteToDiskBytes.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), WriteToDiskBytes.load())), + NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); } for (const auto& SequenceIndexChunksLeftToWriteCounter : SequenceIndexChunksLeftToWriteCounters) @@ -4455,6 +4866,7 @@ namespace { }); // Move all files we will reuse to cache folder + // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; @@ -5186,6 +5598,7 @@ namespace { std::span BuildPartNames, const std::filesystem::path& Path, bool AllowMultiparts, + bool AllowPartialBlockRequests, bool WipeTargetFolder, bool PostDownloadVerify) { @@ -5202,8 +5615,8 @@ namespace { CreateDirectories(Path / ZenTempBlockFolderName); CreateDirectories(Path / ZenTempChunkFolderName); // TODO: Don't clear this - pick up files -> chunks to use CreateDirectories(Path / - ZenTempCacheFolderName); // TODO: Don't clear this - pick up files and use as sequences (non .tmp extension) and - // delete .tmp (maybe?) - chunk them? How do we know the file is worth chunking? + ZenTempCacheFolderName); // TODO: Don't clear this - pick up files and use as sequences (non .tmp extension) + // and delete .tmp (maybe?) - chunk them? How do we know the file is worth chunking? std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -5311,6 +5724,7 @@ namespace { RemoteContent, BlockDescriptions, LooseChunkHashes, + AllowPartialBlockRequests, WipeTargetFolder, LocalFolderState); @@ -5705,6 +6119,12 @@ BuildsCommand::BuildsCommand() "Allow large attachments to be transfered using multipart protocol. Defaults to true.", cxxopts::value(m_AllowMultiparts), ""); + m_DownloadOptions.add_option("", + "", + "allow-partial-block-requests", + "Allow request for partial chunk blocks. Defaults to true.", + cxxopts::value(m_AllowPartialBlockRequests), + ""); m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), ""); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); @@ -5728,6 +6148,18 @@ BuildsCommand::BuildsCommand() AddOutputOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); + m_TestOptions.add_option("", + "", + "allow-multipart", + "Allow large attachments to be transfered using multipart protocol. Defaults to true.", + cxxopts::value(m_AllowMultiparts), + ""); + m_TestOptions.add_option("", + "", + "allow-partial-block-requests", + "Allow request for partial chunk blocks. Defaults to true.", + cxxopts::value(m_AllowPartialBlockRequests), + ""); m_TestOptions.parse_positional({"local-path"}); m_TestOptions.positional_help("local-path"); @@ -6037,7 +6469,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Bucket, GeneratedBuildId ? "Generated " : "", BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } @@ -6175,7 +6606,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } @@ -6190,7 +6620,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } - DownloadFolder(*Storage, BuildId, BuildPartIds, m_BuildPartNames, m_Path, m_AllowMultiparts, m_Clean, m_PostDownloadVerify); + DownloadFolder(*Storage, + BuildId, + BuildPartIds, + m_BuildPartNames, + m_Path, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + m_Clean, + m_PostDownloadVerify); if (false) { @@ -6275,7 +6713,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } @@ -6335,7 +6772,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, true, true); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -6347,7 +6784,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -6449,7 +6886,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -6487,7 +6924,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, false, true); + DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -6495,7 +6932,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false, true); + DownloadFolder(*Storage, + BuildId2, + {BuildPartId2}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -6503,7 +6948,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, m_AllowMultiparts, false, true); + DownloadFolder(*Storage, + BuildId2, + {BuildPartId2}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -6549,7 +7002,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } @@ -6617,7 +7069,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - CreateDirectories(m_Path / ZenTempStorageFolderName); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index c54fb4db1..838a17807 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -49,6 +49,7 @@ private: bool m_Clean = false; uint8_t m_BlockReuseMinPercentLimit = 85; bool m_AllowMultiparts = true; + bool m_AllowPartialBlockRequests = true; std::filesystem::path m_ManifestPath; // Direct access token (may expire) diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index e4c6d243d..30711a432 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -1148,6 +1148,30 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold return true; }; + uint64_t RequestedContentLength = (uint64_t)-1; + if (auto RangeIt = AdditionalHeader.Entries.find("Range"); RangeIt != AdditionalHeader.Entries.end()) + { + if (RangeIt->second.starts_with("bytes")) + { + size_t RangeStartPos = RangeIt->second.find('=', 5); + if (RangeStartPos != std::string::npos) + { + RangeStartPos++; + size_t RangeSplitPos = RangeIt->second.find('-', RangeStartPos); + if (RangeSplitPos != std::string::npos) + { + std::optional RequestedRangeStart = + ParseInt(RangeIt->second.substr(RangeStartPos, RangeSplitPos - RangeStartPos)); + std::optional RequestedRangeEnd = ParseInt(RangeIt->second.substr(RangeStartPos + 1)); + if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value()) + { + RequestedContentLength = RequestedRangeEnd.value() - 1; + } + } + } + } + } + cpr::Response Response; { std::vector> ReceivedHeaders; @@ -1155,10 +1179,10 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold std::pair Header = GetHeader(header); if (Header.first == "Content-Length"sv) { - std::optional ContentSize = ParseInt(Header.second); - if (ContentSize.has_value()) + std::optional ContentLength = ParseInt(Header.second); + if (ContentLength.has_value()) { - if (ContentSize.value() > 1024 * 1024) + if (ContentLength.value() > 1024 * 1024) { PayloadFile = std::make_unique(); std::error_code Ec = PayloadFile->Open(TempFolderPath); @@ -1172,7 +1196,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold } else { - PayloadString.reserve(ContentSize.value()); + PayloadString.reserve(ContentLength.value()); } } } @@ -1218,85 +1242,90 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold auto It = Response.header.find("Content-Length"); if (It != Response.header.end()) { - std::optional ContentLength = ParseInt(It->second); - if (ContentLength) - { - std::vector> ReceivedHeaders; + std::vector> ReceivedHeaders; - auto HeaderCallback = [&](std::string header, intptr_t) { - std::pair Header = GetHeader(header); - if (!Header.first.empty()) - { - ReceivedHeaders.emplace_back(std::move(Header)); - } + auto HeaderCallback = [&](std::string header, intptr_t) { + std::pair Header = GetHeader(header); + if (!Header.first.empty()) + { + ReceivedHeaders.emplace_back(std::move(Header)); + } - if (Header.first == "Content-Range"sv) + if (Header.first == "Content-Range"sv) + { + if (Header.second.starts_with("bytes "sv)) { - if (Header.second.starts_with("bytes "sv)) + size_t RangeStartEnd = Header.second.find('-', 6); + if (RangeStartEnd != std::string::npos) { - size_t RangeStartEnd = Header.second.find('-', 6); - if (RangeStartEnd != std::string::npos) + const auto Start = ParseInt(Header.second.substr(6, RangeStartEnd - 6)); + if (Start) { - const auto Start = ParseInt(Header.second.substr(6, RangeStartEnd - 6)); - if (Start) + uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); + if (Start.value() == DownloadedSize) { - uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); - if (Start.value() == DownloadedSize) - { - return 1; - } - else if (Start.value() > DownloadedSize) - { - return 0; - } - if (PayloadFile) - { - PayloadFile->ResetWritePos(Start.value()); - } - else - { - PayloadString = PayloadString.substr(0, Start.value()); - } return 1; } + else if (Start.value() > DownloadedSize) + { + return 0; + } + if (PayloadFile) + { + PayloadFile->ResetWritePos(Start.value()); + } + else + { + PayloadString = PayloadString.substr(0, Start.value()); + } + return 1; } } - return 0; } - return 1; - }; + return 0; + } + return 1; + }; - KeyValueMap HeadersWithRange(AdditionalHeader); - do - { - uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); + KeyValueMap HeadersWithRange(AdditionalHeader); + do + { + uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); - std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength.value() - 1); - if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) + uint64_t ContentLength = RequestedContentLength; + if (ContentLength == uint64_t(-1)) + { + if (auto ParsedContentLength = ParseInt(It->second); ParsedContentLength.has_value()) { - if (RangeIt->second == Range) - { - // If we didn't make any progress, abort - break; - } + ContentLength = ParsedContentLength.value(); } - HeadersWithRange.Entries.insert_or_assign("Range", Range); - - Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, - Url, - m_ConnectionSettings, - HeadersWithRange, - {}, - m_SessionId, - GetAccessToken()); - Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); - for (const std::pair& H : ReceivedHeaders) + } + + std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength - 1); + if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) + { + if (RangeIt->second == Range) { - Response.header.insert_or_assign(H.first, H.second); + // If we didn't make any progress, abort + break; } - ReceivedHeaders.clear(); - } while (ShouldResume(Response)); - } + } + HeadersWithRange.Entries.insert_or_assign("Range", Range); + + Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, + Url, + m_ConnectionSettings, + HeadersWithRange, + {}, + m_SessionId, + GetAccessToken()); + Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); + for (const std::pair& H : ReceivedHeaders) + { + Response.header.insert_or_assign(H.first, H.second); + } + ReceivedHeaders.clear(); + } while (ShouldResume(Response)); } } } diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index a4bb759e7..e57109006 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -325,7 +325,7 @@ public: return {}; } - virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override { ZEN_UNUSED(BuildId); SimulateLatency(0, 0); @@ -337,10 +337,19 @@ public: if (std::filesystem::is_regular_file(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); - IoBuffer Payload = File.ReadAll(); - ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload)))); - m_Stats.TotalBytesRead += Payload.GetSize(); + IoBuffer Payload; + if (RangeOffset != 0 || RangeBytes != (uint64_t)-1) + { + Payload = IoBuffer(RangeBytes); + File.Read(Payload.GetMutableView().GetData(), RangeBytes, RangeOffset); + } + else + { + Payload = File.ReadAll(); + ZEN_ASSERT_SLOW(ValidateCompressedBuffer(RawHash, CompositeBuffer(SharedBuffer(Payload)))); + } Payload.SetContentType(ZenContentType::kCompressedBinary); + m_Stats.TotalBytesRead += Payload.GetSize(); SimulateLatency(0, Payload.GetSize()); return Payload; } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 9c236310f..9d2bab170 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -40,7 +40,10 @@ public: std::function&& Transmitter, std::function&& OnSentBytes) = 0; - virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; virtual std::vector> GetLargeBuildBlob( const Oid& BuildId, const IoHash& RawHash, diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 852271868..2c5fc73b8 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -123,7 +123,9 @@ public: std::string_view BucketId, const Oid& BuildId, const IoHash& Hash, - std::filesystem::path TempFolderPath); + std::filesystem::path TempFolderPath, + uint64_t Offset = 0, + uint64_t Size = (uint64_t)-1); JupiterResult PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 309885b05..bf89ce785 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -217,13 +217,15 @@ public: return WorkList; } - virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) override + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override { ZEN_TRACE_CPU("Jupiter::GetBuildBlob"); - Stopwatch ExecutionTimer; - auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - JupiterResult GetBuildBlobResult = m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + CreateDirectories(m_TempFolderPath); + JupiterResult GetBuildBlobResult = + m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes); AddStatistic(GetBuildBlobResult); if (!GetBuildBlobResult.Success) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 06ac6ae36..68f214c06 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -698,11 +698,19 @@ JupiterSession::GetBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoHash& Hash, - std::filesystem::path TempFolderPath) + std::filesystem::path TempFolderPath, + uint64_t Offset, + uint64_t Size) { + HttpClient::KeyValueMap Headers; + if (Offset != 0 || Size != (uint64_t)-1) + { + Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)}); + } HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()), - TempFolderPath); + TempFolderPath, + Headers); return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv); } -- cgit v1.2.3 From 7de3d4218ee5969af6147f9ab20bda538a136d9a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 10 Mar 2025 18:33:24 +0100 Subject: pick up existing cache (#299) - Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run - Improvement: Faster abort during stream compression - Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write - Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails --- src/zen/cmds/builds_cmd.cpp | 2438 +++++++++++++++++++----------- src/zen/cmds/builds_cmd.h | 8 +- src/zencore/basicfile.cpp | 37 +- src/zencore/compress.cpp | 51 +- src/zencore/include/zencore/basicfile.h | 2 +- src/zencore/include/zencore/compress.h | 2 +- src/zencore/workthreadpool.cpp | 2 +- src/zenutil/chunkblock.cpp | 51 +- src/zenutil/include/zenutil/chunkblock.h | 9 +- 9 files changed, 1682 insertions(+), 918 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1c9476b96..2c7a73fb3 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -83,15 +83,24 @@ namespace { const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const std::string ZenFolderName = ".zen"; - const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); - const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); - const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); - const std::string ZenTempCacheFolderName = fmt::format("{}/cache", ZenTempFolderName); - const std::string ZenTempStorageFolderName = fmt::format("{}/storage", ZenTempFolderName); - const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); - const std::string ZenTempChunkFolderName = fmt::format("{}/chunks", ZenTempFolderName); - const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; + const std::string ZenFolderName = ".zen"; + const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); + const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); + const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + + const std::string ZenTempCacheFolderName = + fmt::format("{}/cache", ZenTempFolderName); // Decompressed and verified data - chunks & sequences + const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); // Temp storage for whole and partial blocks + const std::string ZenTempChunkFolderName = + fmt::format("{}/chunks", ZenTempFolderName); // Temp storage for decompressed and validated chunks + + const std::string ZenTempDownloadFolderName = + fmt::format("{}/download", ZenTempFolderName); // Temp storage for unverfied downloaded blobs + + const std::string ZenTempStorageFolderName = + fmt::format("{}/storage", ZenTempFolderName); // Temp storage folder for BuildStorage implementations + + const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; const std::string UnsyncFolderName = ".unsync"; @@ -231,21 +240,31 @@ namespace { return AuthToken; } - CompositeBuffer WriteToTempFileIfNeeded(const CompositeBuffer& Buffer, - const std::filesystem::path& TempFolderPath, - const IoHash& Hash, - const std::string& Suffix = {}) + bool IsBufferDiskBased(const IoBuffer& Buffer) { - // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory - std::span Segments = Buffer.GetSegments(); - ZEN_ASSERT(Buffer.GetSegments().size() > 0); IoBufferFileReference FileRef; - if (Segments.back().GetFileReference(FileRef)) + if (Buffer.GetFileReference(FileRef)) { - return Buffer; + return true; } + return false; + } + + bool IsBufferDiskBased(const CompositeBuffer& Buffer) + { + // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory + std::span Segments = Buffer.GetSegments(); + ZEN_ASSERT(Buffer.GetSegments().size() > 0); + return IsBufferDiskBased(Segments.back().AsIoBuffer()); + } + + IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, + const std::filesystem::path& TempFolderPath, + const IoHash& Hash, + const std::string& Suffix = {}) + { std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); - return CompositeBuffer(WriteToTempFile(Buffer, TempFilePath)); + return WriteToTempFile(std::move(Buffer), TempFilePath); } class FilteredRate @@ -1109,12 +1128,22 @@ namespace { IoHashStream Hash; bool CouldDecompress = Compressed.DecompressToStream(0, RawSize, [&Hash](uint64_t, const CompositeBuffer& RangeBuffer) { - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + if (!AbortFlag) { - Hash.Append(Segment.GetView()); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + return true; } + return false; }); + if (AbortFlag) + { + return CompositeBuffer{}; + } + if (!CouldDecompress) { throw std::runtime_error( @@ -1299,14 +1328,14 @@ namespace { { Work.ScheduleWork( VerifyPool, - [&, Payload = std::move(Payload), ChunkAttachment](std::atomic&) { + [&, Payload = std::move(Payload), ChunkAttachment](std::atomic&) mutable { if (!AbortFlag) { FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(IoBuffer(Payload), ChunkAttachment, CompressedSize, DecompressedSize); + ValidateBlob(std::move(Payload), ChunkAttachment, CompressedSize, DecompressedSize); ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid", ChunkAttachment, NiceBytes(CompressedSize), @@ -1349,14 +1378,14 @@ namespace { { Work.ScheduleWork( VerifyPool, - [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) { + [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { if (!AbortFlag) { FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateChunkBlock(IoBuffer(Payload), BlockAttachment, CompressedSize, DecompressedSize); + ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); ZEN_CONSOLE_VERBOSE("Chunk block {} ({} -> {}) is valid", BlockAttachment, NiceBytes(CompressedSize), @@ -1564,8 +1593,12 @@ namespace { { throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash)); } - CompositeBuffer TempPayload = WriteToTempFileIfNeeded(CompressedBlob.GetCompressed(), TempFolderPath, ChunkHash); - return CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)).GetCompressed(); + if (!IsBufferDiskBased(CompressedBlob.GetCompressed())) + { + IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFolderPath, ChunkHash); + CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); + } + return std::move(CompressedBlob).GetCompressed(); } struct GeneratedBlocks @@ -1637,9 +1670,13 @@ namespace { OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompressedBlock.GetCompressed(), - Path / ZenTempBlockFolderName, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash); + if (!IsBufferDiskBased(CompressedBlock.GetCompressed())) + { + IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), + Path / ZenTempBlockFolderName, + OutBlocks.BlockDescriptions[BlockIndex].BlockHash); + CompressedBlock = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); + } { CbObjectWriter Writer; Writer.AddString("createdBy", "zen"); @@ -1663,7 +1700,7 @@ namespace { PendingUploadCount++; Work.ScheduleWork( UploadBlocksPool, - [&, BlockIndex, Payload = std::move(Payload)](std::atomic&) { + [&, BlockIndex, Payload = std::move(CompressedBlock).GetCompressed()](std::atomic&) mutable { if (!AbortFlag) { if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) @@ -1682,12 +1719,16 @@ namespace { BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], OutBlocks.BlockMetaDatas[BlockIndex]); - const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; - Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); - UploadStats.BlocksBytes += Payload.GetSize(); + const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + const uint64_t CompressedBlockSize = Payload.GetSize(); + Storage.PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload)); + UploadStats.BlocksBytes += CompressedBlockSize; ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(Payload.GetSize()), + NiceBytes(CompressedBlockSize), OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); Storage.PutBlockMetadata(BuildId, @@ -1817,7 +1858,7 @@ namespace { auto AsyncUploadBlock = [&](const size_t BlockIndex, const IoHash BlockHash, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) { + [&, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) mutable { if (!AbortFlag) { FilteredUploadedBytesPerSecond.Start(); @@ -1855,7 +1896,7 @@ namespace { auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) { + [&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { if (!AbortFlag) { const uint64_t PayloadSize = Payload.GetSize(); @@ -1868,7 +1909,7 @@ namespace { ZenContentType::kCompressedBinary, PayloadSize, [Payload = std::move(Payload), &FilteredUploadedBytesPerSecond](uint64_t Offset, - uint64_t Size) -> IoBuffer { + uint64_t Size) mutable -> IoBuffer { FilteredUploadedBytesPerSecond.Start(); IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer(); @@ -1974,9 +2015,11 @@ namespace { } ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompressedBlock.GetCompressed(), - Path / ZenTempBlockFolderName, - BlockDescription.BlockHash); + CompositeBuffer Payload = IsBufferDiskBased(CompressedBlock.GetCompressed()) + ? std::move(CompressedBlock).GetCompressed() + : CompositeBuffer(WriteToTempFile(std::move(CompressedBlock).GetCompressed(), + Path / ZenTempBlockFolderName, + BlockDescription.BlockHash)); GenerateBlocksStats.GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; GenerateBlocksStats.GeneratedBlockCount++; @@ -3289,33 +3332,40 @@ namespace { uint64_t FileOffset, uint64_t TargetFinalSize) { + ZEN_TRACE_CPU("WriteFileCache_WriteToFile"); if (!SeenTargetIndexes.empty() && SeenTargetIndexes.back() == TargetIndex) { + ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); ZEN_ASSERT(OpenFileWriter); OpenFileWriter->Write(Buffer, FileOffset); } else { - Flush(); - const std::filesystem::path& TargetPath = GetTargetPath(TargetIndex); - CreateDirectories(TargetPath.parent_path()); - uint32_t Tries = 5; - std::unique_ptr NewOutputFile( - std::make_unique(TargetPath, BasicFile::Mode::kWrite, [&Tries, TargetPath](std::error_code& Ec) { - if (Tries < 3) - { - ZEN_CONSOLE("Failed opening file '{}': {}{}", TargetPath, Ec.message(), Tries > 1 ? " Retrying"sv : ""sv); - } - if (Tries > 1) - { - Sleep(100); - } - return --Tries > 0; - })); + std::unique_ptr NewOutputFile; + { + ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Open"); + Flush(); + const std::filesystem::path& TargetPath = GetTargetPath(TargetIndex); + CreateDirectories(TargetPath.parent_path()); + uint32_t Tries = 5; + NewOutputFile = + std::make_unique(TargetPath, BasicFile::Mode::kWrite, [&Tries, TargetPath](std::error_code& Ec) { + if (Tries < 3) + { + ZEN_CONSOLE("Failed opening file '{}': {}{}", TargetPath, Ec.message(), Tries > 1 ? " Retrying"sv : ""sv); + } + if (Tries > 1) + { + Sleep(100); + } + return --Tries > 0; + }); + } const bool CacheWriter = TargetFinalSize > Buffer.GetSize(); if (CacheWriter) { + ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); OutputFile = std::move(NewOutputFile); @@ -3325,6 +3375,7 @@ namespace { } else { + ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Write"); NewOutputFile->Write(Buffer, FileOffset); } } @@ -3332,6 +3383,7 @@ namespace { void Flush() { + ZEN_TRACE_CPU("WriteFileCache_Flush"); OpenFileWriter = {}; OutputFile = {}; } @@ -3376,7 +3428,7 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& Lookup, std::span> SequenceIndexChunksLeftToWriteCounters, - const BlockWriteOps Ops, + const BlockWriteOps& Ops, std::atomic& OutChunksComplete, std::atomic& OutBytesWritten) { @@ -3420,13 +3472,19 @@ namespace { if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) { - throw std::runtime_error( - fmt::format("Written hunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + ZEN_TRACE_CPU("VerifyChunkHash"); + const IoHash VerifyChunkHash = IoHash::HashBuffer( + IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } } + ZEN_TRACE_CPU("VerifyChunkHashes_rename"); + ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } @@ -3437,139 +3495,38 @@ namespace { bool GetBlockWriteOps(const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& Lookup, + std::span ChunkRawHashes, + std::span ChunkCompressedLengths, + std::span ChunkRawLengths, std::span> SequenceIndexChunksLeftToWriteCounters, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - const CompositeBuffer& DecompressedBlockBuffer, + CompositeBuffer&& PartialBlockBuffer, + uint32_t FirstIncludedBlockChunkIndex, + uint32_t LastIncludedBlockChunkIndex, BlockWriteOps& OutOps) { ZEN_TRACE_CPU("GetBlockWriteOps"); - SharedBuffer BlockBuffer = DecompressedBlockBuffer.Flatten(); - uint64_t HeaderSize = 0; - if (IterateChunkBlock( - BlockBuffer, - [&](CompressedBuffer&& Chunk, const IoHash& ChunkHash) { - if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) - { - const uint32_t ChunkIndex = It->second; - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, Lookup, ChunkIndex); - - if (!ChunkTargetPtrs.empty()) - { - bool NeedsWrite = true; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) - { - CompositeBuffer Decompressed = Chunk.DecompressToComposite(); - if (!Decompressed) - { - throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); - } - ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); - ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) - { - OutOps.WriteOps.push_back( - BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()}); - } - OutOps.ChunkBuffers.emplace_back(std::move(Decompressed)); - } - } - } - }, - HeaderSize)) - { - std::sort(OutOps.WriteOps.begin(), - OutOps.WriteOps.end(), - [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - return Lhs.Target->Offset < Rhs.Target->Offset; - }); - - return true; + MemoryView BlockMemoryView; + UniqueBuffer BlockMemoryBuffer; + IoBufferFileReference FileRef = {}; + if (PartialBlockBuffer.GetSegments().size() == 1 && PartialBlockBuffer.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + { + BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize); + BasicFile Reader; + Reader.Attach(FileRef.FileHandle); + Reader.Read(BlockMemoryBuffer.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); + BlockMemoryView = BlockMemoryBuffer.GetView(); + Reader.Detach(); } else { - return false; - } - } - - bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& RemoteContent, - const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - const CompositeBuffer& BlockBuffer, - const ChunkedContentLookup& Lookup, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - std::atomic& OutChunksComplete, - std::atomic& OutBytesWritten) - { - ZEN_TRACE_CPU("WriteBlockToDisk"); - - IoHash BlockRawHash; - uint64_t BlockRawSize; - CompressedBuffer CompressedBlockBuffer = CompressedBuffer::FromCompressed(BlockBuffer, BlockRawHash, BlockRawSize); - if (!CompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is not a compressed buffer", BlockDescription.BlockHash)); - } - - if (BlockRawHash != BlockDescription.BlockHash) - { - throw std::runtime_error( - fmt::format("Block {} header has a mismatching raw hash {}", BlockDescription.BlockHash, BlockRawHash)); - } - - CompositeBuffer DecompressedBlockBuffer = CompressedBlockBuffer.DecompressToComposite(); - if (!DecompressedBlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} failed to decompress", BlockDescription.BlockHash)); - } - - ZEN_ASSERT_SLOW(BlockDescription.BlockHash == IoHash::HashBuffer(DecompressedBlockBuffer)); - - BlockWriteOps Ops; - if (GetBlockWriteOps(RemoteContent, - Lookup, - SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DecompressedBlockBuffer, - Ops)) - { - WriteBlockChunkOps(CacheFolderPath, - RemoteContent, - Lookup, - SequenceIndexChunksLeftToWriteCounters, - Ops, - OutChunksComplete, - OutBytesWritten); - return true; + BlockMemoryView = PartialBlockBuffer.ViewOrCopyRange(0, PartialBlockBuffer.GetSize(), BlockMemoryBuffer); } - return false; - } - - bool GetPartialBlockWriteOps(const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& Lookup, - const ChunkBlockDescription& BlockDescription, - std::span> SequenceIndexChunksLeftToWriteCounters, - std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - const CompositeBuffer& PartialBlockBuffer, - uint32_t FirstIncludedBlockChunkIndex, - uint32_t LastIncludedBlockChunkIndex, - BlockWriteOps& OutOps) - { - ZEN_TRACE_CPU("GetPartialBlockWriteOps"); uint32_t OffsetInBlock = 0; for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++) { - const uint32_t ChunkCompressedSize = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + const uint32_t ChunkCompressedSize = ChunkCompressedLengths[ChunkBlockIndex]; + const IoHash& ChunkHash = ChunkRawHashes[ChunkBlockIndex]; if (auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); It != Lookup.ChunkHashToChunkIndex.end()) { const uint32_t ChunkIndex = It->second; @@ -3581,7 +3538,9 @@ namespace { bool NeedsWrite = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) { - CompositeBuffer Chunk = PartialBlockBuffer.Mid(OffsetInBlock, ChunkCompressedSize); + // CompositeBuffer Chunk = PartialBlockBuffer.Mid(OffsetInBlock, ChunkCompressedSize); + MemoryView ChunkMemory = BlockMemoryView.Mid(OffsetInBlock, ChunkCompressedSize); + CompositeBuffer Chunk = CompositeBuffer(IoBuffer(IoBuffer::Wrap, ChunkMemory.GetData(), ChunkMemory.GetSize())); IoHash VerifyChunkHash; uint64_t VerifyRawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Chunk, VerifyChunkHash, VerifyRawSize); @@ -3593,9 +3552,12 @@ namespace { { ZEN_ASSERT(false); } - if (VerifyRawSize != BlockDescription.ChunkRawLengths[ChunkBlockIndex]) + if (!ChunkRawLengths.empty()) { - ZEN_ASSERT(false); + if (VerifyRawSize != ChunkRawLengths[ChunkBlockIndex]) + { + ZEN_ASSERT(false); + } } CompositeBuffer Decompressed = Compressed.DecompressToComposite(); if (!Decompressed) @@ -3632,11 +3594,85 @@ namespace { return true; } + bool WriteBlockToDisk(const std::filesystem::path& CacheFolderPath, + const ChunkedFolderContent& RemoteContent, + const ChunkBlockDescription& BlockDescription, + std::span> SequenceIndexChunksLeftToWriteCounters, + CompositeBuffer&& BlockBuffer, + const ChunkedContentLookup& Lookup, + std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + std::atomic& OutChunksComplete, + std::atomic& OutBytesWritten) + { + ZEN_TRACE_CPU("WriteBlockToDisk"); + + BlockWriteOps Ops; + if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty()) + { + ZEN_TRACE_CPU("WriteBlockToDisk_Legacy"); + + UniqueBuffer CopyBuffer; + const MemoryView BlockView = BlockBuffer.ViewOrCopyRange(0, BlockBuffer.GetSize(), CopyBuffer); + uint64_t HeaderSize; + const std::vector ChunkCompressedLengths = ReadChunkBlockHeader(BlockView, HeaderSize); + + CompositeBuffer PartialBlockBuffer = std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize); + + if (GetBlockWriteOps(RemoteContent, + Lookup, + BlockDescription.ChunkRawHashes, + ChunkCompressedLengths, + BlockDescription.ChunkRawLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + std::move(PartialBlockBuffer), + 0, + gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), + Ops)) + { + WriteBlockChunkOps(CacheFolderPath, + RemoteContent, + Lookup, + SequenceIndexChunksLeftToWriteCounters, + Ops, + OutChunksComplete, + OutBytesWritten); + return true; + } + return false; + } + + CompositeBuffer PartialBlockBuffer = + std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + if (GetBlockWriteOps(RemoteContent, + Lookup, + BlockDescription.ChunkRawHashes, + BlockDescription.ChunkCompressedLengths, + BlockDescription.ChunkRawLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + std::move(PartialBlockBuffer), + 0, + gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), + Ops)) + { + WriteBlockChunkOps(CacheFolderPath, + RemoteContent, + Lookup, + SequenceIndexChunksLeftToWriteCounters, + Ops, + OutChunksComplete, + OutBytesWritten); + return true; + } + return false; + } + bool WritePartialBlockToDisk(const std::filesystem::path& CacheFolderPath, const ChunkedFolderContent& RemoteContent, const ChunkBlockDescription& BlockDescription, std::span> SequenceIndexChunksLeftToWriteCounters, - const CompositeBuffer& PartialBlockBuffer, + CompositeBuffer&& PartialBlockBuffer, uint32_t FirstIncludedBlockChunkIndex, uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, @@ -3646,15 +3682,17 @@ namespace { { ZEN_TRACE_CPU("WritePartialBlockToDisk"); BlockWriteOps Ops; - if (GetPartialBlockWriteOps(RemoteContent, - Lookup, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - RemoteChunkIndexNeedsCopyFromSourceFlags, - PartialBlockBuffer, - FirstIncludedBlockChunkIndex, - LastIncludedBlockChunkIndex, - Ops)) + if (GetBlockWriteOps(RemoteContent, + Lookup, + BlockDescription.ChunkRawHashes, + BlockDescription.ChunkCompressedLengths, + BlockDescription.ChunkRawLengths, + SequenceIndexChunksLeftToWriteCounters, + RemoteChunkIndexNeedsCopyFromSourceFlags, + std::move(PartialBlockBuffer), + FirstIncludedBlockChunkIndex, + LastIncludedBlockChunkIndex, + Ops)) { WriteBlockChunkOps(CacheFolderPath, RemoteContent, @@ -3671,7 +3709,7 @@ namespace { } } - SharedBuffer Decompress(const CompositeBuffer& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) + SharedBuffer Decompress(CompositeBuffer&& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) { ZEN_TRACE_CPU("Decompress"); @@ -3709,7 +3747,7 @@ namespace { const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span ChunkTargets, - const CompositeBuffer& ChunkData, + CompositeBuffer&& ChunkData, WriteFileCache& OpenFileCache, std::atomic& OutBytesWritten) { @@ -3734,8 +3772,8 @@ namespace { } } - bool CanStreamDecompress(const ChunkedFolderContent& RemoteContent, - const std::vector Locations) + bool CanDecompressDirectToSequence(const ChunkedFolderContent& RemoteContent, + const std::vector Locations) { if (Locations.size() == 1) { @@ -3776,13 +3814,25 @@ namespace { } IoHashStream Hash; bool CouldDecompress = Compressed.DecompressToStream(0, (uint64_t)-1, [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { - DecompressedTemp.Write(RangeBuffer, Offset); - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + ZEN_TRACE_CPU("StreamDecompress_Write"); + if (!AbortFlag) { - Hash.Append(Segment.GetView()); + DecompressedTemp.Write(RangeBuffer, Offset); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + WriteToDiskBytes += RangeBuffer.GetSize(); + return true; } - WriteToDiskBytes += RangeBuffer.GetSize(); + return false; }); + + if (AbortFlag) + { + return; + } + if (!CouldDecompress) { throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash)); @@ -3801,9 +3851,82 @@ namespace { } } + bool WriteCompressedChunk(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + const IoHash& ChunkHash, + const std::vector& ChunkTargetPtrs, + IoBuffer&& CompressedPart, + std::atomic& WriteToDiskBytes) + { + auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + if (CanDecompressDirectToSequence(RemoteContent, ChunkTargetPtrs)) + { + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; + StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), WriteToDiskBytes); + } + else + { + const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; + SharedBuffer Chunk = + Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + if (!AbortFlag) + { + WriteFileCache OpenFileCache; + + WriteChunkToDisk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkTargetPtrs, + CompositeBuffer(std::move(Chunk)), + OpenFileCache, + WriteToDiskBytes); + return true; + } + } + return false; + } + + void CompleteChunkTargets(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + const IoHash& ChunkHash, + const std::vector& ChunkTargetPtrs, + std::span> SequenceIndexChunksLeftToWriteCounters, + const bool NeedHashVerify) + { + ZEN_TRACE_CPU("CompleteChunkTargets"); + + for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) + { + const uint32_t RemoteSequenceIndex = Location->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + if (NeedHashVerify) + { + ZEN_TRACE_CPU("VerifyChunkHash"); + + const IoHash VerifyChunkHash = + IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + if (VerifyChunkHash != ChunkHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, ChunkHash)); + } + } + + ZEN_TRACE_CPU("RenameToFinal"); + ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + } + } + } + void DownloadLargeBlob(BuildStorage& Storage, - const std::filesystem::path& TempFolderPath, - const std::filesystem::path& CacheFolderPath, + const std::filesystem::path& Path, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, const Oid& BuildId, @@ -3818,6 +3941,7 @@ namespace { std::atomic& BytesDownloaded, std::atomic& MultipartAttachmentCount, std::function&& OnDownloadComplete, + std::function&& OnWriteStart, std::function&& OnWriteComplete) { ZEN_TRACE_CPU("DownloadLargeBlob"); @@ -3828,8 +3952,11 @@ namespace { }; std::shared_ptr Workload(std::make_shared()); + std::filesystem::path DownloadFolder = Path / ZenTempDownloadFolderName; + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::error_code Ec; - Workload->TempFile.CreateTemporary(TempFolderPath, Ec); + Workload->TempFile.CreateTemporary(DownloadFolder, Ec); if (Ec) { throw std::runtime_error( @@ -3839,7 +3966,8 @@ namespace { BuildId, ChunkHash, PreferredMultipartChunkSize, - [&CacheFolderPath, + [DownloadFolder, + TargetFolder, &RemoteContent, &RemoteLookup, &Work, @@ -3849,6 +3977,7 @@ namespace { &BytesDownloaded, OnDownloadComplete = std::move(OnDownloadComplete), OnWriteComplete = std::move(OnWriteComplete), + OnWriteStart = std::move(OnWriteStart), &WriteToDiskBytes, SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs = std::vector( @@ -3857,6 +3986,7 @@ namespace { if (!AbortFlag.load()) { + ZEN_TRACE_CPU("DownloadLargeBlob_Save"); Workload->TempFile.Write(Chunk.GetView(), Offset); if (Chunk.GetSize() == BytesRemaining) { @@ -3864,103 +3994,61 @@ namespace { Work.ScheduleWork( WritePool, // GetSyncWorkerPool(),// - [&CacheFolderPath, + [DownloadFolder, + TargetFolder, &RemoteContent, &RemoteLookup, ChunkHash, Workload, Offset, OnWriteComplete = std::move(OnWriteComplete), + OnWriteStart = std::move(OnWriteStart), &WriteToDiskBytes, SequenceIndexChunksLeftToWriteCounters, ChunkTargetPtrs](std::atomic&) { - ZEN_TRACE_CPU("DownloadLargeBlob_Work"); + ZEN_TRACE_CPU("DownloadLargeBlob_Write"); if (!AbortFlag) { - uint64_t CompressedSize = Workload->TempFile.FileSize(); - void* FileHandle = Workload->TempFile.Detach(); - IoBuffer CompressedPart = IoBuffer(IoBuffer::File, - FileHandle, - 0, - CompressedSize, - /*IsWholeFile*/ true); - if (!CompressedPart) + const std::filesystem::path CompressedChunkPath = DownloadFolder / ChunkHash.ToHexString(); + std::error_code Ec; + Workload->TempFile.MoveTemporaryIntoPlace(CompressedChunkPath, Ec); + if (Ec) { - throw std::runtime_error( - fmt::format("Multipart build blob {} is not a compressed buffer", ChunkHash)); + throw std::runtime_error(fmt::format("Failed moving downloaded chunk {} file to {}. Reason: {}", + ChunkHash, + CompressedChunkPath, + Ec.message())); } - CompressedPart.SetDeleteOnClose(true); - auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); - bool NeedHashVerify = true; - if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) { - const IoHash& SequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; - StreamDecompress(CacheFolderPath, - SequenceRawHash, - CompositeBuffer(std::move(CompressedPart)), - WriteToDiskBytes); - NeedHashVerify = false; - OnWriteComplete(); + throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); } - else - { - const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; - SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), - ChunkHash, - RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); - - // ZEN_ASSERT_SLOW(ChunkHash == - // IoHash::HashBuffer(Chunk.AsIoBuffer())); - if (!AbortFlag) - { - WriteFileCache OpenFileCache; + OnWriteStart(); - WriteChunkToDisk(CacheFolderPath, - RemoteContent, - RemoteLookup, - ChunkTargetPtrs, - CompositeBuffer(Chunk), - OpenFileCache, - WriteToDiskBytes); - OnWriteComplete(); - } - } + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + WriteToDiskBytes); if (!AbortFlag) { - // Write tracking, updating this must be done without any files open (WriteFileCache) - for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) - { - const uint32_t RemoteSequenceIndex = Location->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) - { - const IoHash& SequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - if (NeedHashVerify) - { - ZEN_TRACE_CPU("VerifyChunkHash"); - - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != ChunkHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - ChunkHash)); - } - } + std::filesystem::remove(CompressedChunkPath); - ZEN_TRACE_CPU("VerifyChunkHashes_rename"); - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); - } - } + CompleteChunkTargets(TargetFolder, + RemoteContent, + ChunkHash, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + NeedHashVerify); } } }, @@ -3977,6 +4065,7 @@ namespace { Work.ScheduleWork( NetworkPool, // GetSyncWorkerPool(),// [WorkItem = std::move(WorkItem)](std::atomic&) { + ZEN_TRACE_CPU("DownloadLargeBlob_Work"); if (!AbortFlag) { WorkItem(); @@ -4023,115 +4112,256 @@ namespace { Stopwatch CacheMappingTimer; - uint64_t CacheMappedBytesForReuse = 0; std::vector> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); // std::vector RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false); - std::vector RemoteChunkIndexIsCachedFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + std::vector RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); - // Pick up all whole files we can use from current local state - for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); - RemoteSequenceIndex++) + tsl::robin_map CachedChunkHashesFound; + tsl::robin_map CachedSequenceHashesFound; + uint64_t CachedChunkHashesByteCountFound = 0; + uint64_t CachedSequenceHashesByteCountFound = 0; { - const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) - { - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0; - const uint32_t RemotePathIndex = GetFirstPathIndexForRawHash(RemoteLookup, RemoteSequenceRawHash); - CacheMappedBytesForReuse += RemoteContent.RawSizes[RemotePathIndex]; - } - else + ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); + + DirectoryContent CacheDirContent; + GetDirectoryContent(CacheFolderPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + CacheDirContent); + for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++) { - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + IoHash FileHash; + if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash)) + { + if (auto ChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(FileHash); + ChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t ChunkIndex = ChunkIt->second; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; + if (ChunkSize == CacheDirContent.FileSizes[Index]) + { + CachedChunkHashesFound.insert({FileHash, ChunkIndex}); + CachedChunkHashesByteCountFound += ChunkSize; + continue; + } + } + else if (auto SequenceIt = RemoteLookup.RawHashToSequenceIndex.find(FileHash); + SequenceIt != RemoteLookup.RawHashToSequenceIndex.end()) + { + const uint32_t SequenceIndex = SequenceIt->second; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex]; + if (SequenceSize == CacheDirContent.FileSizes[Index]) + { + CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); + CachedSequenceHashesByteCountFound += SequenceSize; + continue; + } + } + } + std::filesystem::remove(CacheDirContent.Files[Index]); } } - // Pick up all chunks in current local state - struct CacheCopyData + tsl::robin_map CachedBlocksFound; + uint64_t CachedBlocksByteCountFound = 0; { - uint32_t LocalSequenceIndex; - std::vector TargetChunkLocationPtrs; - struct ChunkTarget - { - uint32_t TargetChunkLocationCount; - uint64_t ChunkRawSize; - uint64_t CacheFileOffset; - }; - std::vector ChunkTargets; - }; - - tsl::robin_map RawHashToCacheCopyDataIndex; - std::vector CacheCopyDatas; + ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); - for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size(); - LocalSequenceIndex++) - { - const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; - const uint32_t LocalOrderOffset = LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; + tsl::robin_map AllBlockSizes; + AllBlockSizes.reserve(BlockDescriptions.size()); + for (uint32_t BlockIndex = 0; BlockIndex < BlockDescriptions.size(); BlockIndex++) + { + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex}); + } + DirectoryContent BlockDirContent; + GetDirectoryContent(Path / ZenTempBlockFolderName, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, + BlockDirContent); + CachedBlocksFound.reserve(BlockDirContent.Files.size()); + for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++) { - uint64_t SourceOffset = 0; - const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; - for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) + IoHash FileHash; + if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash)) { - const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; - const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; - const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; - - if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); - RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end()) { - const uint32_t RemoteChunkIndex = RemoteChunkIt->second; - if (!RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) + const uint32_t BlockIndex = BlockIt->second; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize; + for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths) { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + BlockSize += ChunkSize; + } - if (!ChunkTargetPtrs.empty()) - { - CacheCopyData::ChunkTarget Target = { - .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), - .ChunkRawSize = LocalChunkRawSize, - .CacheFileOffset = SourceOffset}; - if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalSequenceRawHash); - CopySourceIt != RawHashToCacheCopyDataIndex.end()) - { - CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; - Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), - ChunkTargetPtrs.begin(), - ChunkTargetPtrs.end()); - Data.ChunkTargets.push_back(Target); - } - else + if (BlockSize == BlockDirContent.FileSizes[Index]) + { + CachedBlocksFound.insert({FileHash, BlockIndex}); + CachedBlocksByteCountFound += BlockSize; + continue; + } + } + } + std::filesystem::remove(BlockDirContent.Files[Index]); + } + } + + std::vector LocalPathIndexesMatchingSequenceIndexes; + uint64_t LocalPathIndexesByteCountMatchingSequenceIndexes = 0; + // Pick up all whole files we can use from current local state + { + ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash); + CacheSequenceIt != CachedSequenceHashesFound.end()) + { + // const uint32_t RemoteSequenceIndex = CacheSequenceIt->second; + // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); + // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + } + else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); + CacheChunkIt != CachedChunkHashesFound.end()) + { + // const uint32_t RemoteChunkIndex = CacheChunkIt->second; + // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); + // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + } + else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); + It != LocalLookup.RawHashToSequenceIndex.end()) + { + const uint32_t LocalSequenceIndex = It->second; + const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); + uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); + LocalPathIndexesByteCountMatchingSequenceIndexes += RawSize; + } + else + { + // We must write the sequence + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + } + } + } + // Pick up all chunks in current local state + struct CacheCopyData + { + uint32_t LocalSequenceIndex; + std::vector TargetChunkLocationPtrs; + struct ChunkTarget + { + uint32_t TargetChunkLocationCount; + uint64_t ChunkRawSize; + uint64_t CacheFileOffset; + }; + std::vector ChunkTargets; + }; + + tsl::robin_map RawHashToCacheCopyDataIndex; + std::vector CacheCopyDatas; + uint64_t LocalChunkHashesMatchingRemoteCount = 0; + uint64_t LocalChunkHashesMatchingRemoteByteCount = 0; + + { + ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); + + for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size(); + LocalSequenceIndex++) + { + const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; + const uint32_t LocalOrderOffset = LocalLookup.SequenceIndexChunkOrderOffset[LocalSequenceIndex]; + + { + uint64_t SourceOffset = 0; + const uint32_t LocalChunkCount = LocalContent.ChunkedContent.ChunkCounts[LocalSequenceIndex]; + for (uint32_t LocalOrderIndex = 0; LocalOrderIndex < LocalChunkCount; LocalOrderIndex++) + { + const uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[LocalOrderOffset + LocalOrderIndex]; + const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex]; + const uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex]; + + if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(LocalChunkHash); + RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = RemoteChunkIt->second; + if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + + if (!ChunkTargetPtrs.empty()) { - RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); - CacheCopyDatas.push_back( - CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, - .TargetChunkLocationPtrs = ChunkTargetPtrs, - .ChunkTargets = std::vector{Target}}); + CacheCopyData::ChunkTarget Target = { + .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), + .ChunkRawSize = LocalChunkRawSize, + .CacheFileOffset = SourceOffset}; + if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalSequenceRawHash); + CopySourceIt != RawHashToCacheCopyDataIndex.end()) + { + CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; + if (Data.TargetChunkLocationPtrs.size() > 1024) + { + RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); + CacheCopyDatas.push_back( + CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + else + { + Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), + ChunkTargetPtrs.begin(), + ChunkTargetPtrs.end()); + Data.ChunkTargets.push_back(Target); + } + } + else + { + RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); + CacheCopyDatas.push_back( + CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + LocalChunkHashesMatchingRemoteByteCount += LocalChunkRawSize; + LocalChunkHashesMatchingRemoteCount++; + RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; } - CacheMappedBytesForReuse += LocalChunkRawSize; - RemoteChunkIndexIsCachedFlags[RemoteChunkIndex] = true; } } + SourceOffset += LocalChunkRawSize; } - SourceOffset += LocalChunkRawSize; } } } - if (CacheMappedBytesForReuse > 0) + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() || + !LocalPathIndexesMatchingSequenceIndexes.empty() || LocalChunkHashesMatchingRemoteCount > 0) { - ZEN_CONSOLE("Mapped {} cached data for reuse in {}", - NiceBytes(CacheMappedBytesForReuse), - NiceTimeSpanMs(CacheMappingTimer.GetElapsedTimeMs())); + ZEN_CONSOLE( + "Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks", + CachedSequenceHashesFound.size(), + NiceBytes(CachedSequenceHashesByteCountFound), + CachedChunkHashesFound.size(), + NiceBytes(CachedChunkHashesByteCountFound), + CachedBlocksFound.size(), + NiceBytes(CachedBlocksByteCountFound), + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(LocalPathIndexesByteCountMatchingSequenceIndexes), + LocalChunkHashesMatchingRemoteCount, + NiceBytes(LocalChunkHashesMatchingRemoteByteCount)); } uint32_t ChunkCountToWrite = 0; for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) { - if (RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) + if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { ChunkCountToWrite++; } @@ -4154,7 +4384,7 @@ namespace { std::atomic WritePartsComplete = 0; { - ZEN_TRACE_CPU("HandleChunks"); + ZEN_TRACE_CPU("WriteChunks"); Stopwatch WriteTimer; @@ -4169,17 +4399,21 @@ namespace { std::atomic BytesDownloaded = 0; - for (const IoHash ChunkHash : LooseChunkHashes) + struct LooseChunkHashWorkData { - if (AbortFlag) - { - break; - } + std::vector ChunkTargetPtrs; + uint32_t RemoteChunkIndex; + }; + std::vector LooseChunkHashWorks; + TotalPartWriteCount += CacheCopyDatas.size(); + + for (const IoHash ChunkHash : LooseChunkHashes) + { auto RemoteChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(RemoteChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - if (RemoteChunkIndexIsCachedFlags[RemoteChunkIndex]) + if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); continue; @@ -4198,173 +4432,192 @@ namespace { { TotalRequestCount++; TotalPartWriteCount++; - Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(),// - [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { - ZEN_TRACE_CPU("UpdateFolder_LooseChunk"); + LooseChunkHashWorks.push_back( + LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); + } + } + } - if (!AbortFlag) + uint32_t BlockCount = gsl::narrow(BlockDescriptions.size()); + + std::vector ChunkIsPickedUpByBlock(RemoteContent.ChunkedContent.ChunkHashes.size(), false); + auto GetNeededChunkBlockIndexes = [&RemoteContent, + &RemoteLookup, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) { + ZEN_TRACE_CPU("UpdateFolder_GetNeededChunkBlockIndexes"); + std::vector NeededBlockChunkIndexes; + for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) + { + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) + { + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + { + ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; + NeededBlockChunkIndexes.push_back(ChunkBlockIndex); + } + } + } + } + return NeededBlockChunkIndexes; + }; + + std::vector CachedChunkBlockIndexes; + + struct BlockRangeDescriptor + { + uint32_t BlockIndex = (uint32_t)-1; + uint64_t RangeStart = 0; + uint64_t RangeLength = 0; + uint32_t ChunkBlockIndexStart = 0; + uint32_t ChunkBlockIndexCount = 0; + }; + std::vector BlockRangeWorks; + + std::vector FullBlockWorks; + + size_t BlocksNeededCount = 0; + uint64_t AllBlocksSize = 0; + uint64_t AllBlocksFetch = 0; + uint64_t AllBlocksSlack = 0; + uint64_t AllBlockRequests = 0; + uint64_t AllBlockChunksSize = 0; + for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) + { + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); + if (!BlockChunkIndexNeeded.empty()) + { + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + { + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); + + TotalPartWriteCount++; + + std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + if (std::filesystem::exists(BlockPath)) + { + CachedChunkBlockIndexes.push_back(BlockIndex); + UsingCachedBlock = true; + } + } + + if (!UsingCachedBlock) + { + bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = + (BlockDescription.HeaderSize > 0) && + (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); + if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + { + std::vector BlockRanges; + + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); + + uint32_t NeedBlockChunkIndexOffset = 0; + uint32_t ChunkBlockIndex = 0; + uint32_t CurrentOffset = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && + ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) + { + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - FilteredDownloadedBytesPerSecond.Start(); - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + if (NextRange.RangeLength > 0) { - DownloadLargeBlob( - Storage, - Path / ZenTempChunkFolderName, - CacheFolderPath, - RemoteContent, - RemoteLookup, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - NetworkPool, - WriteToDiskBytes, - BytesDownloaded, - MultipartAttachmentCount, - [&](uint64_t BytesDownloaded) { - LooseChunksBytes += BytesDownloaded; - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - }, - [&]() { - ChunkCountWritten++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - }); + BlockRanges.push_back(NextRange); + NextRange = {.BlockIndex = BlockIndex}; } - else + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + AllBlockChunksSize += ChunkCompressedLength; + if (NextRange.RangeLength == 0) { - IoBuffer CompressedPart = Storage.GetBuildBlob(BuildId, ChunkHash); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - BytesDownloaded += CompressedPart.GetSize(); - LooseChunksBytes += CompressedPart.GetSize(); - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(CompressedPart)), - Path / ZenTempChunkFolderName, - ChunkHash); - DownloadedChunks++; + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; + } + else + { + ZEN_ASSERT(false); + } + } + AllBlocksSize += CurrentOffset; + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&, ChunkHash, RemoteChunkIndex, ChunkTargetPtrs, CompressedPart = std::move(Payload)]( - std::atomic&) { - ZEN_TRACE_CPU("UpdateFolder_WriteBlob"); + ZEN_ASSERT(!BlockRanges.empty()); + std::vector CollapsedBlockRanges; + auto It = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*It++); + uint64_t TotalSlack = 0; + while (It != BlockRanges.end()) + { + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; + if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + { + LastRange.ChunkBlockIndexCount = + (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + TotalSlack += Slack; + } + else + { + CollapsedBlockRanges.push_back(*It); + } + ++It; + } - if (!AbortFlag) - { - FilteredWrittenBytesPerSecond.Start(); + uint64_t TotalFetch = 0; + for (const BlockRangeDescriptor& Range : CollapsedBlockRanges) + { + TotalFetch += Range.RangeLength; + } - bool NeedHashVerify = true; - if (CanStreamDecompress(RemoteContent, ChunkTargetPtrs)) - { - const IoHash& SequenceRawHash = - RemoteContent.ChunkedContent - .SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; - StreamDecompress(CacheFolderPath, - SequenceRawHash, - CompositeBuffer(CompressedPart), - WriteToDiskBytes); - ChunkCountWritten++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - NeedHashVerify = false; - } - else - { - SharedBuffer Chunk = - Decompress(CompressedPart, - ChunkHash, - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]); - - { - WriteFileCache OpenFileCache; - WriteChunkToDisk(CacheFolderPath, - RemoteContent, - RemoteLookup, - ChunkTargetPtrs, - CompositeBuffer(Chunk), - OpenFileCache, - WriteToDiskBytes); - ChunkCountWritten++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - } - if (!AbortFlag) - { - WritePartsComplete++; - - // Write tracking, updating this must be done without any files open - // (WriteFileCache) - for (const ChunkedContentLookup::ChunkSequenceLocation* Location : - ChunkTargetPtrs) - { - const uint32_t RemoteSequenceIndex = Location->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub( - 1) == 1) - { - const IoHash& SequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - if (NeedHashVerify) - { - ZEN_TRACE_CPU("UpdateFolder_VerifyHash"); - - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, - SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match " - "expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - } - - ZEN_TRACE_CPU("UpdateFolder_rename"); - std::filesystem::rename( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); - } - } - } - } - }, - Work.DefaultErrorFunction()); - } - } - } - }, - Work.DefaultErrorFunction()); + AllBlocksFetch += TotalFetch; + AllBlocksSlack += TotalSlack; + BlocksNeededCount++; + AllBlockRequests += CollapsedBlockRanges.size(); + + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); + + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + } + else + { + BlocksNeededCount++; + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } } } + else + { + ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + } } for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) @@ -4374,14 +4627,12 @@ namespace { break; } - TotalPartWriteCount++; - Work.ScheduleWork( WritePool, // GetSyncWorkerPool(),// [&, CopyDataIndex](std::atomic&) { if (!AbortFlag) { - ZEN_TRACE_CPU("UpdateFolder_Copy"); + ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); FilteredWrittenBytesPerSecond.Start(); const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; @@ -4403,39 +4654,45 @@ namespace { }; std::vector WriteOps; - WriteOps.reserve(AllTargets.size()); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + if (!AbortFlag) { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) + ZEN_TRACE_CPU("Sort"); + WriteOps.reserve(AllTargets.size()); + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkSize = ChunkTarget.ChunkRawSize}); + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) + { + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkSize = ChunkTarget.ChunkRawSize}); + } + TargetStart += ChunkTarget.TargetChunkLocationCount; } - TargetStart += ChunkTarget.TargetChunkLocationCount; - } - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } - return false; - }); + }); + } if (!AbortFlag) { + ZEN_TRACE_CPU("Write"); + BufferedOpenFile SourceFile(LocalFilePath); WriteFileCache OpenFileCache; for (const WriteOp& Op : WriteOps) @@ -4473,23 +4730,26 @@ namespace { // Write tracking, updating this must be done without any files open (WriteFileCache) for (const WriteOp& Op : WriteOps) { - ZEN_TRACE_CPU("UpdateFolder_Copy_VerifyHash"); - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) { const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); + ZEN_TRACE_CPU("VerifyHash"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } } - ZEN_TRACE_CPU("UpdateFolder_Copy_rename"); + ZEN_TRACE_CPU("rename"); + ZEN_ASSERT_SLOW( + !std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } @@ -4508,300 +4768,643 @@ namespace { Work.DefaultErrorFunction()); } - size_t BlockCount = BlockDescriptions.size(); + for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) + { + if (AbortFlag) + { + break; + } - std::vector ChunkIsPickedUpByBlock(RemoteContent.ChunkedContent.ChunkHashes.size(), false); - auto GetNeededChunkBlockIndexes = [&RemoteContent, - &RemoteLookup, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &ChunkIsPickedUpByBlock](const ChunkBlockDescription& BlockDescription) { - std::vector NeededBlockChunkIndexes; - for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) - { - const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; - if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = It->second; - if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) - { - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) - { - ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; - NeededBlockChunkIndexes.push_back(ChunkBlockIndex); - } - } - } - } - return NeededBlockChunkIndexes; - }; + LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex]; - size_t BlocksNeededCount = 0; - uint64_t AllBlocksSize = 0; - uint64_t AllBlocksFetch = 0; - uint64_t AllBlocksSlack = 0; - uint64_t AllBlockRequests = 0; - uint64_t AllBlockChunksSize = 0; - for (size_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) + std::vector ChunkTargetPtrs = + std::move(LooseChunkHashWork.ChunkTargetPtrs); + const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; + + Work.ScheduleWork( + NetworkPool, // NetworkPool, // GetSyncWorkerPool(),// + [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { + if (!AbortFlag) + { + std::filesystem::path ExistingCompressedChunkPath; + { + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + if (std::filesystem::exists(CompressedChunkPath)) + { + IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); + if (ExistingCompressedPart) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) + { + LooseChunksBytes += ExistingCompressedPart.GetSize(); + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + ExistingCompressedChunkPath = std::move(CompressedChunkPath); + } + else + { + std::error_code DummyEc; + std::filesystem::remove(CompressedChunkPath, DummyEc); + } + } + } + } + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + WritePool, // WritePool, GetSyncWorkerPool() + [&Path, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &WriteToDiskBytes, + &ChunkCountWritten, + &WritePartsComplete, + &TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); + + FilteredWrittenBytesPerSecond.Start(); + + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } + + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + WriteToDiskBytes); + + if (!AbortFlag) + { + ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + + std::filesystem::remove(CompressedChunkPath); + + CompleteChunkTargets(TargetFolder, + RemoteContent, + ChunkHash, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + NeedHashVerify); + } + } + }, + Work.DefaultErrorFunction()); + } + else + { + FilteredDownloadedBytesPerSecond.Start(); + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + { + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + Storage, + Path, + RemoteContent, + RemoteLookup, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + NetworkPool, + WriteToDiskBytes, + BytesDownloaded, + MultipartAttachmentCount, + [&](uint64_t BytesDownloaded) { + LooseChunksBytes += BytesDownloaded; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + }, + [&]() { FilteredWrittenBytesPerSecond.Start(); }, + [&]() { + ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + }); + } + else + { + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + + IoBuffer BuildBlob = Storage.GetBuildBlob(BuildId, ChunkHash); + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + uint64_t BlobSize = BuildBlob.GetSize(); + BytesDownloaded += BlobSize; + LooseChunksBytes += BlobSize; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + std::filesystem::path CompressedChunkPath; + + // Check if the dowloaded file is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BuildBlob.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlobSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempChunk"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + BuildBlob.SetDeleteOnClose(false); + BuildBlob = {}; + CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + if (Ec) + { + CompressedChunkPath = std::filesystem::path{}; + + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BuildBlob = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlobSize, true); + BuildBlob.SetDeleteOnClose(true); + } + } + } + } + + if (CompressedChunkPath.empty() && (BlobSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempChunk"); + // Could not be moved and rather large, lets store it on disk + CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + TemporaryFile::SafeWriteFile(CompressedChunkPath, BuildBlob); + BuildBlob = {}; + } + DownloadedChunks++; + + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, // WritePool, GetSyncWorkerPool() + [&Path, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &WriteToDiskBytes, + &ChunkCountWritten, + &WritePartsComplete, + &TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath, + CompressedPart = std::move(BuildBlob)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteChunk"); + + FilteredWrittenBytesPerSecond.Start(); + + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (CompressedChunkPath.empty()) + { + ZEN_ASSERT(CompressedPart); + } + else + { + ZEN_ASSERT(!CompressedPart); + CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } + } + + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + WriteToDiskBytes); + + if (!AbortFlag) + { + ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + + if (!CompressedChunkPath.empty()) + { + std::filesystem::remove(CompressedChunkPath); + } + + CompleteChunkTargets(TargetFolder, + RemoteContent, + ChunkHash, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + NeedHashVerify); + } + } + }, + Work.DefaultErrorFunction()); + } + } + } + } + }, + Work.DefaultErrorFunction()); + } + + for (uint32_t BlockIndex : CachedChunkBlockIndexes) { - if (Work.IsAborted()) + if (AbortFlag) { break; } - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - const std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); - if (!BlockChunkIndexNeeded.empty()) - { - bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); - bool CanDoPartialBlockDownload = (BlockDescription.HeaderSize > 0) && (BlockDescription.ChunkCompressedLengths.size() == - BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) - { - struct BlockRangeDescriptor - { - uint64_t RangeStart = 0; - uint64_t RangeLength = 0; - uint32_t ChunkBlockIndexStart = 0; - uint32_t ChunkBlockIndexCount = 0; - }; - std::vector BlockRanges; - - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalisys"); - - uint32_t NeedBlockChunkIndexOffset = 0; - uint32_t ChunkBlockIndex = 0; - uint32_t CurrentOffset = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); - - BlockRangeDescriptor NextRange; - while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && - ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) + + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(), // WritePool, + [&, BlockIndex](std::atomic&) mutable { + if (!AbortFlag) { - const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + FilteredWrittenBytesPerSecond.Start(); + + std::filesystem::path BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) { - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - NextRange = {}; - } - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; + throw std::runtime_error( + fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); } - else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + ChunkCountWritten, + WriteToDiskBytes)) { - AllBlockChunksSize += ChunkCompressedLength; - if (NextRange.RangeLength == 0) - { - NextRange.RangeStart = CurrentOffset; - NextRange.ChunkBlockIndexStart = ChunkBlockIndex; - } - NextRange.RangeLength += ChunkCompressedLength; - NextRange.ChunkBlockIndexCount++; - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - NeedBlockChunkIndexOffset++; + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } - else + WritePartsComplete++; + std::filesystem::remove(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) { - ZEN_ASSERT(false); + FilteredWrittenBytesPerSecond.Stop(); } } - AllBlocksSize += CurrentOffset; - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - NextRange = {}; - } + }, + Work.DefaultErrorFunction()); + } - ZEN_ASSERT(!BlockRanges.empty()); - std::vector CollapsedBlockRanges; - auto It = BlockRanges.begin(); - CollapsedBlockRanges.push_back(*It++); - uint64_t TotalSlack = 0; - while (It != BlockRanges.end()) + for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) + { + if (AbortFlag) + { + break; + } + const BlockRangeDescriptor BlockRange = BlockRangeWorks[BlockRangeIndex]; + ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); + const uint32_t BlockIndex = BlockRange.BlockIndex; + Work.ScheduleWork( + NetworkPool, // NetworkPool, // GetSyncWorkerPool() + [&, BlockIndex, BlockRange](std::atomic&) { + if (!AbortFlag) { - BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); - uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); - uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; - if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer = + Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + if (!BlockBuffer) { - LastRange.ChunkBlockIndexCount = - (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; - LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; - TotalSlack += Slack; + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } - else + uint64_t BlockSize = BlockBuffer.GetSize(); + BytesDownloaded += BlockSize; + BlockBytes += BlockSize; + DownloadedBlocks++; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) { - CollapsedBlockRanges.push_back(*It); + FilteredDownloadedBytesPerSecond.Stop(); } - ++It; - } - uint64_t TotalFetch = 0; - for (const BlockRangeDescriptor& Range : CollapsedBlockRanges) - { - TotalFetch += Range.RangeLength; - } + std::filesystem::path BlockChunkPath; - AllBlocksFetch += TotalFetch; - AllBlocksSlack += TotalSlack; - BlocksNeededCount++; - AllBlockRequests += CollapsedBlockRanges.size(); + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - for (size_t BlockRangeIndex = 0; BlockRangeIndex < CollapsedBlockRanges.size(); BlockRangeIndex++) - { - TotalRequestCount++; - TotalPartWriteCount++; - const BlockRangeDescriptor BlockRange = CollapsedBlockRanges[BlockRangeIndex]; - // Partial block schedule - Work.ScheduleWork( - NetworkPool, // NetworkPool, // GetSyncWorkerPool() - [&, BlockIndex, BlockRange](std::atomic&) { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialGet"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - BytesDownloaded += BlockBuffer.GetSize(); - BlockBytes += BlockBuffer.GetSize(); - DownloadedBlocks++; - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), - Path / ZenTempBlockFolderName, - BlockDescription.BlockHash, - fmt::format("_{}", BlockRange.RangeStart)); - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - FilteredDownloadedBytesPerSecond.Stop(); + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; + + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } + } + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, // GetSyncWorkerPool(), - [&, BlockIndex, BlockRange, BlockPartialBuffer = std::move(Payload)](std::atomic&) { - if (!AbortFlag) - { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = + Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - FilteredWrittenBytesPerSecond.Start(); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, // WritePool, // GetSyncWorkerPool(), + [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( + std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - BlockPartialBuffer, - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - ChunkCountWritten, - WriteToDiskBytes)) - { - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); } - }, - [&, BlockIndex](const std::exception& Ex, std::atomic&) { - ZEN_ERROR("Failed writing block {}. Reason: {}", - BlockDescriptions[BlockIndex].BlockHash, - Ex.what()); - AbortFlag = true; - }); - } - }, - Work.DefaultErrorFunction()); + } + + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + ChunkCountWritten, + WriteToDiskBytes)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } + + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } } - } - else - { - BlocksNeededCount++; - TotalRequestCount++; - TotalPartWriteCount++; + }, + Work.DefaultErrorFunction()); + } - Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(), // NetworkPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_Get"); + for (uint32_t BlockIndex : FullBlockWorks) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + NetworkPool, // GetSyncWorkerPool(), // NetworkPool, + [&, BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); - // Full block schedule + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - BytesDownloaded += BlockBuffer.GetSize(); - BlockBytes += BlockBuffer.GetSize(); - DownloadedBlocks++; - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + uint64_t BlockSize = BlockBuffer.GetSize(); + BytesDownloaded += BlockSize; + BlockBytes += BlockSize; + DownloadedBlocks++; + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - CompositeBuffer Payload = WriteToTempFileIfNeeded(CompositeBuffer(std::move(BlockBuffer)), - Path / ZenTempBlockFolderName, - BlockDescription.BlockHash); - if (!AbortFlag) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - Work.ScheduleWork( - WritePool, - [&, BlockIndex, BlockBuffer = std::move(Payload)](std::atomic&) { - if (!AbortFlag) - { - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - BlockBuffer, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - ChunkCountWritten, - WriteToDiskBytes)) - { - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } - }, - Work.DefaultErrorFunction()); - } - } - else - { - ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); - } + } + + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } + + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, // WritePool, GetSyncWorkerPool() + [&RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &ChunkCountWritten, + &WriteToDiskBytes, + &WritePartsComplete, + &TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } + } + + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + ChunkCountWritten, + WriteToDiskBytes)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } + + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + } + }, + Work.DefaultErrorFunction()); } + ZEN_DEBUG("Fetching {} with {} slack (ideal {}) out of {} using {} requests for {} blocks", NiceBytes(AllBlocksFetch), NiceBytes(AllBlocksSlack), @@ -4809,28 +5412,30 @@ namespace { NiceBytes(AllBlocksSize), AllBlockRequests, BlocksNeededCount); - ZEN_TRACE_CPU("HandleChunks_Wait"); - - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); - FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load()); - FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load()); - std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", - RequestsComplete.load(), - TotalRequestCount, - NiceBytes(BytesDownloaded.load()), - NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - ChunkCountWritten.load(), - ChunkCountToWrite, - NiceBytes(WriteToDiskBytes.load()), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState({.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, - false); - }); + { + ZEN_TRACE_CPU("WriteChunks_Wait"); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); + FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load()); + FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load()); + std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", + RequestsComplete.load(), + TotalRequestCount, + NiceBytes(BytesDownloaded.load()), + NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), + ChunkCountWritten.load(), + ChunkCountToWrite, + NiceBytes(WriteToDiskBytes.load()), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + WriteProgressBar.UpdateState({.Task = "Writing chunks ", + .Details = Details, + .TotalCount = gsl::narrow(ChunkCountToWrite), + .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, + false); + }); + } FilteredWrittenBytesPerSecond.Stop(); FilteredDownloadedBytesPerSecond.Stop(); @@ -4842,19 +5447,32 @@ namespace { WriteProgressBar.Finish(); - ZEN_CONSOLE("Downloaded {} ({}bits/s). Wrote {} ({}B/s). Completed in {}", + uint32_t RawSequencesMissingWriteCount = 0; + for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) + { + const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; + if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + { + RawSequencesMissingWriteCount++; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; + ZEN_ASSERT(!IncompletePath.empty()); + const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + } + } + ZEN_ASSERT(RawSequencesMissingWriteCount == 0); + + ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", NiceBytes(BytesDownloaded.load()), NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), BytesDownloaded * 8)), + NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), NiceBytes(WriteToDiskBytes.load()), NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), WriteToDiskBytes.load())), + NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); } - for (const auto& SequenceIndexChunksLeftToWriteCounter : SequenceIndexChunksLeftToWriteCounters) - { - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() == 0); - } - std::vector> Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) @@ -4867,17 +5485,24 @@ namespace { // Move all files we will reuse to cache folder // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + if (!LocalPathIndexesMatchingSequenceIndexes.empty()) { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + ZEN_TRACE_CPU("UpdateFolder_CacheReused"); + uint64_t TotalFullFileSizeCached = 0; + for (uint32_t LocalPathIndex : LocalPathIndexesMatchingSequenceIndexes) { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); SetFileReadOnly(LocalFilePath, false); + ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); std::filesystem::rename(LocalFilePath, CacheFilePath); + TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); } + ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(TotalFullFileSizeCached)); } if (WipeTargetFolder) @@ -4923,6 +5548,8 @@ namespace { } { + ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); + WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // ProgressBar RebuildProgressBar(UsePlainProgress); @@ -4943,8 +5570,6 @@ namespace { break; } - ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); - size_t TargetCount = 1; const IoHash& RawHash = Targets[TargetOffset].first; while (Targets[TargetOffset + TargetCount].first == RawHash) @@ -5038,17 +5663,19 @@ namespace { TargetOffset += TargetCount; } - ZEN_TRACE_CPU("FinalizeTree_Wait"); + { + ZEN_TRACE_CPU("FinalizeTree_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); - std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); - RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", - .Details = Details, - .TotalCount = gsl::narrow(Targets.size()), - .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, - false); - }); + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); + RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", + .Details = Details, + .TotalCount = gsl::narrow(Targets.size()), + .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, + false); + }); + } if (AbortFlag) { @@ -5608,15 +6235,10 @@ namespace { const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; CreateDirectories(ZenTempFolder); - auto _ = MakeGuard([&]() { - CleanDirectory(ZenTempFolder, {}); - std::filesystem::remove(ZenTempFolder); - }); + CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempChunkFolderName); // TODO: Don't clear this - pick up files -> chunks to use - CreateDirectories(Path / - ZenTempCacheFolderName); // TODO: Don't clear this - pick up files and use as sequences (non .tmp extension) - // and delete .tmp (maybe?) - chunk them? How do we know the file is worth chunking? + CreateDirectories(Path / ZenTempCacheFolderName); + CreateDirectories(Path / ZenTempDownloadFolderName); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -5748,6 +6370,8 @@ namespace { ZEN_CONSOLE("Downloaded build in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); } } + CleanDirectory(ZenTempFolder, {}); + std::filesystem::remove(ZenTempFolder); } void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) @@ -6191,6 +6815,15 @@ BuildsCommand::BuildsCommand() ""); m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); + + AddCloudOptions(m_MultiTestDownloadOptions); + AddFileOptions(m_MultiTestDownloadOptions); + AddOutputOptions(m_MultiTestDownloadOptions); + m_MultiTestDownloadOptions + .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); + m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); + m_MultiTestDownloadOptions.parse_positional({"local-path"}); + m_MultiTestDownloadOptions.positional_help("local-path"); } BuildsCommand::~BuildsCommand() = default; @@ -6667,6 +7300,79 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return AbortFlag ? 11 : 0; } + if (SubOption == &m_MultiTestDownloadOptions) + { + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + } + + ParseStorageOptions(); + ParseAuthOptions(); + + HttpClient Http(m_BuildsUrl, ClientSettings); + // m_StoragePath = "D:\\buildstorage"; + // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; + // std::vector BuildIdStrings{"07d3942f0e7f4ca1b13b0587", + // "07d394eed89d769f2254e75d", + // "07d3953f22fa3f8000fa6f0a", + // "07d3959df47ed1f42ddbe44c", + // "07d395fa7803d50804f14417", + // "07d3964f919d577a321a1fdd", + // "07d396a6ce875004e16b9528"}; + + BuildStorage::Statistics StorageStats; + std::unique_ptr Storage; + std::string StorageName; + if (!m_BuildsUrl.empty()) + { + ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + FormatArray(m_BuildIds, " "sv), + m_Path, + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + StorageName = "Cloud DDC"; + } + else if (!m_StoragePath.empty()) + { + ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray(m_BuildIds, " "sv), m_Path, m_StoragePath); + Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + + for (const std::string& BuildIdString : m_BuildIds) + { + Oid BuildId = Oid::FromHexString(BuildIdString); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); + } + DownloadFolder(*Storage, + BuildId, + {}, + {}, + m_Path, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + BuildIdString == m_BuildIds.front(), + true); + if (AbortFlag) + { + ZEN_CONSOLE("Download cancelled"); + return 11; + } + ZEN_CONSOLE("\n"); + } + return 0; + } + if (SubOption == &m_TestOptions) { ParseStorageOptions(); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 838a17807..167a5d29f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -92,18 +92,22 @@ private: cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; + cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; + std::vector m_BuildIds; + cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; std::string m_BlobHash; cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; - cxxopts::Options* m_SubCommands[7] = {&m_ListOptions, + cxxopts::Options* m_SubCommands[8] = {&m_ListOptions, &m_UploadOptions, &m_DownloadOptions, &m_DiffOptions, &m_TestOptions, &m_FetchBlobOptions, - &m_ValidateBuildPartOptions}; + &m_ValidateBuildPartOptions, + &m_MultiTestDownloadOptions}; }; } // namespace zen diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 6e879ca0d..95876cff4 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -858,7 +858,7 @@ BasicFileWriter::Flush() } IoBuffer -WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path) +WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) { TemporaryFile Temp; std::error_code Ec; @@ -868,6 +868,7 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path throw std::system_error(Ec, fmt::format("Failed to create temp file for blob at '{}'", Path)); } + uint64_t BufferSize = Buffer.GetSize(); { uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; @@ -899,21 +900,31 @@ WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path Temp.MoveTemporaryIntoPlace(Path, Ec); if (Ec) { - IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); - if (TmpBuffer) + Ec.clear(); + BasicFile OpenTemp(Path, BasicFile::Mode::kDelete, Ec); + if (Ec) { - IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); - const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); - if (ExistingHash == ExpectedHash) - { - TmpBuffer.SetDeleteOnClose(true); - return TmpBuffer; - } + throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); } - throw std::system_error(Ec, fmt::format("Failed to move temp file to '{}'", Path)); - } + if (OpenTemp.FileSize() != BufferSize) + { + throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file size already exists", Path)); + } + IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); - IoBuffer TmpBuffer = IoBufferBuilder::MakeFromFile(Path); + IoHash ExistingHash = IoHash::HashBuffer(TmpBuffer); + const IoHash ExpectedHash = IoHash::HashBuffer(Buffer); + if (ExistingHash != ExpectedHash) + { + throw std::runtime_error(fmt::format("Failed to move temp file to '{}' - mismatching file hash already exists", Path)); + } + Buffer = CompositeBuffer{}; + TmpBuffer.SetDeleteOnClose(true); + return TmpBuffer; + } + Buffer = CompositeBuffer{}; + BasicFile OpenTemp(Path, BasicFile::Mode::kDelete); + IoBuffer TmpBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BufferSize, true); TmpBuffer.SetDeleteOnClose(true); return TmpBuffer; } diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 1844f6a63..88c3bb5b9 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -193,7 +193,7 @@ public: const CompositeBuffer& CompressedData, uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const = 0; + std::function&& Callback) const = 0; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -287,13 +287,16 @@ public: const CompositeBuffer& CompressedData, uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const final + std::function&& Callback) const final { if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() && Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize && (RawOffset + RawSize) <= Header.TotalRawSize) { - Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)); + if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize))) + { + return false; + } return true; } return false; @@ -616,7 +619,7 @@ public: const CompositeBuffer& CompressedData, uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const final; + std::function&& Callback) const final; protected: virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; @@ -744,7 +747,7 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, const CompositeBuffer& CompressedData, uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const + std::function&& Callback) const { if (Header.TotalCompressedSize != CompressedData.GetSize()) { @@ -814,14 +817,23 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, Source.Detach(); return false; } - Callback(BlockIndex * BlockSize + OffsetInFirstBlock, - CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))); + if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)))) + { + Source.Detach(); + return false; + } } else { - Callback(BlockIndex * BlockSize + OffsetInFirstBlock, - CompositeBuffer( - IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))); + if (!Callback( + BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer( + IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)))) + { + Source.Detach(); + return false; + } } OffsetInFirstBlock = 0; @@ -858,14 +870,21 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, { return false; } - Callback(BlockIndex * BlockSize + OffsetInFirstBlock, - CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress))); + if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)))) + { + return false; + } } else { - Callback(BlockIndex * BlockSize + OffsetInFirstBlock, - CompositeBuffer( - IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress))); + if (!Callback( + BlockIndex * BlockSize + OffsetInFirstBlock, + CompositeBuffer( + IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)))) + { + return false; + } } OffsetInFirstBlock = 0; @@ -1978,7 +1997,7 @@ CompressedBuffer::DecompressToComposite() const bool CompressedBuffer::DecompressToStream(uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const + std::function&& Callback) const { using namespace detail; if (CompressedData) diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index a78132879..57798b6f4 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -186,7 +186,7 @@ private: uint64_t m_BufferEnd; }; -IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path); +IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path); ZENCORE_API void basicfile_forcelink(); diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 3969e9dbd..74fd5f767 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -209,7 +209,7 @@ public: */ [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const; + std::function&& Callback) const; /** A null compressed buffer. */ static const CompressedBuffer Null; diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp index d15fb2e83..445fe939e 100644 --- a/src/zencore/workthreadpool.cpp +++ b/src/zencore/workthreadpool.cpp @@ -274,7 +274,7 @@ WorkerThreadPool::ScheduleWork(Ref Work) void WorkerThreadPool::ScheduleWork(std::function&& Work) { - ScheduleWork(Ref(new detail::LambdaWork(Work))); + ScheduleWork(Ref(new detail::LambdaWork(std::move(Work)))); } [[nodiscard]] size_t diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index a19cf5c1b..f3c14edc4 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -187,31 +187,54 @@ GenerateChunkBlock(std::vector>&& FetchChunks, return CompressedBlock; } -bool -IterateChunkBlock(const SharedBuffer& BlockPayload, - std::function Visitor, - uint64_t& OutHeaderSize) +std::vector +ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize) { - ZEN_ASSERT(BlockPayload); - if (BlockPayload.GetSize() < 1) - { - return false; - } - - MemoryView BlockView = BlockPayload.GetView(); - const uint8_t* ReadPtr = reinterpret_cast(BlockView.GetData()); + const uint8_t* ReadPtr = reinterpret_cast(BlockView.GetData()); uint32_t NumberSize; uint64_t ChunkCount = ReadVarUInt(ReadPtr, NumberSize); ReadPtr += NumberSize; - std::vector ChunkSizes; + std::vector ChunkSizes; ChunkSizes.reserve(ChunkCount); while (ChunkCount--) { - ChunkSizes.push_back(ReadVarUInt(ReadPtr, NumberSize)); + if (ReadPtr >= BlockView.GetDataEnd()) + { + throw std::runtime_error("Invalid block header, block data ended unexpectedly"); + } + uint64_t ChunkSize = ReadVarUInt(ReadPtr, NumberSize); + if (ChunkSize > std::numeric_limits::max()) + { + throw std::runtime_error("Invalid block header, header data is corrupt"); + } + if (ChunkSize < 1) + { + throw std::runtime_error("Invalid block header, header data is corrupt"); + } + ChunkSizes.push_back(gsl::narrow(ChunkSize)); ReadPtr += NumberSize; } uint64_t Offset = std::distance((const uint8_t*)BlockView.GetData(), ReadPtr); OutHeaderSize = Offset; + return ChunkSizes; +} + +bool +IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor, + uint64_t& OutHeaderSize) +{ + ZEN_ASSERT(BlockPayload); + if (BlockPayload.GetSize() < 1) + { + return false; + } + + MemoryView BlockView = BlockPayload.GetView(); + + std::vector ChunkSizes = ReadChunkBlockHeader(BlockView, OutHeaderSize); + uint64_t Offset = OutHeaderSize; + OutHeaderSize = Offset; for (uint64_t ChunkSize : ChunkSizes) { IoBuffer Chunk(BlockPayload.AsIoBuffer(), Offset, ChunkSize); diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h index 21107fb7c..277580c74 100644 --- a/src/zenutil/include/zenutil/chunkblock.h +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -31,9 +31,10 @@ CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function(const IoHash& RawHash)> FetchChunkFunc; -CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); -bool IterateChunkBlock(const SharedBuffer& BlockPayload, - std::function Visitor, - uint64_t& OutHeaderSize); +CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); +bool IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor, + uint64_t& OutHeaderSize); +std::vector ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize); } // namespace zen -- cgit v1.2.3 From 90db3ced033d4e06da2739e5d97cdeff2b0ba3b9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 11 Mar 2025 09:58:07 +0100 Subject: Build command tweaks (#301) - Improvement: Don't chunk up .mp4 files as they generally won't benefit from deduplication or partial in-place-updates - Improvement: Emit build name to console output when downloading a build - Improvement: Added some debug logging - Bugfix: Logging setup would previously not function correctly when not logging to file --- src/zen/cmds/builds_cmd.cpp | 19 ++++++--- src/zen/zen.cpp | 53 +++++++++--------------- src/zencore/include/zencore/compactbinaryfmt.h | 23 ++++++++++ src/zenhttp/include/zenhttp/formatters.h | 17 ++++---- src/zenutil/include/zenutil/chunkingcontroller.h | 2 +- src/zenutil/include/zenutil/logging.h | 1 + src/zenutil/logging.cpp | 27 ++++++------ 7 files changed, 83 insertions(+), 59 deletions(-) create mode 100644 src/zencore/include/zencore/compactbinaryfmt.h (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2c7a73fb3..56c3c3c4f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -5699,10 +5700,14 @@ namespace { std::vector> AvailableParts; CbObject BuildObject = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Payload size: {}", + + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), + BuildObject["BuildName"sv].AsString(), NiceBytes(BuildObject.GetSize())); + ZEN_DEBUG("Build object: {}", BuildObject); + CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) { @@ -5782,11 +5787,13 @@ namespace { { ZEN_TRACE_CPU("GetRemoteContent"); - Stopwatch GetBuildPartTimer; - CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildParts[0].first); + Stopwatch GetBuildPartTimer; + const Oid BuildPartId = BuildParts[0].first; + const std::string_view BuildPartName = BuildParts[0].second; + CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", - BuildParts[0].first, - BuildParts[0].second, + BuildPartId, + BuildPartName, NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), NiceBytes(BuildPartManifest.GetSize())); @@ -5915,7 +5922,7 @@ namespace { OutPartContents.resize(1); ParseBuildPartManifest(Storage, BuildId, - BuildParts[0].first, + BuildPartId, BuildPartManifest, OutPartContents[0], OutBlockDescriptions, diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 713a0d66d..4e6161e86 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -377,31 +378,6 @@ main(int argc, char** argv) using namespace zen; using namespace std::literals; - zen::logging::InitializeLogging(); - - // Set output mode to handle virtual terminal sequences - zen::logging::EnableVTMode(); - std::set_terminate([]() { - void* Frames[8]; - uint32_t FrameCount = GetCallstack(2, 8, Frames); - CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); - ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, " ")); - FreeCallstack(Callstack); - }); - - LoggerRef DefaultLogger = zen::logging::Default(); - auto& Sinks = DefaultLogger.SpdLogger->sinks(); - - Sinks.clear(); - auto ConsoleSink = std::make_shared(); - Sinks.push_back(ConsoleSink); - - zen::MaximizeOpenFileCount(); - - ////////////////////////////////////////////////////////////////////////// - - auto _ = zen::MakeGuard([] { spdlog::shutdown(); }); - AttachCommand AttachCmd; BenchCommand BenchCmd; BuildsCommand BuildsCmd; @@ -674,14 +650,25 @@ main(int argc, char** argv) exit(0); } - if (GlobalOptions.IsDebug) - { - logging::SetLogLevel(logging::level::Debug); - } - if (GlobalOptions.IsVerbose) - { - logging::SetLogLevel(logging::level::Trace); - } + zen::LoggingOptions LogOptions; + LogOptions.IsDebug = GlobalOptions.IsDebug; + LogOptions.IsVerbose = GlobalOptions.IsVerbose; + LogOptions.AllowAsync = false; + zen::InitializeLogging(LogOptions); + + std::set_terminate([]() { + void* Frames[8]; + uint32_t FrameCount = GetCallstack(2, 8, Frames); + CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); + ZEN_CRITICAL("Program exited abnormally via std::terminate()\n{}", CallstackToString(Callstack, " ")); + FreeCallstack(Callstack); + }); + + zen::MaximizeOpenFileCount(); + + ////////////////////////////////////////////////////////////////////////// + + auto _ = zen::MakeGuard([] { zen::ShutdownLogging(); }); #if ZEN_WITH_TRACE if (TraceHost.size()) diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h new file mode 100644 index 000000000..ae0c3eb42 --- /dev/null +++ b/src/zencore/include/zencore/compactbinaryfmt.h @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include + +template +requires DerivedFrom +struct fmt::formatter : fmt::formatter +{ + auto format(const zen::CbObject& a, format_context& ctx) const + { + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(a, ObjStr); + return fmt::formatter::format(ObjStr.ToView(), ctx); + } +}; diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index 0fa5dc6da..74da9ab05 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -7,6 +7,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -67,15 +68,17 @@ struct fmt::formatter { using namespace std::literals; - if (Response.status_code == 200 || Response.status_code == 201) + zen::NiceTimeSpanMs NiceResponseTime(uint64_t(Response.elapsed * 1000)); + + if (zen::IsHttpSuccessCode(Response.status_code)) { return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s", + "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}", Response.url.str(), Response.status_code, Response.uploaded_bytes, Response.downloaded_bytes, - Response.elapsed); + NiceResponseTime.c_str()); } else { @@ -90,12 +93,12 @@ struct fmt::formatter std::string_view Json = Obj.ToJson(Sb).ToView(); return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s, Response: '{}', Reason: '{}'", + "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", Response.url.str(), Response.status_code, Response.uploaded_bytes, Response.downloaded_bytes, - Response.elapsed, + NiceResponseTime.c_str(), Json, Response.reason); } @@ -104,12 +107,12 @@ struct fmt::formatter zen::BodyLogFormatter Body(Response.text); return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s, Reponse: '{}', Reason: '{}'", + "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", Response.url.str(), Response.status_code, Response.uploaded_bytes, Response.downloaded_bytes, - Response.elapsed, + NiceResponseTime.c_str(), Body.GetText(), Response.reason); } diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index ebc80e207..246f4498a 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -11,7 +11,7 @@ namespace zen { -const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; +const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h index ebf6372fc..d64eef207 100644 --- a/src/zenutil/include/zenutil/logging.h +++ b/src/zenutil/include/zenutil/logging.h @@ -32,6 +32,7 @@ struct LoggingOptions bool IsDebug = false; bool IsVerbose = false; bool IsTest = false; + bool AllowAsync = true; bool NoConsoleOutput = false; std::filesystem::path AbsLogFile; // Absolute path to main log file std::string LogId; diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index 0444fa2c4..762b75a59 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -49,7 +49,7 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) zen::logging::InitializeLogging(); zen::logging::EnableVTMode(); - bool IsAsync = true; + bool IsAsync = LogOptions.AllowAsync; if (LogOptions.IsDebug) { @@ -181,7 +181,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) LogLevel = logging::level::Debug; } - if (LogOptions.IsTest) + if (LogOptions.IsTest || LogOptions.IsVerbose) { LogLevel = logging::level::Trace; } @@ -194,18 +194,21 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) spdlog::set_formatter( std::make_unique(LogOptions.LogId, std::chrono::system_clock::now())); // default to duration prefix - if (LogOptions.AbsLogFile.extension() == ".json") - { - g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); - } - else + if (g_FileSink) { - g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix - } + if (LogOptions.AbsLogFile.extension() == ".json") + { + g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); + } + else + { + g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix + } - const std::string StartLogTime = zen::DateTime::Now().ToIso8601(); + const std::string StartLogTime = zen::DateTime::Now().ToIso8601(); - spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); }); + spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); }); + } g_IsLoggingInitialized = true; } @@ -213,7 +216,7 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) void ShutdownLogging() { - if (g_IsLoggingInitialized) + if (g_IsLoggingInitialized && g_FileSink) { auto DefaultLogger = zen::logging::Default(); ZEN_LOG_INFO(DefaultLogger, "log ending at {}", zen::DateTime::Now().ToIso8601()); -- cgit v1.2.3 From 5f251575449fff1985a465607ebe03cc320ac8c8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 11 Mar 2025 14:32:25 +0100 Subject: async find blocks (#300) * put/get build and find blocks while scanning local folder when uploading * changelog * remove redundant move --- src/zen/cmds/builds_cmd.cpp | 122 +++++++++++++++++++++++++++++--------------- 1 file changed, 80 insertions(+), 42 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 56c3c3c4f..e03175256 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -81,6 +81,8 @@ namespace { const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; + const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u; + const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; @@ -2309,6 +2311,61 @@ namespace { CbObject ChunkerParameters; + struct PrepareBuildResult + { + std::vector KnownBlocks; + uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize; + uint64_t PayloadSize = 0; + uint64_t PrepareBuildTimeMs = 0; + uint64_t FindBlocksTimeMs = 0; + uint64_t ElapsedTimeMs = 0; + }; + + FindBlocksStatistics FindBlocksStats; + + std::future PrepBuildResultFuture = + GetSmallWorkerPool(EWorkloadType::Burst) + .EnqueueTask(std::packaged_task{ + [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + PrepareBuildResult Result; + Stopwatch Timer; + if (CreateBuild) + { + Stopwatch PutBuildTimer; + CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); + Result.PayloadSize = MetaData.GetSize(); + } + else + { + Stopwatch GetBuildTimer; + CbObject Build = Storage.GetBuild(BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); + } + } + + if (!IgnoreExistingBlocks) + { + Stopwatch KnownBlocksTimer; + Result.KnownBlocks = Storage.FindBlocks(BuildId); + FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); + FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; + }}); + ChunkedFolderContent LocalContent; GetFolderContentStatistics LocalFolderScanStats; @@ -2517,43 +2574,29 @@ namespace { NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); } - const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); - std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; - - if (CreateBuild) - { - Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); - ZEN_CONSOLE("PutBuild took {}. Payload size: {}", - NiceTimeSpanMs(PutBuildTimer.GetElapsedTimeMs()), - NiceBytes(MetaData.GetSize())); - PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(PreferredMultipartChunkSize); - } - else - { - Stopwatch GetBuildTimer; - CbObject Build = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), NiceBytes(Build.GetSize())); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - PreferredMultipartChunkSize = ChunkSize; - } - else if (AllowMultiparts) - { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(PreferredMultipartChunkSize)); - } - } - - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent); - FindBlocksStatistics FindBlocksStats; GenerateBlocksStatistics GenerateBlocksStats; LooseChunksStatistics LooseChunksStats; - std::vector KnownBlocks; - std::vector ReuseBlockIndexes; - std::vector NewBlockChunkIndexes; - Stopwatch BlockArrangeTimer; + std::vector ReuseBlockIndexes; + std::vector NewBlockChunkIndexes; + + PrepareBuildResult PrepBuildResult = PrepBuildResultFuture.get(); + + ZEN_CONSOLE("Build prepare took {}. {} took {}, payload size {}{}", + NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs), + CreateBuild ? "PutBuild" : "GetBuild", + NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs), + NiceBytes(PrepBuildResult.PayloadSize), + IgnoreExistingBlocks ? "" + : fmt::format(". Found {} blocks in {}", + PrepBuildResult.KnownBlocks.size(), + NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; + + Stopwatch BlockArrangeTimer; std::vector LooseChunkIndexes; { @@ -2583,12 +2626,7 @@ namespace { } else { - Stopwatch KnownBlocksTimer; - KnownBlocks = Storage.FindBlocks(BuildId); - FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); - FindBlocksStats.FoundBlockCount = KnownBlocks.size(); - - ReuseBlockIndexes = FindReuseBlocks(KnownBlocks, + ReuseBlockIndexes = FindReuseBlocks(PrepBuildResult.KnownBlocks, LocalContent.ChunkedContent.ChunkHashes, BlockChunkIndexes, BlockReuseMinPercentLimit, @@ -2596,7 +2634,7 @@ namespace { FindBlocksStats); FindBlocksStats.AcceptedBlockCount = ReuseBlockIndexes.size(); - for (const ChunkBlockDescription& Description : KnownBlocks) + for (const ChunkBlockDescription& Description : PrepBuildResult.KnownBlocks) { for (uint32_t ChunkRawLength : Description.ChunkRawLengths) { @@ -2708,8 +2746,8 @@ namespace { AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size()); for (size_t ReuseBlockIndex : ReuseBlockIndexes) { - AllChunkBlockDescriptions.push_back(KnownBlocks[ReuseBlockIndex]); - AllChunkBlockHashes.push_back(KnownBlocks[ReuseBlockIndex].BlockHash); + AllChunkBlockDescriptions.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex]); + AllChunkBlockHashes.push_back(PrepBuildResult.KnownBlocks[ReuseBlockIndex].BlockHash); } AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(), NewBlocks.BlockDescriptions.begin(), -- cgit v1.2.3 From fb09d861fd76e459ac86bec388bd406aaca8e681 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 10:51:57 +0100 Subject: improved block gen logic (#302) - Improvement: Reduced memory usage during upload and part upload validation - Improvement: Reduced I/O usage during upload and download - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild - Improvement: More trace scopes for build upload operations - Bugfix: Fixed crash during download when trying to write outside a file range --- src/zen/cmds/builds_cmd.cpp | 1007 ++++++++++++++------------ src/zen/zen.cpp | 3 + src/zenutil/chunkedcontent.cpp | 15 +- src/zenutil/chunkedfile.cpp | 6 + src/zenutil/chunkingcontroller.cpp | 4 + src/zenutil/filebuildstorage.cpp | 17 + src/zenutil/include/zenutil/chunkedcontent.h | 4 +- 7 files changed, 599 insertions(+), 457 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e03175256..baa46dda8 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -191,15 +191,34 @@ namespace { return SB.ToString(); } - void CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) + bool CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) { ZEN_TRACE_CPU("CleanDirectory"); + bool CleanWipe = true; + DirectoryContent LocalDirectoryContent; GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) { - std::filesystem::remove(LocalFilePath); + try + { + std::filesystem::remove(LocalFilePath); + } + catch (const std::exception&) + { + // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time + Sleep(200); + try + { + std::filesystem::remove(LocalFilePath); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); + CleanWipe = false; + } + } } for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) @@ -215,10 +234,28 @@ namespace { } if (!Leave) { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); + try + { + zen::CleanDirectory(LocalDirPath); + std::filesystem::remove(LocalDirPath); + } + catch (const std::exception&) + { + Sleep(200); + try + { + zen::CleanDirectory(LocalDirPath); + std::filesystem::remove(LocalDirPath); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); + CleanWipe = false; + } + } } } + return CleanWipe; } std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) @@ -1188,6 +1225,7 @@ namespace { uint64_t& OutCompressedSize, uint64_t& OutDecompressedSize) { + ZEN_TRACE_CPU("ValidateBlob"); IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash); if (!Payload) { @@ -1214,6 +1252,7 @@ namespace { const IoHash& ChunkHash, ReadFileCache& OpenFileCache) { + ZEN_TRACE_CPU("FetchChunk"); auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end()); uint32_t ChunkIndex = It->second; @@ -1233,6 +1272,7 @@ namespace { ChunkBlockDescription& OutBlockDescription, DiskStatistics& DiskStats) { + ZEN_TRACE_CPU("GenerateBlock"); ReadFileCache OpenFileCache(DiskStats, Path, Content, Lookup, 4); std::vector> BlockContent; @@ -1255,6 +1295,103 @@ namespace { return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription); }; + CompressedBuffer RebuildBlock(const std::filesystem::path& Path, + const ChunkedFolderContent& Content, + const ChunkedContentLookup& Lookup, + CompositeBuffer&& HeaderBuffer, + const std::vector& ChunksInBlock, + DiskStatistics& DiskStats) + { + ZEN_TRACE_CPU("RebuildBlock"); + ReadFileCache OpenFileCache(DiskStats, Path, Content, Lookup, 4); + + std::vector ResultBuffers; + ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size()); + ResultBuffers.insert(ResultBuffers.end(), HeaderBuffer.GetSegments().begin(), HeaderBuffer.GetSegments().end()); + for (uint32_t ChunkIndex : ChunksInBlock) + { + std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); + ZEN_ASSERT(!ChunkLocations.empty()); + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + ChunkLocations[0].Offset, + Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } + return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); + }; + + void DownloadLargeBlob(BuildStorage& Storage, + const std::filesystem::path& DownloadFolder, + const Oid& BuildId, + const IoHash& ChunkHash, + const std::uint64_t PreferredMultipartChunkSize, + ParallellWork& Work, + WorkerThreadPool& NetworkPool, + std::atomic& BytesDownloaded, + std::atomic& MultipartAttachmentCount, + std::function&& OnDownloadComplete) + { + ZEN_TRACE_CPU("DownloadLargeBlob"); + + struct WorkloadData + { + TemporaryFile TempFile; + }; + std::shared_ptr Workload(std::make_shared()); + + std::error_code Ec; + Workload->TempFile.CreateTemporary(DownloadFolder, Ec); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); + } + std::vector> WorkItems = Storage.GetLargeBuildBlob( + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + [Workload, &BytesDownloaded, OnDownloadComplete = std::move(OnDownloadComplete)](uint64_t Offset, + const IoBuffer& Chunk, + uint64_t BytesRemaining) { + BytesDownloaded += Chunk.GetSize(); + + if (!AbortFlag.load()) + { + ZEN_TRACE_CPU("DownloadLargeBlob_Save"); + Workload->TempFile.Write(Chunk.GetView(), Offset); + if (Chunk.GetSize() == BytesRemaining) + { + uint64_t PayloadSize = Workload->TempFile.FileSize(); + void* FileHandle = Workload->TempFile.Detach(); + ZEN_ASSERT(FileHandle != nullptr); + IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true); + Payload.SetDeleteOnClose(true); + OnDownloadComplete(std::move(Payload)); + } + } + }); + if (!WorkItems.empty()) + { + MultipartAttachmentCount++; + } + for (auto& WorkItem : WorkItems) + { + Work.ScheduleWork( + NetworkPool, // GetSyncWorkerPool(),// + [WorkItem = std::move(WorkItem)](std::atomic&) { + ZEN_TRACE_CPU("DownloadLargeBlob_Work"); + if (!AbortFlag) + { + WorkItem(); + } + }, + Work.DefaultErrorFunction()); + } + } + void ValidateBuildPart(BuildStorage& Storage, const Oid& BuildId, Oid BuildPartId, const std::string_view BuildPartName) { Stopwatch Timer; @@ -1274,6 +1411,11 @@ namespace { throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", BuildId, BuildPartName)); } } + uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize; + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + PreferredMultipartChunkSize = ChunkSize; + } CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); std::vector ChunkAttachments; @@ -1295,9 +1437,20 @@ namespace { } WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& ReadPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // ParallellWork Work(AbortFlag); + const std::filesystem::path TempFolder = ".zen-tmp"; + + CreateDirectories(TempFolder); + auto __ = MakeGuard([&TempFolder]() { + if (CleanDirectory(TempFolder, {})) + { + std::filesystem::remove(TempFolder); + } + }); + ProgressBar ProgressBar(UsePlainProgress); uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); @@ -1308,51 +1461,58 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredVerifiedBytesPerSecond; + std::atomic MultipartAttachmentCount = 0; + for (const IoHash& ChunkAttachment : ChunkAttachments) { Work.ScheduleWork( - NetworkPool, + ReadPool, [&, ChunkAttachment](std::atomic&) { if (!AbortFlag) { - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer Payload = Storage.GetBuildBlob(BuildId, ChunkAttachment); - DownloadedAttachmentCount++; - DownloadedByteCount += Payload.GetSize(); - if (DownloadedAttachmentCount.load() == AttachmentsToVerifyCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (!Payload) - { - throw std::runtime_error(fmt::format("Chunk attachment {} could not be found", ChunkAttachment)); - } - if (!AbortFlag) - { - Work.ScheduleWork( - VerifyPool, - [&, Payload = std::move(Payload), ChunkAttachment](std::atomic&) mutable { - if (!AbortFlag) - { - FilteredVerifiedBytesPerSecond.Start(); + ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(std::move(Payload), ChunkAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid", - ChunkAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); - VerifiedAttachmentCount++; - VerifiedByteCount += DecompressedSize; - if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) - { - FilteredVerifiedBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); - } + FilteredDownloadedBytesPerSecond.Start(); + DownloadLargeBlob(Storage, + TempFolder, + BuildId, + ChunkAttachment, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadedByteCount, + MultipartAttachmentCount, + [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) { + Payload.SetContentType(ZenContentType::kCompressedBinary); + if (!AbortFlag) + { + Work.ScheduleWork( + VerifyPool, + [&, Payload = std::move(Payload), ChunkHash](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_Validate"); + + FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); + ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid", + ChunkHash, + NiceBytes(CompressedSize), + NiceBytes(DecompressedSize)); + VerifiedAttachmentCount++; + VerifiedByteCount += DecompressedSize; + if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredVerifiedBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + }); } }, Work.DefaultErrorFunction()); @@ -1365,6 +1525,8 @@ namespace { [&, BlockAttachment](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); + FilteredDownloadedBytesPerSecond.Start(); IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); DownloadedAttachmentCount++; @@ -1384,6 +1546,8 @@ namespace { [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { if (!AbortFlag) { + ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); + FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; @@ -1442,6 +1606,7 @@ namespace { std::vector& ChunkIndexes, std::vector>& OutBlocks) { + ZEN_TRACE_CPU("ArrangeChunksIntoBlocks"); std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) { const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0]; const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0]; @@ -1535,6 +1700,7 @@ namespace { uint32_t ChunkIndex, const std::filesystem::path& TempFolderPath) { + ZEN_TRACE_CPU("CompressChunk"); ZEN_ASSERT(!TempFolderPath.empty()); const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; @@ -1608,7 +1774,7 @@ namespace { { std::vector BlockDescriptions; std::vector BlockSizes; - std::vector BlockBuffers; + std::vector BlockHeaders; std::vector BlockMetaDatas; std::vector MetaDataHasBeenUploaded; tsl::robin_map BlockHashToBlockIndex; @@ -1625,6 +1791,7 @@ namespace { UploadStatistics& UploadStats, GenerateBlocksStatistics& GenerateBlocksStats) { + ZEN_TRACE_CPU("GenerateBuildBlocks"); const std::size_t NewBlockCount = NewBlockChunks.size(); if (NewBlockCount > 0) { @@ -1632,22 +1799,23 @@ namespace { OutBlocks.BlockDescriptions.resize(NewBlockCount); OutBlocks.BlockSizes.resize(NewBlockCount); - OutBlocks.BlockBuffers.resize(NewBlockCount); OutBlocks.BlockMetaDatas.resize(NewBlockCount); + OutBlocks.BlockHeaders.resize(NewBlockCount); OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, false); OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount); RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// - WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& GenerateBlobsPool = + GetMediumWorkerPool(EWorkloadType::Burst); // GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; ParallellWork Work(AbortFlag); - std::atomic PendingUploadCount(0); + std::atomic QueuedPendingBlocksForUpload = 0; for (size_t BlockIndex = 0; BlockIndex < NewBlockCount; BlockIndex++) { @@ -1661,6 +1829,8 @@ namespace { [&, BlockIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); + FilteredGeneratedBytesPerSecond.Start(); // TODO: Convert ScheduleWork body to function @@ -1672,14 +1842,6 @@ namespace { OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); - - if (!IsBufferDiskBased(CompressedBlock.GetCompressed())) - { - IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlock).GetCompressed(), - Path / ZenTempBlockFolderName, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash); - CompressedBlock = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); - } { CbObjectWriter Writer; Writer.AddString("createdBy", "zen"); @@ -1693,66 +1855,88 @@ namespace { BlockIndex); }); + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { FilteredGeneratedBytesPerSecond.Stop(); } - if (!AbortFlag) + if (QueuedPendingBlocksForUpload.load() > 16) { - PendingUploadCount++; - Work.ScheduleWork( - UploadBlocksPool, - [&, BlockIndex, Payload = std::move(CompressedBlock).GetCompressed()](std::atomic&) mutable { - if (!AbortFlag) - { - if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) - { - FilteredUploadedBytesPerSecond.Stop(); - OutBlocks.BlockBuffers[BlockIndex] = std::move(Payload); - } - else + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + if (!AbortFlag) + { + QueuedPendingBlocksForUpload++; + + Work.ScheduleWork( + UploadBlocksPool, + [&, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { + auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); + if (!AbortFlag) { - FilteredUploadedBytesPerSecond.Start(); - // TODO: Convert ScheduleWork body to function - - PendingUploadCount--; - - const CbObject BlockMetaData = - BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], - OutBlocks.BlockMetaDatas[BlockIndex]); - - const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; - const uint64_t CompressedBlockSize = Payload.GetSize(); - Storage.PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload)); - UploadStats.BlocksBytes += CompressedBlockSize; - ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(CompressedBlockSize), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - - Storage.PutBlockMetadata(BuildId, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(BlockMetaData.GetSize())); - - OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - UploadStats.BlockCount++; - if (UploadStats.BlockCount == NewBlockCount) + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { + ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); + FilteredUploadedBytesPerSecond.Stop(); + std::span Segments = Payload.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); + + FilteredUploadedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function + + const CbObject BlockMetaData = + BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], + OutBlocks.BlockMetaDatas[BlockIndex]); + + const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); + + Storage.PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); + UploadStats.BlocksBytes += CompressedBlockSize; + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlockSize), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + + Storage.PutBlockMetadata(BuildId, + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(BlockMetaData.GetSize())); + + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + UploadStats.BlockCount++; + if (UploadStats.BlockCount == NewBlockCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -1783,6 +1967,8 @@ namespace { false); }); + ZEN_ASSERT(AbortFlag || QueuedPendingBlocksForUpload.load() == 0); + ProgressBar.Finish(); GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS(); @@ -1805,6 +1991,7 @@ namespace { GenerateBlocksStatistics& GenerateBlocksStats, LooseChunksStatistics& LooseChunksStats) { + ZEN_TRACE_CPU("UploadPartBlobs"); { ProgressBar ProgressBar(UsePlainProgress); @@ -1858,12 +2045,37 @@ namespace { const size_t UploadBlockCount = BlockIndexes.size(); const uint32_t UploadChunkCount = gsl::narrow(LooseChunkOrderIndexes.size()); - auto AsyncUploadBlock = [&](const size_t BlockIndex, const IoHash BlockHash, CompositeBuffer&& Payload) { + auto AsyncUploadBlock = [&](const size_t BlockIndex, + const IoHash BlockHash, + CompositeBuffer&& Payload, + std::atomic& QueuedPendingInMemoryBlocksForUpload) { + bool IsInMemoryBlock = true; + if (QueuedPendingInMemoryBlocksForUpload.load() > 16) + { + ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), Path / ZenTempBlockFolderName, BlockHash)); + IsInMemoryBlock = false; + } + else + { + QueuedPendingInMemoryBlocksForUpload++; + } + Work.ScheduleWork( UploadChunkPool, - [&, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) mutable { + [&, IsInMemoryBlock, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) mutable { + auto _ = MakeGuard([IsInMemoryBlock, &QueuedPendingInMemoryBlocksForUpload] { + if (IsInMemoryBlock) + { + QueuedPendingInMemoryBlocksForUpload--; + } + }); if (!AbortFlag) { + ZEN_TRACE_CPU("AsyncUploadBlock"); + + const uint64_t PayloadSize = Payload.GetSize(); + FilteredUploadedBytesPerSecond.Start(); const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); @@ -1871,10 +2083,10 @@ namespace { Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(Payload.GetSize()), + NiceBytes(PayloadSize), NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - UploadedBlockSize += Payload.GetSize(); - UploadStats.BlocksBytes += Payload.GetSize(); + UploadedBlockSize += PayloadSize; + UploadStats.BlocksBytes += PayloadSize; Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", @@ -1902,9 +2114,13 @@ namespace { [&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { if (!AbortFlag) { + ZEN_TRACE_CPU("AsyncUploadLooseChunk"); + const uint64_t PayloadSize = Payload.GetSize(); + ; if (PayloadSize >= LargeAttachmentSize) { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); UploadStats.MultipartAttachmentCount++; std::vector> MultipartWork = Storage.PutLargeBuildBlob( BuildId, @@ -1938,6 +2154,7 @@ namespace { Work.ScheduleWork( UploadChunkPool, [Work = std::move(WorkPart)](std::atomic&) { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work"); if (!AbortFlag) { Work(); @@ -1949,6 +2166,7 @@ namespace { } else { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); UploadStats.ChunksBytes += Payload.GetSize(); @@ -1971,26 +2189,10 @@ namespace { std::atomic GeneratedBlockCount = 0; std::atomic GeneratedBlockByteCount = 0; - // Start upload of any pre-built blocks - for (const size_t BlockIndex : BlockIndexes) - { - if (CompositeBuffer BlockPayload = std::move(NewBlocks.BlockBuffers[BlockIndex]); BlockPayload) - { - const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (!AbortFlag) - { - AsyncUploadBlock(BlockIndex, BlockHash, std::move(BlockPayload)); - } - // GeneratedBlockCount++; - } - else - { - GenerateBlockIndexes.push_back(BlockIndex); - } - } - std::vector CompressLooseChunkOrderIndexes; + std::atomic QueuedPendingInMemoryBlocksForUpload = 0; + // Start upload of any pre-compressed loose chunks for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { @@ -1998,31 +2200,43 @@ namespace { } // Start generation of any non-prebuilt blocks and schedule upload - for (const size_t BlockIndex : GenerateBlockIndexes) + for (const size_t BlockIndex : BlockIndexes) { const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; if (!AbortFlag) { Work.ScheduleWork( - ReadChunkPool, + ReadChunkPool, // GetSyncWorkerPool() [&, BlockIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); + FilteredGenerateBlockBytesPerSecond.Start(); - ChunkBlockDescription BlockDescription; - CompressedBuffer CompressedBlock = - GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); - if (!CompressedBlock) + + CompositeBuffer Payload; + if (NewBlocks.BlockHeaders[BlockIndex]) { - throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + Payload = RebuildBlock(Path, + Content, + Lookup, + std::move(NewBlocks.BlockHeaders[BlockIndex]), + NewBlockChunks[BlockIndex], + DiskStats) + .GetCompressed(); + } + else + { + ChunkBlockDescription BlockDescription; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); + if (!CompressedBlock) + { + throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + } + ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); + Payload = std::move(CompressedBlock).GetCompressed(); } - ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); - - CompositeBuffer Payload = IsBufferDiskBased(CompressedBlock.GetCompressed()) - ? std::move(CompressedBlock).GetCompressed() - : CompositeBuffer(WriteToTempFile(std::move(CompressedBlock).GetCompressed(), - Path / ZenTempBlockFolderName, - BlockDescription.BlockHash)); GenerateBlocksStats.GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; GenerateBlocksStats.GeneratedBlockCount++; @@ -2034,11 +2248,11 @@ namespace { } if (!AbortFlag) { - AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload)); + AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); } ZEN_CONSOLE_VERBOSE("Regenerated block {} ({}) containing {} chunks", NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(CompressedBlock.GetCompressedSize()), + NiceBytes(NewBlocks.BlockSizes[BlockIndex]), NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); } }, @@ -2059,6 +2273,8 @@ namespace { [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); + FilteredCompressedBytesPerSecond.Start(); CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", @@ -2117,6 +2333,8 @@ namespace { false); }); + ZEN_ASSERT(AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0); + ProgressBar.Finish(); UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTimeUS(); @@ -2131,6 +2349,8 @@ namespace { std::vector& OutUnusedChunkIndexes, FindBlocksStatistics& FindBlocksStats) { + ZEN_TRACE_CPU("FindReuseBlocks"); + // Find all blocks with a usage level higher than MinPercentLimit // Pick out the blocks with usage higher or equal to MinPercentLimit // Sort them with highest size usage - most usage first @@ -2303,8 +2523,10 @@ namespace { CreateDirectories(ZenTempFolder); CleanDirectory(ZenTempFolder, {}); auto _ = MakeGuard([&]() { - CleanDirectory(ZenTempFolder, {}); - std::filesystem::remove(ZenTempFolder); + if (CleanDirectory(ZenTempFolder, {})) + { + std::filesystem::remove(ZenTempFolder); + } }); CreateDirectories(Path / ZenTempBlockFolderName); CreateDirectories(Path / ZenTempChunkFolderName); @@ -2327,10 +2549,14 @@ namespace { GetSmallWorkerPool(EWorkloadType::Burst) .EnqueueTask(std::packaged_task{ [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + ZEN_TRACE_CPU("PrepareBuild"); + PrepareBuildResult Result; Stopwatch Timer; if (CreateBuild) { + ZEN_TRACE_CPU("CreateBuild"); + Stopwatch PutBuildTimer; CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); @@ -2339,6 +2565,7 @@ namespace { } else { + ZEN_TRACE_CPU("PutBuild"); Stopwatch GetBuildTimer; CbObject Build = Storage.GetBuild(BuildId); Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); @@ -2356,6 +2583,7 @@ namespace { if (!IgnoreExistingBlocks) { + ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; Result.KnownBlocks = Storage.FindBlocks(BuildId); FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); @@ -2523,11 +2751,8 @@ namespace { ChunkerParameters = ChunkParametersWriter.Save(); } - std::uint64_t TotalRawSize = 0; - for (uint64_t RawSize : Content.RawSizes) - { - TotalRawSize += RawSize; - } + std::uint64_t TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); + { ProgressBar ProgressBar(UsePlainProgress); FilteredRate FilteredBytesHashed; @@ -2956,9 +3181,9 @@ namespace { ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]); } - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockBuffers.size(); BlockIndex++) + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++) { - if (NewBlocks.BlockBuffers[BlockIndex]) + if (NewBlocks.BlockHeaders[BlockIndex]) { // Block was not uploaded during generation ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash); @@ -3457,8 +3682,8 @@ namespace { std::vector ChunkBuffers; struct WriteOpData { - const ChunkedContentLookup::ChunkSequenceLocation* Target; - size_t ChunkBufferIndex; + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + size_t ChunkBufferIndex = (size_t)-1; }; std::vector WriteOps; }; @@ -3964,155 +4189,129 @@ namespace { } } - void DownloadLargeBlob(BuildStorage& Storage, - const std::filesystem::path& Path, - const ChunkedFolderContent& RemoteContent, - const ChunkedContentLookup& RemoteLookup, - const Oid& BuildId, - const IoHash& ChunkHash, - const std::uint64_t PreferredMultipartChunkSize, - const std::vector& ChunkTargetPtrs, - std::span> SequenceIndexChunksLeftToWriteCounters, - ParallellWork& Work, - WorkerThreadPool& WritePool, - WorkerThreadPool& NetworkPool, - std::atomic& WriteToDiskBytes, - std::atomic& BytesDownloaded, - std::atomic& MultipartAttachmentCount, - std::function&& OnDownloadComplete, - std::function&& OnWriteStart, - std::function&& OnWriteComplete) + void AsyncWriteDownloadedChunk(const std::filesystem::path& Path, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& RemoteLookup, + uint32_t RemoteChunkIndex, + std::vector&& ChunkTargetPtrs, + ParallellWork& Work, + WorkerThreadPool& WritePool, + IoBuffer&& Payload, + std::span> SequenceIndexChunksLeftToWriteCounters, + std::atomic& WriteToDiskBytes, + std::atomic& ChunkCountWritten, + std::atomic& WritePartsComplete, + std::atomic& TotalPartWriteCount, + std::atomic& LooseChunksBytes, + FilteredRate& FilteredWrittenBytesPerSecond) { - ZEN_TRACE_CPU("DownloadLargeBlob"); + ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); - struct WorkloadData + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + + uint64_t Size = Payload.GetSize(); + LooseChunksBytes += Size; + + std::filesystem::path CompressedChunkPath; + + // Check if the dowloaded chunk is file based and we can move it directly without rewriting it { - TemporaryFile TempFile; - }; - std::shared_ptr Workload(std::make_shared()); + IoBufferFileReference FileRef; + if (Payload.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == Size)) + { + ZEN_TRACE_CPU("MoveTempChunk"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + Payload.SetDeleteOnClose(false); + Payload = {}; + CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + if (Ec) + { + CompressedChunkPath = std::filesystem::path{}; - std::filesystem::path DownloadFolder = Path / ZenTempDownloadFolderName; - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + Payload = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, Size, true); + Payload.SetDeleteOnClose(true); + } + } + } + } - std::error_code Ec; - Workload->TempFile.CreateTemporary(DownloadFolder, Ec); - if (Ec) + if (CompressedChunkPath.empty() && (Size > 512u * 1024u)) { - throw std::runtime_error( - fmt::format("Failed opening temporary file '{}': {} ({})", Workload->TempFile.GetPath(), Ec.message(), Ec.value())); + ZEN_TRACE_CPU("WriteTempChunk"); + // Could not be moved and rather large, lets store it on disk + CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload); + Payload = {}; } - std::vector> WorkItems = Storage.GetLargeBuildBlob( - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - [DownloadFolder, - TargetFolder, - &RemoteContent, - &RemoteLookup, - &Work, - &WritePool, - Workload, - ChunkHash, - &BytesDownloaded, - OnDownloadComplete = std::move(OnDownloadComplete), - OnWriteComplete = std::move(OnWriteComplete), - OnWriteStart = std::move(OnWriteStart), - &WriteToDiskBytes, + + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(),// + [&, SequenceIndexChunksLeftToWriteCounters, - ChunkTargetPtrs = std::vector( - ChunkTargetPtrs)](uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining) { - BytesDownloaded += Chunk.GetSize(); + CompressedChunkPath, + RemoteChunkIndex, + ChunkTargetPtrs = std::move(ChunkTargetPtrs), + CompressedPart = std::move(Payload)](std::atomic&) mutable { + ZEN_TRACE_CPU("UpdateFolder_WriteChunk"); - if (!AbortFlag.load()) + if (!AbortFlag) { - ZEN_TRACE_CPU("DownloadLargeBlob_Save"); - Workload->TempFile.Write(Chunk.GetView(), Offset); - if (Chunk.GetSize() == BytesRemaining) + FilteredWrittenBytesPerSecond.Start(); + + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (CompressedChunkPath.empty()) { - OnDownloadComplete(Workload->TempFile.FileSize()); - - Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// - [DownloadFolder, - TargetFolder, - &RemoteContent, - &RemoteLookup, - ChunkHash, - Workload, - Offset, - OnWriteComplete = std::move(OnWriteComplete), - OnWriteStart = std::move(OnWriteStart), - &WriteToDiskBytes, - SequenceIndexChunksLeftToWriteCounters, - ChunkTargetPtrs](std::atomic&) { - ZEN_TRACE_CPU("DownloadLargeBlob_Write"); + ZEN_ASSERT(CompressedPart); + } + else + { + ZEN_ASSERT(!CompressedPart); + CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath)); + } + } - if (!AbortFlag) - { - const std::filesystem::path CompressedChunkPath = DownloadFolder / ChunkHash.ToHexString(); - std::error_code Ec; - Workload->TempFile.MoveTemporaryIntoPlace(CompressedChunkPath, Ec); - if (Ec) - { - throw std::runtime_error(fmt::format("Failed moving downloaded chunk {} file to {}. Reason: {}", - ChunkHash, - CompressedChunkPath, - Ec.message())); - } + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + WriteToDiskBytes); - OnWriteStart(); + if (!AbortFlag) + { + ChunkCountWritten++; + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - WriteToDiskBytes); + std::filesystem::remove(CompressedChunkPath); - if (!AbortFlag) - { - std::filesystem::remove(CompressedChunkPath); - - CompleteChunkTargets(TargetFolder, - RemoteContent, - ChunkHash, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - NeedHashVerify); - } - } - }, - Work.DefaultErrorFunction()); + CompleteChunkTargets(TargetFolder, + RemoteContent, + ChunkHash, + ChunkTargetPtrs, + SequenceIndexChunksLeftToWriteCounters, + NeedHashVerify); } } - }); - if (!WorkItems.empty()) - { - MultipartAttachmentCount++; - } - for (auto& WorkItem : WorkItems) - { - Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(),// - [WorkItem = std::move(WorkItem)](std::atomic&) { - ZEN_TRACE_CPU("DownloadLargeBlob_Work"); - if (!AbortFlag) - { - WorkItem(); - } - }, - Work.DefaultErrorFunction()); - } - } + }, + Work.DefaultErrorFunction()); + }; void UpdateFolder(BuildStorage& Storage, const Oid& BuildId, @@ -4291,13 +4490,13 @@ namespace { // Pick up all chunks in current local state struct CacheCopyData { - uint32_t LocalSequenceIndex; + uint32_t LocalSequenceIndex = (uint32_t)-1; std::vector TargetChunkLocationPtrs; struct ChunkTarget { - uint32_t TargetChunkLocationCount; - uint64_t ChunkRawSize; - uint64_t CacheFileOffset; + uint32_t TargetChunkLocationCount = (uint32_t)-1; + uint64_t ChunkRawSize = (uint64_t)-1; + uint64_t CacheFileOffset = (uint64_t)-1; }; std::vector ChunkTargets; }; @@ -4419,8 +4618,8 @@ namespace { uint64_t TotalRequestCount = 0; std::atomic RequestsComplete = 0; std::atomic ChunkCountWritten = 0; - std::atomic TotalPartWriteCount = 0; - std::atomic WritePartsComplete = 0; + std::atomic TotalPartWriteCount = 0; + std::atomic WritePartsComplete = 0; { ZEN_TRACE_CPU("WriteChunks"); @@ -4441,7 +4640,7 @@ namespace { struct LooseChunkHashWorkData { std::vector ChunkTargetPtrs; - uint32_t RemoteChunkIndex; + uint32_t RemoteChunkIndex = (uint32_t)-1; }; std::vector LooseChunkHashWorks; @@ -4687,9 +4886,9 @@ namespace { struct WriteOp { - const ChunkedContentLookup::ChunkSequenceLocation* Target; - uint64_t CacheFileOffset; - uint64_t ChunkSize; + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + uint64_t CacheFileOffset = (uint64_t)-1; + uint64_t ChunkSize = (uint64_t)-1; }; std::vector WriteOps; @@ -4821,10 +5020,11 @@ namespace { const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; Work.ScheduleWork( - NetworkPool, // NetworkPool, // GetSyncWorkerPool(),// - [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) { + WritePool, // NetworkPool, // GetSyncWorkerPool(),// + [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { if (!AbortFlag) { + ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; @@ -4925,39 +5125,37 @@ namespace { if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob( - Storage, - Path, - RemoteContent, - RemoteLookup, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - NetworkPool, - WriteToDiskBytes, - BytesDownloaded, - MultipartAttachmentCount, - [&](uint64_t BytesDownloaded) { - LooseChunksBytes += BytesDownloaded; - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - }, - [&]() { FilteredWrittenBytesPerSecond.Start(); }, - [&]() { - ChunkCountWritten++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - }); + DownloadLargeBlob(Storage, + Path / ZenTempDownloadFolderName, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + BytesDownloaded, + MultipartAttachmentCount, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + RequestsComplete++; + if (RequestsComplete == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WriteToDiskBytes, + ChunkCountWritten, + WritePartsComplete, + TotalPartWriteCount, + LooseChunksBytes, + FilteredWrittenBytesPerSecond); + }); } else { @@ -4970,129 +5168,27 @@ namespace { } uint64_t BlobSize = BuildBlob.GetSize(); BytesDownloaded += BlobSize; - LooseChunksBytes += BlobSize; + RequestsComplete++; if (RequestsComplete == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } - - std::filesystem::path CompressedChunkPath; - - // Check if the dowloaded file is file based and we can move it directly without rewriting it - { - IoBufferFileReference FileRef; - if (BuildBlob.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlobSize)) - { - ZEN_TRACE_CPU("UpdateFolder_MoveTempChunk"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) - { - BuildBlob.SetDeleteOnClose(false); - BuildBlob = {}; - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); - std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); - if (Ec) - { - CompressedChunkPath = std::filesystem::path{}; - - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BuildBlob = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlobSize, true); - BuildBlob.SetDeleteOnClose(true); - } - } - } - } - - if (CompressedChunkPath.empty() && (BlobSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempChunk"); - // Could not be moved and rather large, lets store it on disk - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); - TemporaryFile::SafeWriteFile(CompressedChunkPath, BuildBlob); - BuildBlob = {}; - } - DownloadedChunks++; - - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Path, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &WriteToDiskBytes, - &ChunkCountWritten, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath, - CompressedPart = std::move(BuildBlob)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteChunk"); - - FilteredWrittenBytesPerSecond.Start(); - - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (CompressedChunkPath.empty()) - { - ZEN_ASSERT(CompressedPart); - } - else - { - ZEN_ASSERT(!CompressedPart); - CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error( - fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } - } - - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - WriteToDiskBytes); - - if (!AbortFlag) - { - ChunkCountWritten++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - - if (!CompressedChunkPath.empty()) - { - std::filesystem::remove(CompressedChunkPath); - } - - CompleteChunkTargets(TargetFolder, - RemoteContent, - ChunkHash, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - NeedHashVerify); - } - } - }, - Work.DefaultErrorFunction()); - } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WriteToDiskBytes, + ChunkCountWritten, + WritePartsComplete, + TotalPartWriteCount, + LooseChunksBytes, + FilteredWrittenBytesPerSecond); } } } @@ -5375,7 +5471,7 @@ namespace { WritePool, // WritePool, GetSyncWorkerPool() [&RemoteContent, &RemoteLookup, - &CacheFolderPath, + CacheFolderPath, &RemoteChunkIndexNeedsCopyFromSourceFlags, &SequenceIndexChunksLeftToWriteCounters, BlockIndex, @@ -5550,7 +5646,10 @@ namespace { // Clean target folder ZEN_CONSOLE("Wiping {}", Path); - CleanDirectory(Path, DefaultExcludeFolders); + if (!CleanDirectory(Path, DefaultExcludeFolders)) + { + ZEN_WARN("Some files in {} could not be removed", Path); + } } else { @@ -6415,8 +6514,10 @@ namespace { ZEN_CONSOLE("Downloaded build in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); } } - CleanDirectory(ZenTempFolder, {}); - std::filesystem::remove(ZenTempFolder); + if (CleanDirectory(ZenTempFolder, {})) + { + std::filesystem::remove(ZenTempFolder); + } } void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) @@ -7445,7 +7546,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_StoragePath = GetRunningExecutablePath().parent_path() / ".tmpstore"; CreateDirectories(m_StoragePath); - CleanDirectory(m_StoragePath); + CleanDirectory(m_StoragePath, {}); } auto _ = MakeGuard([&]() { if (m_BuildsUrl.empty() && m_StoragePath.empty()) diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 4e6161e86..0fcf9d871 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -577,12 +577,15 @@ main(int argc, char** argv) GlobalOptions.PassthroughArgs = PassthroughArgs; GlobalOptions.PassthroughArgV = PassthroughArgV; + std::string MemoryOptions; + std::string SubCommand = ""; cxxopts::Options Options("zen", "Zen management tool"); Options.add_options()("d, debug", "Enable debugging", cxxopts::value(GlobalOptions.IsDebug)); Options.add_options()("v, verbose", "Enable verbose logging", cxxopts::value(GlobalOptions.IsVerbose)); + Options.add_options()("malloc", "Configure memory allocator subsystem", cxxopts::value(MemoryOptions)->default_value("mimalloc")); Options.add_options()("help", "Show command line help"); Options.add_options()("c, command", "Sub command", cxxopts::value(SubCommand)); diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 4ca89d996..bb1ee5183 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -96,6 +96,8 @@ namespace { uint32_t PathIndex, std::atomic& AbortFlag) { + ZEN_TRACE_CPU("ChunkFolderContent"); + const uint64_t RawSize = OutChunkedContent.RawSizes[PathIndex]; const std::filesystem::path& Path = OutChunkedContent.Paths[PathIndex]; @@ -136,6 +138,8 @@ namespace { } else { + ZEN_TRACE_CPU("HashOnly"); + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); @@ -228,6 +232,7 @@ FolderContent::operator==(const FolderContent& Rhs) const bool FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const { + ZEN_TRACE_CPU("FolderContent::AreKnownFilesEqual"); tsl::robin_map RhsPathToIndex; const size_t RhsPathCount = Rhs.Paths.size(); RhsPathToIndex.reserve(RhsPathCount); @@ -259,6 +264,7 @@ FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const void FolderContent::UpdateState(const FolderContent& Rhs, std::vector& OutPathIndexesOufOfDate) { + ZEN_TRACE_CPU("FolderContent::UpdateState"); tsl::robin_map RhsPathToIndex; const uint32_t RhsPathCount = gsl::narrow(Rhs.Paths.size()); RhsPathToIndex.reserve(RhsPathCount); @@ -297,6 +303,7 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector& OutP FolderContent GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) { + ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); FolderContent Result = {.Platform = Old.Platform}; tsl::robin_map NewPathToIndex; const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); @@ -332,6 +339,7 @@ GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vecto void SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output) { + ZEN_TRACE_CPU("SaveFolderContentToCompactBinary"); Output.AddString("platform"sv, ToString(Content.Platform)); compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output); compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output); @@ -342,6 +350,7 @@ SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output) FolderContent LoadFolderContentToCompactBinary(CbObjectView Input) { + ZEN_TRACE_CPU("LoadFolderContentToCompactBinary"); FolderContent Content; Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform()); compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths); @@ -494,6 +503,7 @@ GetFolderContent(GetFolderContentStatistics& Stats, void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output) { + ZEN_TRACE_CPU("SaveChunkedFolderContentToCompactBinary"); Output.AddString("platform"sv, ToString(Content.Platform)); compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output); compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output); @@ -512,6 +522,7 @@ SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbW ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input) { + ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary"); ChunkedFolderContent Content; Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform()); compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths); @@ -788,7 +799,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) struct ChunkLocationReference { - uint32_t ChunkIndex; + uint32_t ChunkIndex = (uint32_t)-1; ChunkedContentLookup::ChunkSequenceLocation Location; }; @@ -853,7 +864,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) { Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex}); uint32_t Count = 0; - while (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex) + while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex)) { Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location); Count++; diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp index 4f9344039..a2c041ffd 100644 --- a/src/zenutil/chunkedfile.cpp +++ b/src/zenutil/chunkedfile.cpp @@ -3,6 +3,7 @@ #include #include +#include #include "chunking.h" @@ -33,6 +34,7 @@ namespace { IoBuffer SerializeChunkedInfo(const ChunkedInfo& Info) { + ZEN_TRACE_CPU("SerializeChunkedInfo"); size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) + RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16); IoBuffer HeaderData(HeaderSize); @@ -65,6 +67,7 @@ SerializeChunkedInfo(const ChunkedInfo& Info) ChunkedInfo DeserializeChunkedInfo(IoBuffer& Buffer) { + ZEN_TRACE_CPU("DeserializeChunkedInfo"); MemoryView View = Buffer.GetView(); ChunkedHeader Header; { @@ -99,6 +102,7 @@ DeserializeChunkedInfo(IoBuffer& Buffer) void Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function GetChunk) { + ZEN_TRACE_CPU("Reconstruct"); BasicFile Reconstructed; Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate); BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024); @@ -119,6 +123,8 @@ ChunkData(BasicFile& RawData, std::atomic* BytesProcessed, std::atomic* AbortFlag) { + ZEN_TRACE_CPU("ChunkData"); + ChunkedInfoWithSource Result; tsl::robin_map FoundChunks; diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index 017d12433..2a7057a46 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -4,6 +4,7 @@ #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -61,6 +62,7 @@ public: std::atomic& BytesProcessed, std::atomic& AbortFlag) const override { + ZEN_TRACE_CPU("BasicChunkingController::ProcessFile"); const bool ExcludeFromChunking = std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) != m_ChunkExcludeExtensions.end(); @@ -136,6 +138,7 @@ public: std::atomic& BytesProcessed, std::atomic& AbortFlag) const override { + ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile"); if (RawSize < m_ChunkFileSizeLimit) { return false; @@ -145,6 +148,7 @@ public: if (FixedChunking) { + ZEN_TRACE_CPU("FixedChunking"); IoHashStream FullHash; IoBuffer Source = IoBufferBuilder::MakeFromFile(InputPath); uint64_t Offset = 0; diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index e57109006..47a4e1cc4 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace zen { @@ -36,6 +37,7 @@ public: virtual CbObject ListBuilds(CbObject Query) override { + ZEN_TRACE_CPU("FileBuildStorage::ListBuilds"); ZEN_UNUSED(Query); SimulateLatency(Query.GetSize(), 0); @@ -72,6 +74,7 @@ public: virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override { + ZEN_TRACE_CPU("FileBuildStorage::PutBuild"); SimulateLatency(MetaData.GetSize(), 0); Stopwatch ExecutionTimer; @@ -93,6 +96,7 @@ public: virtual CbObject GetBuild(const Oid& BuildId) override { + ZEN_TRACE_CPU("FileBuildStorage::GetBuild"); SimulateLatency(0, 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); @@ -105,6 +109,7 @@ public: virtual void FinalizeBuild(const Oid& BuildId) override { + ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild"); SimulateLatency(0, 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); @@ -119,6 +124,7 @@ public: std::string_view PartName, const CbObject& MetaData) override { + ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart"); SimulateLatency(MetaData.GetSize(), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); @@ -164,6 +170,7 @@ public: virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override { + ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart"); SimulateLatency(0, 0); Stopwatch ExecutionTimer; @@ -186,6 +193,7 @@ public: virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override { + ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart"); SimulateLatency(0, 0); Stopwatch ExecutionTimer; @@ -215,6 +223,7 @@ public: ZenContentType ContentType, const CompositeBuffer& Payload) override { + ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob"); ZEN_UNUSED(BuildId); ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); SimulateLatency(Payload.GetSize(), 0); @@ -242,6 +251,7 @@ public: std::function&& Transmitter, std::function&& OnSentBytes) override { + ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob"); ZEN_UNUSED(BuildId); ZEN_UNUSED(ContentType); SimulateLatency(0, 0); @@ -281,6 +291,7 @@ public: uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset); WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() { + ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work"); IoBuffer PartPayload = Workload->Transmitter(Offset, Size); SimulateLatency(PartPayload.GetSize(), 0); @@ -327,6 +338,7 @@ public: virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override { + ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob"); ZEN_UNUSED(BuildId); SimulateLatency(0, 0); Stopwatch ExecutionTimer; @@ -363,6 +375,7 @@ public: uint64_t ChunkSize, std::function&& Receiver) override { + ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob"); ZEN_UNUSED(BuildId); SimulateLatency(0, 0); Stopwatch ExecutionTimer; @@ -392,6 +405,7 @@ public: { uint64_t Size = Min(ChunkSize, BlobSize - Offset); WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() { + ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work"); SimulateLatency(0, 0); IoBuffer PartPayload(Size); Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset); @@ -411,6 +425,7 @@ public: virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { + ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata"); ZEN_UNUSED(BuildId); SimulateLatency(MetaData.GetSize(), 0); @@ -429,6 +444,7 @@ public: virtual std::vector FindBlocks(const Oid& BuildId) override { + ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); SimulateLatency(0, 0); Stopwatch ExecutionTimer; @@ -461,6 +477,7 @@ public: virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override { + ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); SimulateLatency(0, 0); Stopwatch ExecutionTimer; diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 309341550..57b55cb8e 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -124,8 +124,8 @@ struct ChunkedContentLookup { struct ChunkSequenceLocation { - uint32_t SequenceIndex; - uint64_t Offset; + uint32_t SequenceIndex = (uint32_t)-1; + uint64_t Offset = (uint64_t)-1; }; tsl::robin_map ChunkHashToChunkIndex; tsl::robin_map RawHashToSequenceIndex; -- cgit v1.2.3 From 26f718b04bc0fba3831124d9dec705c2febd426e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 11:19:06 +0100 Subject: fix mac/linux builds command (#303) * fix linux/mac version of GetModificationTickFromPath and CopyFile --- src/zencore/filesystem.cpp | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 85feab2f7..9f3f4f7fc 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -590,7 +590,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP ScopedFd $From = {FromFd}; // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666); + int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); if (ToFd < 0) { ThrowLastError(fmt::format("failed to create file {}", ToPath)); @@ -598,9 +598,14 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP fchmod(ToFd, 0666); ScopedFd $To = {ToFd}; + struct stat Stat; + fstat(FromFd, &Stat); + + size_t FileSizeBytes = Stat.st_size; + // Copy impl - static const size_t BufferSize = 64 << 10; - void* Buffer = malloc(BufferSize); + const size_t BufferSize = Min(FileSizeBytes, 64u << 10); + void* Buffer = malloc(BufferSize); while (true) { int BytesRead = read(FromFd, Buffer, BufferSize); @@ -610,7 +615,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP break; } - if (write(ToFd, Buffer, BytesRead) != BufferSize) + if (write(ToFd, Buffer, BytesRead) != BytesRead) { Success = false; break; @@ -621,7 +626,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP if (!Success) { - ThrowLastError("file copy failed"sv); + ThrowLastError(fmt::format("file copy from {} to {} failed", FromPath, ToPath)); } return true; @@ -1483,16 +1488,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); } - auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); -#else - int Fd = open(Filename.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd <= 9) - { - ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); - } - Handle = (void*)uintptr_t(Fd); - auto _ = MakeGuard([Handle]() { close(int(uintptr_t(Handle))); }); -#endif + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); std::error_code Ec; uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) @@ -1500,6 +1496,15 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) ThrowSystemError(Ec.value(), Ec.message()); } return ModificatonTick; +#else + struct stat Stat; + int err = stat(Filename.native().c_str(), &Stat); + if (err) + { + ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + } + return gsl::narrow(Stat.st_mtime); +#endif } std::filesystem::path -- cgit v1.2.3 From 7046fc9dc202307ba92d05a6386bfb52e9db0ab9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 12 Mar 2025 13:54:16 +0100 Subject: fixes for log timestamps (#304) * add GetTimeSinceProcessStart returning time since process start. implemented using https://github.com/maxliani/GetTimeSinceProcessStart/tree/main * fix fractions when using epoch mode. Previously it would show the fraction from the absolute time stamp and not relative to epoch * used GetTimeSinceProcessStart to offset the epoch so that it represents the process spawn time --- src/zencore/include/zencore/timer.h | 4 ++++ src/zencore/timer.cpp | 11 +++++++++++ src/zencore/xmake.lua | 1 + src/zenutil/include/zenutil/logging/fullformatter.h | 7 ++++++- src/zenutil/logging.cpp | 6 ++++-- 5 files changed, 26 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h index e4ddc3505..767dc4314 100644 --- a/src/zencore/include/zencore/timer.h +++ b/src/zencore/include/zencore/timer.h @@ -21,6 +21,10 @@ ZENCORE_API uint64_t GetHifreqTimerFrequency(); ZENCORE_API double GetHifreqTimerToSeconds(); ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init +// Query time since process was spawned (returns time in ms) + +uint64_t GetTimeSinceProcessStart(); + class Stopwatch { public: diff --git a/src/zencore/timer.cpp b/src/zencore/timer.cpp index 1655e912d..95536cb26 100644 --- a/src/zencore/timer.cpp +++ b/src/zencore/timer.cpp @@ -12,8 +12,19 @@ # include #endif +#define GTSPS_IMPLEMENTATION +#include "GetTimeSinceProcessStart.h" + namespace zen { +uint64_t +GetTimeSinceProcessStart() +{ + double TimeInSeconds = ::GetTimeSinceProcessStart(); + + return uint64_t(TimeInSeconds * 1000); +} + uint64_t GetHifreqTimerValue() { diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index b8b14084c..13611a2e9 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -29,6 +29,7 @@ target('zencore') end add_includedirs("include", {public=true}) + add_includedirs("$(projectdir)/thirdparty/GetTimeSinceProcessStart") add_includedirs("$(projectdir)/thirdparty/utfcpp/source") add_includedirs("$(projectdir)/thirdparty/Oodle/include") add_includedirs("$(projectdir)/thirdparty/trace", {public=true}) diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h index 07ad408fa..0326870e5 100644 --- a/src/zenutil/include/zenutil/logging/fullformatter.h +++ b/src/zenutil/include/zenutil/logging/fullformatter.h @@ -45,6 +45,8 @@ public: std::chrono::seconds TimestampSeconds; + std::chrono::milliseconds millis; + if (m_UseFullDate) { TimestampSeconds = std::chrono::duration_cast(msg.time.time_since_epoch()); @@ -69,6 +71,8 @@ public: spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime); m_CachedDatetime.push_back('.'); } + + millis = spdlog::details::fmt_helper::time_fraction(msg.time); } else { @@ -97,6 +101,8 @@ public: spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime); m_CachedDatetime.push_back('.'); } + + millis = std::chrono::duration_cast(ElapsedTime - TimestampSeconds); } { @@ -104,7 +110,6 @@ public: OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); } - auto millis = spdlog::details::fmt_helper::time_fraction(msg.time); spdlog::details::fmt_helper::pad3(static_cast(millis.count()), OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index 762b75a59..cb0fd6679 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -16,6 +16,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include #include #include +#include #include #include #include @@ -191,8 +192,9 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) logging::RefreshLogLevels(LogLevel); spdlog::flush_on(spdlog::level::err); spdlog::flush_every(std::chrono::seconds{2}); - spdlog::set_formatter( - std::make_unique(LogOptions.LogId, std::chrono::system_clock::now())); // default to duration prefix + spdlog::set_formatter(std::make_unique( + LogOptions.LogId, + std::chrono::system_clock::now() - std::chrono::milliseconds(GetTimeSinceProcessStart()))); // default to duration prefix if (g_FileSink) { -- cgit v1.2.3 From cd128dd76526ed01c9a6c125b7ecddd9e3e08e57 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 12 Mar 2025 14:48:46 +0100 Subject: ProgressBar improvements (#305) * changed ProgressBar so it doesn't use printf. printf by default is very slow on Windows due to weird buffering behaviour. During a 2 minute build download I profiled 35 CPU seconds inside printf * changed so ProgressBar uses plain output mode if stdout is not a console/tty --- src/zen/zen.cpp | 43 +++++++++++++++++++++++++++++++++++++++++-- src/zen/zen.h | 1 + 2 files changed, 42 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 0fcf9d871..9d0eab7dc 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,10 @@ ZEN_THIRD_PARTY_INCLUDES_END #include +#ifndef ZEN_PLATFORM_WINDOWS +# include +#endif + ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -261,8 +266,22 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy); } +static bool +IsStdoutTty() +{ +#if ZEN_PLATFORM_WINDOWS + static HANDLE hStdOut = ::GetStdHandle(STD_OUTPUT_HANDLE); + DWORD dwMode = 0; + static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); + return IsConsole; +#else + return isatty(fileno(stdout)); +#endif +} + ProgressBar::ProgressBar(bool PlainProgress, bool ShowDetails) -: m_PlainProgress(PlainProgress) +: m_StdoutIsTty(IsStdoutTty()) +, m_PlainProgress(PlainProgress || !m_StdoutIsTty) , m_ShowDetails(ShowDetails) , m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000) { @@ -323,7 +342,27 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) ETA, NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details)); std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0; - printf("%s%s%s", Output.c_str(), std::string(EraseLength, ' ').c_str(), DoLinebreak ? "\n" : ""); + + ExtendableStringBuilder<128> LineToPrint; + LineToPrint << Output << std::string(EraseLength, ' '); + if (DoLinebreak) + LineToPrint << "\n"; + +#if ZEN_PLATFORM_WINDOWS + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + + if (m_StdoutIsTty) + { + WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0); + } + else + { + ::WriteFile(hStdOut, (LPCVOID)LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0); + } +#else + fwrite(LineToPrint.c_str(), 1, LineToPrint.Size(), stdout); +#endif + m_LastOutputLength = DoLinebreak ? 0 : Output.length(); m_State = NewState; } diff --git a/src/zen/zen.h b/src/zen/zen.h index 835c2b6ac..6765101db 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -94,6 +94,7 @@ public: bool HasActiveTask() const; private: + const bool m_StdoutIsTty = true; const bool m_PlainProgress; const bool m_ShowDetails; Stopwatch m_SW; -- cgit v1.2.3 From 9268991f2cedd999d0b0e6175b6e32c89acddf38 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 12 Mar 2025 10:40:20 +0100 Subject: Remove spurious '4' in conditional code block --- src/zen/cmds/builds_cmd.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index baa46dda8..d3536768b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3049,18 +3049,18 @@ namespace { std::vector OutLooseChunkHashes; std::vector OutLooseChunkRawSizes; std::vector OutBlockRawHashes; - 4 ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), - VerifyFolderContent.Platform, - VerifyFolderContent.Paths, - VerifyFolderContent.RawHashes, - VerifyFolderContent.RawSizes, - VerifyFolderContent.Attributes, - VerifyFolderContent.ChunkedContent.SequenceRawHashes, - VerifyFolderContent.ChunkedContent.ChunkCounts, - OutAbsoluteChunkOrders, - OutLooseChunkHashes, - OutLooseChunkRawSizes, - OutBlockRawHashes); + ReadBuildContentFromCompactBinary(PartManifestWriter.Save(), + VerifyFolderContent.Platform, + VerifyFolderContent.Paths, + VerifyFolderContent.RawHashes, + VerifyFolderContent.RawSizes, + VerifyFolderContent.Attributes, + VerifyFolderContent.ChunkedContent.SequenceRawHashes, + VerifyFolderContent.ChunkedContent.ChunkCounts, + OutAbsoluteChunkOrders, + OutLooseChunkHashes, + OutLooseChunkRawSizes, + OutBlockRawHashes); ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes); for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++) -- cgit v1.2.3 From 908f99b749fbbf9754f9485d680914792034334c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 18:58:24 +0100 Subject: fix quoted command lines arguments (#306) Handling of quotes and quotes with leading backslash for command line parsing - UE-231677 --- src/zen/cmds/admin_cmd.cpp | 19 +++-- src/zen/cmds/admin_cmd.h | 8 +- src/zen/cmds/builds_cmd.cpp | 119 +++++++++++++++----------- src/zen/cmds/builds_cmd.h | 38 ++++----- src/zen/cmds/status_cmd.cpp | 9 +- src/zen/cmds/status_cmd.h | 6 +- src/zen/cmds/up_cmd.cpp | 43 ++++++---- src/zen/cmds/up_cmd.h | 28 +++--- src/zen/cmds/workspaces_cmd.cpp | 91 ++++++++++---------- src/zen/cmds/workspaces_cmd.h | 34 ++++---- src/zen/zen.cpp | 18 ++++ src/zencore/filesystem.cpp | 13 +++ src/zencore/include/zencore/filesystem.h | 2 + src/zencore/include/zencore/process.h | 3 + src/zencore/process.cpp | 142 +++++++++++++++++++++++++++++++ 15 files changed, 391 insertions(+), 182 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 995ed4136..835e01151 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -714,26 +714,29 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("data path must be given"); } - if (!std::filesystem::is_directory(m_DataPath)) + std::filesystem::path DataPath = StringToPath(m_DataPath); + std::filesystem::path TargetPath = StringToPath(m_TargetPath); + + if (!std::filesystem::is_directory(DataPath)) { throw OptionParseException("data path must exist"); } - if (m_TargetPath.empty()) + if (TargetPath.empty()) { throw OptionParseException("target path must be given"); } - std::filesystem::path RootManifestPath = m_DataPath / "root_manifest"; - std::filesystem::path TargetRootManifestPath = m_TargetPath / "root_manifest"; + std::filesystem::path RootManifestPath = DataPath / "root_manifest"; + std::filesystem::path TargetRootManifestPath = TargetPath / "root_manifest"; if (!TryCopy(RootManifestPath, TargetRootManifestPath)) { throw OptionParseException("data path is invalid, missing root_manifest"); } - std::filesystem::path CachePath = m_DataPath / "cache"; - std::filesystem::path TargetCachePath = m_TargetPath / "cache"; + std::filesystem::path CachePath = DataPath / "cache"; + std::filesystem::path TargetCachePath = TargetPath / "cache"; // Copy cache state DirectoryContent CacheDirectoryContent; @@ -778,8 +781,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path CasPath = m_DataPath / "cas"; - std::filesystem::path TargetCasPath = m_TargetPath / "cas"; + std::filesystem::path CasPath = DataPath / "cas"; + std::filesystem::path TargetCasPath = TargetPath / "cas"; { std::filesystem::path UCasRootPath = CasPath / ".ucas_root"; diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index c593b2cac..8b6d3e258 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -155,10 +155,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; - std::filesystem::path m_DataPath; - std::filesystem::path m_TargetPath; - bool m_SkipLogs = false; + cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + std::string m_DataPath; + std::string m_TargetPath; + bool m_SkipLogs = false; }; } // namespace zen diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d3536768b..f0ee4904e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -6689,7 +6689,7 @@ BuildsCommand::BuildsCommand() m_Options.add_options()("h,help", "Print help"); auto AddAuthOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); // Direct access token (may expire) Ops.add_option("auth-token", @@ -7025,7 +7025,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : m_SystemRootDir; + std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : StringToPath(m_SystemRootDir); if (m_EncryptionKey.empty()) { @@ -7147,8 +7147,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_StoragePath.empty()) { - ZEN_CONSOLE("Querying builds in folder '{}'.", m_StoragePath); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + ZEN_CONSOLE("Querying builds in folder '{}'.", StoragePath); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); } else { @@ -7199,9 +7200,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } + std::filesystem::path Path = StringToPath(m_Path); + if (m_BuildPartName.empty()) { - m_BuildPartName = m_Path.filename().string(); + m_BuildPartName = Path.filename().string(); } const bool GeneratedBuildId = m_BuildId.empty(); @@ -7241,26 +7244,27 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", m_BuildPartName, - m_Path, + Path, m_BuildsUrl, Http.GetSessionId(), m_Namespace, m_Bucket, GeneratedBuildId ? "Generated " : "", BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } else if (!m_StoragePath.empty()) { + std::filesystem::path StoragePath = StringToPath(m_StoragePath); ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", m_BuildPartName, - m_Path, - m_StoragePath, + Path, + StoragePath, GeneratedBuildId ? "Generated " : "", BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { @@ -7303,7 +7307,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartId, m_BuildPartName, - m_Path, + Path, m_ManifestPath, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -7372,6 +7376,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } + std::filesystem::path Path = StringToPath(m_Path); + BuildStorage::Statistics StorageStats; std::unique_ptr Storage; std::string StorageName; @@ -7379,20 +7385,21 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", BuildId, - m_Path, + Path, m_BuildsUrl, Http.GetSessionId(), m_Namespace, m_Bucket, BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } else if (!m_StoragePath.empty()) { - ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, m_Path, m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { @@ -7403,7 +7410,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartIds, m_BuildPartNames, - m_Path, + Path, m_AllowMultiparts, m_AllowPartialBlockRequests, m_Clean, @@ -7442,7 +7449,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); + std::filesystem::path Path = StringToPath(m_Path); + DiffFolders(Path, m_DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -7467,6 +7475,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; + std::filesystem::path Path = StringToPath(m_Path); + BuildStorage::Statistics StorageStats; std::unique_ptr Storage; std::string StorageName; @@ -7474,19 +7484,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", FormatArray(m_BuildIds, " "sv), - m_Path, + Path, m_BuildsUrl, Http.GetSessionId(), m_Namespace, m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } else if (!m_StoragePath.empty()) { - ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray(m_BuildIds, " "sv), m_Path, m_StoragePath); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray(m_BuildIds, " "sv), Path, StoragePath); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { @@ -7504,7 +7515,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, {}, {}, - m_Path, + Path, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -7531,8 +7542,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } + std::filesystem::path Path = StringToPath(m_Path); + m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = m_Path.filename().string(); + m_BuildPartName = Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; @@ -7542,16 +7555,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr Storage; std::string StorageName; - if (m_BuildsUrl.empty() && m_StoragePath.empty()) + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + + if (m_BuildsUrl.empty() && StoragePath.empty()) { - m_StoragePath = GetRunningExecutablePath().parent_path() / ".tmpstore"; - CreateDirectories(m_StoragePath); - CleanDirectory(m_StoragePath, {}); + m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").generic_string(); + CreateDirectories(StoragePath); + CleanDirectory(StoragePath, {}); } auto _ = MakeGuard([&]() { - if (m_BuildsUrl.empty() && m_StoragePath.empty()) + if (m_BuildsUrl.empty() && StoragePath.empty()) { - DeleteDirectories(m_StoragePath); + DeleteDirectories(StoragePath); } }); @@ -7559,24 +7574,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - m_Path, + Path, m_BuildsUrl, Http.GetSessionId(), m_Namespace, m_Bucket, BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } - else if (!m_StoragePath.empty()) + else if (!StoragePath.empty()) { ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - m_Path, - m_StoragePath, + Path, + StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { @@ -7608,7 +7623,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartId, m_BuildPartName, - m_Path, + Path, {}, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -7622,7 +7637,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 11; } - const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_download"); + const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); if (AbortFlag) @@ -7846,6 +7861,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr Storage; std::string StorageName; + std::filesystem::path Path = StringToPath(m_Path); + if (!m_BuildsUrl.empty()) { ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", @@ -7854,14 +7871,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } else if (!m_StoragePath.empty()) { - ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { @@ -7913,6 +7931,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr Storage; std::string StorageName; + std::filesystem::path Path = StringToPath(m_Path); + if (!m_BuildsUrl.empty()) { ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", @@ -7921,14 +7941,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Namespace, m_Bucket, BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, m_Path / ZenTempStorageFolderName); + Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); StorageName = "Cloud DDC"; } else if (!m_StoragePath.empty()) { - ZEN_CONSOLE("Using folder {}. BuildId '{}'", m_StoragePath, BuildId); - Storage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + std::filesystem::path StoragePath = StringToPath(m_StoragePath); + ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); + Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + StorageName = fmt::format("Disk {}", StoragePath.stem()); } else { diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 167a5d29f..60953efad 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -25,7 +25,7 @@ public: private: cxxopts::Options m_Options{Name, Description}; - std::filesystem::path m_SystemRootDir; + std::string m_SystemRootDir; bool m_PlainProgress = false; bool m_Verbose = false; @@ -37,20 +37,20 @@ private: std::string m_Bucket; // file storage - std::filesystem::path m_StoragePath; - bool m_WriteMetadataAsJson = false; - - std::string m_BuildId; - bool m_CreateBuild = false; - std::string m_BuildMetadataPath; - std::string m_BuildMetadata; - std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path - std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName - bool m_Clean = false; - uint8_t m_BlockReuseMinPercentLimit = 85; - bool m_AllowMultiparts = true; - bool m_AllowPartialBlockRequests = true; - std::filesystem::path m_ManifestPath; + std::string m_StoragePath; + bool m_WriteMetadataAsJson = false; + + std::string m_BuildId; + bool m_CreateBuild = false; + std::string m_BuildMetadataPath; + std::string m_BuildMetadata; + std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path + std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_AllowMultiparts = true; + bool m_AllowPartialBlockRequests = true; + std::string m_ManifestPath; // Direct access token (may expire) std::string m_AccessToken; @@ -76,7 +76,7 @@ private: cxxopts::Options m_ListOptions{"list", "List available builds"}; - std::filesystem::path m_Path; + std::string m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; bool m_PostUploadVerify = false; @@ -86,9 +86,9 @@ private: std::vector m_BuildPartIds; bool m_PostDownloadVerify = false; - cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; - std::filesystem::path m_DiffPath; - bool m_OnlyChunked = false; + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; + std::string m_DiffPath; + bool m_OnlyChunked = false; cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 16754e747..4d1534e05 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -32,16 +32,17 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) uint16_t EffectivePort = 0; if (!m_DataDir.empty()) { - if (!std::filesystem::is_regular_file(m_DataDir / ".lock")) + std::filesystem::path DataDir = StringToPath(m_DataDir); + if (!std::filesystem::is_regular_file(DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); return 1; } EffectivePort = Info.EffectiveListenPort; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index 46bda9ee6..00ad0e758 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -20,9 +20,9 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; - uint16_t m_Port = 0; - std::filesystem::path m_DataDir; + cxxopts::Options m_Options{"status", "Show zen status"}; + uint16_t m_Port = 0; + std::string m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index ac2f42a86..44a41146c 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -77,13 +77,15 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - if (m_ProgramBaseDir.empty()) + std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); + + if (ProgramBaseDir.empty()) { std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - m_ProgramBaseDir = ExePath.parent_path(); + ProgramBaseDir = ExePath.parent_path(); } ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(m_ProgramBaseDir); + ServerEnvironment.Initialize(ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); std::string ServerArguments = GlobalOptions.PassthroughCommandLine; if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos)) @@ -153,18 +155,20 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Sweep(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - if (!m_DataDir.empty()) + std::filesystem::path DataDir = StringToPath(m_DataDir); + + if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(m_DataDir / ".lock")) + if (!std::filesystem::is_regular_file(DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -214,24 +218,27 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Initialize(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - if (m_ProgramBaseDir.empty()) + std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); + if (ProgramBaseDir.empty()) { - std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - m_ProgramBaseDir = ExePath.parent_path(); + std::filesystem::path ExePath = GetRunningExecutablePath(); + ProgramBaseDir = ExePath.parent_path(); } - if (!m_DataDir.empty()) + std::filesystem::path DataDir = StringToPath(m_DataDir); + + if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(m_DataDir / ".lock")) + if (!std::filesystem::is_regular_file(DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -244,7 +251,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) try { ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(m_ProgramBaseDir); + ServerEnvironment.Initialize(ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); Server.AttachToRunningServer(EntryPort); @@ -309,7 +316,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_ForceTerminate) { // Try to find the running executable by path name - std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path ServerExePath = ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec) { diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index c9af16749..32d8ddab3 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -18,11 +18,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; - uint16_t m_Port = 0; - bool m_ShowConsole = false; - bool m_ShowLog = false; - std::filesystem::path m_ProgramBaseDir; + cxxopts::Options m_Options{"up", "Bring up zen service"}; + uint16_t m_Port = 0; + bool m_ShowConsole = false; + bool m_ShowLog = false; + std::string m_ProgramBaseDir; }; class AttachCommand : public ZenCmdBase @@ -35,10 +35,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; - uint16_t m_Port = 0; - int m_OwnerPid = 0; - std::filesystem::path m_DataDir; + cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + uint16_t m_Port = 0; + int m_OwnerPid = 0; + std::string m_DataDir; }; class DownCommand : public ZenCmdBase @@ -51,11 +51,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; - uint16_t m_Port = 0; - bool m_ForceTerminate = false; - std::filesystem::path m_ProgramBaseDir; - std::filesystem::path m_DataDir; + cxxopts::Options m_Options{"down", "Bring down zen service"}; + uint16_t m_Port = 0; + bool m_ForceTerminate = false; + std::string m_ProgramBaseDir; + std::string m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 166d4218d..5f3f8f7ca 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -25,16 +25,7 @@ namespace { if (!Path.empty()) { std::u8string PathString = Path.u8string(); - if (PathString.ends_with(std::filesystem::path::preferred_separator) || PathString.ends_with('/')) - { - PathString.pop_back(); - Path = std::filesystem::path(PathString); - } - // Special case if user gives a path with quotes and includes a backslash at the end: - // ="path\" cxxopts strips the leading quote only but not the trailing. - // As we expect paths here and we don't want trailing slashes we strip away the quote - // manually if the string does not start with a quote UE-231677 - else if (PathString[0] != '\"' && PathString[PathString.length() - 1] == '\"') + if (PathString.ends_with(std::filesystem::path::preferred_separator) || PathString.starts_with('/')) { PathString.pop_back(); Path = std::filesystem::path(PathString); @@ -96,7 +87,7 @@ WorkspaceCommand::WorkspaceCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_options()("system-dir", "Specify system root", cxxopts::value(m_SystemRootDir)); + m_Options.add_options()("system-dir", "Specify system root", cxxopts::value(m_SystemRootDir)); m_Options.add_option("", "v", "verb", "Verb for workspace - create, remove, info", cxxopts::value(m_Verb), ""); m_Options.parse_positional({"verb"}); m_Options.positional_help("verb"); @@ -148,16 +139,18 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) + std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); + + if (SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); - if (m_SystemRootDir.empty()) + SystemRootDir = PickDefaultSystemRootDirectory(); + if (SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } - std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; + std::filesystem::path StatePath = SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -171,12 +164,12 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("path is required\n{}", m_CreateOptions.help())); } - RemoveTrailingPathSeparator(m_Path); + std::filesystem::path Path = StringToPath(m_Path); if (m_Id.empty()) { - m_Id = Workspaces::PathToId(m_Path).ToString(); - ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_Id, m_Path); + m_Id = Workspaces::PathToId(Path).ToString(); + ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_Id, Path); } if (Oid::TryFromHexString(m_Id) == Oid::Zero) @@ -187,7 +180,7 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (Workspaces::AddWorkspace( Log(), StatePath, - {.Id = Oid::FromHexString(m_Id), .RootPath = m_Path, .AllowShareCreationFromHttp = m_AllowShareCreationFromHttp})) + {.Id = Oid::FromHexString(m_Id), .RootPath = Path, .AllowShareCreationFromHttp = m_AllowShareCreationFromHttp})) { if (!m_HostName.empty()) { @@ -287,7 +280,7 @@ WorkspaceShareCommand::WorkspaceShareCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); - m_Options.add_options()("system-dir", "Specify system root", cxxopts::value(m_SystemRootDir)); + m_Options.add_options()("system-dir", "Specify system root", cxxopts::value(m_SystemRootDir)); m_Options.add_option("", "v", "verb", "Verb for workspace - create, remove, info", cxxopts::value(m_Verb), ""); m_Options.parse_positional({"verb"}); m_Options.positional_help("verb"); @@ -399,16 +392,18 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** m_HostName = ResolveTargetHostSpec(m_HostName); - if (m_SystemRootDir.empty()) + std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); + + if (SystemRootDir.empty()) { - m_SystemRootDir = PickDefaultSystemRootDirectory(); - if (m_SystemRootDir.empty()) + SystemRootDir = PickDefaultSystemRootDirectory(); + if (SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } - std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; + std::filesystem::path StatePath = SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -417,7 +412,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_CreateOptions) { - if (m_WorkspaceRoot.empty()) + std::filesystem::path WorkspaceRoot = StringToPath(m_WorkspaceRoot); + if (WorkspaceRoot.empty()) { if (m_WorkspaceId.empty()) { @@ -436,15 +432,15 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); return 0; } - m_WorkspaceRoot = WorkspaceConfig.RootPath; + WorkspaceRoot = WorkspaceConfig.RootPath; } else { - RemoveTrailingPathSeparator(m_WorkspaceRoot); + RemoveTrailingPathSeparator(WorkspaceRoot); if (m_WorkspaceId.empty()) { - m_WorkspaceId = Workspaces::PathToId(m_WorkspaceRoot).ToString(); - ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, m_WorkspaceRoot); + m_WorkspaceId = Workspaces::PathToId(WorkspaceRoot).ToString(); + ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, WorkspaceRoot); } else { @@ -453,23 +449,25 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_WorkspaceId)); } } - if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = m_WorkspaceRoot})) + if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = WorkspaceRoot})) { - ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, m_WorkspaceRoot); + ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, WorkspaceRoot); } else { - ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, m_WorkspaceRoot); + ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, WorkspaceRoot); } } - RemoveTrailingPathSeparator(m_SharePath); - RemoveLeadingPathSeparator(m_SharePath); + std::filesystem::path SharePath = StringToPath(m_SharePath); + + RemoveLeadingPathSeparator(SharePath); + RemoveTrailingPathSeparator(SharePath); if (m_ShareId.empty()) { - m_ShareId = Workspaces::PathToId(m_SharePath).ToString(); - ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, m_SharePath); + m_ShareId = Workspaces::PathToId(SharePath).ToString(); + ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, SharePath); } if (Oid::TryFromHexString(m_ShareId) == Oid::Zero) @@ -478,8 +476,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } if (Workspaces::AddWorkspaceShare(Log(), - m_WorkspaceRoot, - {.Id = Oid::FromHexString(m_ShareId), .SharePath = m_SharePath, .Alias = m_Alias})) + WorkspaceRoot, + {.Id = Oid::FromHexString(m_ShareId), .SharePath = SharePath, .Alias = m_Alias})) { if (!m_HostName.empty()) { @@ -531,7 +529,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); return 0; } - m_WorkspaceRoot = WorkspaceConfig.RootPath; + + std::filesystem::path WorkspaceRoot = WorkspaceConfig.RootPath; if (m_ShareId.empty()) { @@ -543,8 +542,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_ShareId)); } - Workspaces::WorkspaceShareConfiguration Share = - Workspaces::FindWorkspaceShare(Log(), m_WorkspaceRoot, Oid::FromHexString(m_ShareId)); + Workspaces::WorkspaceShareConfiguration Share = Workspaces::FindWorkspaceShare(Log(), WorkspaceRoot, Oid::FromHexString(m_ShareId)); if (Share.Id == Oid::Zero) { ZEN_CONSOLE("Workspace share {} does not exist", m_ShareId); @@ -556,6 +554,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_RemoveOptions) { + std::filesystem::path WorkspaceRoot; if (!m_Alias.empty()) { Workspaces::WorkspaceConfiguration WorkspaceConfig; @@ -566,9 +565,9 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace share with alias {} does not exist", m_Alias); return 0; } - m_ShareId = ShareConfig.Id.ToString(); - m_WorkspaceId = WorkspaceConfig.Id.ToString(); - m_WorkspaceRoot = WorkspaceConfig.RootPath; + m_ShareId = ShareConfig.Id.ToString(); + m_WorkspaceId = WorkspaceConfig.Id.ToString(); + WorkspaceRoot = WorkspaceConfig.RootPath; } if (m_WorkspaceId.empty()) @@ -587,7 +586,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); return 0; } - m_WorkspaceRoot = WorkspaceConfig.RootPath; + WorkspaceRoot = WorkspaceConfig.RootPath; if (m_ShareId.empty()) { @@ -599,7 +598,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_ShareId)); } - if (Workspaces::RemoveWorkspaceShare(Log(), m_WorkspaceRoot, Oid::FromHexString(m_ShareId))) + if (Workspaces::RemoveWorkspaceShare(Log(), WorkspaceRoot, Oid::FromHexString(m_ShareId))) { if (!m_HostName.empty()) { diff --git a/src/zen/cmds/workspaces_cmd.h b/src/zen/cmds/workspaces_cmd.h index de0edd061..86452e25e 100644 --- a/src/zen/cmds/workspaces_cmd.h +++ b/src/zen/cmds/workspaces_cmd.h @@ -21,17 +21,17 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::filesystem::path m_SystemRootDir; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_SystemRootDir; std::string m_Verb; // create, info, remove std::string m_Id; - cxxopts::Options m_CreateOptions{"create", "Create a workspace"}; - std::filesystem::path m_Path; - bool m_AllowShareCreationFromHttp = false; + cxxopts::Options m_CreateOptions{"create", "Create a workspace"}; + std::string m_Path; + bool m_AllowShareCreationFromHttp = false; cxxopts::Options m_InfoOptions{"info", "Info about a workspace"}; @@ -53,17 +53,17 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::filesystem::path m_SystemRootDir; - std::string m_WorkspaceId; - std::filesystem::path m_WorkspaceRoot; - std::string m_Verb; // create, info, remove - std::string m_ShareId; - std::string m_Alias; - - cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; - std::filesystem::path m_SharePath; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_SystemRootDir; + std::string m_WorkspaceId; + std::string m_WorkspaceRoot; + std::string m_Verb; // create, info, remove + std::string m_ShareId; + std::string m_Alias; + + cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; + std::string m_SharePath; bool m_Refresh = false; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 9d0eab7dc..6f831349b 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -414,6 +415,23 @@ ProgressBar::HasActiveTask() const int main(int argc, char** argv) { + std::vector Args; +#if ZEN_PLATFORM_WINDOWS + LPWSTR RawCommandLine = GetCommandLine(); + std::string CommandLine = zen::WideToUtf8(RawCommandLine); + Args = zen::ParseCommandLine(CommandLine); +#else + Args.reserve(argc); + for (int I = 0; I < argc; I++) + { + Args.push_back(std::string(argv[I])); + } +#endif + std::vector RawArgs = zen::StripCommandlineQuotes(Args); + + argc = gsl::narrow(RawArgs.size()); + argv = RawArgs.data(); + using namespace zen; using namespace std::literals; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 9f3f4f7fc..05e2bf049 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2043,6 +2043,19 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) return false; } +std::filesystem::path +StringToPath(const std::string_view& Path) +{ + if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') + { + return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred(); + } + else + { + return std::filesystem::path(Path).make_preferred(); + } +} + ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index e020668fc..9a2b15d1d 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -292,6 +292,8 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); +std::filesystem::path StringToPath(const std::string_view& Path); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index d1394cd9a..0c5931ba0 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -100,6 +100,9 @@ int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); + void process_forcelink(); // internal } // namespace zen diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index c51e8f69d..0761521dc 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -1047,6 +1047,118 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand #endif // ZEN_PLATFORM_LINUX } +std::vector +ParseCommandLine(std::string_view CommandLine) +{ + auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { + if (Pos == CommandLine.length()) + { + return true; + } + if (CommandLine[Pos] == ' ') + { + return true; + } + return false; + }; + + bool IsParsingArg = false; + bool IsInQuote = false; + + std::string::size_type Pos = 0; + std::string::size_type ArgStart = 0; + std::vector Args; + while (Pos < CommandLine.length()) + { + if (IsInQuote) + { + if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); + Pos++; + IsInQuote = false; + IsParsingArg = false; + } + else + { + Pos++; + } + } + else if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + if (CommandLine[Pos] == ' ') + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); + Pos++; + IsParsingArg = false; + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + Pos++; + } + else + { + Pos++; + } + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else if (CommandLine[Pos] != ' ') + { + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else + { + Pos++; + } + } + if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + Args.push_back(std::string(CommandLine.substr(ArgStart))); + } + + return Args; +} + +std::vector +StripCommandlineQuotes(std::vector& InOutArgs) +{ + std::vector RawArgs; + RawArgs.reserve(InOutArgs.size()); + for (std::string& Arg : InOutArgs) + { + std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); + while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) + { + Arg.erase(EscapedQuotePos, 1); + EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); + } + + if (Arg.starts_with("\"")) + { + if (Arg.find('"', 1) == Arg.length() - 1) + { + if (Arg.find(' ', 1) == std::string::npos) + { + Arg = Arg.substr(1, Arg.length() - 2); + } + } + } + RawArgs.push_back(const_cast(Arg.c_str())); + } + return RawArgs; +} + #if ZEN_WITH_TESTS void @@ -1123,6 +1235,36 @@ TEST_CASE("BuildArgV") } } +TEST_CASE("CommandLine") +{ + std::vector v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); + CHECK_EQ(v1[0], "c:\\my\\exe.exe"); + CHECK_EQ(v1[1], "\"quoted arg\""); + CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); + + std::vector v2 = ParseCommandLine( + "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " + "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " + "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " + "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); + + std::vector v2Stripped = StripCommandlineQuotes(v2); + CHECK_EQ(v2Stripped[0], std::string("--tracehost")); + CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); + CHECK_EQ(v2Stripped[2], std::string("builds")); + CHECK_EQ(v2Stripped[3], std::string("download")); + CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); + CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); + CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); + CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); + CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\"")); + CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\"")); + CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\"")); + CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); + CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); + CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); +} + TEST_SUITE_END(/* core.process */); #endif -- cgit v1.2.3 From fa4ef162b1dd53cbad135850a8f9cf8fb532f395 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 08:56:40 +0100 Subject: improved post upload/download summary (#308) * added ValidateStatistics and improved post upload summary * improved download statistics * smoother stats update when compressing * better feedback during stream compresss/decompress * don't capture TotalPartWriteCount by reference * disk stats cleanup * multi-test-download overall timer --- src/zen/cmds/builds_cmd.cpp | 962 ++++++++++++++++++++------------- src/zencore/compress.cpp | 130 +++-- src/zencore/include/zencore/compress.h | 18 +- 3 files changed, 690 insertions(+), 420 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f0ee4904e..61e3c0fab 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -567,6 +567,64 @@ namespace { } }; + struct CacheMappingStatistics + { + uint64_t CacheChunkCount = 0; + uint64_t CacheChunkByteCount = 0; + + uint64_t CacheBlockCount = 0; + uint64_t CacheBlocksByteCount = 0; + + uint64_t CacheSequenceHashesCount = 0; + uint64_t CacheSequenceHashesByteCount = 0; + + uint32_t LocalPathsMatchingSequencesCount = 0; + uint64_t LocalPathsMatchingSequencesByteCount = 0; + + uint64_t LocalChunkMatchingRemoteCount = 0; + uint64_t LocalChunkMatchingRemoteByteCount = 0; + }; + + struct DownloadStatistics + { + std::atomic RequestsCompleteCount = 0; + + std::atomic DownloadedChunkCount = 0; + std::atomic DownloadedChunkByteCount = 0; + std::atomic MultipartAttachmentCount = 0; + + std::atomic DownloadedBlockCount = 0; + std::atomic DownloadedBlockByteCount = 0; + + std::atomic DownloadedPartialBlockCount = 0; + std::atomic DownloadedPartialBlockByteCount = 0; + }; + + struct WriteChunkStatistics + { + std::atomic ChunkCountWritten = 0; + std::atomic ChunkBytesWritten = 0; + uint64_t DownloadTimeUs = 0; + uint64_t WriteTimeUs = 0; + uint64_t WriteChunksElapsedWallTimeUs = 0; + }; + + struct RebuildFolderStateStatistics + { + uint64_t CleanFolderElapsedWallTimeUs = 0; + std::atomic FinalizeTreeFilesMovedCount = 0; + std::atomic FinalizeTreeFilesCopiedCount = 0; + uint64_t FinalizeTreeElapsedWallTimeUs = 0; + }; + + struct VerifyFolderStatistics + { + std::atomic FilesVerified = 0; + std::atomic FilesFailed = 0; + std::atomic ReadBytes = 0; + uint64_t VerifyElapsedWallTimeUs = 0; + }; + std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, const std::span LocalChunkOrder, const tsl::robin_map& ChunkHashToLocalChunkIndex, @@ -1016,7 +1074,15 @@ namespace { class BufferedOpenFile { public: - BufferedOpenFile(const std::filesystem::path Path) : Source(Path, BasicFile::Mode::kRead), SourceSize(Source.FileSize()) {} + BufferedOpenFile(const std::filesystem::path Path, DiskStatistics& DiskStats) + : m_Source(Path, BasicFile::Mode::kRead) + , m_SourceSize(m_Source.FileSize()) + , m_DiskStats(DiskStats) + { + m_DiskStats.OpenReadCount++; + m_DiskStats.CurrentOpenFileCount++; + } + ~BufferedOpenFile() { m_DiskStats.CurrentOpenFileCount--; } BufferedOpenFile() = delete; BufferedOpenFile(const BufferedOpenFile&) = delete; BufferedOpenFile(BufferedOpenFile&&) = delete; @@ -1028,10 +1094,10 @@ namespace { { ZEN_TRACE_CPU("BufferedOpenFile::GetRange"); - ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); - auto _ = MakeGuard([&]() { ZEN_ASSERT((CacheBlockIndex == (uint64_t)-1) || Cache); }); + ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); + auto _ = MakeGuard([&]() { ZEN_ASSERT((m_CacheBlockIndex == (uint64_t)-1) || m_Cache); }); - ZEN_ASSERT((Offset + Size) <= SourceSize); + ZEN_ASSERT((Offset + Size) <= m_SourceSize); const uint64_t BlockIndexStart = Offset / BlockSize; const uint64_t BlockIndexEnd = (Offset + Size - 1) / BlockSize; @@ -1042,21 +1108,23 @@ namespace { for (uint64_t BlockIndex = BlockIndexStart; BlockIndex <= BlockIndexEnd; BlockIndex++) { const uint64_t BlockStartOffset = BlockIndex * BlockSize; - if (CacheBlockIndex != BlockIndex) + if (m_CacheBlockIndex != BlockIndex) { - uint64_t CacheSize = Min(BlockSize, SourceSize - BlockStartOffset); + uint64_t CacheSize = Min(BlockSize, m_SourceSize - BlockStartOffset); ZEN_ASSERT(CacheSize > 0); - Cache = IoBuffer(CacheSize); - Source.Read(Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset); - CacheBlockIndex = BlockIndex; + m_Cache = IoBuffer(CacheSize); + m_Source.Read(m_Cache.GetMutableView().GetData(), CacheSize, BlockStartOffset); + m_DiskStats.ReadCount++; + m_DiskStats.ReadByteCount += CacheSize; + m_CacheBlockIndex = BlockIndex; } const uint64_t BytesRead = ReadOffset - Offset; ZEN_ASSERT(BlockStartOffset <= ReadOffset); const uint64_t OffsetIntoBlock = ReadOffset - BlockStartOffset; - ZEN_ASSERT(OffsetIntoBlock < Cache.GetSize()); - const uint64_t BlockBytes = Min(Cache.GetSize() - OffsetIntoBlock, Size - BytesRead); - BufferRanges.emplace_back(SharedBuffer(IoBuffer(Cache, OffsetIntoBlock, BlockBytes))); + ZEN_ASSERT(OffsetIntoBlock < m_Cache.GetSize()); + const uint64_t BlockBytes = Min(m_Cache.GetSize() - OffsetIntoBlock, Size - BytesRead); + BufferRanges.emplace_back(SharedBuffer(IoBuffer(m_Cache, OffsetIntoBlock, BlockBytes))); ReadOffset += BlockBytes; } CompositeBuffer Result(std::move(BufferRanges)); @@ -1065,10 +1133,11 @@ namespace { } private: - BasicFile Source; - const uint64_t SourceSize; - uint64_t CacheBlockIndex = (uint64_t)-1; - IoBuffer Cache; + BasicFile m_Source; + const uint64_t m_SourceSize; + DiskStatistics& m_DiskStats; + uint64_t m_CacheBlockIndex = (uint64_t)-1; + IoBuffer m_Cache; }; class ReadFileCache @@ -1087,11 +1156,7 @@ namespace { { m_OpenFiles.reserve(MaxOpenFileCount); } - ~ReadFileCache() - { - m_DiskStats.CurrentOpenFileCount -= m_OpenFiles.size(); - m_OpenFiles.clear(); - } + ~ReadFileCache() { m_OpenFiles.clear(); } CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size) { @@ -1109,7 +1174,6 @@ namespace { m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile))); } CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); - m_DiskStats.ReadByteCount += Result.GetSize(); return Result; } const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex]; @@ -1117,20 +1181,15 @@ namespace { if (Size == m_LocalContent.RawSizes[LocalPathIndex]) { IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath); - m_DiskStats.OpenReadCount++; - m_DiskStats.ReadByteCount += Result.GetSize(); return CompositeBuffer(SharedBuffer(Result)); } if (m_OpenFiles.size() == m_OpenFiles.capacity()) { m_OpenFiles.pop_back(); - m_DiskStats.CurrentOpenFileCount--; } - m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::make_unique(LocalFilePath))); + m_OpenFiles.insert(m_OpenFiles.begin(), + std::make_pair(SequenceIndex, std::make_unique(LocalFilePath, m_DiskStats))); CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size); - m_DiskStats.ReadByteCount += Result.GetSize(); - m_DiskStats.OpenReadCount++; - m_DiskStats.CurrentOpenFileCount++; return Result; } @@ -1167,17 +1226,21 @@ namespace { } IoHashStream Hash; - bool CouldDecompress = Compressed.DecompressToStream(0, RawSize, [&Hash](uint64_t, const CompositeBuffer& RangeBuffer) { - if (!AbortFlag) - { - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + bool CouldDecompress = Compressed.DecompressToStream( + 0, + RawSize, + [&Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset, SourceSize, Offset); + if (!AbortFlag) { - Hash.Append(Segment.GetView()); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + return true; } - return true; - } - return false; - }); + return false; + }); if (AbortFlag) { @@ -1330,8 +1393,7 @@ namespace { const std::uint64_t PreferredMultipartChunkSize, ParallellWork& Work, WorkerThreadPool& NetworkPool, - std::atomic& BytesDownloaded, - std::atomic& MultipartAttachmentCount, + DownloadStatistics& DownloadStats, std::function&& OnDownloadComplete) { ZEN_TRACE_CPU("DownloadLargeBlob"); @@ -1353,10 +1415,10 @@ namespace { BuildId, ChunkHash, PreferredMultipartChunkSize, - [Workload, &BytesDownloaded, OnDownloadComplete = std::move(OnDownloadComplete)](uint64_t Offset, - const IoBuffer& Chunk, - uint64_t BytesRemaining) { - BytesDownloaded += Chunk.GetSize(); + [Workload, &DownloadStats, OnDownloadComplete = std::move(OnDownloadComplete)](uint64_t Offset, + const IoBuffer& Chunk, + uint64_t BytesRemaining) { + DownloadStats.DownloadedChunkByteCount += Chunk.GetSize(); if (!AbortFlag.load()) { @@ -1364,6 +1426,7 @@ namespace { Workload->TempFile.Write(Chunk.GetView(), Offset); if (Chunk.GetSize() == BytesRemaining) { + DownloadStats.DownloadedChunkCount++; uint64_t PayloadSize = Workload->TempFile.FileSize(); void* FileHandle = Workload->TempFile.Detach(); ZEN_ASSERT(FileHandle != nullptr); @@ -1375,7 +1438,7 @@ namespace { }); if (!WorkItems.empty()) { - MultipartAttachmentCount++; + DownloadStats.MultipartAttachmentCount++; } for (auto& WorkItem : WorkItems) { @@ -1392,7 +1455,23 @@ namespace { } } - void ValidateBuildPart(BuildStorage& Storage, const Oid& BuildId, Oid BuildPartId, const std::string_view BuildPartName) + struct ValidateStatistics + { + uint64_t BuildBlobSize = 0; + uint64_t BuildPartSize = 0; + uint64_t ChunkAttachmentCount = 0; + uint64_t BlockAttachmentCount = 0; + std::atomic VerifiedAttachmentCount = 0; + std::atomic VerifiedByteCount = 0; + uint64_t ElapsedWallTimeUS = 0; + }; + + void ValidateBuildPart(BuildStorage& Storage, + const Oid& BuildId, + Oid BuildPartId, + const std::string_view BuildPartName, + ValidateStatistics& ValidateStats, + DownloadStatistics& DownloadStats) { Stopwatch Timer; auto _ = MakeGuard([&]() { @@ -1411,23 +1490,27 @@ namespace { throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", BuildId, BuildPartName)); } } + ValidateStats.BuildBlobSize = Build.GetSize(); uint64_t PreferredMultipartChunkSize = DefaultPreferredMultipartChunkSize; if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) { PreferredMultipartChunkSize = ChunkSize; } - CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); + CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); + ValidateStats.BuildPartSize = BuildPart.GetSize(); ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); std::vector ChunkAttachments; for (CbFieldView LooseFileView : BuildPart["chunkAttachments"sv].AsObjectView()["rawHashes"sv]) { ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment()); } + ValidateStats.ChunkAttachmentCount = ChunkAttachments.size(); std::vector BlockAttachments; for (CbFieldView BlocksView : BuildPart["blockAttachments"sv].AsObjectView()["rawHashes"sv]) { BlockAttachments.push_back(BlocksView.AsBinaryAttachment()); } + ValidateStats.BlockAttachmentCount = BlockAttachments.size(); std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); if (VerifyBlockDescriptions.size() != BlockAttachments.size()) @@ -1453,13 +1536,9 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); - std::atomic DownloadedAttachmentCount = 0; - std::atomic VerifiedAttachmentCount = 0; - std::atomic DownloadedByteCount = 0; - std::atomic VerifiedByteCount = 0; - FilteredRate FilteredDownloadedBytesPerSecond; - FilteredRate FilteredVerifiedBytesPerSecond; + uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredVerifiedBytesPerSecond; std::atomic MultipartAttachmentCount = 0; @@ -1480,8 +1559,7 @@ namespace { PreferredMultipartChunkSize, Work, NetworkPool, - DownloadedByteCount, - MultipartAttachmentCount, + DownloadStats, [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) { Payload.SetContentType(ZenContentType::kCompressedBinary); if (!AbortFlag) @@ -1493,6 +1571,12 @@ namespace { { ZEN_TRACE_CPU("ValidateBuildPart_Validate"); + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == + AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; @@ -1502,9 +1586,9 @@ namespace { ChunkHash, NiceBytes(CompressedSize), NiceBytes(DecompressedSize)); - VerifiedAttachmentCount++; - VerifiedByteCount += DecompressedSize; - if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + ValidateStats.VerifiedAttachmentCount++; + ValidateStats.VerifiedByteCount += DecompressedSize; + if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) { FilteredVerifiedBytesPerSecond.Stop(); } @@ -1529,9 +1613,9 @@ namespace { FilteredDownloadedBytesPerSecond.Start(); IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); - DownloadedAttachmentCount++; - DownloadedByteCount += Payload.GetSize(); - if (DownloadedAttachmentCount.load() == AttachmentsToVerifyCount) + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -1557,9 +1641,9 @@ namespace { BlockAttachment, NiceBytes(CompressedSize), NiceBytes(DecompressedSize)); - VerifiedAttachmentCount++; - VerifiedByteCount += DecompressedSize; - if (VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + ValidateStats.VerifiedAttachmentCount++; + ValidateStats.VerifiedByteCount += DecompressedSize; + if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) { FilteredVerifiedBytesPerSecond.Stop(); } @@ -1575,17 +1659,20 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); + const uint64_t DownloadedAttachmentCount = DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount; + const uint64_t DownloadedByteCount = DownloadStats.DownloadedChunkByteCount + DownloadStats.DownloadedBlockByteCount; + FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount); - FilteredVerifiedBytesPerSecond.Update(VerifiedByteCount); + FilteredVerifiedBytesPerSecond.Update(ValidateStats.VerifiedByteCount); std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)", - DownloadedAttachmentCount.load(), + DownloadedAttachmentCount, AttachmentsToVerifyCount, - NiceBytes(DownloadedByteCount.load()), + NiceBytes(DownloadedByteCount), NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - VerifiedAttachmentCount.load(), + ValidateStats.VerifiedAttachmentCount.load(), AttachmentsToVerifyCount, - NiceBytes(VerifiedByteCount.load()), + NiceBytes(ValidateStats.VerifiedByteCount.load()), NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent())); ProgressBar.UpdateState( @@ -1593,11 +1680,12 @@ namespace { .Details = Details, .TotalCount = gsl::narrow(AttachmentsToVerifyCount * 2), .RemainingCount = gsl::narrow(AttachmentsToVerifyCount * 2 - - (DownloadedAttachmentCount.load() + VerifiedAttachmentCount.load()))}, + (DownloadedAttachmentCount + ValidateStats.VerifiedAttachmentCount.load()))}, false); }); ProgressBar.Finish(); + ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); } void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, @@ -1698,7 +1786,9 @@ namespace { const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, uint32_t ChunkIndex, - const std::filesystem::path& TempFolderPath) + const std::filesystem::path& TempFolderPath, + std::atomic& ReadRawBytes, + LooseChunksStatistics& LooseChunksStats) { ZEN_TRACE_CPU("CompressChunk"); ZEN_ASSERT(!TempFolderPath.empty()); @@ -1731,7 +1821,12 @@ namespace { bool CouldCompress = CompressedBuffer::CompressToStream( CompositeBuffer(SharedBuffer(RawSource)), - [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { CompressedFile.Write(RangeBuffer, Offset); }); + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + ReadRawBytes += SourceSize; + CompressedFile.Write(RangeBuffer, Offset); + LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); + }); if (CouldCompress) { uint64_t CompressedSize = CompressedFile.FileSize(); @@ -1749,6 +1844,9 @@ namespace { ZEN_ASSERT(Compressed); ZEN_ASSERT(RawHash == ChunkHash); ZEN_ASSERT(RawSize == ChunkSize); + + LooseChunksStats.CompressedChunkCount++; + return Compressed.GetCompressed(); } CompressedFile.Close(); @@ -1988,7 +2086,6 @@ namespace { const std::uint64_t LargeAttachmentSize, DiskStatistics& DiskStats, UploadStatistics& UploadStats, - GenerateBlocksStatistics& GenerateBlocksStats, LooseChunksStatistics& LooseChunksStats) { ZEN_TRACE_CPU("UploadPartBlobs"); @@ -2238,8 +2335,6 @@ namespace { Payload = std::move(CompressedBlock).GetCompressed(); } - GenerateBlocksStats.GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; - GenerateBlocksStats.GeneratedBlockCount++; GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; GeneratedBlockCount++; if (GeneratedBlockCount == GenerateBlockIndexes.size()) @@ -2260,9 +2355,7 @@ namespace { } } - std::atomic CompressedLooseChunkCount = 0; - std::atomic CompressedLooseChunkByteCount = 0; - std::atomic RawLooseChunkByteCount = 0; + std::atomic RawLooseChunkByteCount = 0; // Start compression of any non-precompressed loose chunks and schedule upload for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) @@ -2276,19 +2369,20 @@ namespace { ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); FilteredCompressedBytesPerSecond.Start(); - CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName); + CompositeBuffer Payload = CompressChunk(Path, + Content, + Lookup, + ChunkIndex, + Path / ZenTempChunkFolderName, + RawLooseChunkByteCount, + LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), NiceBytes(Payload.GetSize())); const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; UploadStats.ReadFromDiskBytes += ChunkRawSize; - LooseChunksStats.CompressedChunkBytes += Payload.GetSize(); - LooseChunksStats.CompressedChunkCount++; - CompressedLooseChunkByteCount += Payload.GetSize(); - CompressedLooseChunkCount++; - RawLooseChunkByteCount += ChunkRawSize; - if (CompressedLooseChunkCount == CompressLooseChunkOrderIndexes.size()) + if (LooseChunksStats.CompressedChunkCount == CompressLooseChunkOrderIndexes.size()) { FilteredCompressedBytesPerSecond.Stop(); } @@ -2303,20 +2397,21 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - FilteredCompressedBytesPerSecond.Update(CompressedLooseChunkByteCount.load()); + FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load(); std::string Details = fmt::format( - "Compressed {}/{} ({}/{}) chunks. " + "Compressed {}/{} ({}/{} {}B/s) chunks. " "Uploaded {}/{} ({}/{}) blobs " "({} {}bits/s)", - CompressedLooseChunkCount.load(), + LooseChunksStats.CompressedChunkCount.load(), CompressLooseChunkOrderIndexes.size(), NiceBytes(RawLooseChunkByteCount), NiceBytes(TotalLooseChunksSize), + NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()), UploadedBlockCount.load() + UploadedChunkCount.load(), UploadBlockCount + UploadChunkCount, @@ -2336,9 +2431,8 @@ namespace { ZEN_ASSERT(AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0); ProgressBar.Finish(); - UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); - GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGenerateBlockBytesPerSecond.GetElapsedTimeUS(); - LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); + LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); } } @@ -2531,6 +2625,8 @@ namespace { CreateDirectories(Path / ZenTempBlockFolderName); CreateDirectories(Path / ZenTempChunkFolderName); + std::uint64_t TotalRawSize = 0; + CbObject ChunkerParameters; struct PrepareBuildResult @@ -2751,7 +2847,7 @@ namespace { ChunkerParameters = ChunkParametersWriter.Save(); } - std::uint64_t TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); + TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); { ProgressBar ProgressBar(UsePlainProgress); @@ -3120,21 +3216,16 @@ namespace { { ZEN_CONSOLE_VERBOSE("Uploading attachments: {}", FormatArray(RawHashes, "\n "sv)); - UploadStatistics TempUploadStats; - GenerateBlocksStatistics TempGenerateBlocksStats; - LooseChunksStatistics TempLooseChunksStats; + UploadStatistics TempUploadStats; + LooseChunksStatistics TempLooseChunksStats; Stopwatch TempUploadTimer; auto __ = MakeGuard([&]() { uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs(); ZEN_CONSOLE( - "Generated {} ({} {}B/s) and uploaded {} ({}) blocks. " + "Uploaded {} ({}) blocks. " "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. " "Transferred {} ({}bits/s) in {}", - TempGenerateBlocksStats.GeneratedBlockCount.load(), - NiceBytes(TempGenerateBlocksStats.GeneratedBlockByteCount.load()), - NiceNum(GetBytesPerSecond(TempGenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, - TempGenerateBlocksStats.GeneratedBlockByteCount)), TempUploadStats.BlockCount.load(), NiceBytes(TempUploadStats.BlocksBytes), @@ -3161,11 +3252,9 @@ namespace { LargeAttachmentSize, DiskStats, TempUploadStats, - TempGenerateBlocksStats, TempLooseChunksStats); UploadStats += TempUploadStats; LooseChunksStats += TempLooseChunksStats; - GenerateBlocksStats += TempGenerateBlocksStats; } }; if (IgnoreExistingBlocks) @@ -3247,18 +3336,25 @@ namespace { } } + ValidateStatistics ValidateStats; + DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { - ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName); + ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } - const double DeltaByteCountPercent = - ChunkingStats.BytesHashed > 0 - ? (100.0 * (FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes)) / (ChunkingStats.BytesHashed) - : 0.0; - - const std::string LargeAttachmentStats = - (LargeAttachmentSize != (uint64_t)-1) ? fmt::format(" ({} as multipart)", UploadStats.MultipartAttachmentCount.load()) : ""; + struct ValidateStatistics + { + uint64_t BuildBlobSize = 0; + uint64_t BuildPartSize = 0; + uint64_t ChunkAttachmentCount = 0; + uint64_t BlockAttachmentCount = 0; + std::atomic DownloadedAttachmentCount = 0; + std::atomic VerifiedAttachmentCount = 0; + std::atomic DownloadedByteCount = 0; + std::atomic VerifiedByteCount = 0; + uint64_t ElapsedWallTimeUS = 0; + }; ZEN_CONSOLE_VERBOSE( "Folder scanning stats:" @@ -3382,42 +3478,122 @@ namespace { UploadStats.MultipartAttachmentCount.load(), NiceLatencyNs(UploadStats.ElapsedWallTimeUS * 1000)); + if (PostUploadVerify) + { + ZEN_CONSOLE_VERBOSE( + "Validate stats:" + "\n BuildBlobSize: {}" + "\n BuildPartSize: {}" + "\n ChunkAttachmentCount: {}" + "\n BlockAttachmentCount: {}" + "\n VerifiedAttachmentCount: {}" + "\n VerifiedByteCount: {}" + "\n ElapsedWallTimeUS: {}", + NiceBytes(ValidateStats.BuildBlobSize), + NiceBytes(ValidateStats.BuildPartSize), + ValidateStats.ChunkAttachmentCount, + ValidateStats.BlockAttachmentCount, + ValidateStats.VerifiedAttachmentCount.load(), + NiceBytes(ValidateStats.VerifiedByteCount.load()), + NiceLatencyNs(ValidateStats.ElapsedWallTimeUS * 1000)); + + ZEN_CONSOLE_VERBOSE( + "Validate download stats:" + "\n RequestsCompleteCount: {}" + "\n DownloadedChunkCount: {}" + "\n DownloadedChunkByteCount: {}" + "\n MultipartAttachmentCount: {}" + "\n DownloadedBlockCount: {}" + "\n DownloadedBlockByteCount: {}" + "\n DownloadedPartialBlockCount: {}" + "\n DownloadedPartialBlockByteCount: {}", + ValidateDownloadStats.RequestsCompleteCount.load(), + ValidateDownloadStats.DownloadedChunkCount.load(), + NiceBytes(ValidateDownloadStats.DownloadedChunkByteCount.load()), + ValidateDownloadStats.MultipartAttachmentCount.load(), + ValidateDownloadStats.DownloadedBlockCount.load(), + NiceBytes(ValidateDownloadStats.DownloadedBlockByteCount.load()), + ValidateDownloadStats.DownloadedPartialBlockCount.load(), + NiceBytes(ValidateDownloadStats.DownloadedPartialBlockByteCount.load())); + } + + const double DeltaByteCountPercent = + ChunkingStats.BytesHashed > 0 + ? (100.0 * (FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes)) / (ChunkingStats.BytesHashed) + : 0.0; + + const std::string MultipartAttachmentStats = + (LargeAttachmentSize != (uint64_t)-1) ? fmt::format(" ({} as multipart)", UploadStats.MultipartAttachmentCount.load()) : ""; + + std::string ValidateInfo; + if (PostUploadVerify) + { + const uint64_t DownloadedCount = ValidateDownloadStats.DownloadedChunkCount + ValidateDownloadStats.DownloadedBlockCount; + const uint64_t DownloadedByteCount = + ValidateDownloadStats.DownloadedChunkByteCount + ValidateDownloadStats.DownloadedBlockByteCount; + ValidateInfo = fmt::format("\n Verified: {} ({}) {}B/sec in {}", + DownloadedCount, + NiceBytes(DownloadedByteCount), + NiceNum(GetBytesPerSecond(ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), + NiceTimeSpanMs(ValidateStats.ElapsedWallTimeUS / 1000)); + } + ZEN_CONSOLE( - "Uploaded {}\n" - " Delta: {}/{} ({:.1f}%)\n" - " Blocks: {} ({})\n" - " Chunks: {} ({}){}\n" - " Rate: {}bits/sec", - NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), + "Uploaded part {} ('{}') to build {} in {} \n" + " Scanned files: {} ({}) {}B/sec in {}\n" + " New data: {} ({:.1f}%)\n" + " New blocks: {} ({}) {}B/sec\n" + " New chunks: {} ({} -> {}) {}B/sec\n" + " Uploaded: {} ({}) {}bits/sec\n" + " Blocks: {} ({})\n" + " Chunks: {} ({}){}" + "{}", + BuildPartId, + BuildPartName, + BuildId, + NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs()), + + LocalFolderScanStats.FoundFileCount.load(), + NiceBytes(LocalFolderScanStats.FoundFileByteCount.load()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), + NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), - NiceBytes(ChunkingStats.BytesHashed), DeltaByteCountPercent, + GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), + NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), + + LooseChunksStats.CompressedChunkCount.load(), + NiceBytes(LooseChunksStats.ChunkByteCount), + NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), + NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), + + NiceBytes(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load()), + NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8))), + UploadStats.BlockCount.load(), - NiceBytes(UploadStats.BlocksBytes), - UploadStats.ChunkCount.load(), - NiceBytes(UploadStats.ChunksBytes), - LargeAttachmentStats, + NiceBytes(UploadStats.BlocksBytes.load()), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8)))); + UploadStats.ChunkCount.load(), + NiceBytes(UploadStats.ChunksBytes.load()), + MultipartAttachmentStats, - ZEN_CONSOLE("Uploaded ({}) build {} part {} ({}) in {}", - NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), - BuildId, - BuildPartName, - BuildPartId, - NiceTimeSpanMs(ProcessTimer.GetElapsedTimeMs())); + ValidateInfo); } - void VerifyFolder(const ChunkedFolderContent& Content, const std::filesystem::path& Path, bool VerifyFileHash) + void VerifyFolder(const ChunkedFolderContent& Content, + const std::filesystem::path& Path, + bool VerifyFileHash, + VerifyFolderStatistics& VerifyFolderStats) { ZEN_TRACE_CPU("VerifyFolder"); - ProgressBar ProgressBar(UsePlainProgress); - std::atomic FilesVerified(0); - std::atomic FilesFailed(0); - std::atomic ReadBytes(0); + Stopwatch Timer; + + ProgressBar ProgressBar(UsePlainProgress); WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // @@ -3473,7 +3649,7 @@ namespace { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); }); - FilesFailed++; + VerifyFolderStats.FilesFailed++; } else { @@ -3485,7 +3661,7 @@ namespace { Errors.push_back( fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value())); }); - FilesFailed++; + VerifyFolderStats.FilesFailed++; } else if (SizeOnDisk < ExpectedSize) { @@ -3495,7 +3671,7 @@ namespace { ExpectedSize, SizeOnDisk)); }); - FilesFailed++; + VerifyFolderStats.FilesFailed++; } else if (SizeOnDisk > ExpectedSize) { @@ -3505,7 +3681,7 @@ namespace { ExpectedSize, SizeOnDisk)); }); - FilesFailed++; + VerifyFolderStats.FilesFailed++; } else if (SizeOnDisk > 0 && VerifyFileHash) { @@ -3540,13 +3716,13 @@ namespace { } FileOffset += ChunkSize; } - FilesFailed++; + VerifyFolderStats.FilesFailed++; } - ReadBytes += SizeOnDisk; + VerifyFolderStats.ReadBytes += SizeOnDisk; } } } - FilesVerified++; + VerifyFolderStats.FilesVerified++; } }, [&, PathIndex](const std::exception& Ex, std::atomic&) { @@ -3555,23 +3731,25 @@ namespace { (Path / Content.Paths[PathIndex]).make_preferred(), Ex.what())); }); - FilesFailed++; + VerifyFolderStats.FilesFailed++; }); } Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", - FilesVerified.load(), + VerifyFolderStats.FilesVerified.load(), PathCount, - NiceBytes(ReadBytes.load()), - FilesFailed.load()); + NiceBytes(VerifyFolderStats.ReadBytes.load()), + VerifyFolderStats.FilesFailed.load()); ProgressBar.UpdateState({.Task = "Verifying files ", .Details = Details, .TotalCount = gsl::narrow(PathCount), - .RemainingCount = gsl::narrow(PathCount - FilesVerified.load())}, + .RemainingCount = gsl::narrow(PathCount - VerifyFolderStats.FilesVerified.load())}, false); }); + VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + ProgressBar.Finish(); for (const std::string& Error : Errors) { @@ -3586,7 +3764,7 @@ namespace { class WriteFileCache { public: - WriteFileCache() {} + WriteFileCache(DiskStatistics& DiskStats) : m_DiskStats(DiskStats) {} ~WriteFileCache() { Flush(); } template @@ -3602,6 +3780,8 @@ namespace { ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); ZEN_ASSERT(OpenFileWriter); OpenFileWriter->Write(Buffer, FileOffset); + m_DiskStats.WriteCount++; + m_DiskStats.WriteByteCount += Buffer.GetSize(); } else { @@ -3624,6 +3804,8 @@ namespace { } return --Tries > 0; }); + m_DiskStats.OpenWriteCount++; + m_DiskStats.CurrentOpenFileCount++; } const bool CacheWriter = TargetFinalSize > Buffer.GetSize(); @@ -3635,12 +3817,18 @@ namespace { OutputFile = std::move(NewOutputFile); OpenFileWriter = std::make_unique(*OutputFile, Min(TargetFinalSize, 256u * 1024u)); OpenFileWriter->Write(Buffer, FileOffset); + m_DiskStats.WriteCount++; + m_DiskStats.WriteByteCount += Buffer.GetSize(); SeenTargetIndexes.push_back(TargetIndex); } else { ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Write"); NewOutputFile->Write(Buffer, FileOffset); + m_DiskStats.WriteCount++; + m_DiskStats.WriteByteCount += Buffer.GetSize(); + NewOutputFile = {}; + m_DiskStats.CurrentOpenFileCount--; } } } @@ -3648,9 +3836,15 @@ namespace { void Flush() { ZEN_TRACE_CPU("WriteFileCache_Flush"); + if (OutputFile) + { + m_DiskStats.CurrentOpenFileCount--; + } + OpenFileWriter = {}; OutputFile = {}; } + DiskStatistics& m_DiskStats; std::vector SeenTargetIndexes; std::unique_ptr OutputFile; std::unique_ptr OpenFileWriter; @@ -3693,12 +3887,12 @@ namespace { const ChunkedContentLookup& Lookup, std::span> SequenceIndexChunksLeftToWriteCounters, const BlockWriteOps& Ops, - std::atomic& OutChunksComplete, - std::atomic& OutBytesWritten) + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("WriteBlockChunkOps"); { - WriteFileCache OpenFileCache; + WriteFileCache OpenFileCache(DiskStats); for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) { if (AbortFlag) @@ -3724,8 +3918,13 @@ namespace { Chunk, FileOffset, RemoteContent.RawSizes[PathIndex]); - OutBytesWritten += ChunkSize; } + WriteChunkStats.ChunkCountWritten += gsl::narrow(Ops.ChunkBuffers.size()); + WriteChunkStats.ChunkBytesWritten += + std::accumulate(Ops.ChunkBuffers.begin(), + Ops.ChunkBuffers.end(), + uint64_t(0), + [](uint64_t Current, const CompositeBuffer& Buffer) -> uint64_t { return Current + Buffer.GetSize(); }); } if (!AbortFlag) { @@ -3753,7 +3952,6 @@ namespace { GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } } - OutChunksComplete += gsl::narrow(Ops.ChunkBuffers.size()); } } @@ -3865,8 +4063,8 @@ namespace { CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - std::atomic& OutChunksComplete, - std::atomic& OutBytesWritten) + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("WriteBlockToDisk"); @@ -3899,8 +4097,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, - OutChunksComplete, - OutBytesWritten); + DiskStats, + WriteChunkStats); return true; } return false; @@ -3925,8 +4123,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, - OutChunksComplete, - OutBytesWritten); + DiskStats, + WriteChunkStats); return true; } return false; @@ -3941,8 +4139,8 @@ namespace { uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - std::atomic& OutChunksComplete, - std::atomic& OutBytesWritten) + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); BlockWriteOps Ops; @@ -3963,8 +4161,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, - OutChunksComplete, - OutBytesWritten); + DiskStats, + WriteChunkStats); return true; } else @@ -4012,8 +4210,7 @@ namespace { const ChunkedContentLookup& Lookup, std::span ChunkTargets, CompositeBuffer&& ChunkData, - WriteFileCache& OpenFileCache, - std::atomic& OutBytesWritten) + WriteFileCache& OpenFileCache) { ZEN_TRACE_CPU("WriteChunkToDisk"); @@ -4032,7 +4229,6 @@ namespace { ChunkData, FileOffset, Content.RawSizes[PathIndex]); - OutBytesWritten += ChunkData.GetSize(); } } @@ -4053,7 +4249,8 @@ namespace { void StreamDecompress(const std::filesystem::path& CacheFolderPath, const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart, - std::atomic& WriteToDiskBytes) + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("StreamDecompress"); const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); @@ -4076,21 +4273,29 @@ namespace { { throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); } + IoHashStream Hash; - bool CouldDecompress = Compressed.DecompressToStream(0, (uint64_t)-1, [&](uint64_t Offset, const CompositeBuffer& RangeBuffer) { - ZEN_TRACE_CPU("StreamDecompress_Write"); - if (!AbortFlag) - { - DecompressedTemp.Write(RangeBuffer, Offset); - for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + bool CouldDecompress = Compressed.DecompressToStream( + 0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + ZEN_TRACE_CPU("StreamDecompress_Write"); + DiskStats.ReadByteCount += SourceSize; + if (!AbortFlag) { - Hash.Append(Segment.GetView()); + WriteChunkStats.ChunkBytesWritten += RangeBuffer.GetSize(); + DecompressedTemp.Write(RangeBuffer, Offset); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + DiskStats.WriteByteCount += RangeBuffer.GetSize(); + DiskStats.WriteCount++; + return true; } - WriteToDiskBytes += RangeBuffer.GetSize(); - return true; - } - return false; - }); + return false; + }); if (AbortFlag) { @@ -4113,6 +4318,7 @@ namespace { throw std::runtime_error( fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); } + WriteChunkStats.ChunkCountWritten++; } bool WriteCompressedChunk(const std::filesystem::path& TargetFolder, @@ -4121,32 +4327,34 @@ namespace { const IoHash& ChunkHash, const std::vector& ChunkTargetPtrs, IoBuffer&& CompressedPart, - std::atomic& WriteToDiskBytes) + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; + const uint64_t ChunkRawSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; if (CanDecompressDirectToSequence(RemoteContent, ChunkTargetPtrs)) { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[ChunkTargetPtrs.front()->SequenceIndex]; - StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), WriteToDiskBytes); + const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; + StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats, WriteChunkStats); } else { - const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; - SharedBuffer Chunk = - Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, ChunkRawSize); if (!AbortFlag) { - WriteFileCache OpenFileCache; - + WriteFileCache OpenFileCache(DiskStats); WriteChunkToDisk(TargetFolder, RemoteContent, RemoteLookup, ChunkTargetPtrs, CompositeBuffer(std::move(Chunk)), - OpenFileCache, - WriteToDiskBytes); + OpenFileCache); + WriteChunkStats.ChunkCountWritten++; + WriteChunkStats.ChunkBytesWritten += ChunkRawSize; return true; } } @@ -4198,19 +4406,17 @@ namespace { WorkerThreadPool& WritePool, IoBuffer&& Payload, std::span> SequenceIndexChunksLeftToWriteCounters, - std::atomic& WriteToDiskBytes, - std::atomic& ChunkCountWritten, std::atomic& WritePartsComplete, - std::atomic& TotalPartWriteCount, - std::atomic& LooseChunksBytes, - FilteredRate& FilteredWrittenBytesPerSecond) + const uint64_t TotalPartWriteCount, + FilteredRate& FilteredWrittenBytesPerSecond, + DiskStatistics& DiskStats, + WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - uint64_t Size = Payload.GetSize(); - LooseChunksBytes += Size; + const uint64_t Size = Payload.GetSize(); std::filesystem::path CompressedChunkPath; @@ -4256,6 +4462,7 @@ namespace { SequenceIndexChunksLeftToWriteCounters, CompressedChunkPath, RemoteChunkIndex, + TotalPartWriteCount, ChunkTargetPtrs = std::move(ChunkTargetPtrs), CompressedPart = std::move(Payload)](std::atomic&) mutable { ZEN_TRACE_CPU("UpdateFolder_WriteChunk"); @@ -4288,11 +4495,11 @@ namespace { ChunkHash, ChunkTargetPtrs, std::move(CompressedPart), - WriteToDiskBytes); + DiskStats, + WriteChunkStats); if (!AbortFlag) { - ChunkCountWritten++; WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) { @@ -4324,19 +4531,16 @@ namespace { const std::vector& LooseChunkHashes, bool AllowPartialBlockRequests, bool WipeTargetFolder, - FolderContent& OutLocalFolderState) + FolderContent& OutLocalFolderState, + DiskStatistics& DiskStats, + CacheMappingStatistics& CacheMappingStats, + DownloadStatistics& DownloadStats, + WriteChunkStatistics& WriteChunkStats, + RebuildFolderStateStatistics& RebuildFolderStateStats) { ZEN_TRACE_CPU("UpdateFolder"); ZEN_UNUSED(WipeTargetFolder); - std::atomic DownloadedBlocks = 0; - std::atomic BlockBytes = 0; - std::atomic DownloadedChunks = 0; - std::atomic LooseChunksBytes = 0; - std::atomic WriteToDiskBytes = 0; - std::atomic MultipartAttachmentCount = 0; - - DiskStatistics DiskStats; Stopwatch IndexTimer; @@ -4351,15 +4555,11 @@ namespace { Stopwatch CacheMappingTimer; std::vector> SequenceIndexChunksLeftToWriteCounters(RemoteContent.ChunkedContent.SequenceRawHashes.size()); - // std::vector RemoteSequenceIndexIsCachedFlags(RemoteContent.ChunkedContent.SequenceRawHashes.size(), false); - std::vector RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); - // Guard if he same chunks is in multiple blocks (can happen due to block reuse, cache reuse blocks writes directly) - std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + std::vector RemoteChunkIndexNeedsCopyFromLocalFileFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); + std::vector> RemoteChunkIndexNeedsCopyFromSourceFlags(RemoteContent.ChunkedContent.ChunkHashes.size()); tsl::robin_map CachedChunkHashesFound; tsl::robin_map CachedSequenceHashesFound; - uint64_t CachedChunkHashesByteCountFound = 0; - uint64_t CachedSequenceHashesByteCountFound = 0; { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); @@ -4380,7 +4580,8 @@ namespace { if (ChunkSize == CacheDirContent.FileSizes[Index]) { CachedChunkHashesFound.insert({FileHash, ChunkIndex}); - CachedChunkHashesByteCountFound += ChunkSize; + CacheMappingStats.CacheChunkCount++; + CacheMappingStats.CacheChunkByteCount += ChunkSize; continue; } } @@ -4393,7 +4594,8 @@ namespace { if (SequenceSize == CacheDirContent.FileSizes[Index]) { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CachedSequenceHashesByteCountFound += SequenceSize; + CacheMappingStats.CacheSequenceHashesCount += SequenceSize; + CacheMappingStats.CacheSequenceHashesByteCount++; continue; } } @@ -4403,7 +4605,6 @@ namespace { } tsl::robin_map CachedBlocksFound; - uint64_t CachedBlocksByteCountFound = 0; { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); @@ -4438,7 +4639,8 @@ namespace { if (BlockSize == BlockDirContent.FileSizes[Index]) { CachedBlocksFound.insert({FileHash, BlockIndex}); - CachedBlocksByteCountFound += BlockSize; + CacheMappingStats.CacheBlockCount++; + CacheMappingStats.CacheBlocksByteCount += BlockSize; continue; } } @@ -4448,7 +4650,6 @@ namespace { } std::vector LocalPathIndexesMatchingSequenceIndexes; - uint64_t LocalPathIndexesByteCountMatchingSequenceIndexes = 0; // Pick up all whole files we can use from current local state { ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); @@ -4477,7 +4678,8 @@ namespace { const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); - LocalPathIndexesByteCountMatchingSequenceIndexes += RawSize; + CacheMappingStats.LocalPathsMatchingSequencesCount++; + CacheMappingStats.LocalPathsMatchingSequencesByteCount += RawSize; } else { @@ -4495,7 +4697,7 @@ namespace { struct ChunkTarget { uint32_t TargetChunkLocationCount = (uint32_t)-1; - uint64_t ChunkRawSize = (uint64_t)-1; + uint32_t RemoteChunkIndex = (uint32_t)-1; uint64_t CacheFileOffset = (uint64_t)-1; }; std::vector ChunkTargets; @@ -4503,8 +4705,6 @@ namespace { tsl::robin_map RawHashToCacheCopyDataIndex; std::vector CacheCopyDatas; - uint64_t LocalChunkHashesMatchingRemoteCount = 0; - uint64_t LocalChunkHashesMatchingRemoteByteCount = 0; { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); @@ -4537,7 +4737,7 @@ namespace { { CacheCopyData::ChunkTarget Target = { .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), - .ChunkRawSize = LocalChunkRawSize, + .RemoteChunkIndex = RemoteChunkIndex, .CacheFileOffset = SourceOffset}; if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(LocalSequenceRawHash); CopySourceIt != RawHashToCacheCopyDataIndex.end()) @@ -4567,8 +4767,8 @@ namespace { .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector{Target}}); } - LocalChunkHashesMatchingRemoteByteCount += LocalChunkRawSize; - LocalChunkHashesMatchingRemoteCount++; + CacheMappingStats.LocalChunkMatchingRemoteCount++; + CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize; RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; } } @@ -4580,20 +4780,20 @@ namespace { } if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() || - !LocalPathIndexesMatchingSequenceIndexes.empty() || LocalChunkHashesMatchingRemoteCount > 0) + !LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) { ZEN_CONSOLE( "Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks", CachedSequenceHashesFound.size(), - NiceBytes(CachedSequenceHashesByteCountFound), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), CachedChunkHashesFound.size(), - NiceBytes(CachedChunkHashesByteCountFound), + NiceBytes(CacheMappingStats.CacheChunkByteCount), CachedBlocksFound.size(), - NiceBytes(CachedBlocksByteCountFound), + NiceBytes(CacheMappingStats.CacheBlocksByteCount), LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(LocalPathIndexesByteCountMatchingSequenceIndexes), - LocalChunkHashesMatchingRemoteCount, - NiceBytes(LocalChunkHashesMatchingRemoteByteCount)); + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); } uint32_t ChunkCountToWrite = 0; @@ -4616,9 +4816,7 @@ namespace { } uint64_t TotalRequestCount = 0; - std::atomic RequestsComplete = 0; - std::atomic ChunkCountWritten = 0; - std::atomic TotalPartWriteCount = 0; + uint64_t TotalPartWriteCount = 0; std::atomic WritePartsComplete = 0; { @@ -4635,8 +4833,6 @@ namespace { ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); - std::atomic BytesDownloaded = 0; - struct LooseChunkHashWorkData { std::vector ChunkTargetPtrs; @@ -4718,12 +4914,6 @@ namespace { std::vector FullBlockWorks; - size_t BlocksNeededCount = 0; - uint64_t AllBlocksSize = 0; - uint64_t AllBlocksFetch = 0; - uint64_t AllBlocksSlack = 0; - uint64_t AllBlockRequests = 0; - uint64_t AllBlockChunksSize = 0; for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; @@ -4779,7 +4969,6 @@ namespace { } else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - AllBlockChunksSize += ChunkCompressedLength; if (NextRange.RangeLength == 0) { NextRange.RangeStart = CurrentOffset; @@ -4796,7 +4985,6 @@ namespace { ZEN_ASSERT(false); } } - AllBlocksSize += CurrentOffset; if (NextRange.RangeLength > 0) { BlockRanges.push_back(NextRange); @@ -4806,7 +4994,6 @@ namespace { std::vector CollapsedBlockRanges; auto It = BlockRanges.begin(); CollapsedBlockRanges.push_back(*It++); - uint64_t TotalSlack = 0; while (It != BlockRanges.end()) { BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); @@ -4817,7 +5004,6 @@ namespace { LastRange.ChunkBlockIndexCount = (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; - TotalSlack += Slack; } else { @@ -4826,17 +5012,6 @@ namespace { ++It; } - uint64_t TotalFetch = 0; - for (const BlockRangeDescriptor& Range : CollapsedBlockRanges) - { - TotalFetch += Range.RangeLength; - } - - AllBlocksFetch += TotalFetch; - AllBlocksSlack += TotalSlack; - BlocksNeededCount++; - AllBlockRequests += CollapsedBlockRanges.size(); - TotalRequestCount += CollapsedBlockRanges.size(); TotalPartWriteCount += CollapsedBlockRanges.size(); @@ -4844,7 +5019,6 @@ namespace { } else { - BlocksNeededCount++; TotalRequestCount++; TotalPartWriteCount++; @@ -4888,7 +5062,7 @@ namespace { { const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; uint64_t CacheFileOffset = (uint64_t)-1; - uint64_t ChunkSize = (uint64_t)-1; + uint32_t ChunkIndex = (uint32_t)-1; }; std::vector WriteOps; @@ -4905,7 +5079,7 @@ namespace { { WriteOps.push_back(WriteOp{.Target = Target, .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkSize = ChunkTarget.ChunkRawSize}); + .ChunkIndex = ChunkTarget.RemoteChunkIndex}); } TargetStart += ChunkTarget.TargetChunkLocationCount; } @@ -4931,8 +5105,10 @@ namespace { { ZEN_TRACE_CPU("Write"); - BufferedOpenFile SourceFile(LocalFilePath); - WriteFileCache OpenFileCache; + tsl::robin_set ChunkIndexesWritten; + + BufferedOpenFile SourceFile(LocalFilePath, DiskStats); + WriteFileCache OpenFileCache(DiskStats); for (const WriteOp& Op : WriteOps) { if (AbortFlag) @@ -4944,7 +5120,7 @@ namespace { RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = Op.ChunkSize; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); @@ -4959,7 +5135,13 @@ namespace { ChunkSource, Op.Target->Offset, RemoteContent.RawSizes[RemotePathIndex]); - WriteToDiskBytes += ChunkSize; + + if (ChunkIndexesWritten.insert(Op.ChunkIndex).second) + { + WriteChunkStats.ChunkCountWritten++; + WriteChunkStats.ChunkBytesWritten += ChunkSize; + } + CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? } } @@ -4992,8 +5174,6 @@ namespace { GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); } } - - ChunkCountWritten += gsl::narrow(CopyData.ChunkTargets.size()); ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); } WritePartsComplete++; @@ -5038,9 +5218,8 @@ namespace { uint64_t RawSize; if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) { - LooseChunksBytes += ExistingCompressedPart.GetSize(); - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -5063,8 +5242,8 @@ namespace { &RemoteLookup, &CacheFolderPath, &SequenceIndexChunksLeftToWriteCounters, - &WriteToDiskBytes, - &ChunkCountWritten, + &DiskStats, + &WriteChunkStats, &WritePartsComplete, &TotalPartWriteCount, &FilteredWrittenBytesPerSecond, @@ -5094,12 +5273,15 @@ namespace { ChunkHash, ChunkTargetPtrs, std::move(CompressedPart), - WriteToDiskBytes); + DiskStats, + WriteChunkStats); + WriteChunkStats.ChunkCountWritten++; + WriteChunkStats.ChunkBytesWritten += + RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]; + WritePartsComplete++; if (!AbortFlag) { - ChunkCountWritten++; - WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -5132,11 +5314,10 @@ namespace { PreferredMultipartChunkSize, Work, NetworkPool, - BytesDownloaded, - MultipartAttachmentCount, + DownloadStats, [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -5149,12 +5330,11 @@ namespace { WritePool, std::move(Payload), SequenceIndexChunksLeftToWriteCounters, - WriteToDiskBytes, - ChunkCountWritten, WritePartsComplete, TotalPartWriteCount, - LooseChunksBytes, - FilteredWrittenBytesPerSecond); + FilteredWrittenBytesPerSecond, + DiskStats, + WriteChunkStats); }); } else @@ -5167,10 +5347,10 @@ namespace { throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); } uint64_t BlobSize = BuildBlob.GetSize(); - BytesDownloaded += BlobSize; - - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -5183,12 +5363,11 @@ namespace { WritePool, std::move(BuildBlob), SequenceIndexChunksLeftToWriteCounters, - WriteToDiskBytes, - ChunkCountWritten, WritePartsComplete, TotalPartWriteCount, - LooseChunksBytes, - FilteredWrittenBytesPerSecond); + FilteredWrittenBytesPerSecond, + DiskStats, + WriteChunkStats); } } } @@ -5228,8 +5407,8 @@ namespace { CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, - ChunkCountWritten, - WriteToDiskBytes)) + DiskStats, + WriteChunkStats)) { std::error_code DummyEc; std::filesystem::remove(BlockChunkPath, DummyEc); @@ -5272,11 +5451,10 @@ namespace { throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } uint64_t BlockSize = BlockBuffer.GetSize(); - BytesDownloaded += BlockSize; - BlockBytes += BlockSize; - DownloadedBlocks++; - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -5367,21 +5545,21 @@ namespace { BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, - ChunkCountWritten, - WriteToDiskBytes)) + DiskStats, + WriteChunkStats)) { std::error_code DummyEc; std::filesystem::remove(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } - WritePartsComplete++; if (!BlockChunkPath.empty()) { std::filesystem::remove(BlockChunkPath); } + WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -5417,11 +5595,10 @@ namespace { throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } uint64_t BlockSize = BlockBuffer.GetSize(); - BytesDownloaded += BlockSize; - BlockBytes += BlockSize; - DownloadedBlocks++; - RequestsComplete++; - if (RequestsComplete == TotalRequestCount) + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { FilteredDownloadedBytesPerSecond.Stop(); } @@ -5476,8 +5653,8 @@ namespace { &SequenceIndexChunksLeftToWriteCounters, BlockIndex, &BlockDescriptions, - &ChunkCountWritten, - &WriteToDiskBytes, + &WriteChunkStats, + &DiskStats, &WritePartsComplete, &TotalPartWriteCount, &FilteredWrittenBytesPerSecond, @@ -5513,20 +5690,21 @@ namespace { CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, - ChunkCountWritten, - WriteToDiskBytes)) + DiskStats, + WriteChunkStats)) { std::error_code DummyEc; std::filesystem::remove(BlockChunkPath, DummyEc); throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } - WritePartsComplete++; if (!BlockChunkPath.empty()) { std::filesystem::remove(BlockChunkPath); } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -5540,35 +5718,32 @@ namespace { Work.DefaultErrorFunction()); } - ZEN_DEBUG("Fetching {} with {} slack (ideal {}) out of {} using {} requests for {} blocks", - NiceBytes(AllBlocksFetch), - NiceBytes(AllBlocksSlack), - NiceBytes(AllBlockChunksSize), - NiceBytes(AllBlocksSize), - AllBlockRequests, - BlocksNeededCount); { ZEN_TRACE_CPU("WriteChunks_Wait"); Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= ChunkCountWritten.load()); - FilteredWrittenBytesPerSecond.Update(WriteToDiskBytes.load()); - FilteredDownloadedBytesPerSecond.Update(BytesDownloaded.load()); + ZEN_ASSERT(ChunkCountToWrite >= WriteChunkStats.ChunkCountWritten.load()); + uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + + DownloadStats.DownloadedBlockByteCount.load() + + +DownloadStats.DownloadedPartialBlockByteCount.load(); + FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load()); + FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", - RequestsComplete.load(), + DownloadStats.RequestsCompleteCount.load(), TotalRequestCount, - NiceBytes(BytesDownloaded.load()), + NiceBytes(DownloadedBytes), NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - ChunkCountWritten.load(), + WriteChunkStats.ChunkCountWritten.load(), ChunkCountToWrite, - NiceBytes(WriteToDiskBytes.load()), + NiceBytes(DiskStats.WriteByteCount.load()), NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState({.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - ChunkCountWritten.load())}, - false); + WriteProgressBar.UpdateState( + {.Task = "Writing chunks ", + .Details = Details, + .TotalCount = gsl::narrow(ChunkCountToWrite), + .RemainingCount = gsl::narrow(ChunkCountToWrite - WriteChunkStats.ChunkCountWritten.load())}, + false); }); } @@ -5598,25 +5773,21 @@ namespace { } ZEN_ASSERT(RawSequencesMissingWriteCount == 0); + const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + + +DownloadStats.DownloadedPartialBlockByteCount.load(); ZEN_CONSOLE("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s) in {}. Completed in {}", - NiceBytes(BytesDownloaded.load()), - NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), BytesDownloaded * 8)), + NiceBytes(DownloadedBytes), + NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)), NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000), - NiceBytes(WriteToDiskBytes.load()), - NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), WriteToDiskBytes.load())), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), DiskStats.WriteByteCount.load())), NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000), NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs())); - } - std::vector> Targets; - Targets.reserve(RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) - { - Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs(); + WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(); + WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } - std::sort(Targets.begin(), Targets.end(), [](const std::pair& Lhs, const std::pair& Rhs) { - return Lhs.first < Rhs.first; - }); // Move all files we will reuse to cache folder // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place @@ -5643,6 +5814,7 @@ namespace { if (WipeTargetFolder) { ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); + Stopwatch Timer; // Clean target folder ZEN_CONSOLE("Wiping {}", Path); @@ -5650,10 +5822,12 @@ namespace { { ZEN_WARN("Some files in {} could not be removed", Path); } + RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } else { ZEN_TRACE_CPU("UpdateFolder_RemoveUnused"); + Stopwatch Timer; // Remove unused tracked files tsl::robin_map RemotePathToRemoteIndex; @@ -5683,10 +5857,12 @@ namespace { std::filesystem::remove(LocalFilePath); } } + RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); + Stopwatch Timer; WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // @@ -5700,6 +5876,16 @@ namespace { std::atomic TargetsComplete = 0; + std::vector> Targets; + Targets.reserve(RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) + { + Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + } + std::sort(Targets.begin(), Targets.end(), [](const std::pair& Lhs, const std::pair& Rhs) { + return Lhs.first < Rhs.first; + }); + size_t TargetOffset = 0; while (TargetOffset < Targets.size()) { @@ -5753,6 +5939,7 @@ namespace { SetFileReadOnly(FirstTargetFilePath, false); } std::filesystem::rename(CacheFilePath, FirstTargetFilePath); + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } OutLocalFolderState.Attributes[FirstTargetPathIndex] = @@ -5781,6 +5968,7 @@ namespace { SetFileReadOnly(ExtraTargetFilePath, false); } CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; OutLocalFolderState.Attributes[ExtraTargetPathIndex] = RemoteContent.Attributes.empty() @@ -5815,6 +6003,8 @@ namespace { }); } + RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + if (AbortFlag) { return; @@ -6474,13 +6664,21 @@ namespace { } else { - ExtendableStringBuilder<128> SB; + ExtendableStringBuilder<128> BuildPartString; for (const std::pair& BuildPart : AllBuildParts) { - SB.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); + BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, SB.ToView()); + ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, BuildPartString.ToView()); FolderContent LocalFolderState; + + DiskStatistics DiskStats; + CacheMappingStatistics CacheMappingStats; + DownloadStatistics DownloadStats; + WriteChunkStatistics WriteChunkStats; + RebuildFolderStateStatistics RebuildFolderStateStats; + VerifyFolderStatistics VerifyFolderStats; + UpdateFolder(Storage, BuildId, Path, @@ -6492,11 +6690,16 @@ namespace { LooseChunkHashes, AllowPartialBlockRequests, WipeTargetFolder, - LocalFolderState); + LocalFolderState, + DiskStats, + CacheMappingStats, + DownloadStats, + WriteChunkStats, + RebuildFolderStateStats); if (!AbortFlag) { - VerifyFolder(RemoteContent, Path, PostDownloadVerify); + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); @@ -6510,8 +6713,37 @@ namespace { CompactBinaryToJson(StateObject, SB); WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 + const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + + DownloadStats.DownloadedPartialBlockCount.load(); + const uint64_t DownloadByteCount = DownloadStats.DownloadedChunkByteCount.load() + + DownloadStats.DownloadedBlockByteCount.load() + + DownloadStats.DownloadedPartialBlockByteCount.load(); + const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs(); + + ZEN_CONSOLE( + "Downloaded build {}, parts:{} in {}\n" + " Download: {} ({}) {}bits/s\n" + " Write: {} ({}) {}B/s\n" + " Clean: {}\n" + " Finalize: {}\n" + " Verify: {}", + BuildId, + BuildPartString.ToView(), + NiceTimeSpanMs(DownloadTimeMs), + + DownloadCount, + NiceBytes(DownloadByteCount), + NiceNum(GetBytesPerSecond(WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)), - ZEN_CONSOLE("Downloaded build in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + DiskStats.WriteCount.load(), + NiceBytes(DiskStats.WriteByteCount.load()), + NiceNum(GetBytesPerSecond(WriteChunkStats.WriteTimeUs, DiskStats.WriteByteCount.load())), + + NiceTimeSpanMs(RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000), + + NiceTimeSpanMs(RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000), + + NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); } } if (CleanDirectory(ZenTempFolder, {})) @@ -7504,6 +7736,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } + Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) { Oid BuildId = Oid::FromHexString(BuildIdString); @@ -7527,6 +7760,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\n"); } + ZEN_CONSOLE("Completed in {}", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); return 0; } @@ -7699,27 +7933,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) case 0: { uint64_t SourceSize = *FileSizeIt; - if (SourceSize > 0) + if (SourceSize > 256) { Work.ScheduleWork( GetMediumWorkerPool(EWorkloadType::Burst), [SourceSize, FilePath](std::atomic&) { if (!AbortFlag) { - IoBuffer Scrambled(SourceSize); + bool IsReadOnly = SetFileReadOnly(FilePath, false); { - IoBuffer Source = IoBufferBuilder::MakeFromFile(FilePath); - Scrambled.GetMutableView().CopyFrom( - Source.GetView().Mid(SourceSize / 3, SourceSize / 3)); - Scrambled.GetMutableView() - .Mid(SourceSize / 3) - .CopyFrom(Source.GetView().Mid(0, SourceSize / 3)); - Scrambled.GetMutableView() - .Mid((SourceSize / 3) * 2) - .CopyFrom(Source.GetView().Mid(SourceSize / 2, SourceSize / 3)); + BasicFile Source(FilePath, BasicFile::Mode::kWrite); + uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); + IoBuffer TempBuffer1(RangeSize); + IoBuffer TempBuffer2(RangeSize); + IoBuffer TempBuffer3(RangeSize); + Source.Read(TempBuffer1.GetMutableView().GetData(), RangeSize, 0); + Source.Read(TempBuffer2.GetMutableView().GetData(), RangeSize, SourceSize / 2); + Source.Read(TempBuffer3.GetMutableView().GetData(), + RangeSize, + SourceSize - RangeSize); + Source.Write(TempBuffer1, SourceSize / 2); + Source.Write(TempBuffer2, SourceSize - RangeSize); + Source.Write(TempBuffer3, SourceSize - 0); } - bool IsReadOnly = SetFileReadOnly(FilePath, false); - WriteFile(FilePath, Scrambled); if (IsReadOnly) { SetFileReadOnly(FilePath, true); @@ -7957,7 +8193,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); - ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName); + ValidateStatistics ValidateStats; + DownloadStatistics DownloadStats; + ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); return AbortFlag ? 13 : 0; } diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 88c3bb5b9..ad6b6103c 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -158,9 +158,10 @@ class BaseEncoder { public: [[nodiscard]] virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize = DefaultBlockSize) const = 0; - [[nodiscard]] virtual bool CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - uint64_t BlockSize = DefaultBlockSize) const = 0; + [[nodiscard]] virtual bool CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize = DefaultBlockSize) const = 0; }; class BaseDecoder @@ -189,11 +190,13 @@ public: uint64_t RawOffset, uint64_t RawSize) const = 0; - virtual bool DecompressToStream(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const = 0; + virtual bool DecompressToStream( + const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) + const = 0; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -207,13 +210,14 @@ public: return CompositeBuffer(HeaderData.MoveToShared(), RawData.MakeOwned()); } - [[nodiscard]] virtual bool CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - uint64_t /* BlockSize */) const final + [[nodiscard]] virtual bool CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t /* BlockSize */) const final { UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); - Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); - Callback(HeaderData.GetSize(), RawData); + Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); + Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData); return true; } }; @@ -283,17 +287,19 @@ public: [[nodiscard]] uint64_t GetHeaderSize(const BufferHeader&) const final { return sizeof(BufferHeader); } - virtual bool DecompressToStream(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const final + virtual bool DecompressToStream( + const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) + const final { if (Header.Method == CompressionMethod::None && Header.TotalCompressedSize == CompressedData.GetSize() && Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize && (RawOffset + RawSize) <= Header.TotalRawSize) { - if (!Callback(0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize))) + if (!Callback(sizeof(BufferHeader) + RawOffset, RawSize, 0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize))) { return false; } @@ -309,9 +315,10 @@ class BlockEncoder : public BaseEncoder { public: virtual CompositeBuffer Compress(const CompositeBuffer& RawData, uint64_t BlockSize) const final; - virtual bool CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - uint64_t BlockSize) const final; + virtual bool CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize) const final; protected: virtual CompressionMethod GetMethod() const = 0; @@ -460,9 +467,10 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize) } bool -BlockEncoder::CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - uint64_t BlockSize = DefaultBlockSize) const +BlockEncoder::CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + uint64_t BlockSize = DefaultBlockSize) const { ZEN_ASSERT(IsPow2(BlockSize) && (BlockSize <= (1u << 31))); @@ -504,13 +512,17 @@ BlockEncoder::CompressToStream(const CompositeBuffer& RawData, uint64_t CompressedBlockSize = CompressedBlock.GetSize(); if (RawBlockSize <= CompressedBlockSize) { - Callback(FullHeaderSize + CompressedSize, + Callback(FileRef.FileChunkOffset + RawOffset, + RawBlockSize, + FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlockCopy.GetView().GetData(), RawBlockSize))); CompressedBlockSize = RawBlockSize; } else { - Callback(FullHeaderSize + CompressedSize, + Callback(FileRef.FileChunkOffset + RawOffset, + RawBlockSize, + FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize))); } @@ -540,12 +552,17 @@ BlockEncoder::CompressToStream(const CompositeBuffer& RawData, uint64_t CompressedBlockSize = CompressedBlock.GetSize(); if (RawBlockSize <= CompressedBlockSize) { - Callback(FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize))); + Callback(RawOffset, + RawBlockSize, + FullHeaderSize + CompressedSize, + CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawBlock.GetData(), RawBlockSize))); CompressedBlockSize = RawBlockSize; } else { - Callback(FullHeaderSize + CompressedSize, + Callback(RawOffset, + RawBlockSize, + FullHeaderSize + CompressedSize, CompositeBuffer(IoBuffer(IoBuffer::Wrap, CompressedBlock.GetData(), CompressedBlockSize))); } @@ -582,7 +599,7 @@ BlockEncoder::CompressToStream(const CompositeBuffer& RawData, HeaderBuffer.GetMutableView().Mid(sizeof(BufferHeader), MetaSize).CopyFrom(MakeMemoryView(CompressedBlockSizes)); Header.Write(HeaderBuffer.GetMutableView()); - Callback(0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize()))); + Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderBuffer.GetData(), HeaderBuffer.GetSize()))); return true; } @@ -615,11 +632,13 @@ public: MutableMemoryView RawView, uint64_t RawOffset) const final; - virtual bool DecompressToStream(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const final; + virtual bool DecompressToStream( + const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) + const final; protected: virtual bool DecompressBlock(MutableMemoryView RawData, MemoryView CompressedData) const = 0; @@ -743,11 +762,12 @@ BlockDecoder::DecompressToComposite(const BufferHeader& Header, const CompositeB } bool -BlockDecoder::DecompressToStream(const BufferHeader& Header, - const CompositeBuffer& CompressedData, - uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const +BlockDecoder::DecompressToStream( + const BufferHeader& Header, + const CompositeBuffer& CompressedData, + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const { if (Header.TotalCompressedSize != CompressedData.GetSize()) { @@ -817,7 +837,9 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, Source.Detach(); return false; } - if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + if (!Callback(FileRef.FileChunkOffset + CompressedOffset, + CompressedBlockSize, + BlockIndex * BlockSize + OffsetInFirstBlock, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)))) { Source.Detach(); @@ -827,6 +849,8 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, else { if (!Callback( + FileRef.FileChunkOffset + CompressedOffset, + BytesToUncompress, BlockIndex * BlockSize + OffsetInFirstBlock, CompositeBuffer( IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)))) @@ -870,7 +894,9 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, { return false; } - if (!Callback(BlockIndex * BlockSize + OffsetInFirstBlock, + if (!Callback(CompressedOffset, + UncompressedBlockSize, + BlockIndex * BlockSize + OffsetInFirstBlock, CompositeBuffer(IoBuffer(IoBuffer::Wrap, RawDataBuffer.GetData(), BytesToUncompress)))) { return false; @@ -879,6 +905,8 @@ BlockDecoder::DecompressToStream(const BufferHeader& Header, else { if (!Callback( + CompressedOffset, + BytesToUncompress, BlockIndex * BlockSize + OffsetInFirstBlock, CompositeBuffer( IoBuffer(IoBuffer::Wrap, CompressedBlockCopy.GetView().Mid(OffsetInFirstBlock).GetData(), BytesToUncompress)))) @@ -1778,11 +1806,12 @@ CompressedBuffer::Compress(const SharedBuffer& RawData, } bool -CompressedBuffer::CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - OodleCompressor Compressor, - OodleCompressionLevel CompressionLevel, - uint64_t BlockSize) +CompressedBuffer::CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor, + OodleCompressionLevel CompressionLevel, + uint64_t BlockSize) { using namespace detail; @@ -1995,9 +2024,10 @@ CompressedBuffer::DecompressToComposite() const } bool -CompressedBuffer::DecompressToStream(uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const +CompressedBuffer::DecompressToStream( + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const { using namespace detail; if (CompressedData) diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 74fd5f767..09fa6249d 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -74,11 +74,12 @@ public: OodleCompressor Compressor = OodleCompressor::Mermaid, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, uint64_t BlockSize = 0); - [[nodiscard]] ZENCORE_API static bool CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); + [[nodiscard]] ZENCORE_API static bool CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); /** * Construct from a compressed buffer previously created by Compress(). @@ -207,9 +208,10 @@ public: * * @return True if the buffer is valid and can be decompressed. */ - [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const; + [[nodiscard]] ZENCORE_API bool DecompressToStream( + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const; /** A null compressed buffer. */ static const CompressedBuffer Null; -- cgit v1.2.3 From a8cef9a7e561b6a1781d080e4d851a8dbc5e92a9 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 09:33:00 +0100 Subject: Reduced disk I/O when writing out chunk blocks during download (#309) --- src/zen/cmds/builds_cmd.cpp | 151 ++++++++++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 70 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 61e3c0fab..8e3d50790 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3955,35 +3955,67 @@ namespace { } } + IoBuffer MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer) + { + ZEN_TRACE_CPU("MakeBufferMemoryBased"); + IoBuffer BlockMemoryBuffer; + std::span Segments = PartialBlockBuffer.GetSegments(); + if (Segments.size() == 1) + { + IoBufferFileReference FileRef = {}; + if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef)) + { + BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer(); + BasicFile Reader; + Reader.Attach(FileRef.FileHandle); + auto _ = MakeGuard([&Reader]() { Reader.Detach(); }); + MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); + Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); + return BlockMemoryBuffer; + } + else + { + return PartialBlockBuffer.GetSegments().front().AsIoBuffer(); + } + } + else + { + // Not a homogenous memory buffer, read all to memory + + BlockMemoryBuffer = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer(); + MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView(); + for (const SharedBuffer& Segment : Segments) + { + IoBufferFileReference FileRef = {}; + if (Segment.AsIoBuffer().GetFileReference(FileRef)) + { + BasicFile Reader; + Reader.Attach(FileRef.FileHandle); + Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); + Reader.Detach(); + ReadMem = ReadMem.Mid(FileRef.FileChunkSize); + } + else + { + ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView()); + } + } + return BlockMemoryBuffer; + } + } + bool GetBlockWriteOps(const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& Lookup, std::span ChunkRawHashes, std::span ChunkCompressedLengths, - std::span ChunkRawLengths, std::span> SequenceIndexChunksLeftToWriteCounters, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - CompositeBuffer&& PartialBlockBuffer, + const MemoryView BlockView, uint32_t FirstIncludedBlockChunkIndex, uint32_t LastIncludedBlockChunkIndex, BlockWriteOps& OutOps) { ZEN_TRACE_CPU("GetBlockWriteOps"); - MemoryView BlockMemoryView; - UniqueBuffer BlockMemoryBuffer; - IoBufferFileReference FileRef = {}; - if (PartialBlockBuffer.GetSegments().size() == 1 && PartialBlockBuffer.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) - { - BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize); - BasicFile Reader; - Reader.Attach(FileRef.FileHandle); - Reader.Read(BlockMemoryBuffer.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset); - BlockMemoryView = BlockMemoryBuffer.GetView(); - Reader.Detach(); - } - else - { - BlockMemoryView = PartialBlockBuffer.ViewOrCopyRange(0, PartialBlockBuffer.GetSize(), BlockMemoryBuffer); - } uint32_t OffsetInBlock = 0; for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++) { @@ -4000,32 +4032,9 @@ namespace { bool NeedsWrite = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) { - // CompositeBuffer Chunk = PartialBlockBuffer.Mid(OffsetInBlock, ChunkCompressedSize); - MemoryView ChunkMemory = BlockMemoryView.Mid(OffsetInBlock, ChunkCompressedSize); - CompositeBuffer Chunk = CompositeBuffer(IoBuffer(IoBuffer::Wrap, ChunkMemory.GetData(), ChunkMemory.GetSize())); - IoHash VerifyChunkHash; - uint64_t VerifyRawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(Chunk, VerifyChunkHash, VerifyRawSize); - if (!Compressed) - { - ZEN_ASSERT(false); - } - if (VerifyChunkHash != ChunkHash) - { - ZEN_ASSERT(false); - } - if (!ChunkRawLengths.empty()) - { - if (VerifyRawSize != ChunkRawLengths[ChunkBlockIndex]) - { - ZEN_ASSERT(false); - } - } - CompositeBuffer Decompressed = Compressed.DecompressToComposite(); - if (!Decompressed) - { - throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); - } + MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock + CompressedBuffer::GetHeaderSizeForNoneEncoder(), + ChunkCompressedSize - CompressedBuffer::GetHeaderSizeForNoneEncoder()); + IoBuffer Decompressed(IoBuffer::Wrap, ChunkMemoryView.GetData(), ChunkMemoryView.GetSize()); ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) @@ -4040,19 +4049,22 @@ namespace { OffsetInBlock += ChunkCompressedSize; } - std::sort(OutOps.WriteOps.begin(), - OutOps.WriteOps.end(), - [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - return Lhs.Target->Offset < Rhs.Target->Offset; - }); + { + ZEN_TRACE_CPU("GetBlockWriteOps_sort"); + std::sort(OutOps.WriteOps.begin(), + OutOps.WriteOps.end(), + [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + return Lhs.Target->Offset < Rhs.Target->Offset; + }); + } return true; } @@ -4068,26 +4080,25 @@ namespace { { ZEN_TRACE_CPU("WriteBlockToDisk"); + IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer); + const MemoryView BlockView = BlockMemoryBuffer.GetView(); + BlockWriteOps Ops; if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty()) { ZEN_TRACE_CPU("WriteBlockToDisk_Legacy"); - UniqueBuffer CopyBuffer; - const MemoryView BlockView = BlockBuffer.ViewOrCopyRange(0, BlockBuffer.GetSize(), CopyBuffer); uint64_t HeaderSize; - const std::vector ChunkCompressedLengths = ReadChunkBlockHeader(BlockView, HeaderSize); - - CompositeBuffer PartialBlockBuffer = std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize); + const std::vector ChunkCompressedLengths = + ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize); if (GetBlockWriteOps(RemoteContent, Lookup, BlockDescription.ChunkRawHashes, ChunkCompressedLengths, - BlockDescription.ChunkRawLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, - std::move(PartialBlockBuffer), + BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize), 0, gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), Ops)) @@ -4104,16 +4115,13 @@ namespace { return false; } - CompositeBuffer PartialBlockBuffer = - std::move(BlockBuffer).Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); if (GetBlockWriteOps(RemoteContent, Lookup, BlockDescription.ChunkRawHashes, BlockDescription.ChunkCompressedLengths, - BlockDescription.ChunkRawLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, - std::move(PartialBlockBuffer), + BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize), 0, gsl::narrow(BlockDescription.ChunkRawHashes.size() - 1), Ops)) @@ -4143,15 +4151,18 @@ namespace { WriteChunkStatistics& WriteChunkStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); + + IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer); + const MemoryView BlockView = BlockMemoryBuffer.GetView(); + BlockWriteOps Ops; if (GetBlockWriteOps(RemoteContent, Lookup, BlockDescription.ChunkRawHashes, BlockDescription.ChunkCompressedLengths, - BlockDescription.ChunkRawLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, - std::move(PartialBlockBuffer), + BlockView, FirstIncludedBlockChunkIndex, LastIncludedBlockChunkIndex, Ops)) -- cgit v1.2.3 From 0295cd45276c9b99f72dd0f35f3b5d5ffe7cb0df Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 09:48:52 +0100 Subject: collapse local writes (#310) * collapse read/writes during local data copy --- src/zen/cmds/builds_cmd.cpp | 349 ++++++++++++++++++++++++-------------------- src/zencore/basicfile.cpp | 6 + 2 files changed, 199 insertions(+), 156 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8e3d50790..8566f1540 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1520,7 +1520,6 @@ namespace { } WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& ReadPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // ParallellWork Work(AbortFlag); @@ -1545,7 +1544,7 @@ namespace { for (const IoHash& ChunkAttachment : ChunkAttachments) { Work.ScheduleWork( - ReadPool, + NetworkPool, [&, ChunkAttachment](std::atomic&) { if (!AbortFlag) { @@ -5043,160 +5042,6 @@ namespace { } } - for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) - { - if (AbortFlag) - { - break; - } - - Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// - [&, CopyDataIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); - - FilteredWrittenBytesPerSecond.Start(); - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - - uint64_t CacheLocalFileBytesRead = 0; - - size_t TargetStart = 0; - const std::span AllTargets( - CopyData.TargetChunkLocationPtrs); - - struct WriteOp - { - const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; - uint64_t CacheFileOffset = (uint64_t)-1; - uint32_t ChunkIndex = (uint32_t)-1; - }; - - std::vector WriteOps; - - if (!AbortFlag) - { - ZEN_TRACE_CPU("Sort"); - WriteOps.reserve(AllTargets.size()); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) - { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) - { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkIndex = ChunkTarget.RemoteChunkIndex}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; - } - - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } - return false; - }); - } - - if (!AbortFlag) - { - ZEN_TRACE_CPU("Write"); - - tsl::robin_set ChunkIndexesWritten; - - BufferedOpenFile SourceFile(LocalFilePath, DiskStats); - WriteFileCache OpenFileCache(DiskStats); - for (const WriteOp& Op : WriteOps) - { - if (AbortFlag) - { - break; - } - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); - const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ChunkSize); - - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - - OpenFileCache.WriteToFile( - RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); - - if (ChunkIndexesWritten.insert(Op.ChunkIndex).second) - { - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += ChunkSize; - } - - CacheLocalFileBytesRead += ChunkSize; // TODO: This should be the sum of unique chunk sizes? - } - } - if (!AbortFlag) - { - // Write tracking, updating this must be done without any files open (WriteFileCache) - for (const WriteOp& Op : WriteOps) - { - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) - { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("VerifyHash"); - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - } - - ZEN_TRACE_CPU("rename"); - ZEN_ASSERT_SLOW( - !std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); - } - } - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); - } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); - } - for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) { if (AbortFlag) @@ -5386,6 +5231,198 @@ namespace { Work.DefaultErrorFunction()); } + for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) + { + if (AbortFlag) + { + break; + } + + Work.ScheduleWork( + WritePool, // GetSyncWorkerPool(),// + [&, CopyDataIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); + + FilteredWrittenBytesPerSecond.Start(); + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); + + uint64_t CacheLocalFileBytesRead = 0; + + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + uint64_t CacheFileOffset = (uint64_t)-1; + uint32_t ChunkIndex = (uint32_t)-1; + }; + + std::vector WriteOps; + + if (!AbortFlag) + { + ZEN_TRACE_CPU("Sort"); + WriteOps.reserve(AllTargets.size()); + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + { + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) + { + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkIndex = ChunkTarget.RemoteChunkIndex}); + } + TargetStart += ChunkTarget.TargetChunkLocationCount; + } + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { + return false; + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); + } + + if (!AbortFlag) + { + ZEN_TRACE_CPU("Write"); + + tsl::robin_set ChunkIndexesWritten; + + BufferedOpenFile SourceFile(LocalFilePath, DiskStats); + WriteFileCache OpenFileCache(DiskStats); + for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) + { + if (AbortFlag) + { + break; + } + const WriteOp& Op = WriteOps[WriteOpIndex]; + + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; + + uint64_t ReadLength = ChunkSize; + size_t WriteCount = 1; + uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; + uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; + while ((WriteOpIndex + WriteCount) < WriteOps.size()) + { + const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; + if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) + { + break; + } + if (NextOp.Target->Offset != OpTargetEnd) + { + break; + } + if (NextOp.CacheFileOffset != OpSourceEnd) + { + break; + } + const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; + if (ReadLength + NextChunkLength > 512u * 1024u) + { + break; + } + ReadLength += NextChunkLength; + OpSourceEnd += NextChunkLength; + OpTargetEnd += NextChunkLength; + WriteCount++; + } + + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + + OpenFileCache.WriteToFile( + RemoteSequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName( + CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); + for (size_t WrittenOpIndex = WriteOpIndex; WrittenOpIndex < WriteOpIndex + WriteCount; WrittenOpIndex++) + { + const WriteOp& WrittenOp = WriteOps[WrittenOpIndex]; + if (ChunkIndexesWritten.insert(WrittenOp.ChunkIndex).second) + { + WriteChunkStats.ChunkCountWritten++; + WriteChunkStats.ChunkBytesWritten += + RemoteContent.ChunkedContent.ChunkRawSizes[WrittenOp.ChunkIndex]; + } + } + + CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? + + WriteOpIndex += WriteCount; + } + } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + for (const WriteOp& Op : WriteOps) + { + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + { + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("VerifyHash"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( + GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + } + + ZEN_TRACE_CPU("rename"); + ZEN_ASSERT_SLOW( + !std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); + std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), + GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + } + } + ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + for (uint32_t BlockIndex : CachedChunkBlockIndexes) { if (AbortFlag) diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 95876cff4..a181bbd66 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -796,6 +796,12 @@ BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { if (m_Buffer == nullptr || (Size >= m_BufferSize)) { + if (FileOffset == m_BufferEnd) + { + Flush(); + m_BufferStart = m_BufferEnd = FileOffset + Size; + } + m_Base.Write(Data, Size, FileOffset); return; } -- cgit v1.2.3 From 7bb818631b3d7431b7ac1c58da0e46bb0ddaa797 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 10:00:55 +0100 Subject: If a chunk or block write operation results in more than one completed chunk sequence, do the additional verifications as async work (#311) --- src/zen/cmds/builds_cmd.cpp | 243 ++++++++++++++++++++++++++++---------------- 1 file changed, 155 insertions(+), 88 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8566f1540..7e3f5345f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3870,6 +3870,100 @@ namespace { return ChunkTargetPtrs; }; + void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) + { + ZEN_TRACE_CPU("FinalizeChunkSequence"); + ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + } + + void FinalizeChunkSequences(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + std::span RemoteSequenceIndexes) + { + ZEN_TRACE_CPU("FinalizeChunkSequences"); + for (uint32_t SequenceIndex : RemoteSequenceIndexes) + { + FinalizeChunkSequence(TargetFolder, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + } + } + + void VerifyAndCompleteChunkSequencesAsync(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + std::span RemoteSequenceIndexes, + ParallellWork& Work, + WorkerThreadPool& VerifyPool) + { + if (RemoteSequenceIndexes.empty()) + { + return; + } + ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence"); + for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) + { + const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; + Work.ScheduleWork( + VerifyPool, + [&RemoteContent, TargetFolder, RemoteSequenceIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = IoHash::HashBuffer( + IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", + VerifyChunkHash, + SequenceRawHash)); + } + } + FinalizeChunkSequence(TargetFolder, SequenceRawHash); + } + }, + Work.DefaultErrorFunction()); + } + const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; + + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = + IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + } + FinalizeChunkSequence(TargetFolder, SequenceRawHash); + } + + bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span> SequenceIndexChunksLeftToWriteCounters) + { + return SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1; + } + + std::vector CompleteChunkTargets(const std::vector& ChunkTargetPtrs, + std::span> SequenceIndexChunksLeftToWriteCounters) + { + ZEN_TRACE_CPU("CompleteChunkTargets"); + + std::vector CompletedSequenceIndexes; + for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) + { + const uint32_t RemoteSequenceIndex = Location->SequenceIndex; + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + { + CompletedSequenceIndexes.push_back(RemoteSequenceIndex); + } + } + return CompletedSequenceIndexes; + } + struct BlockWriteOps { std::vector ChunkBuffers; @@ -3886,6 +3980,8 @@ namespace { const ChunkedContentLookup& Lookup, std::span> SequenceIndexChunksLeftToWriteCounters, const BlockWriteOps& Ops, + ParallellWork& Work, + WorkerThreadPool& VerifyPool, DiskStatistics& DiskStats, WriteChunkStatistics& WriteChunkStats) { @@ -3928,29 +4024,16 @@ namespace { if (!AbortFlag) { // Write tracking, updating this must be done without any files open (WriteFileCache) + std::vector CompletedChunkSequences; for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) { const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("VerifyChunkHash"); - const IoHash VerifyChunkHash = IoHash::HashBuffer( - IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - } - ZEN_TRACE_CPU("VerifyChunkHashes_rename"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + CompletedChunkSequences.push_back(RemoteSequenceIndex); } } + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, CompletedChunkSequences, Work, VerifyPool); } } @@ -4071,6 +4154,8 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkBlockDescription& BlockDescription, std::span> SequenceIndexChunksLeftToWriteCounters, + ParallellWork& Work, + WorkerThreadPool& VerifyPool, CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -4107,6 +4192,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + Work, + VerifyPool, DiskStats, WriteChunkStats); return true; @@ -4130,6 +4217,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + Work, + VerifyPool, DiskStats, WriteChunkStats); return true; @@ -4141,6 +4230,8 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkBlockDescription& BlockDescription, std::span> SequenceIndexChunksLeftToWriteCounters, + ParallellWork& Work, + WorkerThreadPool& VerifyPool, CompositeBuffer&& PartialBlockBuffer, uint32_t FirstIncludedBlockChunkIndex, uint32_t LastIncludedBlockChunkIndex, @@ -4171,6 +4262,8 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + Work, + VerifyPool, DiskStats, WriteChunkStats); return true; @@ -4371,42 +4464,6 @@ namespace { return false; } - void CompleteChunkTargets(const std::filesystem::path& TargetFolder, - const ChunkedFolderContent& RemoteContent, - const IoHash& ChunkHash, - const std::vector& ChunkTargetPtrs, - std::span> SequenceIndexChunksLeftToWriteCounters, - const bool NeedHashVerify) - { - ZEN_TRACE_CPU("CompleteChunkTargets"); - - for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs) - { - const uint32_t RemoteSequenceIndex = Location->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) - { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - if (NeedHashVerify) - { - ZEN_TRACE_CPU("VerifyChunkHash"); - - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != ChunkHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, ChunkHash)); - } - } - - ZEN_TRACE_CPU("RenameToFinal"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), - GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); - } - } - } - void AsyncWriteDownloadedChunk(const std::filesystem::path& Path, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, @@ -4518,12 +4575,16 @@ namespace { std::filesystem::remove(CompressedChunkPath); - CompleteChunkTargets(TargetFolder, - RemoteContent, - ChunkHash, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - NeedHashVerify); + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, RemoteContent, CompletedSequences, Work, WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } } } }, @@ -4694,8 +4755,8 @@ namespace { else { // We must write the sequence - SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; } } } @@ -5098,6 +5159,8 @@ namespace { &RemoteLookup, &CacheFolderPath, &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, &DiskStats, &WriteChunkStats, &WritePartsComplete, @@ -5145,12 +5208,20 @@ namespace { std::filesystem::remove(CompressedChunkPath); - CompleteChunkTargets(TargetFolder, - RemoteContent, - ChunkHash, - ChunkTargetPtrs, - SequenceIndexChunksLeftToWriteCounters, - NeedHashVerify); + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } } } }, @@ -5385,32 +5456,20 @@ namespace { if (!AbortFlag) { // Write tracking, updating this must be done without any files open (WriteFileCache) + std::vector CompletedChunkSequences; for (const WriteOp& Op : WriteOps) { const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1) + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("VerifyHash"); - const IoHash VerifyChunkHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile( - GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } - } - - ZEN_TRACE_CPU("rename"); - ZEN_ASSERT_SLOW( - !std::filesystem::exists(GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash), - GetFinalChunkedSequenceFileName(CacheFolderPath, SequenceRawHash)); + CompletedChunkSequences.push_back(RemoteSequenceIndex); } } + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, + RemoteContent, + CompletedChunkSequences, + Work, + WritePool); ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); } WritePartsComplete++; @@ -5452,6 +5511,8 @@ namespace { RemoteContent, BlockDescription, SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -5588,6 +5649,8 @@ namespace { RemoteContent, BlockDescription, SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, CompositeBuffer(std::move(BlockPartialBuffer)), BlockRange.ChunkBlockIndexStart, BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, @@ -5694,7 +5757,9 @@ namespace { { Work.ScheduleWork( WritePool, // WritePool, GetSyncWorkerPool() - [&RemoteContent, + [&Work, + &WritePool, + &RemoteContent, &RemoteLookup, CacheFolderPath, &RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -5735,6 +5800,8 @@ namespace { RemoteContent, BlockDescription, SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, -- cgit v1.2.3 From 4dc972bdaf752676426550b2eed500a9b7dcc4f5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 10:22:52 +0100 Subject: improved reporting on async error (#312) * report async errors as individual errors --- src/zen/cmds/builds_cmd.cpp | 9 ++++++++- src/zenutil/include/zenutil/parallellwork.h | 16 +++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 7e3f5345f..11efc42ec 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4564,7 +4564,6 @@ namespace { std::move(CompressedPart), DiskStats, WriteChunkStats); - if (!AbortFlag) { WritePartsComplete++; @@ -8315,6 +8314,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return AbortFlag ? 13 : 0; } } + catch (const ParallellWorkException& Ex) + { + for (const std::string& Error : Ex.m_Errors) + { + ZEN_ERROR("{}", Error); + } + return 3; + } catch (const std::exception& Ex) { ZEN_ERROR("{}", Ex.what()); diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h index 79798fc8d..8ea77c65d 100644 --- a/src/zenutil/include/zenutil/parallellwork.h +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -9,6 +9,14 @@ #include +class ParallellWorkException : public std::runtime_error +{ +public: + explicit ParallellWorkException(std::vector&& Errors) : std::runtime_error(Errors.front()), m_Errors(std::move(Errors)) {} + + const std::vector m_Errors; +}; + namespace zen { class ParallellWork @@ -95,13 +103,7 @@ public: } else if (m_Errors.size() > 1) { - ExtendableStringBuilder<128> SB; - SB.Append("Multiple errors:"); - for (const std::string& Error : m_Errors) - { - SB.Append(fmt::format("\n {}", Error)); - } - throw std::runtime_error(SB.ToString()); + throw ParallellWorkException(std::move(m_Errors)); } } Latch& PendingWork() { return m_PendingWork; } -- cgit v1.2.3 From 849db45b08d740a113b70273348a686cc573cc2a Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 19 Mar 2025 08:23:45 +0100 Subject: Missing import statment on dashboard's start page (#314) --- src/zenserver/frontend/html.zip | Bin 154898 -> 154940 bytes src/zenserver/frontend/html/pages/start.js | 1 + 2 files changed, 1 insertion(+) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index e0a1888e6..9be6a8f79 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js index 8c9df62f9..472bb27ae 100644 --- a/src/zenserver/frontend/html/pages/start.js +++ b/src/zenserver/frontend/html/pages/start.js @@ -5,6 +5,7 @@ import { ZenPage } from "./page.js" import { Fetcher } from "../util/fetcher.js" import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" import { Table, Toolbar } from "../util/widgets.js" //////////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 764dba27c3183122989a401824c1771249f335da Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 19 Mar 2025 16:12:06 +0100 Subject: build list filters (#313) - Feature: `zen builds list` command has new options - `--query-path` - path to a .json (json format) or .cbo (compact binary object format) with the search query to use - `--result-path` - path to a .json (json format) or .cbo (compact binary object format) to write output result to, if omitted json format will be output to console --- src/zen/cmds/builds_cmd.cpp | 107 ++++++++++++++++++++++++------- src/zen/cmds/builds_cmd.h | 14 ++-- src/zenhttp/include/zenhttp/httpclient.h | 2 +- src/zenhttp/servers/httpsys.cpp | 4 +- src/zenserver-test/zenserver-test.cpp | 4 +- 5 files changed, 96 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 11efc42ec..a1fb1a94a 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -7133,7 +7134,12 @@ BuildsCommand::BuildsCommand() Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); }; - m_Options.add_option("", "v", "verb", "Verb for build - list, upload, download, diff", cxxopts::value(m_Verb), ""); + m_Options.add_option("", + "v", + "verb", + "Verb for build - list, upload, download, diff, fetch-blob, validate-part", + cxxopts::value(m_Verb), + ""); m_Options.parse_positional({"verb"}); m_Options.positional_help("verb"); @@ -7142,6 +7148,20 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_ListOptions); AddOutputOptions(m_ListOptions); m_ListOptions.add_options()("h,help", "Print help"); + m_ListOptions.add_option("", + "", + "query-path", + "Path to json or compactbinary file containing list query", + cxxopts::value(m_ListQueryPath), + ""); + m_ListOptions.add_option("", + "", + "result-path", + "Path to json or compactbinary to write query result to", + cxxopts::value(m_ListResultPath), + ""); + m_ListOptions.parse_positional({"query-path", "result-path"}); + m_ListOptions.positional_help("query-path result-path"); // upload AddCloudOptions(m_UploadOptions); @@ -7418,7 +7438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(StringToPath(m_AccessTokenPath)); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -7469,32 +7489,50 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_BuildsUrl, ClientSettings); - CbObjectWriter QueryWriter; - QueryWriter.BeginObject("query"); + CbObject QueryObject; + if (m_ListQueryPath.empty()) + { + CbObjectWriter QueryWriter; + QueryWriter.BeginObject("query"); + QueryWriter.EndObject(); // query + QueryObject = QueryWriter.Save(); + } + else { - // QueryWriter.BeginObject("platform"); - // { - // QueryWriter.AddString("$eq", "Windows"); - // } - // QueryWriter.EndObject(); // changelist + std::filesystem::path ListQueryPath = StringToPath(m_ListQueryPath); + if (ToLower(ListQueryPath.extension().string()) == ".cbo") + { + QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); + } + else + { + IoBuffer MetaDataJson = ReadFile(ListQueryPath).Flatten(); + std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_ListQueryPath, JsonError)); + } + } } - QueryWriter.EndObject(); // query BuildStorage::Statistics StorageStats; std::unique_ptr Storage; if (!m_BuildsUrl.empty()) { - ZEN_CONSOLE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); + ZEN_CONSOLE_VERBOSE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Http.GetSessionId(), + m_Namespace, + m_Bucket); Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); } else if (!m_StoragePath.empty()) { std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Querying builds in folder '{}'.", StoragePath); + ZEN_CONSOLE_VERBOSE("Querying builds in folder '{}'.", StoragePath); Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); } else @@ -7502,10 +7540,30 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); } - CbObject Response = Storage->ListBuilds(QueryWriter.Save()); - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(Response.GetView(), SB); - ZEN_CONSOLE("{}", SB.ToView()); + CbObject Response = Storage->ListBuilds(QueryObject); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + if (m_ListResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + } + else + { + std::filesystem::path ListResultPath = StringToPath(m_ListResultPath); + if (ToLower(ListResultPath.extension().string()) == ".cbo") + { + MemoryView ResponseView = Response.GetView(); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); + } + else + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + } + } + return 0; } @@ -7622,7 +7680,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath(m_BuildMetadataPath); + std::filesystem::path MetadataPath = StringToPath(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -7654,7 +7712,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, Path, - m_ManifestPath, + StringToPath(m_ManifestPath), m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -7795,8 +7853,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); - DiffFolders(Path, m_DiffPath, m_OnlyChunked); + std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path DiffPath = StringToPath(m_DiffPath); + DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 60953efad..1634975c1 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -75,6 +75,8 @@ private: std::string m_Verb; // list, upload, download cxxopts::Options m_ListOptions{"list", "List available builds"}; + std::string m_ListQueryPath; + std::string m_ListResultPath; std::string m_Path; @@ -90,23 +92,23 @@ private: std::string m_DiffPath; bool m_OnlyChunked = false; - cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; - - cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; - std::vector m_BuildIds; - cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; std::string m_BlobHash; cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; + cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; + + cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; + std::vector m_BuildIds; + cxxopts::Options* m_SubCommands[8] = {&m_ListOptions, &m_UploadOptions, &m_DownloadOptions, &m_DiffOptions, - &m_TestOptions, &m_FetchBlobOptions, &m_ValidateBuildPartOptions, + &m_TestOptions, &m_MultiTestDownloadOptions}; }; diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index a46b9fd83..4d901bdb5 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -97,7 +97,7 @@ public: HttpResponseCode StatusCode = HttpResponseCode::ImATeapot; IoBuffer ResponsePayload; // Note: this also includes the content type - // Contains the reponse headers + // Contains the response headers KeyValueMap Header; // The number of bytes sent as part of the request diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 3bdcdf098..a315ea2df 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -684,7 +684,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ); } - auto EmitReponseDetails = [&](StringBuilderBase& ResponseDetails) -> void { + auto EmitResponseDetails = [&](StringBuilderBase& ResponseDetails) -> void { for (int i = 0; i < ThisRequestChunkCount; ++i) { const HTTP_DATA_CHUNK Chunk = m_HttpDataChunks[ThisRequestChunkOffset + i]; @@ -767,7 +767,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) // Emit diagnostics ExtendableStringBuilder<256> ResponseDetails; - EmitReponseDetails(ResponseDetails); + EmitResponseDetails(ResponseDetails); ZEN_WARN("failed to send HTTP response (error {}: '{}'), request URL: '{}', ({}.{}) response: {}", SendResult, diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 6259c0f37..027a35998 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -3017,9 +3017,9 @@ TEST_CASE("project.remote") { cpr::Response StatusResponse = Session.Get(); CHECK(IsHttpSuccessCode(StatusResponse.status_code)); - CbObject ReponseObject = + CbObject ResponseObject = LoadCompactBinaryObject(IoBuffer(IoBuffer::Wrap, StatusResponse.text.data(), StatusResponse.text.size())); - std::string_view Status = ReponseObject["Status"sv].AsString(); + std::string_view Status = ResponseObject["Status"sv].AsString(); CHECK(Status != "Aborted"sv); if (Status == "Complete"sv) { -- cgit v1.2.3 From b625980599880e1e7d1149a51b9562e79c0f2994 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 19 Mar 2025 16:12:38 +0100 Subject: jupiter builds stats upload (#315) - Improvement: At end of build upload we post statistics to the Jupiter build stats endpoint: - `totalSize` - `reusedRatio` - `reusedBlockCount` - `reusedBlockByteCount` - `newBlockCount` - `newBlockByteCount` - `uploadedCount` - `uploadedByteCount` - `elapsedTimeSec` - `uploadedBytesPerSec` --- src/zen/cmds/builds_cmd.cpp | 32 ++++++++++++---------- src/zenutil/filebuildstorage.cpp | 29 ++++++++++++++++++++ src/zenutil/include/zenutil/buildstorage.h | 6 ++++ .../include/zenutil/jupiter/jupitersession.h | 6 ++++ src/zenutil/jupiter/jupiterbuildstorage.cpp | 24 ++++++++++++++++ src/zenutil/jupiter/jupitersession.cpp | 15 ++++++++++ 6 files changed, 97 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index a1fb1a94a..98031116a 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3343,19 +3343,6 @@ namespace { ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } - struct ValidateStatistics - { - uint64_t BuildBlobSize = 0; - uint64_t BuildPartSize = 0; - uint64_t ChunkAttachmentCount = 0; - uint64_t BlockAttachmentCount = 0; - std::atomic DownloadedAttachmentCount = 0; - std::atomic VerifiedAttachmentCount = 0; - std::atomic DownloadedByteCount = 0; - std::atomic VerifiedByteCount = 0; - uint64_t ElapsedWallTimeUS = 0; - }; - ZEN_CONSOLE_VERBOSE( "Folder scanning stats:" "\n FoundFileCount: {}" @@ -3572,7 +3559,7 @@ namespace { NiceBytes(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load()), NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), - NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes * 8))), + NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), UploadStats.BlockCount.load(), NiceBytes(UploadStats.BlocksBytes.load()), @@ -3582,6 +3569,21 @@ namespace { MultipartAttachmentStats, ValidateInfo); + + Storage.PutBuildPartStats( + BuildId, + BuildPartId, + {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, + {"reusedRatio", AcceptedByteCountPercent / 100.0}, + {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, + {"reusedBlockByteCount", double(FindBlocksStats.AcceptedByteCount)}, + {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, + {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, + {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, + {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())}, + {"uploadedBytesPerSec", + double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, + {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}}); } void VerifyFolder(const ChunkedFolderContent& Content, @@ -7966,7 +7968,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BuildsUrl.empty() && StoragePath.empty()) { - m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").generic_string(); + StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); CleanDirectory(StoragePath, {}); } diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 47a4e1cc4..130fec355 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -502,6 +502,30 @@ public: return Result; } + virtual void PutBuildPartStats(const Oid& BuildId, + const Oid& BuildPartId, + const tsl::robin_map& FloatStats) override + { + CbObjectWriter Request; + Request.BeginObject("floatStats"sv); + for (auto It : FloatStats) + { + Request.AddFloat(It.first, It.second); + } + Request.EndObject(); + CbObject Payload = Request.Save(); + + SimulateLatency(Payload.GetSize(), 0); + + const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId); + CreateDirectories(BuildPartStatsDataPath.parent_path()); + + TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView()); + WriteAsJson(BuildPartStatsDataPath, Payload); + + SimulateLatency(0, 0); + } + protected: std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; } std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; } @@ -520,6 +544,11 @@ protected: return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb"; } + std::filesystem::path GetBuildPartStatsPath(const Oid& BuildId, const Oid& BuildPartId) const + { + return GetBuildPartFolder(BuildId, BuildPartId) / fmt::format("stats_{}.cb", Oid::NewOid()); + } + std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); } std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 9d2bab170..2ebd65a00 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -5,6 +5,10 @@ #include #include +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { class BuildStorage @@ -53,6 +57,8 @@ public: virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual std::vector FindBlocks(const Oid& BuildId) = 0; virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; + + virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 2c5fc73b8..fda4a7bfe 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -155,6 +155,12 @@ public: JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); + JupiterResult PutBuildPartStats(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& BuildPartId, + IoBuffer Payload); + private: inline LoggerRef Log() { return m_Log; } diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index bf89ce785..d70fd8c00 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -348,6 +348,30 @@ public: return SortedBlockDescriptions; } + virtual void PutBuildPartStats(const Oid& BuildId, + const Oid& BuildPartId, + const tsl::robin_map& FloatStats) override + { + ZEN_UNUSED(BuildId, BuildPartId, FloatStats); + CbObjectWriter Request; + Request.BeginObject("floatStats"sv); + for (auto It : FloatStats) + { + Request.AddFloat(It.first, It.second); + } + Request.EndObject(); + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + JupiterResult PutBuildPartStatsResult = m_Session.PutBuildPartStats(m_Namespace, m_Bucket, BuildId, BuildPartId, Payload); + AddStatistic(PutBuildPartStatsResult); + if (!PutBuildPartStatsResult.Success) + { + throw std::runtime_error(fmt::format("Failed posting build part statistics: {} ({})", + PutBuildPartStatsResult.Reason, + PutBuildPartStatsResult.ErrorCode)); + } + } + private: static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 68f214c06..2e4fe5258 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -776,4 +776,19 @@ JupiterSession::GetBlockMetadata(std::string_view Namespace, std::string_view Bu return detail::ConvertResponse(Response, "JupiterSession::GetBlockMetadata"sv); } +JupiterResult +JupiterSession::PutBuildPartStats(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& BuildPartId, + IoBuffer Payload) +{ + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); + HttpClient::Response Response = + m_HttpClient.Put(fmt::format("/api/v2/builds/{}/{}/{}/parts/{}/stats", Namespace, BucketId, BuildId, BuildPartId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + return detail::ConvertResponse(Response, "JupiterSession::PutBuildPartStats"sv); +} + } // namespace zen -- cgit v1.2.3 From d27d91ccc912c2e09c65bdd210d909f5d87694c1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 19 Mar 2025 16:58:09 +0100 Subject: don't let auth env argument block other auth options (#316) * prioritize actual options over implicit env variable for auth token --- src/zen/cmds/builds_cmd.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 98031116a..889ccef0b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -7434,6 +7434,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } + auto GetEnvAccessToken = [](const std::string& AccessTokenEnv) -> std::string { + if (!AccessTokenEnv.empty()) + { + return GetEnvVariable(AccessTokenEnv); + } + return {}; + }; + if (!m_AccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); @@ -7446,14 +7454,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); } } - else if (!m_AccessTokenEnv.empty()) - { - std::string ResolvedAccessToken = GetEnvVariable(m_AccessTokenEnv); - if (!ResolvedAccessToken.empty()) - { - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); - } - } else if (!m_OAuthUrl.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials( @@ -7464,6 +7464,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CreateAuthMgr(); ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName); } + else if (std::string ResolvedAccessToken = GetEnvAccessToken(m_AccessTokenEnv); !ResolvedAccessToken.empty()) + { + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); + } else { CreateAuthMgr(); -- cgit v1.2.3 From 8878156f30e375b93ebe99f02d94c581bcbbf43c Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:53:59 -0600 Subject: Add CookPackageArtifacts attachment to web ui --- src/zenserver/frontend/html/pages/entry.js | 69 ++++++++++++++++++++++++++++- src/zenserver/projectstore/projectstore.cpp | 43 ++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 65a3ef39b..3891239f0 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -59,6 +59,63 @@ export class Page extends ZenPage } } + _find_iohash_field(container, name) + { + const found_field = container.find(name); + if (found_field != undefined) + { + var found_value = found_field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + return ret; + } + } + return null; + } + + async _build_meta(section, entry) + { + var tree = {} + const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); + if (cookart != null) + { + tree["cook"] = { CookPackageArtifacts: cookart}; + } + + if (Object.keys(tree).length == 0) + return; + + const sub_section = section.add_section("meta"); + + for (const cat_name in tree) + { + const cat_section = sub_section.add_section(cat_name); + const table = cat_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + Object.entries(tree[cat_name]).forEach(([key, value]) => + { + const row = table.add_row(key); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + ); + + const do_nothing = () => void(0); + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); + }); + } + } + async _build_page() { var entry = await this._entry; @@ -78,8 +135,16 @@ export class Page extends ZenPage delete tree["$id"]; - const sub_section = section.add_section("deps"); - this._build_deps(sub_section, tree); + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("deps"); + this._build_deps(sub_section, tree); + } + } + + // meta + { + this._build_meta(section, entry); } // data diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 86791e29a..1132e458e 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4812,11 +4812,46 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || + AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(ZenContentType::kBinary); + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; + } + + if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) + { + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + ExtendableStringBuilder<1024> Sb; + if (AcceptType == HttpContentType::kJSON) + { + ContainerObject.ToJson(Sb); + } + else if (AcceptType == HttpContentType::kYAML) + { + ContainerObject.ToYaml(Sb); + } + IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); + OutChunk = SerializedBuffer; + } + else + { + OutChunk = DecompressedBuffer; + } + OutChunk.SetContentType(AcceptType); + } + else + { + OutChunk = DecompressedBuffer; + OutChunk.SetContentType(ZenContentType::kBinary); + } } else { -- cgit v1.2.3 From 91b31318b2390d827d2a0c2a890d54cbf132dc63 Mon Sep 17 00:00:00 2001 From: Florent Devillechabrol Date: Mon, 24 Mar 2025 11:22:40 -0700 Subject: Fixed missing trailing quote when converting binary data from compact binary to json --- src/zencore/compactbinaryjson.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index d8c8a8584..68ed09549 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -293,6 +293,7 @@ private: const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + Builder << '"'; } private: -- cgit v1.2.3 From cc4d4751201bbf83531ea9919f242bfac3add8d3 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:14:47 -0600 Subject: Moved AcceptType Handling From ProjectStore To HttpProjectStore --- src/zenserver/projectstore/httpprojectstore.cpp | 34 +++++++++++++- src/zenserver/projectstore/projectstore.cpp | 61 ++----------------------- src/zenserver/projectstore/projectstore.h | 1 - 3 files changed, 35 insertions(+), 61 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 47748dd90..f9a13220a 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -885,10 +886,39 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) case HttpVerb::kGet: { IoBuffer Value; - std::pair Result = - m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr); + std::pair Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr); if (Result.first == HttpResponseCode::OK) { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("chunk - '{}' WRONGTYPE", Cid)); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } m_ProjectStats.ChunkHitCount++; return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); } diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 1132e458e..ea3f2aad9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4770,7 +4770,6 @@ std::pair ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag) { @@ -4812,51 +4811,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || - AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) - { - return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; - } - - if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) - { - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - ExtendableStringBuilder<1024> Sb; - if (AcceptType == HttpContentType::kJSON) - { - ContainerObject.ToJson(Sb); - } - else if (AcceptType == HttpContentType::kYAML) - { - ContainerObject.ToYaml(Sb); - } - IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); - OutChunk = SerializedBuffer; - } - else - { - OutChunk = DecompressedBuffer; - } - OutChunk.SetContentType(AcceptType); - } - else - { - OutChunk = DecompressedBuffer; - OutChunk.SetContentType(ZenContentType::kBinary); - } - } - else - { - OutChunk.SetContentType(ZenContentType::kCompressedBinary); - } + OutChunk.SetContentType(ZenContentType::kCompressedBinary); return {HttpResponseCode::OK, {}}; } @@ -8551,12 +8506,7 @@ TEST_CASE("project.store.partial.read") uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; @@ -8565,12 +8515,7 @@ TEST_CASE("project.store.partial.read") CHECK(ModificationTag != 0); CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::NotModified); } diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8f2d3ce0d..368da5ea4 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -449,7 +449,6 @@ public: std::pair GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag); -- cgit v1.2.3 From 44c61d6d310a605074df0baa2af27410c589daae Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:18:13 -0600 Subject: Removed do_nothing from entry.js --- src/zenserver/frontend/html/pages/entry.js | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 3891239f0..f127cb0a3 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -107,7 +107,6 @@ export class Page extends ZenPage "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); @@ -193,7 +192,6 @@ export class Page extends ZenPage ); link.first_child().attr("download", `${io_hash}_${base_name}`); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); -- cgit v1.2.3 From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Mar 2025 17:06:23 +0100 Subject: zen build cache service (#318) - **EXPERIMENTAL** `zen builds` - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata - Feature: New `/builds` endpoint for caching build blobs and blob metadata - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs --- src/zen/cmds/builds_cmd.cpp | 1883 +++++++++++--------- src/zen/cmds/builds_cmd.h | 3 + src/zenhttp/httpclient.cpp | 26 +- src/zenhttp/include/zenhttp/formatters.h | 46 +- src/zenhttp/packageformat.cpp | 45 +- src/zenserver/admin/admin.cpp | 16 + src/zenserver/admin/admin.h | 3 + src/zenserver/buildstore/httpbuildstore.cpp | 526 ++++++ src/zenserver/buildstore/httpbuildstore.h | 65 + src/zenserver/config.cpp | 10 + src/zenserver/config.h | 12 + src/zenserver/workspaces/httpworkspaces.cpp | 2 +- src/zenserver/zenserver.cpp | 19 + src/zenserver/zenserver.h | 4 + src/zenstore-test/zenstore-test.cpp | 2 + src/zenstore/blockstore.cpp | 2 +- src/zenstore/buildstore/buildstore.cpp | 1475 +++++++++++++++ src/zenstore/compactcas.cpp | 4 +- src/zenstore/compactcas.h | 4 +- src/zenstore/gc.cpp | 21 +- src/zenstore/include/zenstore/accesstime.h | 47 + src/zenstore/include/zenstore/blockstore.h | 2 +- .../include/zenstore/buildstore/buildstore.h | 186 ++ .../include/zenstore/cache/cachedisklayer.h | 1 + src/zenstore/include/zenstore/cache/cacheshared.h | 38 - src/zenstore/include/zenstore/gc.h | 4 + src/zenutil/buildstoragecache.cpp | 362 ++++ src/zenutil/chunkblock.cpp | 2 +- src/zenutil/filebuildstorage.cpp | 27 +- src/zenutil/include/zenutil/buildstorage.h | 6 +- src/zenutil/include/zenutil/buildstoragecache.h | 52 + .../include/zenutil/logging/rotatingfilesink.h | 1 - src/zenutil/jupiter/jupiterbuildstorage.cpp | 35 +- 33 files changed, 3984 insertions(+), 947 deletions(-) create mode 100644 src/zenserver/buildstore/httpbuildstore.cpp create mode 100644 src/zenserver/buildstore/httpbuildstore.h create mode 100644 src/zenstore/buildstore/buildstore.cpp create mode 100644 src/zenstore/include/zenstore/accesstime.h create mode 100644 src/zenstore/include/zenstore/buildstore/buildstore.h create mode 100644 src/zenutil/buildstoragecache.cpp create mode 100644 src/zenutil/include/zenutil/buildstoragecache.h (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 889ccef0b..b2ad579f1 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,8 @@ ZEN_THIRD_PARTY_INCLUDES_END #define EXTRA_VERIFY 0 +#define ZEN_CLOUD_STORAGE "Cloud Storage" + namespace zen { namespace { static std::atomic AbortFlag = false; @@ -87,6 +90,8 @@ namespace { const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; + const bool SingleThreaded = false; + const std::string ZenFolderName = ".zen"; const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); @@ -204,22 +209,27 @@ namespace { { try { - std::filesystem::remove(LocalFilePath); - } - catch (const std::exception&) - { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Sleep(200); - try - { - std::filesystem::remove(LocalFilePath); - } - catch (const std::exception& Ex) + std::error_code Ec; + std::filesystem::remove(LocalFilePath, Ec); + if (Ec) { - ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); - CleanWipe = false; + // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time + Ec.clear(); + if (std::filesystem::exists(LocalFilePath, Ec) || Ec) + { + Sleep(200); + if (std::filesystem::exists(LocalFilePath)) + { + std::filesystem::remove(LocalFilePath); + } + } } } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); + CleanWipe = false; + } } for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) @@ -425,7 +435,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, [](bool, std::ptrdiff_t) {}, AbortFlag); @@ -439,7 +449,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent FolderContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, Content, ChunkController, @@ -603,11 +613,9 @@ namespace { struct WriteChunkStatistics { - std::atomic ChunkCountWritten = 0; - std::atomic ChunkBytesWritten = 0; - uint64_t DownloadTimeUs = 0; - uint64_t WriteTimeUs = 0; - uint64_t WriteChunksElapsedWallTimeUs = 0; + uint64_t DownloadTimeUs = 0; + uint64_t WriteTimeUs = 0; + uint64_t WriteChunksElapsedWallTimeUs = 0; }; struct RebuildFolderStateStatistics @@ -626,6 +634,15 @@ namespace { uint64_t VerifyElapsedWallTimeUs = 0; }; + struct StorageInstance + { + std::unique_ptr BuildStorageHttp; + std::unique_ptr BuildStorage; + std::string StorageName; + std::unique_ptr CacheHttp; + std::unique_ptr BuildCacheStorage; + }; + std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, const std::span LocalChunkOrder, const tsl::robin_map& ChunkHashToLocalChunkIndex, @@ -772,7 +789,7 @@ namespace { std::span ChunkCounts, std::span LocalChunkHashes, std::span LocalChunkRawSizes, - std::vector AbsoluteChunkOrders, + const std::vector& AbsoluteChunkOrders, const std::span LooseLocalChunkIndexes, const std::span BlockHashes) { @@ -1444,7 +1461,7 @@ namespace { for (auto& WorkItem : WorkItems) { Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(),// + NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic&) { ZEN_TRACE_CPU("DownloadLargeBlob_Work"); if (!AbortFlag) @@ -1513,15 +1530,16 @@ namespace { } ValidateStats.BlockAttachmentCount = BlockAttachments.size(); - std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); + std::vector VerifyBlockDescriptions = + ParseChunkBlockDescriptionList(Storage.GetBlockMetadatas(BuildId, BlockAttachments)); if (VerifyBlockDescriptions.size() != BlockAttachments.size()) { throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", BlockAttachments.size() - VerifyBlockDescriptions.size())); } - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ParallellWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1881,7 +1899,7 @@ namespace { void GenerateBuildBlocks(const std::filesystem::path& Path, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, - BuildStorage& Storage, + StorageInstance& Storage, const Oid& BuildId, const std::vector>& NewBlockChunks, GeneratedBlocks& OutBlocks, @@ -1904,9 +1922,8 @@ namespace { RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = - GetMediumWorkerPool(EWorkloadType::Burst); // GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// - WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& GenerateBlobsPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& UploadBlocksPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; @@ -2005,21 +2022,35 @@ namespace { const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - Storage.PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload).GetCompressed()); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); + } + + Storage.BuildStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); UploadStats.BlocksBytes += CompressedBlockSize; + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(CompressedBlockSize), OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - Storage.PutBlockMetadata(BuildId, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(BlockMetaData.GetSize())); OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2074,7 +2105,7 @@ namespace { } } - void UploadPartBlobs(BuildStorage& Storage, + void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, const ChunkedFolderContent& Content, @@ -2092,8 +2123,8 @@ namespace { { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& ReadChunkPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& UploadChunkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& ReadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& UploadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); FilteredRate FilteredGenerateBlockBytesPerSecond; FilteredRate FilteredCompressedBytesPerSecond; @@ -2177,18 +2208,26 @@ namespace { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + } + Storage.BuildStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(PayloadSize), NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); UploadedBlockSize += PayloadSize; UploadStats.BlocksBytes += PayloadSize; - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(BlockMetaData.GetSize())); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2214,12 +2253,17 @@ namespace { ZEN_TRACE_CPU("AsyncUploadLooseChunk"); const uint64_t PayloadSize = Payload.GetSize(); - ; + + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + } + if (PayloadSize >= LargeAttachmentSize) { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); UploadStats.MultipartAttachmentCount++; - std::vector> MultipartWork = Storage.PutLargeBuildBlob( + std::vector> MultipartWork = Storage.BuildStorage->PutLargeBuildBlob( BuildId, RawHash, ZenContentType::kCompressedBinary, @@ -2232,7 +2276,7 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&, RawSize](uint64_t SentBytes, bool IsComplete) { + [&, Payload, RawSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; UploadedCompressedChunkSize += SentBytes; if (IsComplete) @@ -2264,7 +2308,7 @@ namespace { else { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); - Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + Storage.BuildStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); UploadStats.ChunksBytes += Payload.GetSize(); UploadStats.ChunkCount++; @@ -2303,7 +2347,7 @@ namespace { if (!AbortFlag) { Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool() + SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -2362,7 +2406,7 @@ namespace { { const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool(),// ReadChunkPool, + SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { @@ -2598,7 +2642,7 @@ namespace { return FilteredReuseBlockIndexes; }; - void UploadFolder(BuildStorage& Storage, + void UploadFolder(StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, const std::string_view BuildPartName, @@ -2654,7 +2698,7 @@ namespace { ZEN_TRACE_CPU("CreateBuild"); Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); + CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); Result.PayloadSize = MetaData.GetSize(); @@ -2663,7 +2707,7 @@ namespace { { ZEN_TRACE_CPU("PutBuild"); Stopwatch GetBuildTimer; - CbObject Build = Storage.GetBuild(BuildId); + CbObject Build = Storage.BuildStorage->GetBuild(BuildId); Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); Result.PayloadSize = Build.GetSize(); if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) @@ -2681,7 +2725,7 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = Storage.FindBlocks(BuildId); + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -2796,10 +2840,10 @@ namespace { } return true; }, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { - ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, AbortFlag); } @@ -2855,7 +2899,7 @@ namespace { FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, Content, *ChunkController, @@ -2985,7 +3029,7 @@ namespace { (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) : 0.0; ZEN_CONSOLE( - "Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n" + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" " Accepting {} ({}) redundant chunks ({:.1f}%)\n" " Rejected {} ({}) chunks in {} blocks\n" @@ -3204,7 +3248,8 @@ namespace { } Stopwatch PutBuildPartResultTimer; - std::pair> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); + std::pair> PutBuildPartResult = + Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), @@ -3289,7 +3334,7 @@ namespace { while (!AbortFlag) { Stopwatch FinalizeBuildPartTimer; - std::vector Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); + std::vector Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), Needs.size()); @@ -3304,7 +3349,7 @@ namespace { if (CreateBuild && !AbortFlag) { Stopwatch FinalizeBuildTimer; - Storage.FinalizeBuild(BuildId); + Storage.BuildStorage->FinalizeBuild(BuildId); ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); } @@ -3321,7 +3366,13 @@ namespace { { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); UploadStats.BlocksBytes += BlockMetaData.GetSize(); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; UploadBlockMetadataCount++; @@ -3340,7 +3391,7 @@ namespace { DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { - ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } ZEN_CONSOLE_VERBOSE( @@ -3570,7 +3621,7 @@ namespace { ValidateInfo); - Storage.PutBuildPartStats( + Storage.BuildStorage->PutBuildPartStats( BuildId, BuildPartId, {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, @@ -3597,7 +3648,7 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ParallellWork Work(AbortFlag); @@ -3873,6 +3924,22 @@ namespace { return ChunkTargetPtrs; }; + uint64_t GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) + { + uint64_t WriteCount = 0; + std::span ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) + { + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) + { + WriteCount++; + } + } + return WriteCount; + }; + void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); @@ -3892,8 +3959,39 @@ namespace { } } + void VerifySequence(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + uint32_t RemoteSequenceIndex) + { + ZEN_TRACE_CPU("VerifySequence"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const std::uint32_t RemotePathIndex = Lookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ExpectedSize = RemoteContent.RawSizes[RemotePathIndex]; + IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + const uint64_t VerifySize = VerifyBuffer.GetSize(); + if (VerifySize != ExpectedSize) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}", + SequenceRawHash, + VerifySize, + ExpectedSize)); + } + ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer)); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + } + } + void VerifyAndCompleteChunkSequencesAsync(const std::filesystem::path& TargetFolder, const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, std::span RemoteSequenceIndexes, ParallellWork& Work, WorkerThreadPool& VerifyPool) @@ -3908,40 +4006,24 @@ namespace { const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; Work.ScheduleWork( VerifyPool, - [&RemoteContent, TargetFolder, RemoteSequenceIndex](std::atomic&) { + [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic&) { if (!AbortFlag) { ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); + if (!AbortFlag) { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = IoHash::HashBuffer( - IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(TargetFolder, SequenceRawHash); } - FinalizeChunkSequence(TargetFolder, SequenceRawHash); } }, Work.DefaultErrorFunction()); } const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); - } - } FinalizeChunkSequence(TargetFolder, SequenceRawHash); } @@ -3985,8 +4067,7 @@ namespace { const BlockWriteOps& Ops, ParallellWork& Work, WorkerThreadPool& VerifyPool, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockChunkOps"); { @@ -4017,12 +4098,6 @@ namespace { FileOffset, RemoteContent.RawSizes[PathIndex]); } - WriteChunkStats.ChunkCountWritten += gsl::narrow(Ops.ChunkBuffers.size()); - WriteChunkStats.ChunkBytesWritten += - std::accumulate(Ops.ChunkBuffers.begin(), - Ops.ChunkBuffers.end(), - uint64_t(0), - [](uint64_t Current, const CompositeBuffer& Buffer) -> uint64_t { return Current + Buffer.GetSize(); }); } if (!AbortFlag) { @@ -4036,7 +4111,7 @@ namespace { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, CompletedChunkSequences, Work, VerifyPool); + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, Lookup, CompletedChunkSequences, Work, VerifyPool); } } @@ -4162,8 +4237,7 @@ namespace { CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockToDisk"); @@ -4197,8 +4271,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4222,8 +4295,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4240,8 +4312,7 @@ namespace { uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); @@ -4267,8 +4338,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } else @@ -4355,8 +4425,7 @@ namespace { void StreamDecompress(const std::filesystem::path& CacheFolderPath, const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("StreamDecompress"); const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); @@ -4390,7 +4459,6 @@ namespace { DiskStats.ReadByteCount += SourceSize; if (!AbortFlag) { - WriteChunkStats.ChunkBytesWritten += RangeBuffer.GetSize(); DecompressedTemp.Write(RangeBuffer, Offset); for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) { @@ -4424,7 +4492,7 @@ namespace { throw std::runtime_error( fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); } - WriteChunkStats.ChunkCountWritten++; + // WriteChunkStats.ChunkCountWritten++; } bool WriteCompressedChunk(const std::filesystem::path& TargetFolder, @@ -4433,8 +4501,7 @@ namespace { const IoHash& ChunkHash, const std::vector& ChunkTargetPtrs, IoBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); @@ -4444,7 +4511,7 @@ namespace { { const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; - StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats, WriteChunkStats); + StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats); } else { @@ -4459,8 +4526,6 @@ namespace { ChunkTargetPtrs, CompositeBuffer(std::move(Chunk)), OpenFileCache); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += ChunkRawSize; return true; } } @@ -4479,8 +4544,7 @@ namespace { std::atomic& WritePartsComplete, const uint64_t TotalPartWriteCount, FilteredRate& FilteredWrittenBytesPerSecond, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); @@ -4527,7 +4591,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, SequenceIndexChunksLeftToWriteCounters, CompressedChunkPath, @@ -4565,8 +4629,7 @@ namespace { ChunkHash, ChunkTargetPtrs, std::move(CompressedPart), - DiskStats, - WriteChunkStats); + DiskStats); if (!AbortFlag) { WritePartsComplete++; @@ -4581,7 +4644,12 @@ namespace { CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); if (NeedHashVerify) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, RemoteContent, CompletedSequences, Work, WritePool); + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); } else { @@ -4593,7 +4661,7 @@ namespace { Work.DefaultErrorFunction()); }; - void UpdateFolder(BuildStorage& Storage, + void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, const std::uint64_t LargeAttachmentSize, @@ -4667,8 +4735,8 @@ namespace { if (SequenceSize == CacheDirContent.FileSizes[Index]) { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CacheMappingStats.CacheSequenceHashesCount += SequenceSize; - CacheMappingStats.CacheSequenceHashesByteCount++; + CacheMappingStats.CacheSequenceHashesCount++; + CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; continue; } } @@ -4869,21 +4937,17 @@ namespace { NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); } - uint32_t ChunkCountToWrite = 0; + uint64_t BytesToWrite = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) { - if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + if (ChunkWriteCount > 0) { - ChunkCountToWrite++; - } - else - { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); - if (!ChunkTargetPtrs.empty()) + BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; + if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; - ChunkCountToWrite++; } } } @@ -4900,8 +4964,8 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -4922,7 +4986,7 @@ namespace { const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); continue; } bool NeedsCopy = true; @@ -4933,7 +4997,7 @@ namespace { if (ChunkTargetPtrs.empty()) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); } else { @@ -5025,6 +5089,10 @@ namespace { uint32_t CurrentOffset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) @@ -5064,6 +5132,7 @@ namespace { } ZEN_ASSERT(!BlockRanges.empty()); + std::vector CollapsedBlockRanges; auto It = BlockRanges.begin(); CollapsedBlockRanges.push_back(*It++); @@ -5085,10 +5154,87 @@ namespace { ++It; } - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); + const std::uint64_t WantedSize = std::accumulate( + CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + ZEN_ASSERT(WantedSize <= TotalBlockSize); + if (WantedSize > ((TotalBlockSize * 95) / 100)) + { + ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) + { + ZEN_CONSOLE_VERBOSE("Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) + { + ZEN_CONSOLE_VERBOSE("Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) + { + ZEN_CONSOLE_VERBOSE("Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + { + ZEN_CONSOLE_VERBOSE("Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else + { + if (WantedSize > ((TotalBlockSize * 5) / 10)) + { + ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", + (WantedSize * 100) / TotalBlockSize, + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + } + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); - BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + } } else { @@ -5101,10 +5247,69 @@ namespace { } else { - ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); } } + struct BlobsExistsResult + { + tsl::robin_set ExistingBlobs; + uint64_t ElapsedTimeMs = 0; + }; + + BlobsExistsResult ExistsResult; + + if (Storage.BuildCacheStorage) + { + ZEN_TRACE_CPU("BlobCacheExistCheck"); + Stopwatch Timer; + + tsl::robin_set BlobHashesSet; + + BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size()); + for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks) + { + BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); + } + for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks) + { + const uint32_t BlockIndex = BlockRange.BlockIndex; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + for (uint32_t BlockIndex : FullBlockWorks) + { + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + + if (!BlobHashesSet.empty()) + { + const std::vector BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end()); + const std::vector CacheExistsResult = + Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes); + + if (CacheExistsResult.size() == BlobHashes.size()) + { + ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + { + if (CacheExistsResult[BlobIndex].HasBody) + { + ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]); + } + } + } + ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!ExistsResult.ExistingBlobs.empty()) + { + ZEN_CONSOLE("Found {} out of {} needed blobs in remote cache in {}", + ExistsResult.ExistingBlobs.size(), + BlobHashes.size(), + NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); + } + } + } for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) { if (AbortFlag) @@ -5119,7 +5324,7 @@ namespace { const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; Work.ScheduleWork( - WritePool, // NetworkPool, // GetSyncWorkerPool(),// + WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { if (!AbortFlag) { @@ -5152,151 +5357,202 @@ namespace { } } } - if (!ExistingCompressedChunkPath.empty()) + if (!AbortFlag) + { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Path, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + WritePool, + [&Path, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - FilteredWrittenBytesPerSecond.Start(); + FilteredWrittenBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats, - WriteChunkStats); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]; - WritePartsComplete++; + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + DiskStats); + WritePartsComplete++; - if (!AbortFlag) - { - if (WritePartsComplete == TotalPartWriteCount) + if (!AbortFlag) { - FilteredWrittenBytesPerSecond.Stop(); - } + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } - std::filesystem::remove(CompressedChunkPath); + std::filesystem::remove(CompressedChunkPath); - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - if (NeedHashVerify) + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } + } + } + }, + Work.DefaultErrorFunction()); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&Path, + &Storage, + BuildId, + &RemoteContent, + &RemoteLookup, + &ExistsResult, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + LargeAttachmentSize, + PreferredMultipartChunkSize, + RemoteChunkIndex, + ChunkTargetPtrs, + &DownloadStats](std::atomic&) mutable { + if (!AbortFlag) + { + FilteredDownloadedBytesPerSecond.Start(); + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - CompletedSequences, - Work, - WritePool); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob(*Storage.BuildStorage, + Path / ZenTempDownloadFolderName, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!AbortFlag) + { + AsyncWriteDownloadedChunk( + Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + }); } else { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + IoBuffer BuildBlob; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash)) + { + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + } + if (!BuildBlob) + { + BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + if (BuildBlob && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + BuildBlob.GetContentType(), + CompositeBuffer(SharedBuffer(BuildBlob))); + } + } + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } } } - } - }, - Work.DefaultErrorFunction()); - } - else - { - FilteredDownloadedBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) - { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(Storage, - Path / ZenTempDownloadFolderName, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); - }); - } - else - { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - - IoBuffer BuildBlob = Storage.GetBuildBlob(BuildId, ChunkHash); - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); + }, + Work.DefaultErrorFunction()); } } } @@ -5312,7 +5568,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, CopyDataIndex](std::atomic&) { if (!AbortFlag) { @@ -5439,16 +5695,6 @@ namespace { ChunkSource, Op.Target->Offset, RemoteContent.RawSizes[RemotePathIndex]); - for (size_t WrittenOpIndex = WriteOpIndex; WrittenOpIndex < WriteOpIndex + WriteCount; WrittenOpIndex++) - { - const WriteOp& WrittenOp = WriteOps[WrittenOpIndex]; - if (ChunkIndexesWritten.insert(WrittenOp.ChunkIndex).second) - { - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[WrittenOp.ChunkIndex]; - } - } CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? @@ -5469,10 +5715,13 @@ namespace { } VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, + RemoteLookup, CompletedChunkSequences, Work, WritePool); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", + NiceBytes(CacheLocalFileBytesRead), + LocalContent.Paths[LocalPathIndex]); } WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) @@ -5492,7 +5741,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(), // WritePool, + WritePool, [&, BlockIndex](std::atomic&) mutable { if (!AbortFlag) { @@ -5509,27 +5758,29 @@ namespace { fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); } - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); - if (WritePartsComplete == TotalPartWriteCount) + if (!AbortFlag) { - FilteredWrittenBytesPerSecond.Stop(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + std::filesystem::remove(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } } }, @@ -5546,7 +5797,7 @@ namespace { ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); const uint32_t BlockIndex = BlockRange.BlockIndex; Work.ScheduleWork( - NetworkPool, // NetworkPool, // GetSyncWorkerPool() + NetworkPool, [&, BlockIndex, BlockRange](std::atomic&) { if (!AbortFlag) { @@ -5555,131 +5806,148 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = - Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } - - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!AbortFlag) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + FilteredDownloadedBytesPerSecond.Stop(); + } - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = - Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, // GetSyncWorkerPool(), - [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( - std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( + std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else - { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -5693,7 +5961,7 @@ namespace { break; } Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(), // NetworkPool, + NetworkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -5702,133 +5970,152 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); + + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (BlockBuffer && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlockBuffer))); + } } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } + if (!AbortFlag) + { + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - std::filesystem::path BlockChunkPath; + std::filesystem::path BlockChunkPath; - // Check if the dowloaded block is file based and we can move it directly without rewriting it - { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + // Check if the dowloaded block is file based and we can move it directly without rewriting it { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; + WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -5840,27 +6127,28 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= WriteChunkStats.ChunkCountWritten.load()); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load()); FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); - std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", + std::string DownloadRateString = + (DownloadStats.RequestsCompleteCount == TotalRequestCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); + std::string Details = fmt::format("{}/{} ({}{}) downloaded. {}/{} ({}B/s) written.", DownloadStats.RequestsCompleteCount.load(), TotalRequestCount, NiceBytes(DownloadedBytes), - NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - WriteChunkStats.ChunkCountWritten.load(), - ChunkCountToWrite, + DownloadRateString, NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState( - {.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - WriteChunkStats.ChunkCountWritten.load())}, - false); + WriteProgressBar.UpdateState({.Task = "Writing chunks ", + .Details = Details, + .TotalCount = gsl::narrow(BytesToWrite), + .RemainingCount = gsl::narrow(BytesToWrite - DiskStats.WriteByteCount.load())}, + false); }); } @@ -5981,7 +6269,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); Stopwatch Timer; - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ProgressBar RebuildProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -6019,7 +6307,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { if (!AbortFlag) { @@ -6070,6 +6358,10 @@ namespace { TargetsComplete++; while (TargetOffset < (BaseTargetOffset + TargetCount)) { + if (AbortFlag) + { + break; + } ZEN_TRACE_CPU("FinalizeTree_Copy"); ZEN_ASSERT(Targets[TargetOffset].first == RawHash); @@ -6140,17 +6432,15 @@ namespace { std::vector> Result; { Stopwatch GetBuildTimer; - - std::vector> AvailableParts; - - CbObject BuildObject = Storage.GetBuild(BuildId); - - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", + CbObject BuildObject = Storage.GetBuild(BuildId); + ZEN_CONSOLE("GetBuild took {}. Name: '{}' ({}, {}), Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["BuildName"sv].AsString(), + BuildObject["name"sv].AsString(), + BuildObject["type"sv].AsString(), + BuildObject["Configuration"sv].AsString(), NiceBytes(BuildObject.GetSize())); - ZEN_DEBUG("Build object: {}", BuildObject); + ZEN_CONSOLE_VERBOSE("Build object: {}", BuildObject); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -6160,6 +6450,8 @@ namespace { OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + std::vector> AvailableParts; + for (CbFieldView PartView : PartsObject) { const std::string BuildPartName = std::string(PartView.GetName()); @@ -6221,7 +6513,7 @@ namespace { return Result; } - ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, + ChunkedFolderContent GetRemoteContent(StorageInstance& Storage, const Oid& BuildId, const std::vector>& BuildParts, std::unique_ptr& OutChunkController, @@ -6234,7 +6526,7 @@ namespace { Stopwatch GetBuildPartTimer; const Oid BuildPartId = BuildParts[0].first; const std::string_view BuildPartName = BuildParts[0].second; - CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildPartId); + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", BuildPartId, BuildPartName, @@ -6248,7 +6540,7 @@ namespace { OutChunkController = CreateChunkingController(ChunkerName, Parameters); } - auto ParseBuildPartManifest = [](BuildStorage& Storage, + auto ParseBuildPartManifest = [](StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, CbObject BuildPartManifest, @@ -6274,12 +6566,103 @@ namespace { // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them - Stopwatch GetBlockMetadataTimer; - OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes); - ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); + { + Stopwatch GetBlockMetadataTimer; + + std::vector UnorderedList; + tsl::robin_map BlockDescriptionLookup; + if (Storage.BuildCacheStorage) + { + std::vector CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes); + UnorderedList.reserve(CacheBlockMetadatas.size()); + for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size(); + CacheBlockMetadataIndex++) + { + const CbObject& CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex]; + ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block"); + } + else + { + UnorderedList.emplace_back(std::move(Description)); + } + } + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); + } + } + + if (UnorderedList.size() < BlockRawHashes.size()) + { + std::vector RemainingBlockHashes; + RemainingBlockHashes.reserve(BlockRawHashes.size() - UnorderedList.size()); + for (const IoHash& BlockRawHash : BlockRawHashes) + { + if (!BlockDescriptionLookup.contains(BlockRawHash)) + { + RemainingBlockHashes.push_back(BlockRawHash); + } + } + CbObject BlockMetadatas = Storage.BuildStorage->GetBlockMetadatas(BuildId, RemainingBlockHashes); + std::vector RemainingList; + { + CbArrayView BlocksArray = BlockMetadatas["blocks"sv].AsArrayView(); + std::vector FoundBlockHashes; + std::vector FoundBlockMetadatas; + for (CbFieldView Block : BlocksArray) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); + + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block"); + } + else + { + if (Storage.BuildCacheStorage) + { + UniqueBuffer MetaBuffer = UniqueBuffer::Alloc(Block.GetSize()); + Block.CopyTo(MetaBuffer.GetMutableView()); + CbObject BlockMetadata(MetaBuffer.MoveToShared()); + + FoundBlockHashes.push_back(Description.BlockHash); + FoundBlockMetadatas.push_back(BlockMetadata); + } + RemainingList.emplace_back(std::move(Description)); + } + } + if (Storage.BuildCacheStorage && !FoundBlockHashes.empty()) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, FoundBlockHashes, FoundBlockMetadatas); + } + } + + for (size_t DescriptionIndex = 0; DescriptionIndex < RemainingList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = RemainingList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, UnorderedList.size() + DescriptionIndex); + } + UnorderedList.insert(UnorderedList.end(), RemainingList.begin(), RemainingList.end()); + } + + OutBlockDescriptions.reserve(BlockDescriptionLookup.size()); + for (const IoHash& BlockHash : BlockRawHashes) + { + if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) + { + OutBlockDescriptions.push_back(std::move(UnorderedList[It->second])); + } + } + + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } if (OutBlockDescriptions.size() != BlockRawHashes.size()) { @@ -6292,7 +6675,8 @@ namespace { ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); std::vector AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); - std::vector FoundBlocks = Storage.FindBlocks(BuildId); + std::vector FoundBlocks = + ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); for (const IoHash& BlockHash : BlockRawHashes) { @@ -6315,7 +6699,7 @@ namespace { } else { - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockHash); + IoBuffer BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockHash); if (!BlockBuffer) { throw std::runtime_error(fmt::format("Block {} could not be found", BlockHash)); @@ -6381,7 +6765,7 @@ namespace { const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; Stopwatch GetOverlayBuildPartTimer; - CbObject OverlayBuildPartManifest = Storage.GetBuildPart(BuildId, OverlayBuildPartId); + CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", OverlayBuildPartId, OverlayBuildPartName, @@ -6486,9 +6870,11 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, + [&](bool, std::ptrdiff_t) { + ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + }, AbortFlag); if (AbortFlag) { @@ -6557,7 +6943,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, UpdatedContent, ChunkController, @@ -6609,8 +6995,6 @@ namespace { { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } - - ZEN_CONSOLE("Using cached local state"); } ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); ScanContent = false; @@ -6636,7 +7020,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, CurrentLocalFolderContent, ChunkController, @@ -6670,7 +7054,7 @@ namespace { return LocalContent; } - void DownloadFolder(BuildStorage& Storage, + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector& BuildPartIds, std::span BuildPartNames, @@ -6694,7 +7078,7 @@ namespace { std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; std::vector> AllBuildParts = - ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); std::vector PartContents; @@ -6786,7 +7170,7 @@ namespace { { BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, BuildPartString.ToView()); + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path); FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -7131,6 +7515,10 @@ BuildsCommand::BuildsCommand() ""); }; + auto AddCacheOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), ""); + }; + auto AddOutputOptions = [this](cxxopts::Options& Ops) { Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); @@ -7169,6 +7557,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_UploadOptions); AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); + AddCacheOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7232,6 +7621,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_DownloadOptions); AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); + AddCacheOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7284,6 +7674,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); + AddCacheOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_TestOptions.add_option("", @@ -7304,6 +7695,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); + AddCacheOptions(m_FetchBlobOptions); m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_FetchBlobOptions .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); @@ -7333,6 +7725,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); + AddCacheOptions(m_MultiTestDownloadOptions); m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); @@ -7385,6 +7778,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); } } + else if (m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } }; std::unique_ptr Auth; @@ -7393,7 +7790,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : StringToPath(m_SystemRootDir); + std::filesystem::path DataRoot = m_SystemRootDir.empty() + ? PickDefaultSystemRootDirectory() + : std::filesystem::absolute(StringToPath(m_SystemRootDir)).make_preferred(); if (m_EncryptionKey.empty()) { @@ -7448,7 +7847,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(StringToPath(m_AccessTokenPath)); + std::string ResolvedAccessToken = + ReadAccessTokenFromFile(std::filesystem::absolute(StringToPath(m_AccessTokenPath)).make_preferred()); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -7486,15 +7886,63 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseOutputOptions(); + auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + const std::filesystem::path& TempPath) -> StorageInstance { + ParseStorageOptions(); + + StorageInstance Result; + + if (!m_BuildsUrl.empty()) + { + ParseAuthOptions(); + Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Using cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); + Result.BuildStorage = + CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); + Result.StorageName = ZEN_CLOUD_STORAGE; + } + else if (!m_StoragePath.empty()) + { + std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + ZEN_CONSOLE("Using folder {}", StoragePath); + Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + if (!m_ZenCacheHost.empty()) + { + Result.CacheHttp = std::make_unique(m_ZenCacheHost, + HttpClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = false, + .AllowResume = true, + .RetryCount = 0}); + if (Result.CacheHttp->Get("/health").IsSuccess()) + { + Result.BuildCacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); + } + else + { + Result.CacheHttp.reset(); + } + } + return Result; + }; + try { if (SubOption == &m_ListOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - CbObject QueryObject; if (m_ListQueryPath.empty()) { @@ -7505,7 +7953,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = StringToPath(m_ListQueryPath); + std::filesystem::path ListQueryPath = std::filesystem::absolute(StringToPath(m_ListQueryPath)).make_preferred(); if (ToLower(ListQueryPath.extension().string()) == ".cbo") { QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); @@ -7525,28 +7973,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE_VERBOSE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE_VERBOSE("Querying builds in folder '{}'.", StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; - CbObject Response = Storage->ListBuilds(QueryObject); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderName); + + CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); if (m_ListResultPath.empty()) { @@ -7556,7 +7987,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = StringToPath(m_ListResultPath); + std::filesystem::path ListResultPath = std::filesystem::absolute(StringToPath(m_ListResultPath)).make_preferred(); if (ToLower(ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); @@ -7575,11 +8006,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); @@ -7610,7 +8036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)); if (m_BuildPartName.empty()) { @@ -7645,48 +8071,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); } + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", - m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", - m_BuildPartName, - Path, - StoragePath, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); CbObject MetaData; if (m_CreateBuild) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath = StringToPath(m_BuildMetadataPath); + std::filesystem::path MetadataPath = std::filesystem::absolute(StringToPath(m_BuildMetadataPath)); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -7713,12 +8111,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, Path, - StringToPath(m_ManifestPath), + std::filesystem::absolute(StringToPath(m_ManifestPath)).make_preferred(), m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -7735,7 +8133,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7751,11 +8149,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -7786,37 +8179,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - BuildId, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, @@ -7835,7 +8205,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7859,8 +8229,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); - std::filesystem::path DiffPath = StringToPath(m_DiffPath); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path DiffPath = std::filesystem::absolute(StringToPath(m_DiffPath)).make_preferred(); DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -7872,10 +8242,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); // m_StoragePath = "D:\\buildstorage"; // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; // std::vector BuildIdStrings{"07d3942f0e7f4ca1b13b0587", @@ -7886,34 +8252,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - FormatArray(m_BuildIds, " "sv), - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray(m_BuildIds, " "sv), Path, StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -7923,7 +8267,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); } - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId, {}, {}, @@ -7945,36 +8289,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = StringToPath(m_StoragePath); + std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); if (m_BuildsUrl.empty() && StoragePath.empty()) { StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); CleanDirectory(StoragePath, {}); + m_StoragePath = StoragePath.generic_string(); } auto _ = MakeGuard([&]() { if (m_BuildsUrl.empty() && StoragePath.empty()) @@ -7983,33 +8320,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!StoragePath.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - StoragePath, - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -8032,7 +8346,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, @@ -8052,7 +8366,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -8064,7 +8378,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -8115,7 +8429,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SourceSize > 256) { Work.ScheduleWork( - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), [SourceSize, FilePath](std::atomic&) { if (!AbortFlag) { @@ -8168,7 +8482,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -8187,7 +8501,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId2, BuildPartId2, m_BuildPartName, @@ -8206,7 +8520,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8214,7 +8528,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, @@ -8230,7 +8544,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, @@ -8250,11 +8564,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -8266,44 +8575,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); } - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } + const Oid BuildId = Oid::FromHexString(m_BuildId); - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - std::unique_ptr Storage; - std::string StorageName; + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); - std::filesystem::path Path = StringToPath(m_Path); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); + ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); if (AbortFlag) { return 11; @@ -8317,10 +8600,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); + // HttpClient Http(m_BuildsUrl, ClientSettings); if (m_BuildsUrl.empty() && m_StoragePath.empty()) { @@ -8342,39 +8622,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; + BuildStorageCache::Statistics StorageCacheStats; - std::filesystem::path Path = StringToPath(m_Path); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); ValidateStatistics ValidateStats; DownloadStatistics DownloadStats; - ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); return AbortFlag ? 13 : 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 1634975c1..b5af236e1 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -40,6 +40,9 @@ private: std::string m_StoragePath; bool m_WriteMetadataAsJson = false; + // cache + std::string m_ZenCacheHost; + std::string m_BuildId; bool m_CreateBuild = false; std::string m_BuildMetadataPath; diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 30711a432..fe5232d89 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -45,6 +45,7 @@ namespace detail { TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {} ~TempPayloadFile() { + ZEN_TRACE_CPU("TempPayloadFile::Close"); try { if (m_FileHandle) @@ -87,6 +88,7 @@ namespace detail { std::error_code Open(const std::filesystem::path& TempFolderPath) { + ZEN_TRACE_CPU("TempPayloadFile::Open"); ZEN_ASSERT(m_FileHandle == nullptr); std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) | @@ -131,6 +133,7 @@ namespace detail { std::error_code Write(std::string_view DataString) { + ZEN_TRACE_CPU("TempPayloadFile::Write"); const uint8_t* DataPtr = (const uint8_t*)DataString.data(); size_t DataSize = DataString.size(); if (DataSize >= CacheBufferSize) @@ -165,6 +168,7 @@ namespace detail { IoBuffer DetachToIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::DetachToIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -180,6 +184,7 @@ namespace detail { IoBuffer BorrowIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::BorrowIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -193,6 +198,7 @@ namespace detail { uint64_t GetSize() const { return m_WriteOffset; } void ResetWritePos(uint64_t WriteOffset) { + ZEN_TRACE_CPU("TempPayloadFile::ResetWritePos"); Flush(); m_WriteOffset = WriteOffset; } @@ -200,6 +206,7 @@ namespace detail { private: std::error_code Flush() { + ZEN_TRACE_CPU("TempPayloadFile::Flush"); if (m_CacheBufferOffset == 0) { return {}; @@ -211,6 +218,7 @@ namespace detail { std::error_code AppendData(const void* Data, uint64_t Size) { + ZEN_TRACE_CPU("TempPayloadFile::AppendData"); ZEN_ASSERT(m_FileHandle != nullptr); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; @@ -314,7 +322,11 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) { - ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT && + HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED) + { + ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + } // Client side failure code return HttpClient::Response{ @@ -376,6 +388,7 @@ ShouldRetry(const cpr::Response& Response) static bool ValidatePayload(cpr::Response& Response, std::unique_ptr& PayloadFile) { + ZEN_TRACE_CPU("ValidatePayload"); IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile) ? PayloadFile->BorrowIoBuffer() : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size()); @@ -535,12 +548,14 @@ struct HttpClient::Impl : public RefCounted inline cpr::Session* operator->() const { return CprSession; } inline cpr::Response Get() { + ZEN_TRACE_CPU("HttpClient::Impl::Get"); cpr::Response Result = CprSession->Get(); ZEN_TRACE("GET {}", Result); return Result; } inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional&& Header = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Download"); if (Header) { CprSession->SetHeaderCallback(std::move(Header.value())); @@ -553,12 +568,14 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Head() { + ZEN_TRACE_CPU("HttpClient::Impl::Head"); cpr::Response Result = CprSession->Head(); ZEN_TRACE("HEAD {}", Result); return Result; } inline cpr::Response Put(std::optional&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Put"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -570,6 +587,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Post(std::optional&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Post"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -581,6 +599,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Delete() { + ZEN_TRACE_CPU("HttpClient::Impl::Delete"); cpr::Response Result = CprSession->Delete(); ZEN_TRACE("DELETE {}", Result); return Result; @@ -620,6 +639,7 @@ HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log) HttpClient::Impl::~Impl() { + ZEN_TRACE_CPU("HttpClient::Impl::~Impl"); m_SessionLock.WithExclusiveLock([&] { for (auto CprSession : m_Sessions) { @@ -638,6 +658,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, const std::string_view SessionId, std::optional AccessToken) { + ZEN_TRACE_CPU("HttpClient::Impl::AllocSession"); cpr::Session* CprSession = nullptr; m_SessionLock.WithExclusiveLock([&] { if (!m_Sessions.empty()) @@ -694,6 +715,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, void HttpClient::Impl::ReleaseSession(cpr::Session* CprSession) { + ZEN_TRACE_CPU("HttpClient::Impl::ReleaseSession"); CprSession->SetUrl({}); CprSession->SetHeader({}); CprSession->SetBody({}); @@ -718,6 +740,7 @@ HttpClient::~HttpClient() bool HttpClient::Authenticate() { + ZEN_TRACE_CPU("HttpClient::Authenticate"); std::optional Token = GetAccessToken(); if (!Token) { @@ -729,6 +752,7 @@ HttpClient::Authenticate() const std::optional HttpClient::GetAccessToken() { + ZEN_TRACE_CPU("HttpClient::GetAccessToken"); if (!m_ConnectionSettings.AccessTokenProvider.has_value()) { return {}; diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index 74da9ab05..05a23d675 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -73,9 +73,11 @@ struct fmt::formatter if (zen::IsHttpSuccessCode(Response.status_code)) { return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}", + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}", Response.url.str(), Response.status_code, + Response.error.message, + int(Response.error.code), Response.uploaded_bytes, Response.downloaded_bytes, NiceResponseTime.c_str()); @@ -92,29 +94,35 @@ struct fmt::formatter zen::ExtendableStringBuilder<256> Sb; std::string_view Json = Obj.ToJson(Sb).ToView(); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Json, - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}). Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Json, + Response.reason); } else { zen::BodyLogFormatter Body(Response.text); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Body.GetText(), - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Body.GetText(), + Response.reason); } } } diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index ae80851e4..9d423ecbc 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -279,11 +279,10 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar { IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); ZEN_ASSERT(ObjIoBuffer.GetSize() > 0); - ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); - *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Attachment.GetHash()}; + ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); } else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary()) { @@ -500,30 +499,25 @@ ParsePackageMessage(IoBuffer Payload, std::function 0) diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 2888f5450..0da6e31ad 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include "config.h" @@ -105,6 +106,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) @@ -112,6 +114,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, , m_CacheStore(CacheStore) , m_CidStore(CidStore) , m_ProjectStore(ProjectStore) +, m_BuildStore(BuildStore) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { @@ -306,6 +309,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "Interval" << ToTimeSpan(State.Config.Interval); Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); @@ -401,6 +405,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } } + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt(Param)) @@ -782,6 +794,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, { m_ProjectStore->Flush(); } + if (m_BuildStore) + { + m_BuildStore->Flush(); + } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index 563c4f536..e7821dead 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -12,6 +12,7 @@ class JobQueue; class ZenCacheStore; class CidStore; class ProjectStore; +class BuildStore; struct ZenServerOptions; class HttpAdminService : public zen::HttpService @@ -28,6 +29,7 @@ public: ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions); ~HttpAdminService(); @@ -42,6 +44,7 @@ private: ZenCacheStore* m_CacheStore; CidStore* m_CidStore; ProjectStore* m_ProjectStore; + BuildStore* m_BuildStore; LogPaths m_LogPaths; const ZenServerOptions& m_ServerOptions; }; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..06bfea423 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -0,0 +1,526 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_StatsService.RegisterHandler("builds", *this); + + m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize >= BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector BlobHashes; + std::vector MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector Attachments; + tsl::robin_set AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h new file mode 100644 index 000000000..a59aa882a --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 809092378..0da98210c 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -377,6 +377,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); @@ -1031,6 +1034,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value>(BucketConfigs), ""); + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("stats", "", "statsd", diff --git a/src/zenserver/config.h b/src/zenserver/config.h index c7781aada..a87b6f8b3 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -59,11 +59,17 @@ struct ZenProjectStoreEvictionPolicy int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; }; +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + struct ZenGcConfig { // ZenCasEvictionPolicy Cas; ZenCacheEvictionPolicy Cache; ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; int32_t MonitorIntervalSeconds = 30; int32_t IntervalSeconds = 0; bool CollectSmallObjects = true; @@ -130,6 +136,11 @@ struct ZenProjectStoreConfig bool StoreProjectAttachmentMetaData = false; }; +struct ZenBuildStoreConfig +{ + bool Enabled = false; +}; + struct ZenWorkspacesConfig { bool Enabled = false; @@ -145,6 +156,7 @@ struct ZenServerOptions zen::HttpServerConfig HttpServerConfig; ZenStructuredCacheConfig StructuredCacheConfig; ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; std::filesystem::path SystemRootDir; // System root directory (used for machine level config) diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 8a4b977ad..0b7fd0400 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -84,7 +84,7 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con HttpWorkspacesService::~HttpWorkspacesService() { - m_StatsService.UnregisterHandler("prj", *this); + m_StatsService.UnregisterHandler("ws", *this); } const char* diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..03e269d49 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_Workspaces)); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + BuildStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "builds"; + m_BuildStore = std::make_unique(std::move(ObjCfg), m_GcManager); + } + if (ServerOptions.StructuredCacheConfig.Enabled) { InitializeStructuredCache(ServerOptions); @@ -310,6 +318,12 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_ObjStoreService); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_Http->RegisterService(*m_BuildStoreService); + } + #if ZEN_WITH_VFS m_VfsService = std::make_unique(); m_VfsService->AddService(Ref(m_ProjectStore)); @@ -327,6 +341,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, .Enabled = ServerOptions.GcConfig.Enabled, .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, @@ -347,6 +362,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_CacheStore.Get(), m_CidStore.get(), m_ProjectStore, + m_BuildStore.get(), HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, @@ -801,6 +817,9 @@ ZenServer::Cleanup() m_ObjStoreService.reset(); m_FrontendService.reset(); + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_StructuredCacheService.reset(); m_UpstreamService.reset(); m_UpstreamCache.reset(); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 80054dc35..5cfa04ba1 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include #include #include "admin/admin.h" +#include "buildstore/httpbuildstore.h" #include "cache/httpstructuredcache.h" #include "diag/diagsvcs.h" #include "frontend/frontend.h" @@ -127,6 +128,8 @@ private: Ref m_CacheStore; std::unique_ptr m_OpenProcessCache; HttpTestService m_TestService; + std::unique_ptr m_BuildStore; + #if ZEN_WITH_TESTS HttpTestingService m_TestingService; #endif @@ -140,6 +143,7 @@ private: HttpHealthService m_HealthService; std::unique_ptr m_FrontendService; std::unique_ptr m_ObjStoreService; + std::unique_ptr m_BuildStoreService; std::unique_ptr m_VfsService; std::unique_ptr m_JobQueue; std::unique_ptr m_AdminService; diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index e5b312984..c56971520 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,6 +19,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) zen::zenstore_forcelinktests(); zen::logging::InitializeLogging(); + zen::buildstore_forcelink(); zen::MaximizeOpenFileCount(); return ZEN_RUN_TESTS(argc, argv); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e976c061d..63c0388fa 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -578,7 +578,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons } void -BlockStore::WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback) +BlockStore::WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback) { ZEN_MEMSCOPE(GetBlocksTag()); ZEN_TRACE_CPU("BlockStore::WriteChunks"); diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp new file mode 100644 index 000000000..8674aab75 --- /dev/null +++ b/src/zenstore/buildstore/buildstore.cpp @@ -0,0 +1,1475 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_WITH_TESTS +# include +# include +# include +# include +#endif // ZEN_WITH_TESTS + +namespace zen { +const FLLMTag& +GetBuildstoreTag() +{ + static FLLMTag _("store", FLLMTag("builds")); + + return _; +} + +using namespace std::literals; + +namespace blobstore::impl { + + const std::string BaseName = "builds"; + const char* IndexExtension = ".uidx"; + const char* LogExtension = ".slog"; + + std::filesystem::path GetBlobIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + IndexExtension); + } + + std::filesystem::path GetBlobLogPath(const std::filesystem::path& RootDirectory) { return RootDirectory / (BaseName + LogExtension); } + + std::filesystem::path GetMetaIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + IndexExtension); + } + + std::filesystem::path GetMetaLogPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + LogExtension); + } +} // namespace blobstore::impl + +BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) +: m_Config(Config) +, m_Gc(Gc) +, m_LargeBlobStore(m_Gc) +, m_SmallBlobStore(Gc) +, m_MetadataBlockStore() +{ + ZEN_TRACE_CPU("BuildStore::BuildStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + try + { + std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); + std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); + bool IsNew = !(std::filesystem::exists(BlobLogPath) && std::filesystem::exists(MetaLogPath)); + + if (!IsNew) + { + m_BlobLogFlushPosition = ReadPayloadLog(RwLock::ExclusiveLockScope(m_Lock), BlobLogPath, 0); + m_MetaLogFlushPosition = ReadMetadataLog(RwLock::ExclusiveLockScope(m_Lock), MetaLogPath, 0); + } + m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew); + m_SmallBlobStore.Initialize(Config.RootDirectory, + "blob_cas", + m_Config.SmallBlobBlockStoreMaxBlockSize, + m_Config.SmallBlobBlockStoreAlignement, + IsNew); + m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20); + { + BlockStore::BlockIndexSet KnownBlocks; + for (const BlobEntry& Blob : m_BlobEntries) + { + if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) + { + const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; + KnownBlocks.insert(Metadata.Location.BlockIndex); + } + } + m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + } + + m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite); + m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite); + + m_Gc.AddGcReferencer(*this); + m_Gc.AddGcReferenceLocker(*this); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what()); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + } +} + +BuildStore::~BuildStore() +{ + try + { + ZEN_TRACE_CPU("BuildStore::~BuildStore"); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + Flush(); + m_MetadatalogFile.Close(); + m_PayloadlogFile.Close(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BuildStore() threw exception: {}", Ex.what()); + } +} + +void +BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) +{ + ZEN_TRACE_CPU("BuildStore::PutBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCompressedBinary); + { + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex BlobIndex = It->second; + if (m_BlobEntries[BlobIndex].Payload) + { + return; + } + } + } + + PayloadEntry Entry; + if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize) + { + CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = PayloadEntry::kStandalone}; + } + else + { + CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = 0}; + } + m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Payload) + { + m_PayloadEntries[Blob.Payload] = Entry; + } + else + { + Blob.Payload = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + // we only remove during GC and compact this then... + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } +} + +IoBuffer +BuildStore::GetBlob(const IoHash& BlobHash) +{ + ZEN_TRACE_CPU("BuildStore::GetBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + RwLock::SharedLockScope Lock(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + Blob.LastAccessTime = GcClock::TickCount(); + if (Blob.Payload) + { + const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload]; + const bool IsStandalone = (Entry.Flags & PayloadEntry::kStandalone) != 0; + Lock.ReleaseNow(); + + IoBuffer Chunk; + if (IsStandalone) + { + ZEN_TRACE_CPU("GetLarge"); + Chunk = m_LargeBlobStore.FindChunk(BlobHash); + } + else + { + ZEN_TRACE_CPU("GetSmall"); + Chunk = m_SmallBlobStore.FindChunk(BlobHash); + } + if (Chunk) + { + Chunk.SetContentType(ZenContentType::kCompressedBinary); + return Chunk; + } + else + { + ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block"); + } + } + } + return {}; +} + +std::vector +BuildStore::BlobsExists(std::span BlobHashes) +{ + ZEN_TRACE_CPU("BuildStore::BlobsExists"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector Result; + Result.reserve(BlobHashes.size()); + RwLock::SharedLockScope _(m_Lock); + for (const IoHash& BlobHash : BlobHashes) + { + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + bool HasPayload = !!Blob.Payload; + bool HasMetadata = !!Blob.Metadata; + Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata}); + } + else + { + Result.push_back({}); + } + } + return Result; +} + +void +BuildStore::PutMetadatas(std::span BlobHashes, std::span MetaDatas) +{ + ZEN_TRACE_CPU("BuildStore::PutMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + size_t WriteBlobIndex = 0; + m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span Locations) { + RwLock::ExclusiveLockScope _(m_Lock); + for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++) + { + const IoBuffer& Data = MetaDatas[WriteBlobIndex]; + const IoHash& BlobHash = BlobHashes[WriteBlobIndex]; + const BlockStoreLocation& Location = Locations[LocationIndex]; + + MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0}; + m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Metadata) + { + m_MetadataEntries[Blob.Metadata] = Entry; + } + else + { + Blob.Metadata = MetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + MetadataIndex NewMetadataIndex = MetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } + WriteBlobIndex++; + if (m_TrackedCacheKeys) + { + m_TrackedCacheKeys->insert(BlobHash); + } + } + }); +} + +std::vector +BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* OptionalWorkerPool) +{ + ZEN_TRACE_CPU("BuildStore::GetMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector MetaLocations; + std::vector MetaLocationResultIndexes; + MetaLocations.reserve(BlobHashes.size()); + MetaLocationResultIndexes.reserve(BlobHashes.size()); + tsl::robin_set ReferencedBlocks; + + std::vector Result; + std::vector ResultContentTypes; + Result.resize(BlobHashes.size()); + ResultContentTypes.resize(BlobHashes.size(), ZenContentType::kUnknownContentType); + { + RwLock::SharedLockScope _(m_Lock); + for (size_t Index = 0; Index < BlobHashes.size(); Index++) + { + const IoHash& BlobHash = BlobHashes[Index]; + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlobEntry = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlobEntry.Metadata) + { + const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata]; + MetaLocations.push_back(ExistingMetadataEntry.Location); + MetaLocationResultIndexes.push_back(Index); + ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); + ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; + } + ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::Tick()); + } + } + } + + auto DoOneBlock = [&](std::span ChunkIndexes) { + if (ChunkIndexes.size() < 4) + { + for (size_t ChunkIndex : ChunkIndexes) + { + IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]); + if (Chunk) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = std::move(Chunk); + } + } + return true; + } + return m_MetadataBlockStore.IterateBlock( + MetaLocations, + ChunkIndexes, + [&](size_t ChunkIndex, const void* Data, uint64_t Size) { + if (Data != nullptr) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size); + } + return true; + }, + [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = File.GetChunk(Offset, Size); + return true; + }, + 8u * 1024u); + }; + + if (!MetaLocations.empty()) + { + Latch WorkLatch(1); + + m_MetadataBlockStore.IterateChunks(MetaLocations, [&](uint32_t BlockIndex, std::span ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + { + return DoOneBlock(ChunkIndexes); + } + else + { + ZEN_ASSERT(OptionalWorkerPool != nullptr); + WorkLatch.AddCount(1); + try + { + OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector(ChunkIndexes.begin(), ChunkIndexes.end())]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + try + { + DoOneBlock(ChunkIndexes); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + } + catch (const std::exception& Ex) + { + WorkLatch.CountDown(); + ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + return true; + } + }); + + WorkLatch.CountDown(); + WorkLatch.Wait(); + } + for (size_t Index = 0; Index < Result.size(); Index++) + { + if (Result[Index]) + { + Result[Index].SetContentType(ResultContentTypes[Index]); + } + } + return Result; +} + +void +BuildStore::Flush() +{ + ZEN_TRACE_CPU("BuildStore::Flush"); + try + { + m_LargeBlobStore.Flush(); + m_SmallBlobStore.Flush(); + m_MetadataBlockStore.Flush(false); + + m_PayloadlogFile.Flush(); + m_MetadatalogFile.Flush(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("BuildStore::Flush failed. Reason: {}", Ex.what()); + } +} + +void +BuildStore::CompactState() +{ + ZEN_TRACE_CPU("BuildStore::CompactState"); + + std::vector BlobEntries; + std::vector PayloadEntries; + std::vector MetadataEntries; + + tsl::robin_map BlobLookup; + + RwLock::ExclusiveLockScope _(m_Lock); + const size_t EntryCount = m_BlobLookup.size(); + BlobLookup.reserve(EntryCount); + const size_t PayloadCount = m_PayloadEntries.size(); + PayloadEntries.reserve(PayloadCount); + const size_t MetadataCount = m_MetadataEntries.size(); + MetadataEntries.reserve(MetadataCount); + + for (auto LookupIt : m_BlobLookup) + { + const IoHash& BlobHash = LookupIt.first; + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const BlobIndex WriteBlobIndex(gsl::narrow(BlobEntries.size())); + BlobEntries.push_back(ReadBlobEntry); + BlobEntry& WriteBlobEntry = BlobEntries.back(); + + if (WriteBlobEntry.Payload) + { + const PayloadEntry& ReadPayloadEntry = m_PayloadEntries[ReadBlobEntry.Payload]; + WriteBlobEntry.Payload = PayloadIndex(gsl::narrow(PayloadEntries.size())); + PayloadEntries.push_back(ReadPayloadEntry); + } + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_MetadataEntries[ReadBlobEntry.Metadata]; + WriteBlobEntry.Metadata = MetadataIndex(gsl::narrow(MetadataEntries.size())); + MetadataEntries.push_back(ReadMetadataEntry); + } + + BlobLookup.insert({BlobHash, WriteBlobIndex}); + } + m_BlobEntries.swap(BlobEntries); + m_PayloadEntries.swap(PayloadEntries); + m_MetadataEntries.swap(MetadataEntries); + m_BlobLookup.swap(BlobLookup); +} + +uint64_t +BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); + if (!std::filesystem::is_regular_file(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' payload log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile CasLog; + if (!CasLog.IsValid(LogPath)) + { + std::filesystem::remove(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full payload log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const PayloadDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & PayloadEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + m_BlobLookup.erase(Record.BlobHash); + return; + } + + if (!ValidatePayloadDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid payload entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Payload) + { + const PayloadIndex ExistingPayloadIndex = ExistingBlob.Payload; + m_PayloadEntries[ExistingPayloadIndex] = Record.Entry; + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + ExistingBlob.Payload = NewPayloadIndex; + } + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid payload entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +uint64_t +BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); + if (!std::filesystem::is_regular_file(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' metadata log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile CasLog; + if (!CasLog.IsValid(LogPath)) + { + std::filesystem::remove(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full metadata log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const MetadataDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & MetadataEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + m_BlobLookup.erase(Record.BlobHash); + return; + } + + if (!ValidateMetadataDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid metadata entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Metadata) + { + const MetadataIndex ExistingMetadataIndex = ExistingBlob.Metadata; + m_MetadataEntries[ExistingMetadataIndex] = Record.Entry; + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + ExistingBlob.Metadata = NewMetadataIndex; + } + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid metadata entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +bool +BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) + { + OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.Flags, Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & PayloadEntry::kTombStone) + { + return true; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0 || Entry.Entry.Reserved3 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +bool +BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Location.Size == 0) + { + OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size); + return false; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & MetadataEntry::kTombStone) + { + return true; + } + if (Entry.Entry.ContentType == ZenContentType::kCOUNT) + { + OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +class BuildStoreGcReferenceChecker : public GcReferenceChecker +{ +public: + BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {} + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + + virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); } + + virtual void UpdateLockedState(GcCtx& Ctx) override + { + ZEN_TRACE_CPU("Builds::UpdateLockedState"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + m_References.reserve(m_Store.m_BlobLookup.size()); + for (const auto& It : m_Store.m_BlobLookup) + { + const BuildStore::BlobIndex ExistingBlobIndex = It.second; + if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload) + { + m_References.push_back(It.first); + } + } + FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References); + } + + virtual std::span GetUnusedReferences(GcCtx& Ctx, std::span IoCids) override + { + ZEN_UNUSED(Ctx); + ZEN_TRACE_CPU("Builds::GetUnusedReferences"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + size_t InitialCount = IoCids.size(); + size_t UsedCount = InitialCount; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}", + "buildstore", + UsedCount, + InitialCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + std::span UnusedReferences = KeepUnusedReferences(m_References, IoCids); + UsedCount = IoCids.size() - UnusedReferences.size(); + return UnusedReferences; + } + +private: + BuildStore& m_Store; + std::vector m_References; +}; + +std::string +BuildStore::GetGcName(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string()); +} + +class BuildStoreGcCompator : public GcStoreCompactor +{ + using BlobEntry = BuildStore::BlobEntry; + using PayloadEntry = BuildStore::PayloadEntry; + using MetadataEntry = BuildStore::MetadataEntry; + using MetadataDiskEntry = BuildStore::MetadataDiskEntry; + using BlobIndex = BuildStore::BlobIndex; + using PayloadIndex = BuildStore::PayloadIndex; + using MetadataIndex = BuildStore::MetadataIndex; + +public: + BuildStoreGcCompator(BuildStore& Store, std::vector&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {} + + virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function& ClaimDiskReserveCallback) override + { + ZEN_UNUSED(ClaimDiskReserveCallback); + ZEN_TRACE_CPU("Builds::CompactStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}", + m_Store.m_Config.RootDirectory, + NiceBytes(Stats.RemovedDisk), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + if (!m_RemovedBlobs.empty()) + { + if (Ctx.Settings.CollectSmallObjects) + { + m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique(); }); + auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); }); + + BlockStore::BlockUsageMap BlockUsage; + { + RwLock::SharedLockScope __(m_Store.m_Lock); + + for (auto LookupIt : m_Store.m_BlobLookup) + { + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex]; + + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata]; + + uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex; + uint64_t ChunkSize = RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement); + + if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end()) + { + BlockStore::BlockUsageInfo& Info = BlockUsageIt.value(); + Info.EntryCount++; + Info.DiskUsage += ChunkSize; + } + else + { + BlockUsage.insert_or_assign(BlockIndex, + BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1}); + } + } + } + } + + BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90); + BlockStoreCompactState BlockCompactState; + std::vector BlockCompactStateKeys; + BlockCompactState.IncludeBlocks(BlocksToCompact); + + if (BlocksToCompact.size() > 0) + { + { + RwLock::SharedLockScope ___(m_Store.m_Lock); + for (const auto& Entry : m_Store.m_BlobLookup) + { + BlobIndex Index = Entry.second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location)) + { + BlockCompactStateKeys.push_back(Entry.first); + } + } + } + } + + if (Ctx.Settings.IsDeleteMode) + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + + m_Store.m_MetadataBlockStore.CompactBlocks( + BlockCompactState, + m_Store.m_Config.MetadataBlockStoreAlignement, + [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + std::vector MovedEntries; + MovedEntries.reserve(MovedArray.size()); + RwLock::ExclusiveLockScope _(m_Store.m_Lock); + for (const std::pair& Moved : MovedArray) + { + size_t ChunkIndex = Moved.first; + const IoHash& Key = BlockCompactStateKeys[ChunkIndex]; + + ZEN_ASSERT(m_Store.m_TrackedCacheKeys); + if (m_Store.m_TrackedCacheKeys->contains(Key)) + { + continue; + } + + if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + m_Store.m_MetadataEntries[Meta].Location = Moved.second; + MovedEntries.push_back( + MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); + } + } + } + m_Store.m_MetadatalogFile.Append(MovedEntries); + Stats.RemovedDisk += FreedDiskSpace; + if (Ctx.IsCancelledFlag.load()) + { + return false; + } + return true; + }, + ClaimDiskReserveCallback, + fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory)); + } + else + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + } + } + } + } + } + + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + +private: + BuildStore& m_Store; + const std::vector m_RemovedBlobs; +}; + +GcStoreCompactor* +BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) +{ + ZEN_TRACE_CPU("Builds::RemoveExpiredData"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}", + m_Config.RootDirectory, + Stats.CheckedCount, + Stats.FoundCount, + Stats.DeletedCount, + NiceBytes(Stats.FreedMemory), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + }); + + const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count(); + + std::vector ExpiredBlobs; + { + RwLock::SharedLockScope __(m_Lock); + for (const auto& It : m_BlobLookup) + { + const BlobIndex ReadBlobIndex = It.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + if (AccessTick < ExpireTicks) + { + ExpiredBlobs.push_back(It.first); + } + } + Stats.CheckedCount += m_BlobLookup.size(); + Stats.FoundCount += ExpiredBlobs.size(); + } + + std::vector RemovedBlobs; + if (!ExpiredBlobs.empty()) + { + if (Ctx.Settings.IsDeleteMode) + { + RemovedBlobs.reserve(ExpiredBlobs.size()); + + std::vector RemovedPayloads; + std::vector RemoveMetadatas; + + RwLock::ExclusiveLockScope __(m_Lock); + if (Ctx.IsCancelledFlag.load()) + { + return nullptr; + } + + for (const IoHash& ExpiredBlob : ExpiredBlobs) + { + if (auto It = m_BlobLookup.find(ExpiredBlob); It != m_BlobLookup.end()) + { + const BlobIndex ReadBlobIndex = It->second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + + if (AccessTick < ExpireTicks) + { + if (ReadBlobEntry.Payload) + { + RemovedPayloads.push_back( + PayloadDiskEntry{.Entry = m_PayloadEntries[ReadBlobEntry.Payload], .BlobHash = ExpiredBlob}); + RemovedPayloads.back().Entry.Flags |= PayloadEntry::kTombStone; + m_PayloadEntries[ReadBlobEntry.Payload] = {}; + m_BlobEntries[ReadBlobIndex].Payload = {}; + } + if (ReadBlobEntry.Metadata) + { + RemoveMetadatas.push_back( + MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob}); + RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; + m_MetadataEntries[ReadBlobEntry.Metadata] = {}; + m_BlobEntries[ReadBlobIndex].Metadata = {}; + } + + m_BlobLookup.erase(It); + + RemovedBlobs.push_back(ExpiredBlob); + Stats.DeletedCount++; + } + } + } + if (!RemovedPayloads.empty()) + { + m_PayloadlogFile.Append(RemovedPayloads); + } + if (!RemoveMetadatas.empty()) + { + m_MetadatalogFile.Append(RemoveMetadatas); + } + } + } + + if (!RemovedBlobs.empty()) + { + CompactState(); + } + + return new BuildStoreGcCompator(*this, std::move(RemovedBlobs)); +} + +std::vector +BuildStore::CreateReferenceCheckers(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + return {new BuildStoreGcReferenceChecker(*this)}; +} + +std::vector +BuildStore::CreateReferenceValidators(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + return {}; +} + +std::vector +BuildStore::LockState(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + std::vector Locks; + Locks.emplace_back(RwLock::SharedLockScope(m_Lock)); + return Locks; +} + +/* + ___________ __ + \__ ___/___ _______/ |_ ______ + | |_/ __ \ / ___/\ __\/ ___/ + | |\ ___/ \___ \ | | \___ \ + |____| \___ >____ > |__| /____ > + \/ \/ \/ +*/ + +#if ZEN_WITH_TESTS + +TEST_CASE("BuildStore.Blobs") +{ + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } +} + +namespace blockstore::testing { + IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector>& KeyValues) + { + CbObjectWriter Writer; + Writer.AddHash("rawHash"sv, BlobHash); + Writer.BeginObject("values"); + { + for (const auto& V : KeyValues) + { + Writer.AddString(V.first, V.second); + } + } + Writer.EndObject(); // values + return Writer.Save().GetBuffer().AsIoBuffer(); + }; + +} // namespace blockstore::testing + +TEST_CASE("BuildStore.Metadata") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector BlobHashes; + std::vector MetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I))); + MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); + MetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(BlobHashes, MetaPayloads); + + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + for (const IoHash& BlobHash : BlobHashes) + { + CHECK(!Store.GetBlob(BlobHash)); + } + } + std::vector CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + for (const auto& MetadataIt : MetadataPayloads) + { + CHECK(!MetadataIt); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } + + std::vector BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK_EQ(IoHash::HashBuffer(MetadataPayload), IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + + BlobMetaPayloads.clear(); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back( + MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } +} + +TEST_CASE("BuildStore.GC") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector CompressedBlobsHashes; + std::vector BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), + .CollectSmallObjects = false, + .IsDeleteMode = false, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash); + } + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() + std::chrono::hours(1), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(!Blob); + } + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(!MetadataPayload); + } + } + } +} + +void +buildstore_forcelink() +{ +} + +#endif + +} // namespace zen diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2be0542db..b64bc26dd 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -226,7 +226,7 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) } std::vector -CasContainerStrategy::InsertChunks(std::span Chunks, std::span ChunkHashes) +CasContainerStrategy::InsertChunks(std::span Chunks, std::span ChunkHashes) { ZEN_MEMSCOPE(GetCasContainerTag()); @@ -323,7 +323,7 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks) } bool -CasContainerStrategy::IterateChunks(std::span ChunkHashes, +CasContainerStrategy::IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit) diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 07e620086..2eb4c233a 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -52,11 +52,11 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore ~CasContainerStrategy(); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); - std::vector InsertChunks(std::span Chunks, std::span ChunkHashes); + std::vector InsertChunks(std::span Chunks, std::span ChunkHashes); IoBuffer FindChunk(const IoHash& ChunkHash); bool HaveChunk(const IoHash& ChunkHash); void FilterChunks(HashKeySet& InOutChunks); - bool IterateChunks(std::span ChunkHashes, + bool IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 7ac10d613..fe5ae284b 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1081,7 +1081,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size()); { ZEN_TRACE_CPU("GcV2::LockReferencers"); - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceLockers Latch WorkLeft(1); { @@ -1108,7 +1108,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::UpdateLockedState"); // Locking all references checkers so we have a steady state of which references are used - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceCheckers Latch WorkLeft(1); @@ -1739,6 +1739,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons { Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime); Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime); + Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime); Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects; Writer << "IsDeleteMode"sv << Settings.IsDeleteMode; Writer << "SkipCidDelete"sv << Settings.SkipCidDelete; @@ -1940,6 +1941,7 @@ GcScheduler::SchedulerThread() std::chrono::seconds LightweightGcInterval = m_Config.LightweightInterval; std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; + std::chrono::seconds MaxBuildStoreDuration = m_Config.MaxBuildStoreDuration; uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; bool SkipCid = false; GcVersion UseGCVersion = m_Config.UseGCVersion; @@ -1975,6 +1977,10 @@ GcScheduler::SchedulerThread() { MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration; } + if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max()) + { + MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration; + } if (TriggerParams.DiskSizeSoftLimit != 0) { DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit; @@ -2046,6 +2052,8 @@ GcScheduler::SchedulerThread() MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration; GcClock::TimePoint ProjectStoreExpireTime = MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration; + GcClock::TimePoint BuildStoreExpireTime = + MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration; const GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); @@ -2102,6 +2110,10 @@ GcScheduler::SchedulerThread() { ProjectStoreExpireTime = SizeBasedExpireTime; } + if (SizeBasedExpireTime > BuildStoreExpireTime) + { + BuildStoreExpireTime = SizeBasedExpireTime; + } } std::chrono::seconds RemainingTimeUntilGc = @@ -2227,6 +2239,7 @@ GcScheduler::SchedulerThread() bool GcSuccess = CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, + BuildStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, @@ -2333,6 +2346,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time bool GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, @@ -2416,6 +2430,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcSettings Settings = {.CacheExpireTime = CacheExpireTime, .ProjectStoreExpireTime = ProjectStoreExpireTime, + .BuildStoreExpireTime = BuildStoreExpireTime, .CollectSmallObjects = CollectSmallObjects, .IsDeleteMode = Delete, .SkipCidDelete = SkipCid, @@ -2447,6 +2462,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, } SB.Append(fmt::format(" Cache cutoff time: {}\n", Settings.CacheExpireTime)); SB.Append(fmt::format(" Project store cutoff time: {}\n", Settings.ProjectStoreExpireTime)); + SB.Append(fmt::format(" Build store cutoff time: {}\n", Settings.BuildStoreExpireTime)); }; { @@ -2552,6 +2568,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, if (Delete) { GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime); + KeepRangeStart = Min(KeepRangeStart, BuildStoreExpireTime); m_LastGcExpireTime = KeepRangeStart; std::unique_lock Lock(m_GcMutex); m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count()); diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h new file mode 100644 index 000000000..a28dc908b --- /dev/null +++ b/src/zenstore/include/zenstore/accesstime.h @@ -0,0 +1,47 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen { + +// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch +struct AccessTime +{ + explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} + AccessTime& operator=(GcClock::Tick Tick) noexcept + { + SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); + return *this; + } + operator GcClock::Tick() const noexcept + { + return std::chrono::duration_cast(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) + .count(); + } + + AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime& operator=(AccessTime&& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + AccessTime& operator=(const AccessTime& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + +private: + static uint32_t ToSeconds(GcClock::Tick Tick) + { + return gsl::narrow(std::chrono::duration_cast(GcClock::Duration(Tick)).count()); + } + std::atomic_uint32_t SecondsSinceEpoch; +}; + +} // namespace zen diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 97357e5cb..0c72a13aa 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -156,7 +156,7 @@ public: void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback); typedef std::function Locations)> WriteChunksCallback; - void WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback); + void WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback); IoBuffer TryGetChunk(const BlockStoreLocation& Location) const; void Flush(bool ForceNewBlock); diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h new file mode 100644 index 000000000..302af5f9c --- /dev/null +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -0,0 +1,186 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include "../compactcas.h" +#include "../filecas.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +struct BuildStoreConfig +{ + std::filesystem::path RootDirectory; + uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024; + uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024; + uint32_t SmallBlobBlockStoreAlignement = 16; + uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024; + uint32_t MetadataBlockStoreAlignement = 8; +}; + +class BuildStore : public GcReferencer, public GcReferenceLocker //, public GcStorage +{ +public: + explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc); + virtual ~BuildStore(); + + void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload); + IoBuffer GetBlob(const IoHash& BlobHashes); + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + std::vector BlobsExists(std::span BlobHashes); + + void PutMetadatas(std::span BlobHashes, std::span MetaDatas); + std::vector GetMetadatas(std::span BlobHashes, WorkerThreadPool* OptionalWorkerPool); + + void Flush(); + +private: + void CompactState(); + + uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + + //////// GcReferencer + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual std::vector CreateReferenceCheckers(GcCtx& Ctx) override; + virtual std::vector CreateReferenceValidators(GcCtx& Ctx) override; + + //////// GcReferenceLocker + virtual std::vector LockState(GcCtx& Ctx) override; + +#pragma pack(push) +#pragma pack(1) + struct PayloadEntry + { + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value + + uint8_t Flags = 0; + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + }; + static_assert(sizeof(PayloadEntry) == 4); + + struct PayloadDiskEntry + { + PayloadEntry Entry; // 4 bytes + IoHash BlobHash; // 20 bytes + }; + static_assert(sizeof(PayloadDiskEntry) == 24); + + struct MetadataEntry + { + BlockStoreLocation Location; // 12 bytes + + ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + uint8_t Flags = 0; // 1 byte + + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + }; + static_assert(sizeof(MetadataEntry) == 16); + + struct MetadataDiskEntry + { + MetadataEntry Entry; // 16 bytes + IoHash BlobHash; // 20 bytes + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + uint8_t Reserved4 = 0; + }; + static_assert(sizeof(MetadataDiskEntry) == 40); + +#pragma pack(pop) + + static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason); + static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason); + + struct PayloadIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + PayloadIndex() = default; + explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const PayloadIndex& Other) const = default; + }; + + struct MetadataIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + MetadataIndex() = default; + explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const MetadataIndex& Other) const = default; + }; + + struct BlobIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + BlobIndex() = default; + explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const BlobIndex& Other) const = default; + }; + + struct BlobEntry + { + PayloadIndex Payload; + MetadataIndex Metadata; + AccessTime LastAccessTime; + }; + static_assert(sizeof(BlobEntry) == 12); + + const BuildStoreConfig m_Config; + GcManager& m_Gc; + + RwLock m_Lock; + + std::vector m_PayloadEntries; + std::vector m_MetadataEntries; + + std::vector m_BlobEntries; + tsl::robin_map m_BlobLookup; + + FileCasStrategy m_LargeBlobStore; + CasContainerStrategy m_SmallBlobStore; + BlockStore m_MetadataBlockStore; + + TCasLogFile m_PayloadlogFile; + TCasLogFile m_MetadatalogFile; + uint64_t m_BlobLogFlushPosition = 0; + uint64_t m_MetaLogFlushPosition = 0; + + std::unique_ptr m_TrackedCacheKeys; + + friend class BuildStoreGcReferenceChecker; + friend class BuildStoreGcReferencePruner; + friend class BuildStoreGcCompator; +}; + +void buildstore_forcelink(); + +} // namespace zen diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 05400c784..5a51718d3 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -5,6 +5,7 @@ #include "cacheshared.h" #include +#include #include #include diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 521c78bb1..ef1b803de 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -72,42 +72,4 @@ struct CacheContentStats bool IsKnownBadBucketName(std::string_view BucketName); bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer); -////////////////////////////////////////////////////////////////////////// - -// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch -struct AccessTime -{ - explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} - AccessTime& operator=(GcClock::Tick Tick) noexcept - { - SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); - return *this; - } - operator GcClock::Tick() const noexcept - { - return std::chrono::duration_cast(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) - .count(); - } - - AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime& operator=(AccessTime&& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - AccessTime& operator=(const AccessTime& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - -private: - static uint32_t ToSeconds(GcClock::Tick Tick) - { - return gsl::narrow(std::chrono::duration_cast(GcClock::Duration(Tick)).count()); - } - std::atomic_uint32_t SecondsSinceEpoch; -}; - } // namespace zen diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 3daae0a93..67aadef71 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -55,6 +55,7 @@ struct GcSettings { GcClock::TimePoint CacheExpireTime = GcClock::Now(); GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now(); + GcClock::TimePoint BuildStoreExpireTime = GcClock::Now(); bool CollectSmallObjects = false; bool IsDeleteMode = false; bool SkipCidDelete = false; @@ -412,6 +413,7 @@ struct GcSchedulerConfig std::chrono::seconds Interval{}; std::chrono::seconds MaxCacheDuration{86400}; std::chrono::seconds MaxProjectStoreDuration{604800}; + std::chrono::seconds MaxBuildStoreDuration{604800}; bool CollectSmallObjects = true; bool Enabled = true; uint64_t DiskReserveSize = 1ul << 28; @@ -496,6 +498,7 @@ public: bool CollectSmallObjects = false; std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max(); std::chrono::seconds MaxProjectStoreDuration = std::chrono::seconds::max(); + std::chrono::seconds MaxBuildStoreDuration = std::chrono::seconds::max(); uint64_t DiskSizeSoftLimit = 0; bool SkipCid = false; bool SkipDelete = false; @@ -528,6 +531,7 @@ private: void SchedulerThread(); bool CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp new file mode 100644 index 000000000..c95215889 --- /dev/null +++ b/src/zenutil/buildstoragecache.cpp @@ -0,0 +1,362 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +class ZenBuildStorageCache : public BuildStorageCache +{ +public: + explicit ZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) + : m_HttpClient(HttpClient) + , m_Stats(Stats) + , m_Namespace(Namespace.empty() ? "none" : Namespace) + , m_Bucket(Bucket.empty() ? "none" : Bucket) + , m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred()) + , m_BackgroundWorkPool(1) + , m_PendingBackgroundWorkCount(1) + , m_CancelBackgroundWork(false) + { + } + + virtual ~ZenBuildStorageCache() + { + try + { + m_CancelBackgroundWork.store(true); + m_PendingBackgroundWorkCount.CountDown(); + m_PendingBackgroundWorkCount.Wait(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~ZenBuildStorageCache() failed with: {}", Ex.what()); + } + } + + void ScheduleBackgroundWork(std::function&& Work) + { + m_PendingBackgroundWorkCount.AddCount(1); + try + { + m_BackgroundWorkPool.ScheduleWork([this, Work = std::move(Work)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::BackgroundWork"); + auto _ = MakeGuard([this]() { m_PendingBackgroundWorkCount.CountDown(); }); + if (!m_CancelBackgroundWork) + { + try + { + Work(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed executing background upload to build cache. Reason: {}", Ex.what()); + } + } + }); + } + catch (const std::exception& Ex) + { + m_PendingBackgroundWorkCount.CountDown(); + ZEN_ERROR("Failed scheduling background upload to build cache. Reason: {}", Ex.what()); + } + } + + virtual void PutBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + const CompositeBuffer& Payload) override + { + ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); + ScheduleBackgroundWork( + [this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBuildBlob"); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::Response CacheResponse = + m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + Payload, + ContentType); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlob"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::KeyValueMap Headers; + if (RangeOffset != 0 || RangeBytes != (uint64_t)-1) + { + Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)}); + } + CreateDirectories(m_TempFolderPath); + HttpClient::Response CacheResponse = + m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + m_TempFolderPath, + Headers); + AddStatistic(CacheResponse); + if (CacheResponse.IsSuccess()) + { + return CacheResponse.ResponsePayload; + } + return {}; + } + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) override + { + ScheduleBackgroundWork([this, + BuildId = Oid(BuildId), + BlobRawHashes = std::vector(BlobHashes.begin(), BlobHashes.end()), + MetaDatas = std::vector(MetaDatas.begin(), MetaDatas.end())]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + const uint64_t BlobCount = BlobRawHashes.size(); + + CbPackage RequestPackage; + std::vector Attachments; + tsl::robin_set AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter RequestWriter; + RequestWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + RequestWriter.AddHash(BlobRawHashes[BlockHashIndex]); + } + RequestWriter.EndArray(); // blobHashes + + RequestWriter.BeginArray("metadatas"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + const IoHash ObjectHash = MetaDatas[BlockHashIndex].GetHash(); + RequestWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(MetaDatas[BlockHashIndex], ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + + RequestWriter.EndArray(); // metadatas + + RequestPackage.SetObject(RequestWriter.Save()); + } + RequestPackage.AddAttachments(Attachments); + + CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage); + + HttpClient::Response CacheResponse = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/putBlobMetadata", m_Namespace, m_Bucket, BuildId), + RpcRequestBuffer, + ZenContentType::kCbPackage); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob metadata to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/getBlobMetadata", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + std::vector Result; + + CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload); + CbObject ResponseObject = ResponsePackage.GetObject(); + + CbArrayView BlobHashArray = ResponseObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView(); + Result.reserve(MetadatasArray.Num()); + auto BlobHashesIt = BlobHashes.begin(); + auto BlobHashArrayIt = begin(BlobHashArray); + auto MetadataArrayIt = begin(MetadatasArray); + while (MetadataArrayIt != end(MetadatasArray)) + { + const IoHash BlobHash = (*BlobHashArrayIt).AsHash(); + while (BlobHash != *BlobHashesIt) + { + ZEN_ASSERT(BlobHashesIt != BlobHashes.end()); + BlobHashesIt++; + } + + ZEN_ASSERT(BlobHash == *BlobHashesIt); + + const IoHash MetaHash = (*MetadataArrayIt).AsAttachment(); + const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash); + ZEN_ASSERT(MetaAttachment); + + CbObject Metadata = MetaAttachment->AsObject(); + Result.emplace_back(std::move(Metadata)); + + BlobHashArrayIt++; + MetadataArrayIt++; + BlobHashesIt++; + } + return Result; + } + return {}; + } + + virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::BlobsExists"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/exists", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + CbObject ResponseObject = LoadCompactBinaryObject(Response.ResponsePayload); + if (!ResponseObject) + { + throw std::runtime_error("BlobExists reponse is invalid, failed to load payload as compact binary object"); + } + CbArrayView BlobsExistsArray = ResponseObject["blobExists"sv].AsArrayView(); + if (!BlobsExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'blobExists' array is missing"); + } + if (BlobsExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error(fmt::format("BlobExists reponse is invalid, 'blobExists' array contains {} entries, expected {}", + BlobsExistsArray.Num(), + BlobHashes.size())); + } + + CbArrayView MetadatasExistsArray = ResponseObject["metadataExists"sv].AsArrayView(); + if (!MetadatasExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'metadataExists' array is missing"); + } + if (MetadatasExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error( + fmt::format("BlobExists reponse is invalid, 'metadataExists' array contains {} entries, expected {}", + MetadatasExistsArray.Num(), + BlobHashes.size())); + } + + std::vector Result; + Result.reserve(BlobHashes.size()); + auto BlobExistsIt = begin(BlobsExistsArray); + auto MetadataExistsIt = begin(MetadatasExistsArray); + while (BlobExistsIt != end(BlobsExistsArray)) + { + ZEN_ASSERT(MetadataExistsIt != end(MetadatasExistsArray)); + + const bool HasBody = (*BlobExistsIt).AsBool(); + const bool HasMetadata = (*MetadataExistsIt).AsBool(); + + Result.push_back({.HasBody = HasBody, .HasMetadata = HasMetadata}); + + BlobExistsIt++; + MetadataExistsIt++; + } + return Result; + } + return {}; + } + +private: + void AddStatistic(const HttpClient::Response& Result) + { + m_Stats.TotalBytesWritten += Result.UploadedBytes; + m_Stats.TotalBytesRead += Result.DownloadedBytes; + m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); + m_Stats.TotalRequestCount++; + } + + HttpClient& m_HttpClient; + BuildStorageCache::Statistics& m_Stats; + const std::string m_Namespace; + const std::string m_Bucket; + const std::filesystem::path m_TempFolderPath; + + WorkerThreadPool m_BackgroundWorkPool; + Latch m_PendingBackgroundWorkCount; + std::atomic m_CancelBackgroundWork; +}; + +std::unique_ptr +CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) +{ + return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath); +} + +} // namespace zen diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index f3c14edc4..abfc0fb63 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -52,7 +52,7 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) return {}; } std::vector Result; - CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); + CbArrayView Blocks = BlocksObject["blocks"sv].AsArrayView(); Result.reserve(Blocks.Num()); for (CbFieldView BlockView : Blocks) { diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 130fec355..f040e9ece 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,18 +442,19 @@ public: SimulateLatency(0, 0); } - virtual std::vector FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(BuildId), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); for (const std::filesystem::path& MetaDataFile : Content.Files) { IoHash ChunkHash; @@ -467,24 +468,28 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } } - SimulateLatency(0, sizeof(IoHash) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) override { ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); + for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); @@ -495,10 +500,12 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } - SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 2ebd65a00..f8c7c012c 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -54,9 +54,9 @@ public: uint64_t ChunkSize, std::function&& Receiver) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual std::vector FindBlocks(const Oid& BuildId) = 0; - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h new file mode 100644 index 000000000..08c936bf5 --- /dev/null +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include + +namespace zen { + +class HttpClient; + +class BuildStorageCache +{ +public: + struct Statistics + { + std::atomic TotalBytesRead = 0; + std::atomic TotalBytesWritten = 0; + std::atomic TotalRequestCount = 0; + std::atomic TotalRequestTimeUs = 0; + std::atomic TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorageCache() {} + + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) = 0; + virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) = 0; + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; +}; + +std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath); +} // namespace zen diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 758722156..cd28bdcb2 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -27,7 +27,6 @@ public: { ZEN_MEMSCOPE(ELLMTag::Logging); - ZEN_MEMSCOPE(ELLMTag::Logging); std::error_code Ec; if (RotateOnOpen) { diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index d70fd8c00..b6d9e3990 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -49,7 +49,7 @@ public: { throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode)); } - return PayloadToJson("Failed listing builds"sv, ListResult.Response); + return PayloadToCbObject("Failed listing builds"sv, ListResult.Response); } virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override @@ -66,7 +66,7 @@ public: { throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); + return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); } virtual CbObject GetBuild(const Oid& BuildId) override @@ -81,7 +81,7 @@ public: { throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); } virtual void FinalizeBuild(const Oid& BuildId) override @@ -134,7 +134,7 @@ public: GetBuildPartResult.Reason, GetBuildPartResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); } virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override @@ -289,7 +289,7 @@ public: } } - virtual std::vector FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); @@ -301,10 +301,10 @@ public: { throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode)); } - return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response)); + return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response); } - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) override { ZEN_TRACE_CPU("Jupiter::GetBlockMetadata"); @@ -328,24 +328,7 @@ public: throw std::runtime_error( fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); } - std::vector UnorderedList = - ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response)); - tsl::robin_map BlockDescriptionLookup; - for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) - { - const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; - BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); - } - std::vector SortedBlockDescriptions; - SortedBlockDescriptions.reserve(BlockDescriptionLookup.size()); - for (const IoHash& BlockHash : BlockHashes) - { - if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) - { - SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second])); - } - } - return SortedBlockDescriptions; + return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response); } virtual void PutBuildPartStats(const Oid& BuildId, @@ -373,7 +356,7 @@ public: } private: - static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload) + static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload) { if (Payload.GetContentType() == ZenContentType::kJSON) { -- cgit v1.2.3 From df9bbea4fb1dad9e8c853f14600259a4af12d334 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Wed, 26 Mar 2025 16:57:48 -0600 Subject: Descriptive type conversion messages Handling decompression or validation errors with more descriptive messages --- src/zenserver/projectstore/httpprojectstore.cpp | 54 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index f9a13220a..6313fd69e 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -896,27 +896,51 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || - AcceptType == ZenContentType::kCbObject) + if (DecompressedBuffer) { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) { - m_ProjectStats.BadRequestCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); - return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, - HttpContentType::kText, - fmt::format("chunk - '{}' WRONGTYPE", Cid)); - } + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } - m_ProjectStats.ChunkHitCount++; - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } } else { - Value = DecompressedBuffer; - Value.SetContentType(ZenContentType::kBinary); + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); } } m_ProjectStats.ChunkHitCount++; -- cgit v1.2.3 From a0a0dba13317533f882a85b7f4087588cfa09066 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 14:09:01 +0100 Subject: optional compress of block chunks (#326) - Feature: zenserver: Add command line option `--gc-buildstore-duration-seconds` to control GC life time of build store data - Improvement: ELF and MachO executable files are no longer chunked - Improvement: Compress chunks in blocks that encloses a full file (such as small executables) - Bugfix: Strip path delimiter at end of string in StringToPath --- src/zen/cmds/builds_cmd.cpp | 187 ++++++++++++++++------- src/zencore/filesystem.cpp | 9 +- src/zenhttp/httpclient.cpp | 2 +- src/zenserver/config.cpp | 10 ++ src/zenstore/buildstore/buildstore.cpp | 31 +++- src/zenutil/chunkingcontroller.cpp | 39 ++++- src/zenutil/include/zenutil/chunkingcontroller.h | 4 + 7 files changed, 216 insertions(+), 66 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b2ad579f1..08d30948b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -82,9 +82,10 @@ namespace { size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize; }; - const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, - .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; - + const ChunksBlockParameters DefaultChunksBlockParams{ + .MaxBlockSize = 32u * 1024u * 1024u, + .MaxChunkEmbedSize = 2u * 1024u * 1024u // DefaultChunkedParams.MaxSize + }; const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u; const double DefaultLatency = 0; // .0010; @@ -92,6 +93,8 @@ namespace { const bool SingleThreaded = false; + const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; + const std::string ZenFolderName = ".zen"; const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); @@ -511,6 +514,7 @@ namespace { uint64_t AcceptedBlockCount = 0; uint64_t AcceptedChunkCount = 0; uint64_t AcceptedByteCount = 0; + uint64_t AcceptedRawByteCount = 0; uint64_t RejectedBlockCount = 0; uint64_t RejectedChunkCount = 0; uint64_t RejectedByteCount = 0; @@ -549,6 +553,7 @@ namespace { uint64_t ChunkCount = 0; uint64_t ChunkByteCount = 0; std::atomic CompressedChunkCount = 0; + std::atomic CompressedChunkRawBytes = 0; std::atomic CompressedChunkBytes = 0; uint64_t CompressChunksElapsedWallTimeUS = 0; @@ -557,6 +562,7 @@ namespace { ChunkCount += Rhs.ChunkCount; ChunkByteCount += Rhs.ChunkByteCount; CompressedChunkCount += Rhs.CompressedChunkCount; + CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes; CompressedChunkBytes += Rhs.CompressedChunkBytes; CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; return *this; @@ -1369,7 +1375,15 @@ namespace { ZEN_ASSERT(false); } uint64_t RawSize = Chunk.GetSize(); - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + // Standalone chunk, not part of a sequence + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid)}; + } + else + { + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + } })); } @@ -1393,13 +1407,24 @@ namespace { { std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); - CompositeBuffer CompressedChunk = - CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + + const uint64_t RawSize = Chunk.GetSize(); + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + CompositeBuffer CompressedChunk = CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } + else + { + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } } return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); }; @@ -1805,7 +1830,6 @@ namespace { const ChunkedContentLookup& Lookup, uint32_t ChunkIndex, const std::filesystem::path& TempFolderPath, - std::atomic& ReadRawBytes, LooseChunksStatistics& LooseChunksStats) { ZEN_TRACE_CPU("CompressChunk"); @@ -1841,7 +1865,7 @@ namespace { CompositeBuffer(SharedBuffer(RawSource)), [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { ZEN_UNUSED(SourceOffset); - ReadRawBytes += SourceSize; + LooseChunksStats.CompressedChunkRawBytes += SourceSize; CompressedFile.Write(RangeBuffer, Offset); LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); }); @@ -2330,16 +2354,8 @@ namespace { std::atomic GeneratedBlockCount = 0; std::atomic GeneratedBlockByteCount = 0; - std::vector CompressLooseChunkOrderIndexes; - std::atomic QueuedPendingInMemoryBlocksForUpload = 0; - // Start upload of any pre-compressed loose chunks - for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) - { - CompressLooseChunkOrderIndexes.push_back(LooseChunkOrderIndex); - } - // Start generation of any non-prebuilt blocks and schedule upload for (const size_t BlockIndex : BlockIndexes) { @@ -2399,12 +2415,10 @@ namespace { } } - std::atomic RawLooseChunkByteCount = 0; - // Start compression of any non-precompressed loose chunks and schedule upload - for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) + for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { - const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; + const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; Work.ScheduleWork( SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, ChunkIndex](std::atomic&) { @@ -2413,20 +2427,15 @@ namespace { ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); FilteredCompressedBytesPerSecond.Start(); - CompositeBuffer Payload = CompressChunk(Path, - Content, - Lookup, - ChunkIndex, - Path / ZenTempChunkFolderName, - RawLooseChunkByteCount, - LooseChunksStats); + CompositeBuffer Payload = + CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName, LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), NiceBytes(Payload.GetSize())); const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; UploadStats.ReadFromDiskBytes += ChunkRawSize; - if (LooseChunksStats.CompressedChunkCount == CompressLooseChunkOrderIndexes.size()) + if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) { FilteredCompressedBytesPerSecond.Stop(); } @@ -2441,7 +2450,7 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkBytes.load()); + FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); @@ -2452,8 +2461,8 @@ namespace { "Uploaded {}/{} ({}/{}) blobs " "({} {}bits/s)", LooseChunksStats.CompressedChunkCount.load(), - CompressLooseChunkOrderIndexes.size(), - NiceBytes(RawLooseChunkByteCount), + LooseChunkOrderIndexes.size(), + NiceBytes(LooseChunksStats.CompressedChunkRawBytes), NiceBytes(TotalLooseChunksSize), NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()), @@ -2582,9 +2591,10 @@ namespace { for (size_t KnownBlockIndex : ReuseBlockIndexes) { std::vector FoundChunkIndexes; - size_t BlockSize = 0; - size_t AdjustedReuseSize = 0; - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + size_t BlockSize = 0; + size_t AdjustedReuseSize = 0; + size_t AdjustedRawReuseSize = 0; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; for (size_t BlockChunkIndex = 0; BlockChunkIndex < KnownBlock.ChunkRawHashes.size(); BlockChunkIndex++) { const IoHash& BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex]; @@ -2597,6 +2607,7 @@ namespace { { FoundChunkIndexes.push_back(ChunkIndex); AdjustedReuseSize += KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + AdjustedRawReuseSize += KnownBlock.ChunkRawLengths[BlockChunkIndex]; } } } @@ -2617,6 +2628,7 @@ namespace { } FindBlocksStats.AcceptedChunkCount += FoundChunkIndexes.size(); FindBlocksStats.AcceptedByteCount += AdjustedReuseSize; + FindBlocksStats.AcceptedRawByteCount += AdjustedRawReuseSize; FindBlocksStats.AcceptedReduntantChunkCount += KnownBlock.ChunkRawHashes.size() - FoundChunkIndexes.size(); FindBlocksStats.AcceptedReduntantByteCount += BlockSize - AdjustedReuseSize; } @@ -3020,9 +3032,10 @@ namespace { } FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size(); - const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 - ? (100.0 * FindBlocksStats.AcceptedByteCount / FindBlocksStats.PotentialChunkByteCount) - : 0.0; + const double AcceptedByteCountPercent = + FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * FindBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; const double AcceptedReduntantByteCountPercent = FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / @@ -3042,7 +3055,7 @@ namespace { NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), FindBlocksStats.AcceptedChunkCount, - NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.AcceptedBlockCount, AcceptedByteCountPercent, @@ -3276,8 +3289,8 @@ namespace { TempLooseChunksStats.CompressedChunkCount.load(), NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, - TempLooseChunksStats.CompressedChunkBytes)), + NiceNum( + GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), TempUploadStats.ChunkCount.load(), NiceBytes(TempUploadStats.ChunksBytes), @@ -3435,6 +3448,7 @@ namespace { "\n AcceptedBlockCount: {}" "\n AcceptedChunkCount: {}" "\n AcceptedByteCount: {}" + "\n AcceptedRawByteCount: {}" "\n RejectedBlockCount: {}" "\n RejectedChunkCount: {}" "\n RejectedByteCount: {}" @@ -3452,6 +3466,7 @@ namespace { FindBlocksStats.AcceptedBlockCount, FindBlocksStats.AcceptedChunkCount, NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.RejectedBlockCount, FindBlocksStats.RejectedChunkCount, NiceBytes(FindBlocksStats.RejectedByteCount), @@ -3627,7 +3642,7 @@ namespace { {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, {"reusedRatio", AcceptedByteCountPercent / 100.0}, {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, - {"reusedBlockByteCount", double(FindBlocksStats.AcceptedByteCount)}, + {"reusedBlockByteCount", double(FindBlocksStats.AcceptedRawByteCount)}, {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, @@ -4192,11 +4207,36 @@ namespace { bool NeedsWrite = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) { - MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock + CompressedBuffer::GetHeaderSizeForNoneEncoder(), - ChunkCompressedSize - CompressedBuffer::GetHeaderSizeForNoneEncoder()); - IoBuffer Decompressed(IoBuffer::Wrap, ChunkMemoryView.GetData(), ChunkMemoryView.GetSize()); - ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize); + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); + ZEN_ASSERT(CompressedChunk); + ZEN_ASSERT(VerifyChunkHash == ChunkHash); + ZEN_ASSERT(VerifyChunkSize == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + OodleCompressor ChunkCompressor; + OodleCompressionLevel ChunkCompressionLevel; + uint64_t ChunkBlockSize; + + bool GetCompressParametersSuccess = + CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize); + ZEN_ASSERT(GetCompressParametersSuccess); + + IoBuffer Decompressed; + if (ChunkCompressionLevel == OodleCompressionLevel::None) + { + MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()); + Decompressed = + IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize()); + } + else + { + Decompressed = CompressedChunk.Decompress().AsIoBuffer(); + } ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { OutOps.WriteOps.push_back( @@ -6208,7 +6248,17 @@ namespace { ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); SetFileReadOnly(LocalFilePath, false); ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); - std::filesystem::rename(LocalFilePath, CacheFilePath); + std::error_code Ec; + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); } ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", @@ -6343,7 +6393,17 @@ namespace { { SetFileReadOnly(FirstTargetFilePath, false); } - std::filesystem::rename(CacheFilePath, FirstTargetFilePath); + std::error_code Ec; + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } @@ -7893,15 +7953,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Result; + std::string StorageDescription; + std::string CacheDescription; + if (!m_BuildsUrl.empty()) { ParseAuthOptions(); Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); - ZEN_CONSOLE("Using cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); + StorageDescription = fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); Result.StorageName = ZEN_CLOUD_STORAGE; @@ -7909,7 +7972,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); - ZEN_CONSOLE("Using folder {}", StoragePath); + StorageDescription = fmt::format("folder {}", StoragePath); Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); } @@ -7930,12 +7993,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); + CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(" {}.", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" {}.", m_Bucket); + } } else { Result.CacheHttp.reset(); } } + ZEN_CONSOLE("Remote: {}.{}", + StorageDescription, + CacheDescription.empty() ? std::string("") : fmt::format(" Cache: {}", CacheDescription)); return Result; }; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 05e2bf049..8a369f02e 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2046,14 +2046,17 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) std::filesystem::path StringToPath(const std::string_view& Path) { + std::string_view UnquotedPath = Path; + if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') { - return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred(); + UnquotedPath = Path.substr(1, Path.length() - 2); } - else + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) { - return std::filesystem::path(Path).make_preferred(); + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); } + return std::filesystem::path(UnquotedPath).make_preferred(); } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index fe5232d89..f3baf37ce 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -325,7 +325,7 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT && HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED) { - ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + ZEN_WARN("HttpClient client failure (session: {}): {}", SessionId, HttpResponse); } // Client side failure code diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 0da98210c..d3af0c6a6 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -495,6 +495,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); ////// security LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); @@ -961,6 +964,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), ""); + options.add_option("gc", + "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + options.add_option("gc", "", "disk-reserve-size", diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 8674aab75..eb36be049 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -340,7 +340,7 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; } - ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::Tick()); + ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount()); } } } @@ -543,7 +543,17 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys if (Record.Entry.Flags & PayloadEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation - m_BlobLookup.erase(Record.BlobHash); + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Metadata) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Payload = {}; + } + } return; } @@ -575,7 +585,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys m_PayloadEntries.push_back(Record.Entry); const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); - m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); } }, @@ -635,7 +645,18 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy if (Record.Entry.Flags & MetadataEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation - m_BlobLookup.erase(Record.BlobHash); + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Payload) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Metadata = {}; + } + } return; } @@ -667,7 +688,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy m_MetadataEntries.push_back(Record.Entry); const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); - m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); } }, diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index 2a7057a46..a5ebce193 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -41,9 +41,13 @@ class BasicChunkingController : public ChunkingController { public: BasicChunkingController(std::span ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) : m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end()) + , m_ExcludeElfFiles(ExcludeElfFiles) + , m_ExcludeMachOFiles(ExcludeMachOFiles) , m_ChunkFileSizeLimit(ChunkFileSizeLimit) , m_ChunkingParams(ChunkingParams) { @@ -51,6 +55,8 @@ public: BasicChunkingController(CbObjectView Parameters) : m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView())) + , m_ExcludeElfFiles(Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles)) + , m_ExcludeMachOFiles(Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles)) , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) { @@ -73,6 +79,25 @@ public: } BasicFile Buffer(InputPath, BasicFile::Mode::kRead); + if (m_ExcludeElfFiles && Buffer.FileSize() > 4) + { + uint32_t ElfCheck = 0; + Buffer.Read(&ElfCheck, 4, 0); + if (ElfCheck == 0x464c457f) + { + return false; + } + } + if (m_ExcludeMachOFiles && Buffer.FileSize() > 4) + { + uint32_t MachOCheck = 0; + Buffer.Read(&MachOCheck, 4, 0); + if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe)) + { + return false; + } + } + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag); return true; } @@ -90,6 +115,10 @@ public: } } Writer.EndArray(); // ChunkExcludeExtensions + + Writer.AddBool("ExcludeElfFiles"sv, m_ExcludeElfFiles); + Writer.AddBool("ExcludeMachOFiles"sv, m_ExcludeMachOFiles); + Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); Writer.BeginObject("ChunkingParams"sv); { @@ -106,6 +135,8 @@ public: protected: const std::vector m_ChunkExcludeExtensions; + const bool m_ExcludeElfFiles = false; + const bool m_ExcludeMachOFiles = false; const uint64_t m_ChunkFileSizeLimit; const ChunkedParams m_ChunkingParams; }; @@ -230,10 +261,16 @@ protected: std::unique_ptr CreateBasicChunkingController(std::span ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) { - return std::make_unique(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams); + return std::make_unique(ExcludeExtensions, + ExcludeElfFiles, + ExcludeMachOFiles, + ChunkFileSizeLimit, + ChunkingParams); } std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters) diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 246f4498a..970917fb0 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -12,6 +12,8 @@ namespace zen { const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -40,6 +42,8 @@ public: std::unique_ptr CreateBasicChunkingController( std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, const ChunkedParams& ChunkingParams = DefaultChunkedParams); std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); -- cgit v1.2.3 From 013ac818cd09c1d31bf9411e00b2bbbf02defa3f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 16:08:47 +0100 Subject: build cache prime (#327) - Feature: zen `--boost-workers` option to builds `upload`, `download` and `validate-part` that will increase the number of worker threads, may cause computer to be less responsive - Feature: zen `--cache-prime-only` that uploads referenced data from a part to `--zen-cache-host` if it is not already present. Target folder will be untouched. --- src/zen/cmds/builds_cmd.cpp | 1087 ++++++++++++++--------- src/zen/cmds/builds_cmd.h | 6 +- src/zenutil/buildstoragecache.cpp | 59 +- src/zenutil/include/zenutil/buildstoragecache.h | 7 +- src/zenutil/include/zenutil/workerpools.h | 3 + src/zenutil/workerpools.cpp | 14 +- 6 files changed, 722 insertions(+), 454 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 08d30948b..3a54de935 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -91,7 +91,16 @@ namespace { const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; + const bool SingleThreaded = false; + bool BoostWorkerThreads = false; + + WorkerThreadPool& GetIOWorkerPool() + { + return SingleThreaded ? GetSyncWorkerPool() + : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst) + : GetMediumWorkerPool(EWorkloadType::Burst); + } + WorkerThreadPool& GetNetworkPool() { return SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); } const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; @@ -438,7 +447,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [](bool, std::ptrdiff_t) {}, AbortFlag); @@ -452,7 +461,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent FolderContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, ChunkController, @@ -1563,8 +1572,8 @@ namespace { BlockAttachments.size() - VerifyBlockDescriptions.size())); } - WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1946,8 +1955,8 @@ namespace { RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& UploadBlocksPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& GenerateBlobsPool = GetIOWorkerPool(); + WorkerThreadPool& UploadBlocksPool = GetNetworkPool(); FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; @@ -2147,8 +2156,8 @@ namespace { { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& ReadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& UploadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& ReadChunkPool = GetIOWorkerPool(); + WorkerThreadPool& UploadChunkPool = GetNetworkPool(); FilteredRate FilteredGenerateBlockBytesPerSecond; FilteredRate FilteredCompressedBytesPerSecond; @@ -2363,7 +2372,7 @@ namespace { if (!AbortFlag) { Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, + ReadChunkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -2420,7 +2429,7 @@ namespace { { const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, + ReadChunkPool, [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { @@ -2697,54 +2706,52 @@ namespace { FindBlocksStatistics FindBlocksStats; - std::future PrepBuildResultFuture = - GetSmallWorkerPool(EWorkloadType::Burst) - .EnqueueTask(std::packaged_task{ - [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { - ZEN_TRACE_CPU("PrepareBuild"); + std::future PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task{ + [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + ZEN_TRACE_CPU("PrepareBuild"); - PrepareBuildResult Result; - Stopwatch Timer; - if (CreateBuild) - { - ZEN_TRACE_CPU("CreateBuild"); + PrepareBuildResult Result; + Stopwatch Timer; + if (CreateBuild) + { + ZEN_TRACE_CPU("CreateBuild"); - Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); - Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); - Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); - Result.PayloadSize = MetaData.GetSize(); - } - else - { - ZEN_TRACE_CPU("PutBuild"); - Stopwatch GetBuildTimer; - CbObject Build = Storage.BuildStorage->GetBuild(BuildId); - Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); - Result.PayloadSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - else if (AllowMultiparts) - { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", - NiceBytes(Result.PreferredMultipartChunkSize)); - } - } + Stopwatch PutBuildTimer; + CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); + Result.PayloadSize = MetaData.GetSize(); + } + else + { + ZEN_TRACE_CPU("PutBuild"); + Stopwatch GetBuildTimer; + CbObject Build = Storage.BuildStorage->GetBuild(BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); + } + } - if (!IgnoreExistingBlocks) - { - ZEN_TRACE_CPU("FindBlocks"); - Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); - FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); - FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); - Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - return Result; - }}); + if (!IgnoreExistingBlocks) + { + ZEN_TRACE_CPU("FindBlocks"); + Stopwatch KnownBlocksTimer; + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); + FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; + }}); ChunkedFolderContent LocalContent; @@ -2852,7 +2859,7 @@ namespace { } return true; }, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); @@ -2911,7 +2918,7 @@ namespace { FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, *ChunkController, @@ -3663,7 +3670,7 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); @@ -4712,6 +4719,7 @@ namespace { const std::vector& LooseChunkHashes, bool AllowPartialBlockRequests, bool WipeTargetFolder, + bool PrimeCacheOnly, FolderContent& OutLocalFolderState, DiskStatistics& DiskStats, CacheMappingStatistics& CacheMappingStats, @@ -4721,7 +4729,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder"); - ZEN_UNUSED(WipeTargetFolder); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); Stopwatch IndexTimer; @@ -4741,6 +4749,7 @@ namespace { tsl::robin_map CachedChunkHashesFound; tsl::robin_map CachedSequenceHashesFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); @@ -4786,6 +4795,7 @@ namespace { } tsl::robin_map CachedBlocksFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); @@ -4831,8 +4841,10 @@ namespace { } std::vector LocalPathIndexesMatchingSequenceIndexes; - // Pick up all whole files we can use from current local state + + if (!PrimeCacheOnly) { + // Pick up all whole files we can use from current local state ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) @@ -4870,6 +4882,15 @@ namespace { } } } + else + { + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + } + } // Pick up all chunks in current local state struct CacheCopyData { @@ -4887,6 +4908,7 @@ namespace { tsl::robin_map RawHashToCacheCopyDataIndex; std::vector CacheCopyDatas; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); @@ -5004,8 +5026,8 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; - WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -5097,191 +5119,205 @@ namespace { const std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); if (!BlockChunkIndexNeeded.empty()) { - bool UsingCachedBlock = false; - if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + if (PrimeCacheOnly) { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); - + TotalRequestCount++; TotalPartWriteCount++; - std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) + FullBlockWorks.push_back(BlockIndex); + } + else + { + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) { - CachedChunkBlockIndexes.push_back(BlockIndex); - UsingCachedBlock = true; + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); + + TotalPartWriteCount++; + + std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + if (std::filesystem::exists(BlockPath)) + { + CachedChunkBlockIndexes.push_back(BlockIndex); + UsingCachedBlock = true; + } } - } - if (!UsingCachedBlock) - { - bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); - bool CanDoPartialBlockDownload = - (BlockDescription.HeaderSize > 0) && - (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + if (!UsingCachedBlock) { - std::vector BlockRanges; + bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = + (BlockDescription.HeaderSize > 0) && + (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); + if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + { + std::vector BlockRanges; - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); - uint32_t NeedBlockChunkIndexOffset = 0; - uint32_t ChunkBlockIndex = 0; - uint32_t CurrentOffset = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + uint32_t NeedBlockChunkIndexOffset = 0; + uint32_t ChunkBlockIndex = 0; + uint32_t CurrentOffset = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); - const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(CurrentOffset)); + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); - BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; - while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && - ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) - { - const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && + ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) { - if (NextRange.RangeLength > 0) + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + NextRange = {.BlockIndex = BlockIndex}; + } + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - BlockRanges.push_back(NextRange); - NextRange = {.BlockIndex = BlockIndex}; + if (NextRange.RangeLength == 0) + { + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; } - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - } - else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) - { - if (NextRange.RangeLength == 0) + else { - NextRange.RangeStart = CurrentOffset; - NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + ZEN_ASSERT(false); } - NextRange.RangeLength += ChunkCompressedLength; - NextRange.ChunkBlockIndexCount++; - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - NeedBlockChunkIndexOffset++; } - else + if (NextRange.RangeLength > 0) { - ZEN_ASSERT(false); + BlockRanges.push_back(NextRange); } - } - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - } - ZEN_ASSERT(!BlockRanges.empty()); + ZEN_ASSERT(!BlockRanges.empty()); - std::vector CollapsedBlockRanges; - auto It = BlockRanges.begin(); - CollapsedBlockRanges.push_back(*It++); - while (It != BlockRanges.end()) - { - BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); - uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); - uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; - if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out - { - LastRange.ChunkBlockIndexCount = - (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; - LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; - } - else + std::vector CollapsedBlockRanges; + auto It = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*It++); + while (It != BlockRanges.end()) { - CollapsedBlockRanges.push_back(*It); + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; + if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + { + LastRange.ChunkBlockIndexCount = + (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + } + else + { + CollapsedBlockRanges.push_back(*It); + } + ++It; } - ++It; - } - - const std::uint64_t WantedSize = std::accumulate( - CollapsedBlockRanges.begin(), - CollapsedBlockRanges.end(), - uint64_t(0), - [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - ZEN_ASSERT(WantedSize <= TotalBlockSize); - if (WantedSize > ((TotalBlockSize * 95) / 100)) - { - ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize)); - TotalRequestCount++; - TotalPartWriteCount++; - - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) - { - ZEN_CONSOLE_VERBOSE("Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) - { - ZEN_CONSOLE_VERBOSE("Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + const std::uint64_t WantedSize = std::accumulate( + CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + ZEN_ASSERT(WantedSize <= TotalBlockSize); + if (WantedSize > ((TotalBlockSize * 95) / 100)) + { + ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + TotalRequestCount++; + TotalPartWriteCount++; - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) - { - ZEN_CONSOLE_VERBOSE("Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else + { + if (WantedSize > ((TotalBlockSize * 5) / 10)) + { + ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", + (WantedSize * 100) / TotalBlockSize, + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + } + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); - FullBlockWorks.push_back(BlockIndex); + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + } } - else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + else { - ZEN_CONSOLE_VERBOSE("Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); TotalRequestCount++; TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } - else - { - if (WantedSize > ((TotalBlockSize * 5) / 10)) - { - ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", - (WantedSize * 100) / TotalBlockSize, - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - } - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); - - BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); - } - } - else - { - TotalRequestCount++; - TotalPartWriteCount++; - - FullBlockWorks.push_back(BlockIndex); } } } @@ -5363,6 +5399,12 @@ namespace { std::move(LooseChunkHashWork.ChunkTargetPtrs); const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { @@ -5370,6 +5412,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; + if (!PrimeCacheOnly) { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); @@ -5481,6 +5524,7 @@ namespace { NetworkPool, [&Path, &Storage, + PrimeCacheOnly, BuildId, &RemoteContent, &RemoteLookup, @@ -5503,54 +5547,67 @@ namespace { &DownloadStats](std::atomic&) mutable { if (!AbortFlag) { - FilteredDownloadedBytesPerSecond.Start(); const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(*Storage.BuildStorage, - Path / ZenTempDownloadFolderName, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (!AbortFlag) - { - AsyncWriteDownloadedChunk( - Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - }); + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - else + if (!BuildBlob) { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - IoBuffer BuildBlob; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash)) + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + *Storage.BuildStorage, + Path / ZenTempDownloadFolderName, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); } - if (!BuildBlob) + else { + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); if (BuildBlob && Storage.BuildCacheStorage) { @@ -5560,34 +5617,37 @@ namespace { BuildBlob.GetContentType(), CompositeBuffer(SharedBuffer(BuildBlob))); } - } - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - if (!AbortFlag) - { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BuildBlob) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); } } } @@ -5602,6 +5662,7 @@ namespace { for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -5775,6 +5836,7 @@ namespace { for (uint32_t BlockIndex : CachedChunkBlockIndexes) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -5829,6 +5891,7 @@ namespace { for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -6000,6 +6063,13 @@ namespace { { break; } + + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( NetworkPool, [&, BlockIndex](std::atomic&) { @@ -6011,8 +6081,10 @@ namespace { FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + IoBuffer BlockBuffer; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) { BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); } @@ -6042,119 +6114,123 @@ namespace { FilteredDownloadedBytesPerSecond.Stop(); } - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!PrimeCacheOnly) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; + WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } } @@ -6176,19 +6252,24 @@ namespace { (DownloadStats.RequestsCompleteCount == TotalRequestCount) ? "" : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string Details = fmt::format("{}/{} ({}{}) downloaded. {}/{} ({}B/s) written.", - DownloadStats.RequestsCompleteCount.load(), - TotalRequestCount, - NiceBytes(DownloadedBytes), - DownloadRateString, - NiceBytes(DiskStats.WriteByteCount.load()), - NiceBytes(BytesToWrite), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState({.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(BytesToWrite), - .RemainingCount = gsl::narrow(BytesToWrite - DiskStats.WriteByteCount.load())}, - false); + std::string WriteDetails = PrimeCacheOnly ? "" + : fmt::format(" {}/{} ({}B/s) written.", + NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", + DownloadStats.RequestsCompleteCount.load(), + TotalRequestCount, + NiceBytes(DownloadedBytes), + DownloadRateString, + WriteDetails); + WriteProgressBar.UpdateState( + {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ", + .Details = Details, + .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, + .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) + : (BytesToWrite - DiskStats.WriteByteCount.load())}, + false); }); } @@ -6202,21 +6283,24 @@ namespace { WriteProgressBar.Finish(); - uint32_t RawSequencesMissingWriteCount = 0; - for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) + if (!PrimeCacheOnly) { - const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; - if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + uint32_t RawSequencesMissingWriteCount = 0; + for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) { - RawSequencesMissingWriteCount++; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; - ZEN_ASSERT(!IncompletePath.empty()); - const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; + if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + { + RawSequencesMissingWriteCount++; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; + ZEN_ASSERT(!IncompletePath.empty()); + const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + } } + ZEN_ASSERT(RawSequencesMissingWriteCount == 0); } - ZEN_ASSERT(RawSequencesMissingWriteCount == 0); const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); @@ -6234,6 +6318,11 @@ namespace { WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } + if (PrimeCacheOnly) + { + return; + } + // Move all files we will reuse to cache folder // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place if (!LocalPathIndexesMatchingSequenceIndexes.empty()) @@ -6319,7 +6408,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); Stopwatch Timer; - WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar RebuildProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -6930,7 +7019,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); @@ -7003,7 +7092,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, UpdatedContent, ChunkController, @@ -7080,7 +7169,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, CurrentLocalFolderContent, ChunkController, @@ -7122,10 +7211,13 @@ namespace { bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, - bool PostDownloadVerify) + bool PostDownloadVerify, + bool PrimeCacheOnly) { ZEN_TRACE_CPU("DownloadFolder"); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + Stopwatch DownloadTimer; const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; @@ -7150,27 +7242,30 @@ namespace { ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - if (!ChunkController) - { - ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); - ChunkController = CreateBasicChunkingController(); - } - + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; - if (std::filesystem::is_directory(Path)) + if (!PrimeCacheOnly) { - if (!WipeTargetFolder) + if (std::filesystem::is_directory(Path)) + { + if (!WipeTargetFolder) + { + if (!ChunkController) + { + ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ChunkController = CreateBasicChunkingController(); + } + + LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + } + } + else { - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + CreateDirectories(Path); } } - else - { - CreateDirectories(Path); - } if (AbortFlag) { return; @@ -7251,6 +7346,7 @@ namespace { LooseChunkHashes, AllowPartialBlockRequests, WipeTargetFolder, + PrimeCacheOnly, LocalFolderState, DiskStats, CacheMappingStats, @@ -7260,20 +7356,23 @@ namespace { if (!AbortFlag) { - VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); + if (!PrimeCacheOnly) + { + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); - Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + CreateDirectories((Path / ZenStateFilePath).parent_path()); + TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 + } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + DownloadStats.DownloadedPartialBlockCount.load(); const uint64_t DownloadByteCount = DownloadStats.DownloadedChunkByteCount.load() + @@ -7307,6 +7406,23 @@ namespace { NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); } } + if (PrimeCacheOnly) + { + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + if (Remaining == 0) + { + ZEN_CONSOLE("Build cache upload complete"); + } + else + { + ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + } + return !AbortFlag; + }); + } + } if (CleanDirectory(ZenTempFolder, {})) { std::filesystem::remove(ZenTempFolder); @@ -7584,6 +7700,15 @@ BuildsCommand::BuildsCommand() Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); }; + auto AddWorkerOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "boost-workers", + "Increase the number of worker threads - may cause computer to less responsive", + cxxopts::value(m_BoostWorkerThreads), + ""); + }; + m_Options.add_option("", "v", "verb", @@ -7618,6 +7743,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); AddCacheOptions(m_UploadOptions); + AddWorkerOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7682,6 +7808,15 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); AddCacheOptions(m_DownloadOptions); + + m_DownloadOptions.add_option("cache", + "", + "cache-prime-only", + "Only download blobs missing in cache and upload to cache", + cxxopts::value(m_PrimeCacheOnly), + ""); + + AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7719,6 +7854,7 @@ BuildsCommand::BuildsCommand() m_DownloadOptions.positional_help("local-path build-id build-part-name"); AddOutputOptions(m_DiffOptions); + AddWorkerOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), ""); @@ -7735,6 +7871,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); AddCacheOptions(m_TestOptions); + AddWorkerOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_TestOptions.add_option("", @@ -7765,6 +7902,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); + AddWorkerOptions(m_ValidateBuildPartOptions); m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_ValidateBuildPartOptions.add_option("", "", @@ -7786,6 +7924,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); AddCacheOptions(m_MultiTestDownloadOptions); + AddWorkerOptions(m_MultiTestDownloadOptions); m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); @@ -7991,9 +8130,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .RetryCount = 0}); if (Result.CacheHttp->Get("/health").IsSuccess()) { - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); - CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); + Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + m_PrimeCacheOnly); + CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); if (!m_Namespace.empty()) { CacheDescription += fmt::format(" {}.", m_Namespace); @@ -8014,6 +8157,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return Result; }; + BoostWorkerThreads = m_BoostWorkerThreads; + try { if (SubOption == &m_ListOptions) @@ -8243,6 +8388,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + if (m_PostDownloadVerify && m_PrimeCacheOnly) + { + throw zen::OptionParseException( + fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", m_DownloadOptions.help())); + } + + if (m_Clean && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled"); + } + + if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + } + std::vector BuildPartIds; for (const std::string& BuildPartId : m_BuildPartIds) { @@ -8267,9 +8428,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildPartNames, Path, m_AllowMultiparts, - m_AllowPartialBlockRequests, + m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, - m_PostDownloadVerify); + m_PostDownloadVerify, + m_PrimeCacheOnly); if (false) { @@ -8350,7 +8512,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); @@ -8441,7 +8604,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + true, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -8453,7 +8625,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -8504,7 +8685,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SourceSize > 256) { Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), [SourceSize, FilePath](std::atomic&) { if (!AbortFlag) { @@ -8557,7 +8738,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -8595,7 +8785,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8611,7 +8810,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8627,7 +8827,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index b5af236e1..46257a567 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -27,8 +27,9 @@ private: std::string m_SystemRootDir; - bool m_PlainProgress = false; - bool m_Verbose = false; + bool m_PlainProgress = false; + bool m_Verbose = false; + bool m_BoostWorkerThreads = false; // cloud builds std::string m_BuildsUrl; @@ -42,6 +43,7 @@ private: // cache std::string m_ZenCacheHost; + bool m_PrimeCacheOnly = false; std::string m_BuildId; bool m_CreateBuild = false; diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp index c95215889..f273ac699 100644 --- a/src/zenutil/buildstoragecache.cpp +++ b/src/zenutil/buildstoragecache.cpp @@ -11,6 +11,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -27,13 +28,16 @@ public: BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath) + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) : m_HttpClient(HttpClient) , m_Stats(Stats) , m_Namespace(Namespace.empty() ? "none" : Namespace) , m_Bucket(Bucket.empty() ? "none" : Bucket) , m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred()) - , m_BackgroundWorkPool(1) + , m_BoostBackgroundThreadCount(BoostBackgroundThreadCount) + , m_BackgroundWorkPool(m_BoostBackgroundThreadCount ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)) , m_PendingBackgroundWorkCount(1) , m_CancelBackgroundWork(false) { @@ -44,8 +48,11 @@ public: try { m_CancelBackgroundWork.store(true); - m_PendingBackgroundWorkCount.CountDown(); - m_PendingBackgroundWorkCount.Wait(); + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + m_PendingBackgroundWorkCount.Wait(); + } } catch (const std::exception& Ex) { @@ -86,6 +93,7 @@ public: ZenContentType ContentType, const CompositeBuffer& Payload) override { + ZEN_ASSERT(!IsFlushed); ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); ScheduleBackgroundWork( [this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() { @@ -132,6 +140,7 @@ public: virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) override { + ZEN_ASSERT(!IsFlushed); ScheduleBackgroundWork([this, BuildId = Oid(BuildId), BlobRawHashes = std::vector(BlobHashes.begin(), BlobHashes.end()), @@ -329,6 +338,39 @@ public: return {}; } + virtual void Flush(int32_t UpdateInteralMS, std::function&& UpdateCallback) override + { + if (IsFlushed) + { + return; + } + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + IsFlushed = true; + } + if (m_PendingBackgroundWorkCount.Wait(100)) + { + return; + } + while (true) + { + intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining(); + if (UpdateCallback(Remaining)) + { + if (m_PendingBackgroundWorkCount.Wait(UpdateInteralMS)) + { + UpdateCallback(0); + return; + } + } + else + { + m_CancelBackgroundWork.store(true); + } + } + } + private: void AddStatistic(const HttpClient::Response& Result) { @@ -343,8 +385,10 @@ private: const std::string m_Namespace; const std::string m_Bucket; const std::filesystem::path m_TempFolderPath; + const bool m_BoostBackgroundThreadCount; + bool IsFlushed = false; - WorkerThreadPool m_BackgroundWorkPool; + WorkerThreadPool& m_BackgroundWorkPool; Latch m_PendingBackgroundWorkCount; std::atomic m_CancelBackgroundWork; }; @@ -354,9 +398,10 @@ CreateZenBuildStorageCache(HttpClient& HttpClient, BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath) + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) { - return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath); + return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount); } } // namespace zen diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index 08c936bf5..cab35328d 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -42,11 +42,16 @@ public: }; virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; + + virtual void Flush( + int32_t UpdateInteralMS, + std::function&& UpdateCallback = [](intptr_t) { return true; }) = 0; }; std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath); + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount); } // namespace zen diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h index 9683ad720..df2033bca 100644 --- a/src/zenutil/include/zenutil/workerpools.h +++ b/src/zenutil/include/zenutil/workerpools.h @@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType); // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType); +// Worker pool with minimum number of worker threads, but at least one thread +WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType); + // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread // This is useful for debugging when multiple async thread can make stepping in debugger complicated WorkerThreadPool& GetSyncWorkerPool(); diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp index e3165e838..797034978 100644 --- a/src/zenutil/workerpools.cpp +++ b/src/zenutil/workerpools.cpp @@ -11,9 +11,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace { - const int LargeWorkerThreadPoolTreadCount = gsl::narrow(std::thread::hardware_concurrency()); + const int LargeWorkerThreadPoolTreadCount = gsl::narrow(Max(std::thread::hardware_concurrency() - 1u, 2u)); const int MediumWorkerThreadPoolTreadCount = gsl::narrow(Max((std::thread::hardware_concurrency() / 4u), 2u)); const int SmallWorkerThreadPoolTreadCount = gsl::narrow(Max((std::thread::hardware_concurrency() / 8u), 1u)); + const int TinyWorkerThreadPoolTreadCount = 1; static bool IsShutDown = false; @@ -35,6 +36,9 @@ namespace { WorkerPool BurstSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(burst)"}; WorkerPool BackgroundSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(bkg)"}; + WorkerPool BurstTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(burst)"}; + WorkerPool BackgroundTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(bkg)"}; + WorkerPool SyncWorkerPool = {.TreadCount = 0, .Name = "SyncThreadPool"}; WorkerThreadPool& EnsurePoolPtr(WorkerPool& Pool) @@ -74,6 +78,12 @@ GetSmallWorkerPool(EWorkloadType WorkloadType) return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstSmallWorkerPool : BackgroundSmallWorkerPool); } +WorkerThreadPool& +GetTinyWorkerPool(EWorkloadType WorkloadType) +{ + return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstTinyWorkerPool : BackgroundTinyWorkerPool); +} + WorkerThreadPool& GetSyncWorkerPool() { @@ -91,6 +101,8 @@ ShutdownWorkerPools() BackgroundMediumWorkerPool.Pool.reset(); BurstSmallWorkerPool.Pool.reset(); BackgroundSmallWorkerPool.Pool.reset(); + BurstTinyWorkerPool.Pool.reset(); + BackgroundTinyWorkerPool.Pool.reset(); SyncWorkerPool.Pool.reset(); } } // namespace zen -- cgit v1.2.3 From 8f192ab154ff9de41d4c063138478400fe2aef24 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 28 Mar 2025 14:12:42 +0100 Subject: temp path options and reduced scanning of target folder (#328) - Feature: zen: `--zen-folder-path` added to `builds` command, `list`, `upload`, `download`, `fetch-blob`, `validate-part` to control where `.zen` folder is placed and named - Improvement: Only check known files from remote state when downloading to a target folder with no local state file - Improvement: Don't move existing local to cache and back if they are untouched --- src/zen/cmds/builds_cmd.cpp | 812 ++++++++++++++++++++++++++++------------- src/zen/cmds/builds_cmd.h | 2 + src/zenutil/chunkedcontent.cpp | 18 +- 3 files changed, 580 insertions(+), 252 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 3a54de935..d4add0e04 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -104,22 +104,34 @@ namespace { const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; - const std::string ZenFolderName = ".zen"; - const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); - const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); - const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + const std::string ZenFolderName = ".zen"; + std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } + // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; + // } + std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "tmp"; } - const std::string ZenTempCacheFolderName = - fmt::format("{}/cache", ZenTempFolderName); // Decompressed and verified data - chunks & sequences - const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); // Temp storage for whole and partial blocks - const std::string ZenTempChunkFolderName = - fmt::format("{}/chunks", ZenTempFolderName); // Temp storage for decompressed and validated chunks + std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences + } + std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks + } + std::filesystem::path ZenTempChunkFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "chunks"; // Temp storage for decompressed and validated chunks + } - const std::string ZenTempDownloadFolderName = - fmt::format("{}/download", ZenTempFolderName); // Temp storage for unverfied downloaded blobs + std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks + } - const std::string ZenTempStorageFolderName = - fmt::format("{}/storage", ZenTempFolderName); // Temp storage folder for BuildStorage implementations + // std::filesystem::path ZenTempStorageFolderPath(const std::filesystem::path& ZenFolderPath) + // { + // return ZenTempFolderPath(ZenFolderPath) / "storage"; // Temp storage folder for BuildStorage implementations + // } const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; @@ -150,6 +162,23 @@ namespace { ); + std::filesystem::path MakeSafeAbsolutePath(const std::string Path) + { + std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); +#if ZEN_PLATFORM_WINDOWS && \ + 0 // TODO: We need UNC for long file names but we need to stop using std::filesystem for those paths - std::filesystem::* functions + const std::string_view Prefix = "\\\\?\\"; + const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); + std::u8string PathString = AbsolutePath.u8string(); + if (!PathString.starts_with(PrefixU8)) + { + PathString.insert(0, PrefixU8); + return std::filesystem::path(PathString); + } +#endif + return AbsolutePath; + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -2141,6 +2170,7 @@ namespace { void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span RawHashes, @@ -2214,7 +2244,7 @@ namespace { if (QueuedPendingInMemoryBlocksForUpload.load() > 16) { ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); - Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), Path / ZenTempBlockFolderName, BlockHash)); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), ZenTempBlockFolderPath(ZenFolderPath), BlockHash)); IsInMemoryBlock = false; } else @@ -2437,7 +2467,7 @@ namespace { FilteredCompressedBytesPerSecond.Start(); CompositeBuffer Payload = - CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName, LooseChunksStats); + CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), @@ -2668,6 +2698,7 @@ namespace { const Oid& BuildPartId, const std::string_view BuildPartName, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::filesystem::path& ManifestPath, const uint8_t BlockReuseMinPercentLimit, bool AllowMultiparts, @@ -2678,7 +2709,7 @@ namespace { { Stopwatch ProcessTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); CleanDirectory(ZenTempFolder, {}); auto _ = MakeGuard([&]() { @@ -2687,8 +2718,8 @@ namespace { std::filesystem::remove(ZenTempFolder); } }); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempChunkFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); std::uint64_t TotalRawSize = 0; @@ -3308,6 +3339,7 @@ namespace { UploadPartBlobs(Storage, BuildId, Path, + ZenFolderPath, LocalContent, LocalLookup, RawHashes, @@ -4579,7 +4611,7 @@ namespace { return false; } - void AsyncWriteDownloadedChunk(const std::filesystem::path& Path, + void AsyncWriteDownloadedChunk(const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, uint32_t RemoteChunkIndex, @@ -4613,7 +4645,7 @@ namespace { { Payload.SetDeleteOnClose(false); Payload = {}; - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { @@ -4632,7 +4664,7 @@ namespace { { ZEN_TRACE_CPU("WriteTempChunk"); // Could not be moved and rather large, lets store it on disk - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload); Payload = {}; } @@ -4668,7 +4700,7 @@ namespace { } } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, @@ -4711,6 +4743,7 @@ namespace { void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::uint64_t LargeAttachmentSize, const std::uint64_t PreferredMultipartChunkSize, const ChunkedFolderContent& LocalContent, @@ -4739,7 +4772,7 @@ namespace { ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName; + const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); Stopwatch CacheMappingTimer; @@ -4786,6 +4819,12 @@ namespace { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); CacheMappingStats.CacheSequenceHashesCount++; CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + continue; } } @@ -4808,7 +4847,7 @@ namespace { } DirectoryContent BlockDirContent; - GetDirectoryContent(Path / ZenTempBlockFolderName, + GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath), DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, BlockDirContent); CachedBlocksFound.reserve(BlockDirContent.Files.size()); @@ -4856,6 +4895,8 @@ namespace { // const uint32_t RemoteSequenceIndex = CacheSequenceIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4863,13 +4904,16 @@ namespace { // const uint32_t RemoteChunkIndex = CacheChunkIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; CacheMappingStats.LocalPathsMatchingSequencesByteCount += RawSize; @@ -5135,7 +5179,8 @@ namespace { TotalPartWriteCount++; - std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::path BlockPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); if (std::filesystem::exists(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); @@ -5414,8 +5459,9 @@ namespace { std::filesystem::path ExistingCompressedChunkPath; if (!PrimeCacheOnly) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = + ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); if (std::filesystem::exists(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); @@ -5448,6 +5494,7 @@ namespace { Work.ScheduleWork( WritePool, [&Path, + &ZenFolderPath, &RemoteContent, &RemoteLookup, &CacheFolderPath, @@ -5479,7 +5526,7 @@ namespace { CompressedChunkPath)); } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, RemoteLookup, @@ -5523,9 +5570,10 @@ namespace { Work.ScheduleWork( NetworkPool, [&Path, + &ZenFolderPath, &Storage, - PrimeCacheOnly, BuildId, + &PrimeCacheOnly, &RemoteContent, &RemoteLookup, &ExistsResult, @@ -5563,7 +5611,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); DownloadLargeBlob( *Storage.BuildStorage, - Path / ZenTempDownloadFolderName, + ZenTempDownloadFolderPath(ZenFolderPath), BuildId, ChunkHash, PreferredMultipartChunkSize, @@ -5588,7 +5636,7 @@ namespace { { if (!AbortFlag) { - AsyncWriteDownloadedChunk(Path, + AsyncWriteDownloadedChunk(ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -5633,7 +5681,7 @@ namespace { { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(Path, + AsyncWriteDownloadedChunk(ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -5852,8 +5900,9 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredWrittenBytesPerSecond.Start(); - std::filesystem::path BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + std::filesystem::path BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); if (!BlockBuffer) { throw std::runtime_error( @@ -5955,11 +6004,10 @@ namespace { { BlockBuffer.SetDeleteOnClose(false); BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -5978,11 +6026,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -6130,8 +6177,9 @@ namespace { if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -6150,7 +6198,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -6323,36 +6371,106 @@ namespace { return; } - // Move all files we will reuse to cache folder - // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place - if (!LocalPathIndexesMatchingSequenceIndexes.empty()) + tsl::robin_map RemotePathIndexToLocalPathIndex; + RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size()); + + tsl::robin_map SequenceHashToLocalPathIndex; + std::vector RemoveLocalPathIndexes; + + if (!WipeTargetFolder) { - ZEN_TRACE_CPU("UpdateFolder_CacheReused"); - uint64_t TotalFullFileSizeCached = 0; - for (uint32_t LocalPathIndex : LocalPathIndexesMatchingSequenceIndexes) + tsl::robin_set CachedRemoteSequences; + tsl::robin_map RemotePathToRemoteIndex; + RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); - SetFileReadOnly(LocalFilePath, false); - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); - std::error_code Ec; - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); + } + + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + uint64_t CachedCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + + if (!WipeTargetFolder) { - Sleep(100); - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); + RemotePathIt != RemotePathToRemoteIndex.end()) + { + const uint32_t RemotePathIndex = RemotePathIt->second; + if (RemoteContent.RawHashes[RemotePathIndex] == RawHash) + { + // It is already in it's desired place + RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; + SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); + MatchCount++; + continue; + } + else + { + HashMismatchCount++; + } + } + else + { + PathMismatchCount++; + } } - if (Ec) + if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + { + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + if (!CachedRemoteSequences.contains(RawHash)) + { + // We need it + ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + + std::error_code Ec; + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + CachedRemoteSequences.insert(RawHash); + CachedCount++; + } + else + { + // We already have it + ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + SkippedCount++; + } + } + else if (!WipeTargetFolder) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + // We don't need it + RemoveLocalPathIndexes.push_back(LocalPathIndex); + DeleteCount++; } - TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); } - ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(TotalFullFileSizeCached)); + + ZEN_DEBUG( + "Local state prep: MatchCount: {}, PathMismatchCount: {}, HashMismatchCount: {}, CachedCount: {}, SkippedCount: {}, " + "DeleteCount: {}", + MatchCount, + PathMismatchCount, + HashMismatchCount, + CachedCount, + SkippedCount, + DeleteCount); } if (WipeTargetFolder) @@ -6374,34 +6492,17 @@ namespace { Stopwatch Timer; // Remove unused tracked files - tsl::robin_map RemotePathToRemoteIndex; - RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) - { - RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); - } - std::vector LocalFilesToRemove; - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + + if (!RemoveLocalPathIndexes.empty()) { - if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string())) + ZEN_CONSOLE("Cleaning {} removed files from {}", RemoveLocalPathIndexes.size(), Path); + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) - { - LocalFilesToRemove.emplace_back(std::move(LocalFilePath)); - } - } - } - if (!LocalFilesToRemove.empty()) - { - ZEN_CONSOLE("Cleaning {} removed files from {}", LocalFilesToRemove.size(), Path); - for (const std::filesystem::path& LocalFilePath : LocalFilesToRemove) - { SetFileReadOnly(LocalFilePath, false); std::filesystem::remove(LocalFilePath); } } - RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } { @@ -6420,14 +6521,28 @@ namespace { std::atomic TargetsComplete = 0; - std::vector> Targets; + struct FinalizeTarget + { + IoHash RawHash; + uint32_t RemotePathIndex; + }; + + std::vector Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { - Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); } - std::sort(Targets.begin(), Targets.end(), [](const std::pair& Lhs, const std::pair& Rhs) { - return Lhs.first < Rhs.first; + std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { + if (Lhs.RawHash < Rhs.RawHash) + { + return true; + } + else if (Lhs.RawHash > Rhs.RawHash) + { + return false; + } + return Lhs.RemotePathIndex < Rhs.RemotePathIndex; }); size_t TargetOffset = 0; @@ -6438,9 +6553,8 @@ namespace { break; } - size_t TargetCount = 1; - const IoHash& RawHash = Targets[TargetOffset].first; - while (Targets[TargetOffset + TargetCount].first == RawHash) + size_t TargetCount = 1; + while (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash) { TargetCount++; } @@ -6452,93 +6566,157 @@ namespace { { ZEN_TRACE_CPU("FinalizeTree_Work"); - size_t TargetOffset = BaseTargetOffset; - const IoHash& RawHash = Targets[TargetOffset].first; - const uint32_t FirstTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstTargetPathIndex]; - OutLocalFolderState.Paths[FirstTargetPathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstTargetPathIndex] = RemoteContent.RawSizes[FirstTargetPathIndex]; - const std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; + if (RawHash == IoHash::Zero) { - if (std::filesystem::exists(FirstTargetFilePath)) + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - SetFileReadOnly(FirstTargetFilePath, false); - } - CreateDirectories(FirstTargetFilePath.parent_path()); - { - BasicFile OutputFile; - OutputFile.Open(FirstTargetFilePath, BasicFile::Mode::kTruncate); + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) + { + if (std::filesystem::exists(TargetFilePath)) + { + SetFileReadOnly(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + } + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; } } else { - ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace"); + ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); - CreateDirectories(FirstTargetFilePath.parent_path()); - if (std::filesystem::exists(FirstTargetFilePath)) - { - SetFileReadOnly(FirstTargetFilePath, false); - } - std::error_code Ec; - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - Sleep(100); - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); } - if (Ec) + else { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } + if (std::filesystem::exists(FirstTargetFilePath)) + { + SetFileReadOnly(FirstTargetFilePath, false); + } + else + { + CreateDirectories(FirstTargetFilePath.parent_path()); + } - OutLocalFolderState.Attributes[FirstTargetPathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstTargetPathIndex]); - OutLocalFolderState.ModificationTicks[FirstTargetPathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); + InplaceIt != SequenceHashToLocalPathIndex.end()) + { + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(std::filesystem::exists(SourceFilePath)); - TargetOffset++; - TargetsComplete++; - while (TargetOffset < (BaseTargetOffset + TargetCount)) - { - if (AbortFlag) - { - break; - } - ZEN_TRACE_CPU("FinalizeTree_Copy"); - - ZEN_ASSERT(Targets[TargetOffset].first == RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); - const uint32_t ExtraTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& ExtraTargetPath = RemoteContent.Paths[ExtraTargetPathIndex]; - const std::filesystem::path ExtraTargetFilePath = (Path / ExtraTargetPath).make_preferred(); - OutLocalFolderState.Paths[ExtraTargetPathIndex] = ExtraTargetPath; - OutLocalFolderState.RawSizes[ExtraTargetPathIndex] = RemoteContent.RawSizes[ExtraTargetPathIndex]; - CreateDirectories(ExtraTargetFilePath.parent_path()); - if (std::filesystem::exists(ExtraTargetFilePath)) - { - SetFileReadOnly(ExtraTargetFilePath, false); + CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + + std::error_code Ec; + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + } } - CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false}); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - OutLocalFolderState.Attributes[ExtraTargetPathIndex] = + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + + OutLocalFolderState.Attributes[FirstRemotePathIndex] = RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(ExtraTargetFilePath) - : SetNativeFileAttributes(ExtraTargetFilePath, + ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, RemoteContent.Platform, - RemoteContent.Attributes[ExtraTargetPathIndex]); - OutLocalFolderState.ModificationTicks[ExtraTargetPathIndex] = - GetModificationTickFromPath(ExtraTargetFilePath); + RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = + GetModificationTickFromPath(FirstTargetFilePath); TargetOffset++; TargetsComplete++; + + while (TargetOffset < (BaseTargetOffset + TargetCount)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(std::filesystem::exists(TargetFilePath)); + } + else + { + if (std::filesystem::exists(TargetFilePath)) + { + SetFileReadOnly(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + + ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } } } }, @@ -6980,66 +7158,111 @@ namespace { ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, - ChunkingController& ChunkController) + const std::filesystem::path& ZenFolderPath, + ChunkingController& ChunkController, + const ChunkedFolderContent& ReferenceContent, + FolderContent& OutLocalFolderContent) { - ChunkedFolderContent LocalContent; + { + const uint32_t PathCount = gsl::narrow(ReferenceContent.Paths.size()); + OutLocalFolderContent.Paths.resize(PathCount); + OutLocalFolderContent.RawSizes.resize(PathCount); + OutLocalFolderContent.Attributes.resize(PathCount); + OutLocalFolderContent.ModificationTicks.resize(PathCount); - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string_view& ExcludeFolder : ExcludeFolders) { - if (RelativePath.starts_with(ExcludeFolder)) + Stopwatch Timer; + auto _ = + MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + ProgressBar ProgressBar(UsePlainProgress); + + ParallellWork Work(AbortFlag); + std::atomic CompletedPathCount = 0; + uint32_t PathIndex = 0; + + while (PathIndex < PathCount) { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } + uint32_t PathRangeCount = Min(1024u, PathCount - PathIndex); + Work.ScheduleWork( + GetIOWorkerPool(), + [PathIndex, + PathRangeCount, + &ReferenceContent, + &Path, + &OutLocalFolderContent, + &CompletedPathCount, + &LocalFolderScanStats](std::atomic&) { + for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) + { + const std::filesystem::path& FilePath = ReferenceContent.Paths[PathRangeIndex]; + std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); + if (std::filesystem::exists(LocalFilePath)) + { + const uint64_t FileSize = std::filesystem::file_size(LocalFilePath); + OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; + OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; + OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); + OutLocalFolderContent.ModificationTicks[PathRangeIndex] = GetModificationTickFromPath(LocalFilePath); + LocalFolderScanStats.FoundFileCount++; + LocalFolderScanStats.FoundFileByteCount += FileSize; + LocalFolderScanStats.AcceptedFileCount++; + LocalFolderScanStats.AcceptedFileByteCount += FileSize; + } + CompletedPathCount++; + } + }, + Work.DefaultErrorFunction()); + PathIndex += PathRangeCount; } + Work.Wait(200, [&](bool, ptrdiff_t) { + // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} checked, {} found", + CompletedPathCount.load(), + PathCount, + LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount.load()}, + false); + }); + ProgressBar.Finish(); } - return true; - }; - auto IsAcceptedFile = [ExcludeExtensions = - DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string_view& ExcludeExtension : ExcludeExtensions) + uint32_t WritePathIndex = 0; + for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++) { - if (RelativePath.ends_with(ExcludeExtension)) + if (!OutLocalFolderContent.Paths[ReadPathIndex].empty()) { - return false; + if (WritePathIndex < ReadPathIndex) + { + OutLocalFolderContent.Paths[WritePathIndex] = std::move(OutLocalFolderContent.Paths[ReadPathIndex]); + OutLocalFolderContent.RawSizes[WritePathIndex] = OutLocalFolderContent.RawSizes[ReadPathIndex]; + OutLocalFolderContent.Attributes[WritePathIndex] = OutLocalFolderContent.Attributes[ReadPathIndex]; + OutLocalFolderContent.ModificationTicks[WritePathIndex] = OutLocalFolderContent.ModificationTicks[ReadPathIndex]; + } + WritePathIndex++; } } - return true; - }; - FolderContent CurrentLocalFolderContent = GetFolderContent( - LocalFolderScanStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); - }, - AbortFlag); - if (AbortFlag) - { - return {}; + OutLocalFolderContent.Paths.resize(WritePathIndex); + OutLocalFolderContent.RawSizes.resize(WritePathIndex); + OutLocalFolderContent.Attributes.resize(WritePathIndex); + OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); } - FolderContent LocalFolderState; + FolderContent LocalFolderState; + ChunkedFolderContent LocalContent; bool ScanContent = true; std::vector PathIndexesOufOfDate; - if (std::filesystem::is_regular_file(Path / ZenStateFilePath)) + if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) { try { Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object; + CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; if (CurrentStateObject) { Oid CurrentBuildId; @@ -7066,11 +7289,11 @@ namespace { std::span(SavedPartContents).subspan(1)); } - if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) + if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { const size_t LocaStatePathCount = LocalFolderState.Paths.size(); std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); if (!DeletedPaths.empty()) { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); @@ -7160,27 +7383,27 @@ namespace { if (ScanContent) { uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : CurrentLocalFolderContent.RawSizes) + for (const uint64_t RawSize : OutLocalFolderContent.RawSizes) { ByteCountToScan += RawSize; } ProgressBar ProgressBar(false); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + LocalContent = ChunkFolderContent( ChunkingStats, GetIOWorkerPool(), Path, - CurrentLocalFolderContent, + OutLocalFolderContent, ChunkController, UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), - CurrentLocalFolderContent.Paths.size(), + OutLocalFolderContent.Paths.size(), NiceBytes(ChunkingStats.BytesHashed.load()), - ByteCountToScan, + NiceBytes(ByteCountToScan), NiceNum(FilteredBytesHashed.GetCurrent()), ChunkingStats.UniqueChunksFound.load(), NiceBytes(ChunkingStats.UniqueBytesFound.load())); @@ -7208,6 +7431,7 @@ namespace { const std::vector& BuildPartIds, std::span BuildPartNames, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, @@ -7220,12 +7444,12 @@ namespace { Stopwatch DownloadTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempCacheFolderName); - CreateDirectories(Path / ZenTempDownloadFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath)); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -7246,6 +7470,7 @@ namespace { GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; + FolderContent LocalFolderContent; if (!PrimeCacheOnly) { if (std::filesystem::is_directory(Path)) @@ -7258,7 +7483,13 @@ namespace { ChunkController = CreateBasicChunkingController(); } - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + LocalContent = GetLocalContent(LocalFolderScanStats, + ChunkingStats, + Path, + ZenFolderPath, + *ChunkController, + RemoteContent, + LocalFolderContent); } } else @@ -7317,6 +7548,12 @@ namespace { { ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); } else { @@ -7338,6 +7575,7 @@ namespace { UpdateFolder(Storage, BuildId, Path, + ZenFolderPath, LargeAttachmentSize, PreferredMultipartChunkSize, LocalContent, @@ -7363,14 +7601,14 @@ namespace { Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 ExtendableStringBuilder<1024> SB; CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + @@ -7709,6 +7947,14 @@ BuildsCommand::BuildsCommand() ""); }; + auto AddZenFolderOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "zen-folder-path", + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), + cxxopts::value(m_ZenFolderPath), + ""); + }; m_Options.add_option("", "v", "verb", @@ -7722,6 +7968,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ListOptions); AddFileOptions(m_ListOptions); AddOutputOptions(m_ListOptions); + AddZenFolderOptions(m_ListOptions); m_ListOptions.add_options()("h,help", "Print help"); m_ListOptions.add_option("", "", @@ -7744,6 +7991,7 @@ BuildsCommand::BuildsCommand() AddOutputOptions(m_UploadOptions); AddCacheOptions(m_UploadOptions); AddWorkerOptions(m_UploadOptions); + AddZenFolderOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7808,7 +8056,8 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); AddCacheOptions(m_DownloadOptions); - + AddZenFolderOptions(m_DownloadOptions); + AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_option("cache", "", "cache-prime-only", @@ -7816,7 +8065,6 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_PrimeCacheOnly), ""); - AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7848,6 +8096,7 @@ BuildsCommand::BuildsCommand() "Allow request for partial chunk blocks. Defaults to true.", cxxopts::value(m_AllowPartialBlockRequests), ""); + m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), ""); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); @@ -7893,6 +8142,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); AddCacheOptions(m_FetchBlobOptions); + AddZenFolderOptions(m_FetchBlobOptions); m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_FetchBlobOptions .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); @@ -7903,6 +8153,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); AddWorkerOptions(m_ValidateBuildPartOptions); + AddZenFolderOptions(m_ValidateBuildPartOptions); m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_ValidateBuildPartOptions.add_option("", "", @@ -7989,10 +8240,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() - ? PickDefaultSystemRootDirectory() - : std::filesystem::absolute(StringToPath(m_SystemRootDir)).make_preferred(); - + std::filesystem::path DataRoot = + m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; @@ -8046,8 +8295,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = - ReadAccessTokenFromFile(std::filesystem::absolute(StringToPath(m_AccessTokenPath)).make_preferred()); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath)); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -8110,7 +8358,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_StoragePath.empty()) { - std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); StorageDescription = fmt::format("folder {}", StoragePath); Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); @@ -8173,7 +8421,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = std::filesystem::absolute(StringToPath(m_ListQueryPath)).make_preferred(); + std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath); if (ToLower(ListQueryPath.extension().string()) == ".cbo") { QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); @@ -8195,7 +8443,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -8207,7 +8466,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = std::filesystem::absolute(StringToPath(m_ListResultPath)).make_preferred(); + std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); if (ToLower(ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); @@ -8256,7 +8515,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); if (m_BuildPartName.empty()) { @@ -8297,14 +8556,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); CbObject MetaData; if (m_CreateBuild) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath = std::filesystem::absolute(StringToPath(m_BuildMetadataPath)); + std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -8336,7 +8606,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, Path, - std::filesystem::absolute(StringToPath(m_ManifestPath)).make_preferred(), + ZenFolderPath, + MakeSafeAbsolutePath(m_ManifestPath), m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -8415,18 +8686,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, Path, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, @@ -8466,8 +8741,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); - std::filesystem::path DiffPath = std::filesystem::absolute(StringToPath(m_DiffPath)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath); DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -8489,12 +8764,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -8509,6 +8787,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, {}, Path, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -8532,7 +8811,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = Path.filename().string(); @@ -8542,7 +8821,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const Oid BuildId = Oid::FromHexString(m_BuildId); const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); if (m_BuildsUrl.empty() && StoragePath.empty()) { @@ -8561,7 +8840,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (DownloadPath / ZenFolderName) : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -8589,6 +8872,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, Path, + ZenFolderPath, {}, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -8602,13 +8886,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 11; } - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, @@ -8630,6 +8914,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8743,6 +9028,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8771,6 +9057,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, + ZenFolderPath, {}, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -8790,6 +9077,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8807,6 +9095,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8824,6 +9113,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8853,12 +9143,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const Oid BuildId = Oid::FromHexString(m_BuildId); - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); uint64_t CompressedSize; uint64_t DecompressedSize; @@ -8898,12 +9199,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 46257a567..4a77f8bd7 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -31,6 +31,8 @@ private: bool m_Verbose = false; bool m_BoostWorkerThreads = false; + std::string m_ZenFolderPath; + // cloud builds std::string m_BuildsUrl; bool m_AssumeHttp2 = false; diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index bb1ee5183..1e8447a57 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -304,14 +304,22 @@ FolderContent GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) { ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); - FolderContent Result = {.Platform = Old.Platform}; + + const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); + + FolderContent Result = {.Platform = Old.Platform}; + Result.Paths.reserve(NewPathCount); + Result.RawSizes.reserve(NewPathCount); + Result.Attributes.reserve(NewPathCount); + Result.ModificationTicks.reserve(NewPathCount); + tsl::robin_map NewPathToIndex; - const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); NewPathToIndex.reserve(NewPathCount); for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++) { NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex}); } + uint32_t OldPathCount = gsl::narrow(Old.Paths.size()); for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++) { @@ -667,6 +675,12 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); tsl::robin_map ChunkHashToChunkIndex; + const size_t ExpectedCount = BaseContent.Paths.size() - DeletedPaths.size(); + Result.Paths.reserve(ExpectedCount); + Result.RawSizes.reserve(ExpectedCount); + Result.Attributes.reserve(ExpectedCount); + Result.RawHashes.reserve(ExpectedCount); + tsl::robin_map RawHashToSequenceRawHashIndex; for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) { -- cgit v1.2.3 From 173f722fe773beae525bfca9657f64497cc16b67 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sun, 30 Mar 2025 10:35:01 +0200 Subject: check file from local track state during download (#329) --- src/zen/cmds/builds_cmd.cpp | 265 ++++++++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 118 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d4add0e04..e3dc20621 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -7163,8 +7163,79 @@ namespace { const ChunkedFolderContent& ReferenceContent, FolderContent& OutLocalFolderContent) { + FolderContent LocalFolderState; + ChunkedFolderContent LocalContent; + + bool HasLocalState = false; + if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + { + try + { + Stopwatch ReadStateTimer; + CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; + if (CurrentStateObject) + { + Oid CurrentBuildId; + std::vector SavedBuildPartIds; + std::vector SavedBuildPartsNames; + std::vector SavedPartContents; + if (ReadStateObject(CurrentStateObject, + CurrentBuildId, + SavedBuildPartIds, + SavedBuildPartsNames, + SavedPartContents, + LocalFolderState)) + { + if (!SavedPartContents.empty()) + { + if (SavedPartContents.size() == 1) + { + LocalContent = std::move(SavedPartContents[0]); + } + else + { + LocalContent = + MergeChunkedFolderContents(SavedPartContents[0], + std::span(SavedPartContents).subspan(1)); + } + HasLocalState = true; + } + } + } + ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + } + } + { - const uint32_t PathCount = gsl::narrow(ReferenceContent.Paths.size()); + const uint32_t LocalPathCount = gsl::narrow(ReferenceContent.Paths.size()); + const uint32_t RemotePathCount = gsl::narrow(LocalFolderState.Paths.size()); + + std::vector PathsToCheck; + PathsToCheck.reserve(LocalPathCount + RemotePathCount); + + tsl::robin_set FileSet; + FileSet.reserve(LocalPathCount + RemotePathCount); + + for (const std::filesystem::path& LocalPath : LocalFolderState.Paths) + { + FileSet.insert(LocalPath.generic_string()); + PathsToCheck.push_back(LocalPath); + } + + for (const std::filesystem::path& RemotePath : ReferenceContent.Paths) + { + if (FileSet.insert(RemotePath.generic_string()).second) + { + PathsToCheck.push_back(RemotePath); + } + } + + const uint32_t PathCount = gsl::narrow(PathsToCheck.size()); + OutLocalFolderContent.Paths.resize(PathCount); OutLocalFolderContent.RawSizes.resize(PathCount); OutLocalFolderContent.Attributes.resize(PathCount); @@ -7188,14 +7259,14 @@ namespace { GetIOWorkerPool(), [PathIndex, PathRangeCount, - &ReferenceContent, + &PathsToCheck, &Path, &OutLocalFolderContent, &CompletedPathCount, &LocalFolderScanStats](std::atomic&) { for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) { - const std::filesystem::path& FilePath = ReferenceContent.Paths[PathRangeIndex]; + const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); if (std::filesystem::exists(LocalFilePath)) { @@ -7252,132 +7323,90 @@ namespace { OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); } - FolderContent LocalFolderState; - ChunkedFolderContent LocalContent; - bool ScanContent = true; std::vector PathIndexesOufOfDate; - if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + if (HasLocalState) { - try + if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { - Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; - if (CurrentStateObject) + const size_t LocaStatePathCount = LocalFolderState.Paths.size(); + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); + if (!DeletedPaths.empty()) { - Oid CurrentBuildId; - std::vector SavedBuildPartIds; - std::vector SavedBuildPartsNames; - std::vector SavedPartContents; - if (ReadStateObject(CurrentStateObject, - CurrentBuildId, - SavedBuildPartIds, - SavedBuildPartsNames, - SavedPartContents, - LocalFolderState)) - { - if (!SavedPartContents.empty()) - { - if (SavedPartContents.size() == 1) - { - LocalContent = std::move(SavedPartContents[0]); - } - else - { - LocalContent = - MergeChunkedFolderContents(SavedPartContents[0], - std::span(SavedPartContents).subspan(1)); - } - - if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) - { - const size_t LocaStatePathCount = LocalFolderState.Paths.size(); - std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocaStatePathCount); - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(false); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - ChunkingStats, - GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, - false); - }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set LocalFolderPaths; - LocalFolderPaths.reserve(LocalFolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector DeletedPaths; - for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } - } - ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - ScanContent = false; - } + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocaStatePathCount); + if (UpdatedContent.Paths.size() > 0) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : UpdatedContent.RawSizes) + { + ByteCountToScan += RawSize; } + ProgressBar ProgressBar(false); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetIOWorkerPool(), + Path, + UpdatedContent, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + if (AbortFlag) + { + return {}; + } + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); } } - catch (const std::exception& Ex) + else { - ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + // Remove files from LocalContent no longer in LocalFolderState + tsl::robin_set LocalFolderPaths; + LocalFolderPaths.reserve(LocalFolderState.Paths.size()); + for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) + { + LocalFolderPaths.insert(LocalFolderPath.generic_string()); + } + std::vector DeletedPaths; + for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) + { + if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) + { + DeletedPaths.push_back(LocalContentPath); + } + } + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } } + ScanContent = false; } if (ScanContent) -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zen/cmds/admin_cmd.cpp | 4 +- src/zen/cmds/builds_cmd.cpp | 309 +++++--- src/zen/cmds/cache_cmd.cpp | 2 +- src/zen/cmds/copy_cmd.cpp | 20 +- src/zen/cmds/projectstore_cmd.cpp | 4 +- src/zen/cmds/rpcreplay_cmd.cpp | 2 +- src/zen/cmds/run_cmd.cpp | 2 +- src/zen/cmds/serve_cmd.cpp | 2 +- src/zen/cmds/status_cmd.cpp | 2 +- src/zen/cmds/up_cmd.cpp | 4 +- src/zencore/basicfile.cpp | 12 +- src/zencore/except.cpp | 2 +- src/zencore/filesystem.cpp | 875 ++++++++++++++++++--- src/zencore/include/zencore/filesystem.h | 85 +- src/zencore/process.cpp | 2 +- src/zencore/testutils.cpp | 11 +- src/zenhttp/auth/authmgr.cpp | 2 +- src/zenserver-test/zenserver-test.cpp | 18 +- src/zenserver/admin/admin.cpp | 2 +- src/zenserver/config.cpp | 2 +- src/zenserver/frontend/frontend.cpp | 4 +- src/zenserver/main.cpp | 6 +- src/zenserver/objectstore/objectstore.cpp | 10 +- .../projectstore/fileremoteprojectstore.cpp | 10 +- src/zenserver/projectstore/projectstore.cpp | 106 +-- src/zenserver/projectstore/remoteprojectstore.cpp | 6 +- src/zenserver/workspaces/httpworkspaces.cpp | 2 +- src/zenserver/zenserver.cpp | 6 +- src/zenstore/blockstore.cpp | 18 +- src/zenstore/buildstore/buildstore.cpp | 10 +- src/zenstore/cache/cachedisklayer.cpp | 48 +- src/zenstore/cas.cpp | 2 +- src/zenstore/caslog.cpp | 2 +- src/zenstore/compactcas.cpp | 30 +- src/zenstore/filecas.cpp | 78 +- src/zenstore/gc.cpp | 20 +- src/zenstore/workspaces.cpp | 26 +- src/zenutil/cache/rpcrecording.cpp | 8 +- src/zenutil/filebuildstorage.cpp | 14 +- src/zenutil/zenserverprocess.cpp | 8 +- src/zenvfs/vfsprovider.cpp | 13 +- 41 files changed, 1270 insertions(+), 519 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 835e01151..573639c2d 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -688,7 +688,7 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) static bool TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target) { - if (!std::filesystem::is_regular_file(Source)) + if (!IsFile(Source)) { return false; } @@ -717,7 +717,7 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path DataPath = StringToPath(m_DataPath); std::filesystem::path TargetPath = StringToPath(m_TargetPath); - if (!std::filesystem::is_directory(DataPath)) + if (!IsDir(DataPath)) { throw OptionParseException("data path must exist"); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e3dc20621..d2ba20e78 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -165,12 +165,11 @@ namespace { std::filesystem::path MakeSafeAbsolutePath(const std::string Path) { std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); -#if ZEN_PLATFORM_WINDOWS && \ - 0 // TODO: We need UNC for long file names but we need to stop using std::filesystem for those paths - std::filesystem::* functions +#if ZEN_PLATFORM_WINDOWS && 1 const std::string_view Prefix = "\\\\?\\"; const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); std::u8string PathString = AbsolutePath.u8string(); - if (!PathString.starts_with(PrefixU8)) + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) { PathString.insert(0, PrefixU8); return std::filesystem::path(PathString); @@ -179,6 +178,82 @@ namespace { return AbsolutePath; } + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) + { + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + RenameFile(SourcePath, TargetPath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -250,21 +325,8 @@ namespace { { try { - std::error_code Ec; - std::filesystem::remove(LocalFilePath, Ec); - if (Ec) - { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Ec.clear(); - if (std::filesystem::exists(LocalFilePath, Ec) || Ec) - { - Sleep(200); - if (std::filesystem::exists(LocalFilePath)) - { - std::filesystem::remove(LocalFilePath); - } - } - } + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } catch (const std::exception& Ex) { @@ -273,7 +335,7 @@ namespace { } } - for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; for (const std::string_view ExcludeDirectory : ExcludeDirectories) @@ -288,23 +350,27 @@ namespace { { try { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); - } - catch (const std::exception&) - { - Sleep(200); - try + std::error_code Ec; + zen::CleanDirectory(LocalDirPath, true, Ec); + if (Ec) { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); + Sleep(200); + zen::CleanDirectory(LocalDirPath, true); + Ec.clear(); } - catch (const std::exception& Ex) + + RemoveDir(LocalDirPath, Ec); + if (Ec) { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; + Sleep(200); + RemoveDir(LocalDirPath); } } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); + CleanWipe = false; + } } } return CleanWipe; @@ -312,7 +378,7 @@ namespace { std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -355,7 +421,7 @@ namespace { const IoHash& Hash, const std::string& Suffix = {}) { - std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / (Hash.ToHexString() + Suffix); return WriteToTempFile(std::move(Buffer), TempFilePath); } @@ -453,12 +519,12 @@ namespace { std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred(); + return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); } std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + return CacheFolderPath / RawHash.ToHexString(); } ChunkedFolderContent ScanAndChunkFolder( @@ -1611,7 +1677,8 @@ namespace { auto __ = MakeGuard([&TempFolder]() { if (CleanDirectory(TempFolder, {})) { - std::filesystem::remove(TempFolder); + std::error_code DummyEc; + RemoveDir(TempFolder, DummyEc); } }); @@ -1888,7 +1955,7 @@ namespace { } ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); { - std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString(); BasicFile CompressedFile; std::error_code Ec; @@ -1930,7 +1997,7 @@ namespace { return Compressed.GetCompressed(); } CompressedFile.Close(); - std::filesystem::remove(TempFilePath, Ec); + RemoveFile(TempFilePath, Ec); ZEN_UNUSED(Ec); } @@ -2715,7 +2782,8 @@ namespace { auto _ = MakeGuard([&]() { if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + std::error_code DummyEc; + RemoveDir(ZenTempFolder, DummyEc); } }); CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); @@ -2861,7 +2929,7 @@ namespace { { std::filesystem::path ExcludeManifestPath = Path / ZenExcludeManifestName; tsl::robin_set ExcludeAssetPaths; - if (std::filesystem::is_regular_file(ExcludeManifestPath)) + if (IsFile(ExcludeManifestPath)) { std::vector AssetPaths = ParseManifest(Path, ExcludeManifestPath); ExcludeAssetPaths.reserve(AssetPaths.size()); @@ -2904,12 +2972,13 @@ namespace { for (const std::filesystem::path& AssetPath : AssetPaths) { Content.Paths.push_back(AssetPath); - Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath)); + const std::filesystem::path AssetFilePath = (Path / AssetPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(Path / AssetPath)); + Content.Attributes.push_back(GetFileAttributes(AssetFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(Path / AssetPath)); + Content.Attributes.push_back(GetFileMode(AssetFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); LocalFolderScanStats.AcceptedFileCount++; @@ -2917,12 +2986,13 @@ namespace { if (ManifestPath.is_relative()) { Content.Paths.push_back(ManifestPath); - Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath)); + const std::filesystem::path ManifestFilePath = (Path / ManifestPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(ManifestPath)); + Content.Attributes.push_back(GetFileAttributes(ManifestFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(ManifestPath)); + Content.Attributes.push_back(GetFileMode(ManifestFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); @@ -3751,7 +3821,7 @@ namespace { if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) { const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!std::filesystem::exists(TargetPath)) + if (!IsFile(TargetPath)) { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); @@ -3761,7 +3831,7 @@ namespace { else { std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow(std::filesystem::file_size(TargetPath, Ec)); + uint64_t SizeOnDisk = gsl::narrow(FileSizeFromPath(TargetPath, Ec)); if (Ec) { ErrorLock.WithExclusiveLock([&]() { @@ -3997,9 +4067,15 @@ namespace { void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), - GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::error_code Ec; + RenameFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash), + Ec); + if (Ec) + { + throw std::system_error(Ec); + } } void FinalizeChunkSequences(const std::filesystem::path& TargetFolder, @@ -4646,7 +4722,7 @@ namespace { Payload.SetDeleteOnClose(false); Payload = {}; CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + RenameFile(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { CompressedChunkPath = std::filesystem::path{}; @@ -4717,7 +4793,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -4823,13 +4899,13 @@ namespace { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); continue; } } } - std::filesystem::remove(CacheDirContent.Files[Index]); + RemoveFileWithRetry(CacheDirContent.Files[Index]); } } @@ -4875,7 +4951,7 @@ namespace { } } } - std::filesystem::remove(BlockDirContent.Files[Index]); + RemoveFileWithRetry(BlockDirContent.Files[Index]); } } @@ -4896,7 +4972,7 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4905,14 +4981,14 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; @@ -5181,7 +5257,7 @@ namespace { std::filesystem::path BlockPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) + if (IsFile(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); UsingCachedBlock = true; @@ -5462,7 +5538,7 @@ namespace { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - if (std::filesystem::exists(CompressedChunkPath)) + if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); if (ExistingCompressedPart) @@ -5481,7 +5557,7 @@ namespace { else { std::error_code DummyEc; - std::filesystem::remove(CompressedChunkPath, DummyEc); + RemoveFile(CompressedChunkPath, DummyEc); } } } @@ -5543,7 +5619,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -5923,11 +5999,11 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -6008,7 +6084,7 @@ namespace { BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6079,14 +6155,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6180,7 +6256,7 @@ namespace { BlockBuffer = {}; BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6259,14 +6335,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6398,7 +6474,7 @@ namespace { const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); if (!WipeTargetFolder) { @@ -6430,27 +6506,18 @@ namespace { if (!CachedRemoteSequences.contains(RawHash)) { // We need it - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - std::error_code Ec; - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedRemoteSequences.insert(RawHash); CachedCount++; } else { // We already have it - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); SkippedCount++; } } @@ -6499,8 +6566,8 @@ namespace { for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnly(LocalFilePath, false); - std::filesystem::remove(LocalFilePath); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } } } @@ -6579,9 +6646,9 @@ namespace { std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { @@ -6615,13 +6682,13 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); } else { - if (std::filesystem::exists(FirstTargetFilePath)) + if (IsFile(FirstTargetFilePath)) { - SetFileReadOnly(FirstTargetFilePath, false); + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); } else { @@ -6634,7 +6701,7 @@ namespace { const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(std::filesystem::exists(SourceFilePath)); + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; @@ -6643,19 +6710,10 @@ namespace { { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - std::error_code Ec; - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } } @@ -6685,20 +6743,20 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(TargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(TargetFilePath)); } else { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { CreateDirectories(TargetFilePath.parent_path()); } - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } @@ -7167,7 +7225,7 @@ namespace { ChunkedFolderContent LocalContent; bool HasLocalState = false; - if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + if (IsFile(ZenStateFilePath(ZenFolderPath))) { try { @@ -7268,9 +7326,9 @@ namespace { { const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) + if (IsFile(LocalFilePath)) { - const uint64_t FileSize = std::filesystem::file_size(LocalFilePath); + const uint64_t FileSize = FileSizeFromPath(LocalFilePath); OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); @@ -7502,7 +7560,7 @@ namespace { FolderContent LocalFolderContent; if (!PrimeCacheOnly) { - if (std::filesystem::is_directory(Path)) + if (IsDir(Path)) { if (!WipeTargetFolder) { @@ -7692,7 +7750,7 @@ namespace { } if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + RemoveDirWithRetry(ZenTempFolder); } } @@ -8479,7 +8537,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8592,7 +8650,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8721,7 +8779,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8799,7 +8857,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8871,7 +8929,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (DownloadPath / ZenFolderName) : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -9000,10 +9058,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { Work.ScheduleWork( GetIOWorkerPool(), - [SourceSize, FilePath](std::atomic&) { + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic&) { if (!AbortFlag) { - bool IsReadOnly = SetFileReadOnly(FilePath, false); + bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); { BasicFile Source(FilePath, BasicFile::Mode::kWrite); uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); @@ -9030,7 +9088,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } break; case 1: - std::filesystem::remove(FilePath); + { + (void)SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + } break; default: break; @@ -9184,7 +9245,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -9240,7 +9301,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 6ec6a80db..185edc35d 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -629,7 +629,7 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_OutputPath.empty()) { TargetPath = std::filesystem::path(m_OutputPath); - if (std::filesystem::is_directory(TargetPath)) + if (IsDir(TargetPath)) { TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index cc6ddd505..53e80c896 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -64,8 +64,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); - const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); + const bool IsFileCopy = IsFile(m_CopySource); + const bool IsDirCopy = IsDir(m_CopySource); if (!IsFileCopy && !IsDirCopy) { @@ -79,20 +79,14 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsDirCopy) { - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - const bool IsTargetDir = std::filesystem::is_directory(ToPath); - if (!IsTargetDir) - { - if (std::filesystem::is_regular_file(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - } + throw std::runtime_error("Attempted copy of directory into file"); } - else + + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 6bc499f03..13c7c4b23 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,7 +41,7 @@ namespace { std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -2185,7 +2185,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - std::filesystem::remove_all(TmpPath); + DeleteDirectories(TmpPath); ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 5b88a1f73..4fc38d92a 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -196,7 +196,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("Rpc replay command requires a path"); } - if (!std::filesystem::exists(m_RecordingPath) || !std::filesystem::is_directory(m_RecordingPath)) + if (!IsDir(m_RecordingPath)) { throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); } diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp index a99ba9704..309b8996a 100644 --- a/src/zen/cmds/run_cmd.cpp +++ b/src/zen/cmds/run_cmd.cpp @@ -100,7 +100,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - CleanDirectory(BaseDirectory); + CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); } } diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index f87725e36..64039e4c9 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -67,7 +67,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("command requires a root path"); } - if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + if (!IsDir(m_RootPath)) { throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); } diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 4d1534e05..88c0b22a2 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -33,7 +33,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 44a41146c..aacc115a0 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -159,7 +159,7 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; @@ -229,7 +229,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index a181bbd66..ea526399c 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -590,7 +590,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: // deleting the temporary file BasicFile::Close(); - std::filesystem::rename(m_TempPath, FinalFileName, Ec); + RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { @@ -984,9 +984,9 @@ TEST_CASE("TemporaryFile") TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); - CHECK(std::filesystem::exists(Path)); + CHECK(IsFile(Path)); } - CHECK(std::filesystem::exists(Path) == false); + CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") @@ -997,11 +997,11 @@ TEST_CASE("TemporaryFile") CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); + CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); + CHECK(IsFile(TempPath) == false); + CHECK(IsFile(FinalPath)); } } diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp index d5eabea9d..610b0ced5 100644 --- a/src/zencore/except.cpp +++ b/src/zencore/except.cpp @@ -47,7 +47,7 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string { if (HRESULT_FACILITY(hRes) == FACILITY_WIN32) { - throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message)); + throw std::system_error(std::error_code(HRESULT_CODE(hRes), std::system_category()), std::string(Message)); } else { diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8a369f02e..6ff4dd053 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -86,16 +86,9 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) } bool -CreateDirectories(const wchar_t* Dir) +CreateDirectories(const wchar_t* Path) { - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); + return CreateDirectories(std::filesystem::path(Path)); } // Erase all files and directories in a given directory, leaving an empty directory @@ -212,75 +205,326 @@ DeleteDirectoriesInternal(const wchar_t* DirPath) bool CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles) { - if (std::filesystem::exists(DirPath)) + if (IsDir(DirPath)) { return WipeDirectory(DirPath, KeepDotFiles); } return CreateDirectories(DirPath); } + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; +#endif // ZEN_PLATFORM_WINDOWS + +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; #endif // ZEN_PLATFORM_WINDOWS +} bool -CreateDirectories(const std::filesystem::path& Dir) +IsFileModeReadOnly(uint32_t FileMode) { - if (Dir.string().ends_with(":")) + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + +bool +RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::RemoveDirectory(Path.native().c_str()); + if (!Success) { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } return false; } - while (!std::filesystem::is_directory(Dir)) + return true; +#else + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::DeleteFile(Path.native().c_str()); + if (!Success) { - if (Dir.has_parent_path()) + if (ForceRemoveReadOnlyFiles) { - CreateDirectories(Dir.parent_path()); + DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str()); + if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) + { + ::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(Path.native().c_str()); + } } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) + if (!Success) { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; } - return true; } - return false; + return true; +#else + if (!ForceRemoveReadOnlyFiles) + { + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + const uint32_t Mode = (uint32_t)Stat.st_mode; + if (IsFileModeReadOnly(Mode)) + { + Ec = MakeErrorCode(EACCES); + return false; + } + } + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +static void +WipeDirectoryContentInternal(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsFile(LocalFilePath)) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + } + } + if (Ec) + { + return; + } + } + + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + { + WipeDirectoryContentInternal(LocalDirPath, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + return; + } + + RemoveDirNative(LocalDirPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(LocalDirPath)) + { + RemoveDirNative(LocalDirPath, Ec); + } + } + if (Ec) + { + return; + } + } +} + +bool +CreateDirectory(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::CreateDirectory(Path.native().c_str(), nullptr); + if (!Success) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return Success; +#else + return std::filesystem::create_directory(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS } bool -DeleteDirectories(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path) { - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); + std::error_code Ec; + bool Success = CreateDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to create directories for '{}'", Path.string())); + } + return Success; } bool -CleanDirectory(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec) { - if (std::filesystem::exists(Dir)) + if (Path.string().ends_with(":")) { - bool Success = true; + return false; + } + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return false; + } + if (Exists) + { + return false; + } - for (const auto& Item : std::filesystem::directory_iterator(Dir)) + if (Path.has_parent_path()) + { + bool Result = CreateDirectories(Path.parent_path(), Ec); + if (Ec) { - std::error_code ErrorCode; - const uintmax_t RemovedCount = std::filesystem::remove_all(Item, ErrorCode); - - Success = Success && !ErrorCode && RemovedCount; + return Result; } + } + return CreateDirectory(Path, Ec); +} - return Success; +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles) +{ + std::error_code Ec; + bool Result = CleanDirectory(Path, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to clean directory for '{}'", Path.string())); } + return Result; +} - return CreateDirectories(Dir); +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + if (Exists) + { + WipeDirectoryContentInternal(Path, ForceRemoveReadOnlyFiles, Ec); + return false; + } + return CreateDirectory(Path, Ec); +} + +bool +DeleteDirectories(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = DeleteDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to delete directories for '{}'", Path.string())); + } + return Result; } bool -CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir) +DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + + if (Exists) + { + WipeDirectoryContentInternal(Path, false, Ec); + if (Ec) + { + return false; + } + bool Result = RemoveDirNative(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(Path)) + { + Result = RemoveDirNative(Path, Ec); + } + } + return Result; + } + return false; +} + +bool +CleanDirectoryExceptDotFiles(const std::filesystem::path& Path) { #if ZEN_PLATFORM_WINDOWS const bool KeepDotFiles = true; - return CleanDirectory(Dir.c_str(), KeepDotFiles); + return CleanDirectory(Path.c_str(), KeepDotFiles); #else - ZEN_UNUSED(Dir); + ZEN_UNUSED(Path); ZEN_NOT_IMPLEMENTED(); #endif @@ -637,7 +881,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { // Validate arguments - if (FromPath.empty() || !std::filesystem::is_directory(FromPath)) + if (FromPath.empty() || !IsDir(FromPath)) throw std::runtime_error("invalid CopyTree source directory specified"); if (ToPath.empty()) @@ -646,16 +890,13 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop if (Options.MustClone && !SupportsBlockRefCounting(FromPath)) throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath)); - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - if (!std::filesystem::is_directory(ToPath)) - { - throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); - } + throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); } - else + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } if (Options.MustClone && !SupportsBlockRefCounting(ToPath)) @@ -811,7 +1052,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { Outfile.Close(); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else @@ -819,7 +1060,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { close(Fd); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); } #endif // ZEN_PLATFORM_WINDOWS @@ -1172,7 +1413,7 @@ void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { #if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; + std::vector FileInfoBuffer(8 * 1024); FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; @@ -1183,7 +1424,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr if (FAILED(hRes)) { - if (hRes == ERROR_FILE_NOT_FOUND || hRes == ERROR_PATH_NOT_FOUND) + if (HRESULT_CODE(hRes) == ERROR_FILE_NOT_FOUND || HRESULT_CODE(hRes) == ERROR_PATH_NOT_FOUND) { // Directory no longer exist, treat it as empty return; @@ -1193,8 +1434,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr while (Continue) { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + BOOL Success = + GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer.data(), (DWORD)(FileInfoBuffer.size() * sizeof(uint64_t))); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration uint64_t EntryOffset = 0; @@ -1213,7 +1455,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr for (;;) { const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast(reinterpret_cast(FileInfoBuffer) + EntryOffset); + reinterpret_cast(reinterpret_cast(FileInfoBuffer.data()) + EntryOffset); std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); @@ -1338,6 +1580,172 @@ CanonicalPath(std::filesystem::path InPath, std::error_code& Ec) #endif } +bool +IsFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a file", Path.string())); + } + return Result; +} + +bool +IsFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISREG(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a directory", Path.string())); + } + return Result; +} + +bool +IsDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISDIR(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove file '{}'", Path.string())); + } + return Success; +} + +bool +RemoveFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveFileNative(Path, false, Ec); +#else + bool IsDirectory = std::filesystem::is_directory(Path, Ec); + if (IsDirectory) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveFileNative(Path, false, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove directory '{}'", Path.string())); + } + return Success; +} + +bool +RemoveDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveDirNative(Path, Ec); +#else + bool IsFile = std::filesystem::is_regular_file(Path, Ec); + if (IsFile) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveDirNative(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec) { @@ -1434,6 +1842,49 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec) #endif // ZEN_PLATFORM_WINDOWS } +uint64_t +FileSizeFromPath(const std::filesystem::path& Path) +{ + std::error_code Ec; + uint64_t Size = FileSizeFromPath(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get file size for path '{}'", Path.string())); + } + return Size; +} + +uint64_t +FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + void* Handle = ::CreateFile(Path.native().c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + DWORD LastError = GetLastError(); + Ec = MakeErrorCode(LastError); + return 0; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + LARGE_INTEGER FileSize; + BOOL Success = GetFileSizeEx(Handle, &FileSize); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + return 0; + } + return FileSize.QuadPart; +#else + return std::filesystem::file_size(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + uint64_t FileSizeFromHandle(void* NativeHandle) { @@ -1483,7 +1934,13 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) // PathFromHandle void* Handle; #if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + Handle = CreateFileW(Filename.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); if (Handle == INVALID_HANDLE_VALUE) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); @@ -1493,7 +1950,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) { - ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(Ec, fmt::format("Failed to get modification tick for path '{}'", Filename.string())); } return ModificatonTick; #else @@ -1507,6 +1964,56 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename path from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFileEx(SourcePath.native().c_str(), TargetPath.native().c_str(), MOVEFILE_REPLACE_EXISTING); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameDirectory(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename directory from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFile(SourcePath.native().c_str(), TargetPath.native().c_str()); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path GetRunningExecutablePath() { @@ -1793,7 +2300,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) }; auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool { - bool Exists = std::filesystem::exists(Path, Ec); + bool Exists = IsFile(Path, Ec); if (Ec) { return false; @@ -1802,7 +2309,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) { return true; } - uintmax_t Size = std::filesystem::file_size(Path, Ec); + uintmax_t Size = FileSizeFromPath(Path, Ec); if (Ec) { return false; @@ -1821,17 +2328,17 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) for (auto i = MaxFiles; i > 0; i--) { std::filesystem::path src = GetFileName(i - 1); - if (!std::filesystem::exists(src)) + if (!IsFile(src)) { continue; } std::error_code DummyEc; std::filesystem::path target = GetFileName(i); - if (std::filesystem::exists(target, DummyEc)) + if (IsFile(target, DummyEc)) { - std::filesystem::remove(target, DummyEc); + RemoveFile(target, DummyEc); } - std::filesystem::rename(src, target, DummyEc); + RenameFile(src, target, DummyEc); } } @@ -1868,16 +2375,16 @@ RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDir { const std::filesystem::path SourcePath = GetPathForIndex(i - 1); - if (std::filesystem::exists(SourcePath)) + if (IsDir(SourcePath)) { std::filesystem::path TargetPath = GetPathForIndex(i); std::error_code DummyEc; - if (std::filesystem::exists(TargetPath, DummyEc)) + if (IsDir(TargetPath, DummyEc)) { - std::filesystem::remove_all(TargetPath, DummyEc); + DeleteDirectories(TargetPath, DummyEc); } - std::filesystem::rename(SourcePath, TargetPath, DummyEc); + RenameDirectory(SourcePath, TargetPath, DummyEc); } } @@ -1936,22 +2443,46 @@ PickDefaultSystemRootDirectory() #if ZEN_PLATFORM_WINDOWS uint32_t -GetFileAttributes(const std::filesystem::path& Filename) +GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec) { DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); if (Attributes == INVALID_FILE_ATTRIBUTES) { - ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Attributes; } +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = zen::GetFileAttributes(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to get attributes of file '{}'", Filename.string())); + } + return Result; +} + void -SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec) { if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) { - ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + } +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + std::error_code Ec; + zen::SetFileAttributes(Filename, Attributes, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set attributes of file {}", Filename.string())); } } @@ -1961,88 +2492,104 @@ SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) uint32_t GetFileMode(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = GetFileMode(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get mode of file {}", Filename)); + } + return Result; +} + +uint32_t +GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec) { struct stat Stat; int err = stat(Filename.native().c_str(), &Stat); if (err) { - ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Stat.st_mode; } void -SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode) { - int err = chmod(Filename.native().c_str(), (mode_t)Attributes); - if (err) + std::error_code Ec; + SetFileMode(Filename, Mode, Ec); + if (Ec) { - ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + throw std::system_error(Ec, fmt::format("Failed to set mode of file {}", Filename)); } } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -#if ZEN_PLATFORM_WINDOWS -const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; -#else -const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; -#endif // ZEN_PLATFORM_WINDOWS - -const uint32_t FileModeWriteEnableFlags = 0222; - -bool -IsFileAttributeReadOnly(uint32_t FileAttributes) -{ -#if ZEN_PLATFORM_WINDOWS - return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; -#else - return (FileAttributes & 0x00000001) != 0; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -IsFileModeReadOnly(uint32_t FileMode) -{ - return (FileMode & FileModeWriteEnableFlags) == 0; -} - -uint32_t -MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec) { - return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); + int err = chmod(Filename.native().c_str(), (mode_t)Mode); + if (err) + { + Ec = MakeErrorCodeFromLastError(); + } } -uint32_t -MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) -{ - return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); -} +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool -SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - uint32_t CurrentAttributes = GetFileAttributes(Filename); - uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + uint32_t CurrentAttributes = GetFileAttributes(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); if (CurrentAttributes != NewAttributes) { - SetFileAttributes(Filename, NewAttributes); + SetFileAttributes(Filename, NewAttributes, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - uint32_t CurrentMode = GetFileMode(Filename); - uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + uint32_t CurrentMode = GetFileMode(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); if (CurrentMode != NewMode) { - SetFileMode(Filename, NewMode); + SetFileMode(Filename, NewMode, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC return false; } +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ + std::error_code Ec; + bool Result = SetFileReadOnly(Filename, ReadOnly, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set read only mode of file '{}'", Filename.string())); + } + return Result; +} + std::filesystem::path StringToPath(const std::string_view& Path) { @@ -2079,7 +2626,7 @@ TEST_CASE("filesystem") path BinPath = GetRunningExecutablePath(); const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); + CHECK(IsFile(BinPath)); // PathFromHandle void* Handle; @@ -2132,6 +2679,80 @@ TEST_CASE("filesystem") CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); } +TEST_CASE("Filesystem.Basics") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; + CleanDirectory(TestBaseDir, true); + DeleteDirectories(TestBaseDir); + CHECK(!IsDir(TestBaseDir)); + CHECK(CleanDirectory(TestBaseDir, false)); + CHECK(IsDir(TestBaseDir)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(!IsDir(TestBaseDir / "no_such_thing")); + CHECK(!IsDir("hgjda/cev_/q12")); + CHECK(!IsFile(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "no_such_thing")); + CHECK(!IsFile("hgjda/cev_/q12")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir) == 0); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "no_such_file")); + CHECK(!CreateDirectories(TestBaseDir)); + CHECK(CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(!CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep" / "no")); + CHECK_THROWS(WriteFile(TestBaseDir / "nested" / "a", IoBuffer(20))); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "a" / "yo", IoBuffer(20))); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "nested" / "a" / "yo") == 20); + CHECK(!IsFile(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsDir(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested")); + CHECK(!IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(CreateDirectories(TestBaseDir / "nested" / "deeper")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "deeper" / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK(IsDir(TestBaseDir / "new_place")); + CHECK(!IsFile(TestBaseDir / "new_place")); + CHECK_THROWS(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(!RemoveDir(TestBaseDir / "nested" / "deeper")); + CHECK(RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "new_place" / "yo")); + CHECK(!RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested")); + CHECK_THROWS(RemoveDir(TestBaseDir)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameFile(TestBaseDir / "yo", TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "yo")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK_THROWS(RemoveDir(TestBaseDir / "new_place" / "yo")); + CHECK(DeleteDirectories(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsDir(TestBaseDir)); + CHECK(!IsDir(TestBaseDir / "nested")); + CHECK(CreateDirectories(TestBaseDir / "nested")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK_THROWS(CleanDirectory(TestBaseDir, false)); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", false)); + CHECK(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK(!CleanDirectory(TestBaseDir / "nested", true)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(RemoveDir(TestBaseDir)); +} + TEST_CASE("WriteFile") { std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); @@ -2166,7 +2787,7 @@ TEST_CASE("WriteFile") CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); } - std::filesystem::remove(TempFile); + RemoveFile(TempFile); } TEST_CASE("DiskSpaceInfo") @@ -2223,7 +2844,7 @@ TEST_CASE("PathBuilder") TEST_CASE("RotateDirectories") { std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; - CleanDirectory(TestBaseDir); + CleanDirectory(TestBaseDir, false); std::filesystem::path RotateDir = TestBaseDir / "rotate_dir" / "dir_to_rotate"; IoBuffer DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5); @@ -2237,16 +2858,16 @@ TEST_CASE("RotateDirectories") const int RotateMax = 10; NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); - CHECK(std::filesystem::exists(DirWithSuffix(2))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); + CHECK(IsDir(DirWithSuffix(2))); for (int i = 0; i < RotateMax; ++i) { @@ -2256,16 +2877,16 @@ TEST_CASE("RotateDirectories") CHECK_EQ(IsError, false); } - CHECK(!std::filesystem::exists(RotateDir)); + CHECK(!IsDir(RotateDir)); for (int i = 0; i < RotateMax; ++i) { - CHECK(std::filesystem::exists(DirWithSuffix(i + 1))); + CHECK(IsDir(DirWithSuffix(i + 1))); } for (int i = RotateMax; i < RotateMax + 5; ++i) { - CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1))); + CHECK(!IsDir(DirWithSuffix(RotateMax + i + 1))); } } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 9a2b15d1d..c23f16d03 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -20,21 +20,35 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists. + + Will also create any required parent direCleanDirectoryctories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ @@ -44,6 +58,46 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_ */ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); @@ -56,6 +110,22 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -277,12 +347,16 @@ std::filesystem::path PickDefaultSystemRootDirectory(); #if ZEN_PLATFORM_WINDOWS uint32_t GetFileAttributes(const std::filesystem::path& Filename); +uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec); void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC uint32_t GetFileMode(const std::filesystem::path& Filename); -void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool IsFileAttributeReadOnly(uint32_t FileAttributes); @@ -290,6 +364,7 @@ bool IsFileModeReadOnly(uint32_t FileMode); uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); std::filesystem::path StringToPath(const std::string_view& Path); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 0761521dc..2fe5b8948 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -60,7 +60,7 @@ GetPidStatus(int Pid, std::error_code& OutEc) { std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid); std::filesystem::path StatPath = EntryPath / "stat"; - if (std::filesystem::is_regular_file(StatPath)) + if (IsFile(StatPath)) { FILE* StatFile = fopen(StatPath.c_str(), "r"); if (StatFile) diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 641d5508a..9f50de032 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -4,6 +4,7 @@ #if ZEN_WITH_TESTS +# include # include # include "zencore/string.h" @@ -19,8 +20,8 @@ CreateTemporaryDirectory() std::error_code Ec; std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); - std::filesystem::remove_all(DirPath, Ec); - std::filesystem::create_directories(DirPath); + DeleteDirectories(DirPath, Ec); + CreateDirectories(DirPath); return DirPath; } @@ -32,14 +33,14 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory) { std::error_code Ec; - std::filesystem::remove_all(Directory, Ec); - std::filesystem::create_directories(Directory); + DeleteDirectories(Directory, Ec); + CreateDirectories(Directory); } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { std::error_code Ec; - std::filesystem::remove_all(m_RootPath, Ec); + DeleteDirectories(m_RootPath, Ec); } IoBuffer diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 1a9892d5c..8f7befc80 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -379,7 +379,7 @@ private: AuthState.EndArray(); } - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); std::optional Reason; diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 027a35998..78a735ea0 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -3306,18 +3306,18 @@ GenerateFolderContent(const std::filesystem::path& RootPath) std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -3473,7 +3473,7 @@ TEST_CASE("workspaces.create") while (true) { std::error_code Ec; - std::filesystem::remove_all(Root2Path / Share2Path, Ec); + DeleteDirectories(Root2Path / Share2Path, Ec); if (!Ec) break; } @@ -3630,7 +3630,7 @@ TEST_CASE("workspaces.lifetimes") } // Wipe system config - std::filesystem::remove_all(SystemRootPath); + DeleteDirectories(SystemRootPath); // Restart @@ -3696,8 +3696,8 @@ TEST_CASE("workspaces.share") uint64_t Size = FileObject["size"sv].AsUInt64(); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); - CHECK(std::filesystem::file_size(AbsFilePath) == Size); + CHECK(IsFile(AbsFilePath)); + CHECK(FileSizeFromPath(AbsFilePath) == Size); Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size)); } } @@ -3720,7 +3720,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } } @@ -3740,7 +3740,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 0da6e31ad..73166e608 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -40,7 +40,7 @@ struct DirStats DirStats GetStatsForDirectory(std::filesystem::path Dir) { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) return {}; struct StatsTraversal : public GetDirectoryContentVisitor diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index d3af0c6a6..52f539dcd 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -1116,7 +1116,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (DataDir.empty()) throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); - if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) + if (!IsDir(ServerOptions.BaseSnapshotDir)) throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); } diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 31d9e1c94..104b26954 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -50,7 +50,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { break; } - if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode)) + if (IsFile(ParentPath / "xmake.lua", ErrorCode)) { if (ErrorCode) { @@ -59,7 +59,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html"; - if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + if (IsDir(HtmlDir, ErrorCode)) { m_Directory = HtmlDir; } diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index d5419d342..78ddd39a0 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -406,17 +406,17 @@ main(int argc, char* argv[]) if (!DeleteReason.empty()) { - if (std::filesystem::exists(ServerOptions.DataDir)) + if (IsDir(ServerOptions.DataDir)) { ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } - if (!std::filesystem::exists(ServerOptions.DataDir)) + if (!IsDir(ServerOptions.DataDir)) { ServerOptions.IsFirstRun = true; - std::filesystem::create_directories(ServerOptions.DataDir); + CreateDirectories(ServerOptions.DataDir); } if (!ServerOptions.BaseSnapshotDir.empty()) diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index e757ef84e..5af803617 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -251,7 +251,7 @@ HttpObjectStoreService::Inititalize() ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!fs::exists(BucketsPath)) + if (!IsDir(BucketsPath)) { CreateDirectories(BucketsPath); } @@ -324,7 +324,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { std::lock_guard _(BucketsMutex); - if (!fs::exists(BucketPath)) + if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); @@ -406,7 +406,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); FileSystemTraversal Traversal; - if (std::filesystem::exists(FullPath)) + if (IsDir(FullPath)) { std::lock_guard _(BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); @@ -475,7 +475,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st } const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!fs::exists(FilePath)) + if (!IsFile(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -576,7 +576,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { std::lock_guard _(BucketsMutex); - if (!fs::exists(FileDirectory)) + if (!IsDir(FileDirectory)) { CreateDirectories(FileDirectory); } diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 98e292d91..375e44e59 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -73,7 +73,7 @@ public: ContainerObject.IterateAttachments([&](CbFieldView FieldView) { IoHash AttachmentHash = FieldView.AsBinaryAttachment(); std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash); - if (!std::filesystem::exists(AttachmentPath)) + if (!IsFile(AttachmentPath)) { Result.Needs.insert(AttachmentHash); } @@ -111,7 +111,7 @@ public: Stopwatch Timer; SaveAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::exists(ChunkPath)) + if (!IsFile(ChunkPath)) { try { @@ -182,7 +182,7 @@ public: for (const IoHash& RawHash : BlockHashes) { std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (std::filesystem::is_regular_file(ChunkPath)) + if (IsFile(ChunkPath)) { ExistingBlockHashes.push_back(RawHash); } @@ -203,7 +203,7 @@ public: Stopwatch Timer; LoadAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::is_regular_file(ChunkPath)) + if (!IsFile(ChunkPath)) { Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); @@ -246,7 +246,7 @@ private: LoadContainerResult Result; std::filesystem::path SourcePath = m_OutputPath; SourcePath.append(Name); - if (!std::filesystem::is_regular_file(SourcePath)) + if (!IsFile(SourcePath)) { Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index ea3f2aad9..1966eeef9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -58,7 +58,7 @@ namespace { std::filesystem::path DroppedBucketPath; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return true; } @@ -68,7 +68,7 @@ namespace { std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId); DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { if (!DeleteDirectories(DroppedBucketPath)) { @@ -77,7 +77,7 @@ namespace { Dir); continue; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir); continue; @@ -88,13 +88,13 @@ namespace { do { std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { OutDeleteDir = DroppedBucketPath; return true; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.", Dir, @@ -486,7 +486,7 @@ struct ProjectStore::OplogStorage : public RefCounted [[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); } [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath)); + return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath)); } [[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); } [[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath) @@ -496,13 +496,13 @@ struct ProjectStore::OplogStorage : public RefCounted void WipeState() const { std::error_code Ec; - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); } static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); } - uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); } + uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); } uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); } static uint64_t OpsSize(const std::filesystem::path& BasePath) @@ -510,7 +510,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (Exists(BasePath)) { std::error_code DummyEc; - return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath)); + return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath)); } return 0; } @@ -689,7 +689,7 @@ struct ProjectStore::OplogStorage : public RefCounted m_OpBlobs.Close(); Oplog.Close(); - std::filesystem::rename(OplogPath, GetLogPath(), Ec); + RenameFile(OplogPath, GetLogPath(), Ec); if (Ec) { throw std::system_error( @@ -702,9 +702,9 @@ struct ProjectStore::OplogStorage : public RefCounted if (Ec) { // We failed late - clean everything up as best we can - std::filesystem::remove(OpBlobs.GetPath(), Ec); - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); throw std::system_error(Ec, fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'", OpBlobs.GetPath(), @@ -739,7 +739,7 @@ struct ProjectStore::OplogStorage : public RefCounted } catch (const std::exception& /*Ex*/) { - std::filesystem::remove(OpBlobs.GetPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); throw; } } @@ -1108,7 +1108,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath); m_Storage->WipeState(); std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } } m_Storage->Open(/* IsCreate */ !StoreExists); @@ -1116,7 +1116,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, m_MetaPath = m_BasePath / "ops.meta"sv; m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath()); - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); } ProjectStore::Oplog::~Oplog() @@ -1142,7 +1142,7 @@ ProjectStore::Oplog::Flush() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } uint64_t LogCount = m_Storage->LogCount(); @@ -1238,19 +1238,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = OplogStorage::OpsSize(BasePath); std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::exists(StateFilePath)) + if (IsFile(StateFilePath)) { - Size += std::filesystem::file_size(StateFilePath); + Size += FileSizeFromPath(StateFilePath); } std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv; - if (std::filesystem::exists(MetaFilePath)) + if (IsFile(MetaFilePath)) { - Size += std::filesystem::file_size(MetaFilePath); + Size += FileSizeFromPath(MetaFilePath); } std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv; - if (std::filesystem::exists(IndexFilePath)) + if (IsFile(IndexFilePath)) { - Size += std::filesystem::file_size(IndexFilePath); + Size += FileSizeFromPath(IndexFilePath); } return Size; @@ -1303,7 +1303,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath) using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - return std::filesystem::is_regular_file(StateFilePath); + return IsFile(StateFilePath); } bool @@ -1337,7 +1337,7 @@ ProjectStore::Oplog::Read() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } ReadIndexSnapshot(); @@ -1438,7 +1438,7 @@ ProjectStore::Oplog::Reset() m_Storage = new OplogStorage(this, m_BasePath); m_Storage->Open(true); m_MetaValid = false; - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); Write(); } // Erase content on disk @@ -1457,7 +1457,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::is_regular_file(StateFilePath)) + if (IsFile(StateFilePath)) { // ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath); @@ -1536,7 +1536,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo if (File.Hash == IoHash::Zero) { std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); HasMissingEntries = true; @@ -1625,10 +1625,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp"; // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'", GetOuterProject()->Identifier, @@ -1641,9 +1641,9 @@ ProjectStore::Oplog::WriteIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -1778,11 +1778,11 @@ ProjectStore::Oplog::WriteIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'", @@ -1793,10 +1793,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'", m_OuterProject->Identifier, @@ -1814,7 +1814,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot"); std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint64_t EntryCount = 0; Stopwatch Timer; @@ -3133,7 +3133,7 @@ ProjectStore::Project::~Project() bool ProjectStore::Project::Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(BasePath / "Project.zcb"); + return IsFile(BasePath / "Project.zcb"); } void @@ -3207,7 +3207,7 @@ ProjectStore::Project::ReadAccessTimes() using namespace std::literals; std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - if (!std::filesystem::exists(ProjectAccessTimesFilePath)) + if (!IsFile(ProjectAccessTimesFilePath)) { return; } @@ -3598,14 +3598,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = 0; std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv; - if (std::filesystem::exists(AccessTimesFilePath)) + if (IsFile(AccessTimesFilePath)) { - Size += std::filesystem::file_size(AccessTimesFilePath); + Size += FileSizeFromPath(AccessTimesFilePath); } std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectFilePath)) + if (IsFile(ProjectFilePath)) { - Size += std::filesystem::file_size(ProjectFilePath); + Size += FileSizeFromPath(ProjectFilePath); } return Size; @@ -3717,7 +3717,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName, if (!MarkerPath.empty()) { std::error_code Ec; - if (std::filesystem::exists(MarkerPath, Ec)) + if (IsFile(MarkerPath, Ec)) { if (Ec) { @@ -3870,7 +3870,7 @@ void ProjectStore::DiscoverProjects() { ZEN_MEMSCOPE(GetProjectstoreTag()); - if (!std::filesystem::exists(m_ProjectBasePath)) + if (!IsDir(m_ProjectBasePath)) { return; } @@ -3979,7 +3979,7 @@ ProjectStore::StorageSize() const GcStorageSize Result; { - if (std::filesystem::exists(m_ProjectBasePath)) + if (IsDir(m_ProjectBasePath)) { DirectoryContent ProjectsFolderContent; GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent); @@ -3987,7 +3987,7 @@ ProjectStore::StorageSize() const for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) { std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectStateFilePath)) + if (IsFile(ProjectStateFilePath)) { Result.DiskSize += Project::TotalSize(ProjectBasePath); DirectoryContent DirContent; @@ -7243,7 +7243,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project1FilePath); + RemoveFile(Project1FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7272,7 +7272,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2Oplog1Path); + RemoveFile(Project2Oplog1Path); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7301,7 +7301,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2FilePath); + RemoveFile(Project2FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), @@ -8025,7 +8025,7 @@ TEST_CASE("project.store.rpc.getchunks") CompositeBuffer Buffer = Attachment->AsCompositeBinary(); CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), IoHash::HashBuffer(Buffer)); - CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second)); CHECK(!Chunk.FindView("RawSize")); } { diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index a7263da83..f96b3e185 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -1212,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore, { std::string_view ServerPath = View["serverpath"sv].AsString(); std::filesystem::path FilePath = Project.RootDir / ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { remotestore_impl::ReportMessage( OptionalContext, @@ -3083,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore, OptionalContext]() { auto _ = MakeGuard([&DechunkLatch, &TempFileName] { std::error_code Ec; - if (std::filesystem::exists(TempFileName, Ec)) + if (IsFile(TempFileName, Ec)) { - std::filesystem::remove(TempFileName, Ec); + RemoveFile(TempFileName, Ec); if (Ec) { ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message()); diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 0b7fd0400..ac0aaef8e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -1100,7 +1100,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace } } - if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath)) + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) { return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 03e269d49..45c91d691 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -434,7 +434,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -483,7 +483,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -914,7 +914,7 @@ ZenServer::CheckStateMarker() std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 63c0388fa..7cc09be15 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -85,7 +85,7 @@ BlockStoreFile::Create(uint64_t InitialSize) ZEN_TRACE_CPU("BlockStoreFile::Create"); auto ParentPath = m_Path.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + if (!IsDir(ParentPath)) { CreateDirectories(ParentPath); } @@ -215,7 +215,7 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p } if (MetaWriteTime < BlockWriteTime) { - std::filesystem::remove(MetaPath, Ec); + RemoveFile(MetaPath, Ec); return false; } return true; @@ -239,7 +239,7 @@ BlockStoreFile::MetaSize() const if (IsMetaDataValid(m_Path, MetaPath)) { std::error_code DummyEc; - if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc) + if (uint64_t Size = FileSizeFromPath(MetaPath, DummyEc); !DummyEc) { return Size; } @@ -252,7 +252,7 @@ BlockStoreFile::RemoveMeta() { std::filesystem::path MetaPath = GetMetaPath(); std::error_code DummyEc; - std::filesystem::remove(MetaPath, DummyEc); + RemoveFile(MetaPath, DummyEc); } std::filesystem::path @@ -291,7 +291,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max m_MaxBlockSize = MaxBlockSize; m_MaxBlockCount = MaxBlockCount; - if (std::filesystem::is_directory(m_BlocksBasePath)) + if (IsDir(m_BlocksBasePath)) { uint32_t NextBlockIndex = 0; std::vector FoldersToScan; @@ -500,7 +500,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&, { OutBlockPath = GetBlockPath(m_BlocksBasePath, ProbeIndex); std::error_code Ec; - bool Exists = std::filesystem::exists(OutBlockPath, Ec); + bool Exists = IsFile(OutBlockPath, Ec); if (Ec) { ZEN_WARN("Failed to probe existence of file '{}' when trying to allocate a new block. Reason: '{}'", @@ -1375,14 +1375,14 @@ TEST_CASE("blockstore.blockfile") BoopChunk = File1.GetChunk(5, 5); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); const char* Data = static_cast(DataChunk.GetData()); CHECK(std::string(Data) == "data"); const char* Boop = static_cast(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); { IoBuffer DataChunk; @@ -1401,7 +1401,7 @@ TEST_CASE("blockstore.blockfile") const char* Boop = static_cast(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(!std::filesystem::exists(RootDirectory / "1")); + CHECK(!IsFile(RootDirectory / "1")); } namespace blockstore::impl { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index eb36be049..f26901458 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -71,7 +71,7 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) { std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); - bool IsNew = !(std::filesystem::exists(BlobLogPath) && std::filesystem::exists(MetaLogPath)); + bool IsNew = !(IsFile(BlobLogPath) && IsFile(MetaLogPath)); if (!IsNew) { @@ -501,7 +501,7 @@ uint64_t BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -518,7 +518,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys TCasLogFile CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); @@ -603,7 +603,7 @@ uint64_t BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -620,7 +620,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy TCasLogFile CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..b2d2416be 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -200,21 +200,21 @@ namespace cache::impl { int DropIndex = 0; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return false; } std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { DropIndex++; continue; } std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { DeleteDirectories(DroppedBucketPath); @@ -909,16 +909,16 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, { std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { - if (!std::filesystem::remove(LogPath, Ec) || Ec) + if (!RemoveFile(LogPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); std::error_code RemoveIndexEc; - std::filesystem::remove(IndexPath, RemoveIndexEc); + RemoveFile(IndexPath, RemoveIndexEc); } } } @@ -939,7 +939,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const { ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile"); - if (!std::filesystem::is_regular_file(IndexPath)) + if (!IsFile(IndexPath)) { return 0; } @@ -1023,7 +1023,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std:: { ZEN_TRACE_CPU("Z$::Bucket::ReadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -1103,37 +1103,37 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco if (IsNew) { - fs::remove(LogPath); - fs::remove(IndexPath); - fs::remove_all(m_BlocksBasePath); + RemoveFile(LogPath); + RemoveFile(IndexPath); + DeleteDirectories(m_BlocksBasePath); } CreateDirectories(m_BucketDir); m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexLock, IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition); } - else if (fs::is_regular_file(LogPath)) + else if (IsFile(LogPath)) { ZEN_WARN("removing invalid log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -2146,7 +2146,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); std::error_code Ec; - uintmax_t size = std::filesystem::file_size(DataFilePath.ToPath(), Ec); + uintmax_t size = FileSizeFromPath(DataFilePath.ToPath(), Ec); if (Ec) { ReportBadKey(HashKey); @@ -2287,11 +2287,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) BuildPath(Path, Entry.Key); fs::path FilePath = Path.ToPath(); RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key)); - if (fs::is_regular_file(FilePath)) + if (IsFile(FilePath)) { ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8()); std::error_code Ec; - fs::remove(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... + RemoveFile(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... } } } @@ -2424,7 +2424,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (CleanUpTempFile) { std::error_code Ec; - std::filesystem::remove(DataFile.GetPath(), Ec); + RemoveFile(DataFile.GetPath(), Ec); if (Ec) { ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'", @@ -2452,7 +2452,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); // We do a speculative remove of the file instead of probing with a exists call and check the error code instead - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec) { if (Ec.value() != ENOENT) @@ -2460,7 +2460,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message()); Sleep(100); Ec.clear(); - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir)); @@ -2791,7 +2791,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - if (!fs::remove(FilePath, Ec)) + if (!RemoveFile(FilePath, Ec)) { continue; } @@ -2812,7 +2812,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(FilePath, Ec); + bool Existed = IsFile(FilePath, Ec); if (Ec) { ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'", diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..ed42f254e 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -118,7 +118,7 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Ensure root directory exists - create if it doesn't exist already - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); // Open or create manifest diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..492ce9317 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -37,7 +37,7 @@ CasLogFile::~CasLogFile() bool CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) { - if (!std::filesystem::is_regular_file(FileName)) + if (!IsFile(FileName)) { return false; } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index b64bc26dd..184251da7 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -927,10 +927,10 @@ CasContainerStrategy::MakeIndexSnapshot() fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); return; @@ -939,9 +939,9 @@ CasContainerStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -992,21 +992,21 @@ CasContainerStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); } @@ -1092,7 +1092,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski if (!TCasLogFile::IsValid(LogPath)) { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } @@ -1155,7 +1155,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore) { - std::filesystem::remove_all(BasePath); + DeleteDirectories(BasePath); } CreateDirectories(BasePath); @@ -1165,19 +1165,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); std::filesystem::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { @@ -1186,7 +1186,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..14bdc41f0 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -176,10 +176,10 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN if (IsNewStore) { - std::filesystem::remove(LogPath); - std::filesystem::remove(IndexPath); + RemoveFile(LogPath); + RemoveFile(IndexPath); - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { // We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside // in this folder as well @@ -211,24 +211,24 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor); for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots) { - std::filesystem::remove_all(SharededRoot); + DeleteDirectories(SharededRoot); } } } - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { @@ -237,7 +237,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -327,7 +327,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); std::error_code Ec; - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("unable to move existing CAS file {} to {}", ChunkPath, TempPath)); @@ -452,7 +452,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { PayloadFile.Close(); std::error_code DummyEc; - std::filesystem::remove(ChunkPath, DummyEc); + RemoveFile(ChunkPath, DummyEc); throw; } bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); @@ -503,7 +503,7 @@ FileCasStrategy::SafeOpenChunk(const IoHash& ChunkHash, uint64 ExpectedSize) { std::error_code Ec; std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { Chunk.SetDeleteOnClose(true); @@ -574,7 +574,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) ShardingHelper Name(m_RootDirectory, ChunkHash); const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath(); - uint64_t FileSize = static_cast(std::filesystem::file_size(ChunkPath, Ec)); + uint64_t FileSize = static_cast(FileSizeFromPath(ChunkPath, Ec)); if (Ec) { ZEN_WARN("get file size FAILED, file cas '{}'", ChunkPath); @@ -582,9 +582,9 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) } ZEN_DEBUG("deleting CAS payload file '{}' {}", ChunkPath, NiceBytes(FileSize)); - std::filesystem::remove(ChunkPath, Ec); + RemoveFile(ChunkPath, Ec); - if (!Ec || !std::filesystem::exists(ChunkPath)) + if (!Ec || !IsFile(ChunkPath)) { { RwLock::ExclusiveLockScope _(m_Lock); @@ -941,10 +941,10 @@ FileCasStrategy::MakeIndexSnapshot() fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message()); return; @@ -953,9 +953,9 @@ FileCasStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, STmpIndexPath); + RenameFile(IndexPath, STmpIndexPath); } // Write the current state of the location map to a new index state @@ -1004,21 +1004,21 @@ FileCasStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(STmpIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(STmpIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message()); } } } - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message()); } @@ -1032,7 +1032,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; std::vector Entries; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1077,7 +1077,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& return 0; } - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { ZEN_INFO("missing index for file cas, scanning for cas files in {}", m_RootDirectory); TCasLogFile CasLog; @@ -1116,7 +1116,7 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr using namespace filecas::impl; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { uint64_t LogEntryCount = 0; Stopwatch Timer; @@ -1274,12 +1274,12 @@ public: ChunkPath); } std::error_code Ec; - uint64_t SizeOnDisk = std::filesystem::file_size(ChunkPath, Ec); + uint64_t SizeOnDisk = FileSizeFromPath(ChunkPath, Ec); if (Ec) { SizeOnDisk = 0; } - bool Existed = std::filesystem::remove(ChunkPath, Ec); + bool Existed = RemoveFile(ChunkPath, Ec); if (Ec) { // Target file may be open for read, attempt to move it to a temp file and mark it delete on close @@ -1290,7 +1290,7 @@ public: if (OldChunk) { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { OldChunk.SetDeleteOnClose(true); @@ -1317,7 +1317,7 @@ public: else { std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(ChunkPath, Ec); + bool Existed = IsFile(ChunkPath, Ec); if (Ec) { if (Ctx.Settings.Verbose) @@ -1516,7 +1516,7 @@ TEST_CASE("cas.chunk.moveoverwrite") Payload1.SetDeleteOnClose(true); CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, true); - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1526,9 +1526,9 @@ TEST_CASE("cas.chunk.moveoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1554,7 +1554,7 @@ TEST_CASE("cas.chunk.moveoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; @@ -1598,9 +1598,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, true); - CHECK(std::filesystem::exists(Payload1Path)); + CHECK(IsFile(Payload1Path)); Payload1 = {}; - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1611,9 +1611,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1640,7 +1640,7 @@ TEST_CASE("cas.chunk.copyoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index fe5ae284b..ac4dda83f 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -62,11 +62,11 @@ namespace { { if (Size == 0) { - std::filesystem::remove(Path); + RemoveFile(Path); return std::error_code{}; } CreateDirectories(Path.parent_path()); - if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) + if (IsFile(Path) && FileSizeFromPath(Path) == Size) { return std::error_code(); } @@ -1262,12 +1262,12 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::CompactStores"); auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(Settings.DiskReservePath)) + if (!IsFile(Settings.DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath); - if (std::filesystem::remove(Settings.DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath); + if (RemoveFile(Settings.DiskReservePath)) { return ReclaimedSize; } @@ -1557,7 +1557,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_Config.LightweightInterval = m_Config.MonitorInterval; } - std::filesystem::create_directories(Config.RootDirectory); + CreateDirectories(Config.RootDirectory); std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) @@ -1850,7 +1850,7 @@ GcScheduler::GetState() const if (Result.Config.DiskReserveSize != 0) { Ec.clear(); - Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; + Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; } if (Result.Status != GcSchedulerStatus::kRunning) @@ -2389,12 +2389,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { // We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(DiskReservePath)) + if (!IsFile(DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath); - if (std::filesystem::remove(DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath); + if (RemoveFile(DiskReservePath)) { return ReclaimedSize; } diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..0ca2adab2 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -444,7 +444,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& RootPath = Workspace->GetConfig().RootPath; std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::string Error; std::vector WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); @@ -458,7 +458,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& SharePath = Configuration.SharePath; - if (std::filesystem::is_directory(RootPath / SharePath)) + if (IsDir(RootPath / SharePath)) { DeletedShares.erase(Configuration.Id); @@ -808,7 +808,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work ZEN_DEBUG("Reading workspaces state from {}", WorkspaceStatePath); const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -847,7 +847,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p ZEN_DEBUG("Reading workspace state from {}", WorkspaceRoot); std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -886,7 +886,7 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work { throw std::invalid_argument(fmt::format("invalid root path '{}' for workspace {}", Configuration.RootPath, Configuration.Id)); } - if (!std::filesystem::is_directory(Configuration.RootPath)) + if (!IsDir(Configuration.RootPath)) { throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); @@ -965,7 +965,7 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, throw std::invalid_argument( fmt::format("workspace share path '{}' is not a sub-path of workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); } - if (!std::filesystem::is_directory(WorkspaceRoot / Configuration.SharePath)) + if (!IsDir(WorkspaceRoot / Configuration.SharePath)) { throw std::invalid_argument( fmt::format("workspace share path '{}' does not exist in workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); @@ -1244,7 +1244,7 @@ Workspaces::FindWorkspaceShare(const Oid& WorkspaceId, const Oid& ShareId, bool const Workspaces::WorkspaceConfiguration& WorkspaceConfig = Workspace->GetConfig(); const Workspaces::WorkspaceShareConfiguration& ShareConfig = Share->GetConfig(); std::filesystem::path FullSharePath = WorkspaceConfig.RootPath / ShareConfig.SharePath; - if (std::filesystem::is_directory(FullSharePath)) + if (IsDir(FullSharePath)) { if (ForceRefresh || !Share->IsInitialized()) { @@ -1306,18 +1306,18 @@ namespace { std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -1365,13 +1365,13 @@ TEST_CASE("workspaces.scanfolder") Structure->IterateEntries([&](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::filesystem::path AbsPath = RootPath / Entry.RelativePath; - CHECK(std::filesystem::is_regular_file(AbsPath)); - CHECK(std::filesystem::file_size(AbsPath) == Entry.Size); + CHECK(IsFile(AbsPath)); + CHECK(FileSizeFromPath(AbsPath) == Entry.Size); const FolderStructure::FileEntry* FindEntry = Structure->FindEntry(Id); CHECK(FindEntry); std::filesystem::path Path = RootPath / FindEntry->RelativePath; CHECK(AbsPath == Path); - CHECK(std::filesystem::file_size(AbsPath) == FindEntry->Size); + CHECK(FileSizeFromPath(AbsPath) == FindEntry->Size); }); } diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 1f951167d..380c182b2 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -46,7 +46,7 @@ struct RecordedRequestsWriter void BeginWrite(const std::filesystem::path& BasePath) { m_BasePath = BasePath; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void EndWrite() @@ -426,7 +426,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath, m_BasePath = BasePath; m_SegmentIndex = SegmentIndex; m_RequestBaseIndex = RequestBaseIndex; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void @@ -1051,14 +1051,14 @@ public: static bool IsCompatible(const std::filesystem::path& BasePath) { - if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb")) + if (IsFile(BasePath / "rpc_recording_info.zcb")) { return true; } const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0); - if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin")) + if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin")) { // top-level metadata is missing, possibly because of premature exit // on the recording side diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f040e9ece..0fa05194f 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -235,7 +235,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); @@ -260,7 +260,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); @@ -346,7 +346,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); IoBuffer Payload; @@ -383,7 +383,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { struct WorkloadData { @@ -461,7 +461,7 @@ public: if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) { std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -493,7 +493,7 @@ public: for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); - if (std::filesystem::is_regular_file(MetaDataFile)) + if (IsFile(MetaDataFile)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -623,7 +623,7 @@ protected: BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { NeededAttachments.push_back(AttachmentHash); } diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b36f11741..c0c2a754a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -534,7 +534,7 @@ ZenServerEnvironment::CreateNewTestDir() TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1)); std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); - ZEN_ASSERT(!std::filesystem::exists(TestPath)); + ZEN_ASSERT(!IsDir(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); @@ -568,7 +568,7 @@ ZenServerInstance::~ZenServerInstance() { Shutdown(); std::error_code DummyEc; - std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); + RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); } catch (const std::exception& Err) { @@ -1033,7 +1033,7 @@ std::string ZenServerInstance::GetLogOutput() const { std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - if (std::filesystem::is_regular_file(OutputPath)) + if (IsFile(OutputPath)) { FileContents Contents = ReadFile(OutputPath); if (!Contents.ErrorCode) @@ -1123,7 +1123,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; } - if (!std::filesystem::is_directory(Info.DataDir)) + if (!IsDir(Info.DataDir)) { OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir); return false; diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp index a3cfe9d15..9cec5372a 100644 --- a/src/zenvfs/vfsprovider.cpp +++ b/src/zenvfs/vfsprovider.cpp @@ -373,13 +373,12 @@ VfsProvider::Initialize() std::filesystem::path ManifestPath = Root / ".zen_vfs"; bool HaveManifest = false; - if (std::filesystem::exists(Root)) + if (IsFile(Root)) + { + throw std::runtime_error("specified VFS root exists but is not a directory"); + } + if (IsDir(Root)) { - if (!std::filesystem::is_directory(Root)) - { - throw std::runtime_error("specified VFS root exists but is not a directory"); - } - std::error_code Ec; m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str()); @@ -388,7 +387,7 @@ VfsProvider::Initialize() throw std::system_error(Ec); } - if (std::filesystem::exists(ManifestPath)) + if (IsFile(ManifestPath)) { FileContents ManifestData = zen::ReadFile(ManifestPath); CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten()); -- cgit v1.2.3 From ebe13120c030f8d24c5f05c068d79b2f72fc3c0b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 11:30:28 +0200 Subject: multithreaded clean (#331) - Improvement: Faster cleaning of directories - Improvement: Faster initial scanning of local state --- src/zen/cmds/builds_cmd.cpp | 425 ++++++++++++++++++++++++------- src/zencore/filesystem.cpp | 55 +++- src/zencore/include/zencore/filesystem.h | 5 + 3 files changed, 393 insertions(+), 92 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d2ba20e78..2bbb21012 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -316,25 +316,108 @@ namespace { bool CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) { ZEN_TRACE_CPU("CleanDirectory"); + Stopwatch Timer; - bool CleanWipe = true; + ProgressBar Progress(UsePlainProgress); - DirectoryContent LocalDirectoryContent; - GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); - for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + std::atomic CleanWipe = true; + std::atomic DiscoveredItemCount = 0; + std::atomic DeletedItemCount = 0; + std::atomic DeletedByteCount = 0; + ParallellWork Work(AbortFlag); + + struct AsyncVisitor : public GetDirectoryContentVisitor { - try + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic& InCleanWipe, + std::atomic& InDiscoveredItemCount, + std::atomic& InDeletedItemCount, + std::atomic& InDeletedByteCount, + std::span InExcludeDirectories) + : Path(InPath) + , CleanWipe(InCleanWipe) + , DiscoveredItemCount(InDiscoveredItemCount) + , DeletedItemCount(InDeletedItemCount) + , DeletedByteCount(InDeletedByteCount) + , ExcludeDirectories(InExcludeDirectories) { - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); } - catch (const std::exception& Ex) + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); - CleanWipe = false; + ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); + if (!AbortFlag) + { + if (!Content.FileNames.empty()) + { + DiscoveredItemCount += Content.FileNames.size(); + + const std::string RelativeRootString = RelativeRoot.generic_string(); + bool RemoveContent = true; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (RelativeRootString.starts_with(ExcludeDirectory)) + { + if (RelativeRootString.length() > ExcludeDirectory.length()) + { + const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; + if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || + MaybePathDelimiter == std::filesystem::path::preferred_separator) + { + RemoveContent = false; + break; + } + } + else + { + RemoveContent = false; + break; + } + } + } + if (RemoveContent) + { + ZEN_TRACE_CPU("DeleteFiles"); + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); + try + { + SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + CleanWipe = false; + } + } + } + } + } } - } + const std::filesystem::path& Path; + std::atomic& CleanWipe; + std::atomic& DiscoveredItemCount; + std::atomic& DeletedItemCount; + std::atomic& DeletedByteCount; + std::span ExcludeDirectories; + } Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories); + GetDirectoryContent( + Path, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, + Visitor, + GetIOWorkerPool(), + Work.PendingWork()); + + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + DiscoveredItemCount += LocalDirectoryContent.Directories.size(); + std::vector DirectoriesToDelete; + DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size()); for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; @@ -348,31 +431,78 @@ namespace { } if (!Leave) { - try - { - std::error_code Ec; - zen::CleanDirectory(LocalDirPath, true, Ec); - if (Ec) - { - Sleep(200); - zen::CleanDirectory(LocalDirPath, true); - Ec.clear(); - } + DirectoriesToDelete.emplace_back(std::move(LocalDirPath)); + DiscoveredItemCount++; + } + } - RemoveDir(LocalDirPath, Ec); - if (Ec) - { - Sleep(200); - RemoveDir(LocalDirPath); - } - } - catch (const std::exception& Ex) + uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); + }); + + for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) + { + ZEN_TRACE_CPU("DeleteDirs"); + try + { + std::error_code Ec; + zen::CleanDirectory(DirectoryToDelete, true, Ec); + if (Ec) { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; + Sleep(200); + zen::CleanDirectory(DirectoryToDelete, true); + Ec.clear(); } + + RemoveDirWithRetry(DirectoryToDelete); + + DeletedItemCount++; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + CleanWipe = false; + } + + uint64_t NowMs = Timer.GetElapsedTimeMs(); + if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + { + LastUpdateTimeMs = NowMs; + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); } } + + Progress.Finish(); + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (ElapsedTimeMs >= 200) + { + ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}", + Path, + DiscoveredItemCount.load(), + NiceBytes(DeletedByteCount.load()), + NiceTimeSpanMs(ElapsedTimeMs)); + } return CleanWipe; } @@ -1959,7 +2089,7 @@ namespace { BasicFile CompressedFile; std::error_code Ec; - CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec); + CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec); if (Ec) { throw std::runtime_error( @@ -2406,7 +2536,7 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&, Payload, RawSize](uint64_t SentBytes, bool IsComplete) { + [&, RawSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; UploadedCompressedChunkSize += SentBytes; if (IsComplete) @@ -4159,7 +4289,10 @@ namespace { bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span> SequenceIndexChunksLeftToWriteCounters) { - return SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1; + uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1); + ZEN_ASSERT(PreviousValue >= 1); + ZEN_ASSERT(PreviousValue != (uint32_t)-1); + return PreviousValue == 1; } std::vector CompleteChunkTargets(const std::vector& ChunkTargetPtrs, @@ -5680,7 +5813,31 @@ namespace { { BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - if (!BuildBlob) + if (BuildBlob) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + else { if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { @@ -6420,6 +6577,10 @@ namespace { const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; ZEN_ASSERT(!IncompletePath.empty()); const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_CONSOLE("{}: Max count {}, Current count {}", + IncompletePath, + ExpectedSequenceCount, + SequenceIndexChunksLeftToWriteCounter.load()); ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); } } @@ -6453,8 +6614,14 @@ namespace { tsl::robin_map SequenceHashToLocalPathIndex; std::vector RemoveLocalPathIndexes; - if (!WipeTargetFolder) + if (AbortFlag) + { + return; + } + { + ZEN_TRACE_CPU("UpdateFolder_PrepareTarget"); + tsl::robin_set CachedRemoteSequences; tsl::robin_map RemotePathToRemoteIndex; RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); @@ -6463,14 +6630,21 @@ namespace { RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); } - uint64_t MatchCount = 0; - uint64_t PathMismatchCount = 0; - uint64_t HashMismatchCount = 0; - uint64_t CachedCount = 0; - uint64_t SkippedCount = 0; - uint64_t DeleteCount = 0; + std::vector FilesToCache; + + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + std::atomic CachedCount = 0; + std::atomic CachedByteCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { + if (AbortFlag) + { + break; + } const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; @@ -6502,22 +6676,16 @@ namespace { } if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); if (!CachedRemoteSequences.contains(RawHash)) { + ZEN_TRACE_CPU("MoveToCache"); // We need it - ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); - const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - - RenameFileWithRetry(LocalFilePath, CacheFilePath); - + FilesToCache.push_back(LocalPathIndex); CachedRemoteSequences.insert(RawHash); - CachedCount++; } else { // We already have it - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); SkippedCount++; } } @@ -6529,15 +6697,80 @@ namespace { } } - ZEN_DEBUG( - "Local state prep: MatchCount: {}, PathMismatchCount: {}, HashMismatchCount: {}, CachedCount: {}, SkippedCount: {}, " - "DeleteCount: {}", - MatchCount, - PathMismatchCount, - HashMismatchCount, - CachedCount, - SkippedCount, - DeleteCount); + if (AbortFlag) + { + return; + } + + { + ZEN_TRACE_CPU("UpdateFolder_CopyToCache"); + + Stopwatch Timer; + + WorkerThreadPool& WritePool = GetIOWorkerPool(); + + ProgressBar CacheLocalProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); + + for (uint32_t LocalPathIndex : FilesToCache) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; + } + }, + Work.DefaultErrorFunction()); + } + + { + ZEN_TRACE_CPU("CacheLocal_Wait"); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + const uint64_t WorkTotal = FilesToCache.size(); + const uint64_t WorkComplete = CachedCount.load(); + std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount)); + CacheLocalProgressBar.UpdateState({.Task = "Caching local ", + .Details = Details, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, + false); + }); + } + + if (AbortFlag) + { + return; + } + + CacheLocalProgressBar.Finish(); + + ZEN_DEBUG( + "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " + "Delete: {}", + MatchCount, + PathMismatchCount, + HashMismatchCount, + CachedCount.load(), + NiceBytes(CachedByteCount.load()), + SkippedCount, + DeleteCount); + } } if (WipeTargetFolder) @@ -6546,30 +6779,16 @@ namespace { Stopwatch Timer; // Clean target folder - ZEN_CONSOLE("Wiping {}", Path); if (!CleanDirectory(Path, DefaultExcludeFolders)) { ZEN_WARN("Some files in {} could not be removed", Path); } RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } - else - { - ZEN_TRACE_CPU("UpdateFolder_RemoveUnused"); - Stopwatch Timer; - - // Remove unused tracked files - if (!RemoveLocalPathIndexes.empty()) - { - ZEN_CONSOLE("Cleaning {} removed files from {}", RemoveLocalPathIndexes.size(), Path); - for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) - { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); - } - } + if (AbortFlag) + { + return; } { @@ -6586,6 +6805,28 @@ namespace { OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size()); OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size()); + std::atomic DeletedCount = 0; + + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic&) { + if (!AbortFlag) + { + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); + DeletedCount++; + } + }, + Work.DefaultErrorFunction()); + } + std::atomic TargetsComplete = 0; struct FinalizeTarget @@ -6638,6 +6879,7 @@ namespace { if (RawHash == IoHash::Zero) { + ZEN_TRACE_CPU("ZeroSize"); while (TargetOffset < (BaseTargetOffset + TargetCount)) { const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; @@ -6674,6 +6916,7 @@ namespace { } else { + ZEN_TRACE_CPU("Files"); ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; @@ -6698,16 +6941,19 @@ namespace { if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); InplaceIt != SequenceHashToLocalPathIndex.end()) { + ZEN_TRACE_CPU("Copy"); const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } else { + ZEN_TRACE_CPU("Rename"); const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); @@ -6747,6 +6993,7 @@ namespace { } else { + ZEN_TRACE_CPU("Copy"); if (IsFile(TargetFilePath)) { SetFileReadOnlyWithRetry(TargetFilePath, false); @@ -6757,6 +7004,7 @@ namespace { } ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } @@ -6788,11 +7036,13 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); + const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); + const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); + std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", .Details = Details, - .TotalCount = gsl::narrow(Targets.size()), - .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, false); }); } @@ -7312,7 +7562,7 @@ namespace { while (PathIndex < PathCount) { - uint32_t PathRangeCount = Min(1024u, PathCount - PathIndex); + uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); Work.ScheduleWork( GetIOWorkerPool(), [PathIndex, @@ -7326,17 +7576,16 @@ namespace { { const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (IsFile(LocalFilePath)) + if (TryGetFileProperties(LocalFilePath, + OutLocalFolderContent.RawSizes[PathRangeIndex], + OutLocalFolderContent.ModificationTicks[PathRangeIndex], + OutLocalFolderContent.Attributes[PathRangeIndex])) { - const uint64_t FileSize = FileSizeFromPath(LocalFilePath); - OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; - OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; - OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); - OutLocalFolderContent.ModificationTicks[PathRangeIndex] = GetModificationTickFromPath(LocalFilePath); + OutLocalFolderContent.Paths[PathRangeIndex] = std::move(FilePath); LocalFolderScanStats.FoundFileCount++; - LocalFolderScanStats.FoundFileByteCount += FileSize; + LocalFolderScanStats.FoundFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; LocalFolderScanStats.AcceptedFileCount++; - LocalFolderScanStats.AcceptedFileByteCount += FileSize; + LocalFolderScanStats.AcceptedFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; } CompletedPathCount++; } diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 6ff4dd053..4ec563ba3 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -280,16 +280,17 @@ bool RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - BOOL Success = ::DeleteFile(Path.native().c_str()); + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + BOOL Success = ::DeleteFile(NativePath); if (!Success) { if (ForceRemoveReadOnlyFiles) { - DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str()); + DWORD FileAttributes = ::GetFileAttributes(NativePath); if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) { - ::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false)); - Success = ::DeleteFile(Path.native().c_str()); + ::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(NativePath); } } if (!Success) @@ -1964,6 +1965,52 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +bool +TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + { + void* Handle = CreateFileW(NativePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + return false; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + + BY_HANDLE_FILE_INFORMATION Bhfh = {}; + if (!GetFileInformationByHandle(Handle, &Bhfh)) + { + return false; + } + OutSize = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow; + OutModificationTick = ((uint64_t(Bhfh.ftLastWriteTime.dwHighDateTime) << 32) | Bhfh.ftLastWriteTime.dwLowDateTime); + OutNativeModeOrAttributes = Bhfh.dwFileAttributes; + return true; + } +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err) + { + return false; + } + OutModificationTick = gsl::narrow(Stat.st_mtime); + OutSize = size_t(Stat.st_size); + OutNativeModeOrAttributes = (uint32_t)Stat.st_mode; + return true; +#endif +} + void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index c23f16d03..66deffa6f 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -110,6 +110,11 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes); + /** Move a file, if the files are not on the same drive the function will fail */ ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -- cgit v1.2.3 From 2103b5d1f20f1b46a30eced4130c22a7cf7ae694 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 13:53:01 +0200 Subject: logging improvement (#332) * Cache -> Download cache * cleaned up info regarding local cache/state and remote cache --- src/zen/cmds/builds_cmd.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2bbb21012..eebb6d82b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -5235,21 +5235,24 @@ namespace { } } - if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() || - !LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE( - "Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks", - CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), - CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), - CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + { + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks.", + CachedSequenceHashesFound.size(), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + CachedChunkHashesFound.size(), + NiceBytes(CacheMappingStats.CacheChunkByteCount), + CachedBlocksFound.size(), + NiceBytes(CacheMappingStats.CacheBlocksByteCount)); + } + + if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); } uint64_t BytesToWrite = 0; @@ -5633,7 +5636,7 @@ namespace { ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (!ExistsResult.ExistingBlobs.empty()) { - ZEN_CONSOLE("Found {} out of {} needed blobs in remote cache in {}", + ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}", ExistsResult.ExistingBlobs.size(), BlobHashes.size(), NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); @@ -8737,7 +8740,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("Remote: {}.{}", StorageDescription, - CacheDescription.empty() ? std::string("") : fmt::format(" Cache: {}", CacheDescription)); + CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription)); return Result; }; -- cgit v1.2.3 From bbb7e0c77b9ed114baf428256f347631b3e1092c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 14:26:53 +0200 Subject: verify that we can read input files that are only hashed (#333) * output build and part details by default * output executable and version at start of builds command * verify that we can read files we do not chunk --- src/zen/cmds/builds_cmd.cpp | 121 +++++++++++++++++++++++++++++++++++++++-- src/zenutil/chunkedcontent.cpp | 8 ++- 2 files changed, 122 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index eebb6d82b..689b29db5 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -7061,6 +7063,107 @@ namespace { } } + std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) + { + ExtendableStringBuilder<512> SB; + std::vector> NameStringValuePairs; + for (CbFieldView Field : Object) + { + std::string_view Name = Field.GetName(); + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::String: + NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); + break; + case CbFieldType::IntegerPositive: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); + break; + case CbFieldType::IntegerNegative: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::BoolFalse: + NameStringValuePairs.push_back({std::string(Name), "false"}); + break; + case CbFieldType::BoolTrue: + NameStringValuePairs.push_back({std::string(Name), "true"}); + break; + case CbFieldType::Hash: + { + NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); + } + break; + case CbFieldType::Uuid: + { + StringBuilder Builder; + Accessor.AsUuid().ToString(Builder); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::DateTime: + { + ExtendableStringBuilder<64> Builder; + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::TimeSpan: + { + ExtendableStringBuilder<64> Builder; + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + break; + } + case CbFieldType::ObjectId: + NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); + break; + } + } + std::string::size_type LongestKey = 0; + for (const std::pair& KeyValue : NameStringValuePairs) + { + LongestKey = Max(KeyValue.first.length(), LongestKey); + } + for (const std::pair& KeyValue : NameStringValuePairs) + { + SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); + } + return SB.ToString(); + } + std::vector> ResolveBuildPartNames(BuildStorage& Storage, const Oid& BuildId, const std::vector& BuildPartIds, @@ -7071,14 +7174,12 @@ namespace { { Stopwatch GetBuildTimer; CbObject BuildObject = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Name: '{}' ({}, {}), Payload size: {}", + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), BuildObject["name"sv].AsString(), - BuildObject["type"sv].AsString(), - BuildObject["Configuration"sv].AsString(), NiceBytes(BuildObject.GetSize())); - ZEN_CONSOLE_VERBOSE("Build object: {}", BuildObject); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -7170,6 +7271,7 @@ namespace { BuildPartName, NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), NiceBytes(BuildPartManifest.GetSize())); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); { CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); @@ -8750,6 +8852,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (SubOption == &m_ListOptions) { + if (!m_ListResultPath.empty()) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + } CbObject QueryObject; if (m_ListQueryPath.empty()) { @@ -8824,6 +8930,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); @@ -8979,6 +9087,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -9472,6 +9582,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -9519,7 +9630,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - // HttpClient Http(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BuildsUrl.empty() && m_StoragePath.empty()) { diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 1e8447a57..32ae2d94a 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -140,8 +140,12 @@ namespace { { ZEN_TRACE_CPU("HashOnly"); - IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); - const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); + if (Buffer.GetSize() != RawSize) + { + throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + } + const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); Lock.WithExclusiveLock([&]() { if (!RawHashToSequenceRawHashIndex.contains(Hash)) -- cgit v1.2.3 From 3ecdceceee909b57faa70ac7efa4ff58245b35d0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 17:29:56 +0200 Subject: reduce disk io during gc (#335) * do cache bucket flush/write snapshot as part of compact to reduce disk I/O --- src/zenstore/cache/cachedisklayer.cpp | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index b2d2416be..e4d962b56 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2731,9 +2731,10 @@ class DiskBucketStoreCompactor : public GcStoreCompactor using CacheBucket = ZenCacheDiskLayer::CacheBucket; public: - DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector>&& ExpiredStandaloneKeys) + DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector>&& ExpiredStandaloneKeys, bool FlushBucket) : m_Bucket(Bucket) , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys)) + , m_FlushBucket(FlushBucket) { m_ExpiredStandaloneKeys.shrink_to_fit(); } @@ -2960,6 +2961,10 @@ public: } } } + if (m_FlushBucket) + { + m_Bucket.Flush(); + } } virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); } @@ -2967,6 +2972,7 @@ public: private: ZenCacheDiskLayer::CacheBucket& m_Bucket; std::vector> m_ExpiredStandaloneKeys; + bool m_FlushBucket = false; }; GcStoreCompactor* @@ -2990,24 +2996,6 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) NiceBytes(Stats.FreedMemory), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } - if (Stats.DeletedCount > 0) - { - bool Expected = false; - if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true)) - { - return; - } - auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); }); - - try - { - SaveSnapshot([]() { return 0; }); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what()); - } - } }); const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); @@ -3094,7 +3082,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) return nullptr; } - return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys)); + return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys), /*FlushBucket*/ Stats.DeletedCount > 0); } bool -- cgit v1.2.3 From 571aad8c5d2ed9d975ba2fed2890116e3c8bd6ea Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 18:27:05 +0200 Subject: builds url discovery (#334) - Feature: Added `--host` option to use Jupiters list of cloud host and zen servers to resolve best hosts - Feature: Use local zenserver as builds cache if it has the `builds` service enabled and `--cloud-discovery-host` is provided and no remote zenserver cache hosts can be found - Improvement: Added `--override-host` option as a replacement for `--url` (`--url` still works, but `--override-host` is preferred) --- src/zen/cmds/builds_cmd.cpp | 247 +++++++++++++++++++----- src/zen/cmds/builds_cmd.h | 3 +- src/zenserver/buildstore/httpbuildstore.cpp | 19 +- src/zenserver/buildstore/httpbuildstore.h | 9 +- src/zenserver/frontend/frontend.cpp | 15 +- src/zenserver/frontend/frontend.h | 7 +- src/zenserver/objectstore/objectstore.cpp | 14 +- src/zenserver/objectstore/objectstore.h | 7 +- src/zenserver/projectstore/httpprojectstore.cpp | 22 ++- src/zenserver/projectstore/httpprojectstore.h | 11 +- src/zenserver/vfs/vfsservice.cpp | 15 +- src/zenserver/vfs/vfsservice.h | 9 +- src/zenserver/workspaces/httpworkspaces.cpp | 21 +- src/zenserver/workspaces/httpworkspaces.h | 10 +- src/zenserver/zenserver.cpp | 13 +- 15 files changed, 345 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 689b29db5..1c24a3c5b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -883,6 +883,7 @@ namespace { std::string StorageName; std::unique_ptr CacheHttp; std::unique_ptr BuildCacheStorage; + std::string CacheName; }; std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, @@ -8348,7 +8349,14 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); - Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), ""); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); + Ops.add_option("cloud build", + "", + "url", + "Cloud Builds host url (legacy - use --override-host)", + cxxopts::value(m_OverrideHost), + ""); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), ""); Ops.add_option("cloud build", "", "assume-http2", @@ -8657,7 +8665,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } auto ParseStorageOptions = [&]() { - if (!m_BuildsUrl.empty()) + if (!m_OverrideHost.empty() || !m_Host.empty()) { if (!m_StoragePath.empty()) { @@ -8676,7 +8684,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; std::unique_ptr Auth; - HttpClientSettings ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2}; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; auto CreateAuthMgr = [&]() { if (!Auth) @@ -8762,7 +8773,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); } - if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider) + if (!ClientSettings.AccessTokenProvider) { ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials."); } @@ -8781,21 +8792,176 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Result; + std::string BuildStorageName = ZEN_CLOUD_STORAGE; + std::string BuildCacheName; + bool CacheAssumeHttp2 = false; std::string StorageDescription; std::string CacheDescription; - if (!m_BuildsUrl.empty()) + if (!m_Host.empty() || !m_OverrideHost.empty()) { ParseAuthOptions(); - Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); - StorageDescription = fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, + } + + std::string CloudHost; + + if (!m_Host.empty()) + { + if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) + { + HttpClient DiscoveryHttpClient(m_Host, ClientSettings); + HttpClient::Response ServerInfoResponse = + DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); + if (!ServerInfoResponse.IsSuccess()) + { + throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'", + m_Host, + ServerInfoResponse.ErrorMessage(""))); + } + + std::string_view JsonResponse = ServerInfoResponse.AsText(); + CbObject ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject(); + + if (m_OverrideHost.empty()) + { + CbArrayView ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView(); + std::uint64_t ServerCount = ServerEndpointsArray.Num(); + if (ServerCount == 0) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); + } + for (CbFieldView ServerEndpointView : ServerEndpointsArray) + { + CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView(); + + std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = ServerEndpointObject["name"sv].AsString(); + + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + CloudHost = BaseUrl; + m_AssumeHttp2 = AssumeHttp2; + BuildStorageName = Name; + break; + } + } + } + if (CloudHost.empty()) + { + throw std::runtime_error( + fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host)); + } + } + + auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool { + HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); + if (TestResponse.IsSuccess()) + { + return true; + } + return false; + }; + + if (m_ZenCacheHost.empty()) + { + CbArrayView CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView(); + std::uint64_t CacheCount = CacheEndpointsArray.Num(); + for (CbFieldView CacheEndpointView : CacheEndpointsArray) + { + CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView(); + + std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = CacheEndpointObject["name"sv].AsString(); + + if (TestCacheEndpoint(BaseUrl, AssumeHttp2)) + { + m_ZenCacheHost = BaseUrl; + CacheAssumeHttp2 = AssumeHttp2; + BuildCacheName = Name; + break; + } + } + } + if (m_ZenCacheHost.empty()) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (m_ZenCacheHost.empty()) + { + std::string ZenServerLocalHostUrl = + fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (TestCacheEndpoint(ZenServerLocalHostUrl, false)) + { + m_ZenCacheHost = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + BuildCacheName = "localhost"; + } + } + }); + } + if (m_ZenCacheHost.empty()) + { + ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); + } + } + } + else if (TestCacheEndpoint(m_ZenCacheHost, false)) + { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength); + } + } + } + else + { + CloudHost = m_OverrideHost; + } + + if (!CloudHost.empty()) + { + Result.BuildStorageHttp = std::make_unique(CloudHost, ClientSettings); + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), + CloudHost, Result.BuildStorageHttp->GetSessionId(), m_Namespace, m_Bucket); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); - Result.StorageName = ZEN_CLOUD_STORAGE; + Result.StorageName = BuildStorageName; } else if (!m_StoragePath.empty()) { @@ -8810,39 +8976,39 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } if (!m_ZenCacheHost.empty()) { - Result.CacheHttp = std::make_unique(m_ZenCacheHost, + Result.CacheHttp = std::make_unique(m_ZenCacheHost, HttpClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = false, - .AllowResume = true, - .RetryCount = 0}); - if (Result.CacheHttp->Get("/health").IsSuccess()) + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = CacheAssumeHttp2, + .AllowResume = true, + .RetryCount = 0}); + Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + m_PrimeCacheOnly); + CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", + BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName), + m_ZenCacheHost, + Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) { - Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - m_PrimeCacheOnly); - CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(" {}.", m_Namespace); - } - if (!m_Bucket.empty()) - { - CacheDescription += fmt::format(" {}.", m_Bucket); - } + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); } - else + if (!m_Bucket.empty()) { - Result.CacheHttp.reset(); + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } + Result.CacheName = BuildCacheName; + } + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheName.empty()) + { + ZEN_CONSOLE("Cache : {}", CacheDescription); } - ZEN_CONSOLE("Remote: {}.{}", - StorageDescription, - CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription)); return Result; }; @@ -9272,7 +9438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); @@ -9280,7 +9446,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_StoragePath = StoragePath.generic_string(); } auto _ = MakeGuard([&]() { - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { DeleteDirectories(StoragePath); } @@ -9632,11 +9798,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } - if (m_BuildId.empty()) { throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 4a77f8bd7..bc5cd235f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -34,7 +34,8 @@ private: std::string m_ZenFolderPath; // cloud builds - std::string m_BuildsUrl; + std::string m_OverrideHost; + std::string m_Host; bool m_AssumeHttp2 = false; std::string m_Namespace; std::string m_Bucket; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index 06bfea423..c918f5683 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -19,16 +19,22 @@ using namespace std::literals; ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); -HttpBuildStoreService::HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store) +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) : m_Log(logging::Get("builds")) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_BuildStore(Store) { Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); } HttpBuildStoreService::~HttpBuildStoreService() { + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); } const char* @@ -42,8 +48,6 @@ HttpBuildStoreService::Initialize() { ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); - m_StatsService.RegisterHandler("builds", *this); - m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)"); m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)"); m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); @@ -523,4 +527,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + } // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h index a59aa882a..50cb5db12 100644 --- a/src/zenserver/buildstore/httpbuildstore.h +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -12,16 +13,17 @@ namespace zen { class BuildStore; -class HttpBuildStoreService final : public zen::HttpService, public IHttpStatsProvider +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store); + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); virtual ~HttpBuildStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct BuildStoreStats @@ -55,7 +57,8 @@ private: HttpRequestRouter m_Router; - HttpStatsService& m_StatsService; + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; BuildStore& m_BuildStore; BuildStoreStats m_BuildStoreStats; diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 104b26954..dfa710ae0 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -2,6 +2,7 @@ #include "frontend.h" +#include #include #include #include @@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = { namespace zen { //////////////////////////////////////////////////////////////////////////////// -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService) +: m_Directory(Directory) +, m_StatusService(StatusService) { std::filesystem::path SelfPath = GetRunningExecutablePath(); @@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { ZEN_INFO("front-end is NOT AVAILABLE"); } + m_StatusService.RegisterHandler("dashboard", *this); } HttpFrontendService::~HttpFrontendService() { + m_StatusService.UnregisterHandler("dashboard", *this); } const char* @@ -94,6 +99,14 @@ HttpFrontendService::BaseUri() const } //////////////////////////////////////////////////////////////////////////////// +void +HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 6eac20620..84ffaac42 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -3,23 +3,26 @@ #pragma once #include +#include #include "zipfs.h" #include namespace zen { -class HttpFrontendService final : public zen::HttpService +class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpFrontendService(std::filesystem::path Directory); + HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService); virtual ~HttpFrontendService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: ZipFs m_ZipFs; std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index 5af803617..8faf12165 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -219,13 +219,17 @@ private: StringBuilderBase& Builder; }; -HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg)) +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) { Inititalize(); + m_StatusService.RegisterHandler("obj", *this); } HttpObjectStoreService::~HttpObjectStoreService() { + m_StatusService.UnregisterHandler("obj", *this); } const char* @@ -244,6 +248,14 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) } } +void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpObjectStoreService::Inititalize() { diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index dae979c4c..44e50e208 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -23,14 +24,15 @@ struct ObjectStoreConfig std::vector Buckets; }; -class HttpObjectStoreService final : public zen::HttpService +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpObjectStoreService(ObjectStoreConfig Cfg); + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); virtual ~HttpObjectStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: void Inititalize(); @@ -41,6 +43,7 @@ private: void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); void PutObject(zen::HttpRouterRequest& Request); + HttpStatusService& m_StatusService; ObjectStoreConfig m_Cfg; std::mutex BucketsMutex; HttpRequestRouter m_Router; diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 6313fd69e..317a419eb 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -235,10 +235,15 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr) +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_AuthMgr(AuthMgr) { @@ -246,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, using namespace std::literals; - m_StatsService.RegisterHandler("prj", *this); - m_Router.AddPattern("project", "([[:alnum:]_.]+)"); m_Router.AddPattern("log", "([[:alnum:]_.]+)"); m_Router.AddPattern("op", "([[:digit:]]+?)"); @@ -366,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, "details\\$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); } HttpProjectService::~HttpProjectService() { m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); } const char* @@ -465,6 +472,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) { diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 8e74c57a5..295defa5c 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace zen { @@ -31,16 +32,21 @@ class ProjectStore; // refs: // -class HttpProjectService : public HttpService, public IHttpStatsProvider +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct ProjectStats @@ -89,6 +95,7 @@ private: CidStore& m_CidStore; HttpRequestRouter m_Router; Ref m_ProjectStore; + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; AuthMgr& m_AuthMgr; ProjectStats m_ProjectStats; diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp index d302a10ec..bf761f8d1 100644 --- a/src/zenserver/vfs/vfsservice.cpp +++ b/src/zenserver/vfs/vfsservice.cpp @@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { m_Impl = new Impl; @@ -136,10 +136,12 @@ VfsService::VfsService() } }, HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); } VfsService::~VfsService() { + m_StatusService.UnregisterHandler("vfs", *this); delete m_Impl; } @@ -169,8 +171,9 @@ VfsService::AddService(Ref&& Z$) #else -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { + ZEN_UNUSED(StatusService); } VfsService::~VfsService() @@ -208,6 +211,14 @@ VfsService::BaseUri() const return "/vfs/"; } +void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) { diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h index dcdc71e81..0d0168e23 100644 --- a/src/zenserver/vfs/vfsservice.h +++ b/src/zenserver/vfs/vfsservice.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -24,10 +25,10 @@ class ZenCacheStore; */ -class VfsService : public HttpService +class VfsService : public HttpService, public IHttpStatusProvider { public: - VfsService(); + explicit VfsService(HttpStatusService& StatusService); ~VfsService(); void Mount(std::string_view MountPoint); @@ -39,12 +40,14 @@ public: protected: virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct Impl; Impl* m_Impl = nullptr; - HttpRequestRouter m_Router; + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; friend struct VfsServiceDataSource; }; diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index ac0aaef8e..7ef84743e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -73,8 +73,12 @@ namespace { } // namespace -HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces) +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) : m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_Config(Cfg) , m_Workspaces(Workspaces) @@ -85,6 +89,7 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con HttpWorkspacesService::~HttpWorkspacesService() { m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); } const char* @@ -148,6 +153,15 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpWorkspacesService::Initialize() { @@ -155,8 +169,6 @@ HttpWorkspacesService::Initialize() ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); - m_StatsService.RegisterHandler("ws", *this); - m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); @@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize() HttpVerb::kGet); RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); } std::filesystem::path diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h index f01f58b86..89a8e8bdc 100644 --- a/src/zenserver/workspaces/httpworkspaces.h +++ b/src/zenserver/workspaces/httpworkspaces.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace zen { @@ -16,16 +17,20 @@ struct WorkspacesServeConfig bool AllowConfigurationChanges = false; }; -class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces); + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); virtual ~HttpWorkspacesService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct WorkspacesStats @@ -80,6 +85,7 @@ private: void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; const WorkspacesServeConfig m_Config; HttpRequestRouter m_Router; diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 45c91d691..c7cb2ba6e 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -251,13 +251,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); if (ServerOptions.WorksSpacesConfig.Enabled) { m_Workspaces.reset(new Workspaces()); m_HttpWorkspacesService.reset( - new HttpWorkspacesService(m_StatsService, + new HttpWorkspacesService(m_StatusService, + m_StatsService, {.SystemRootDir = ServerOptions.SystemRootDir, .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, *m_Workspaces)); @@ -295,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_HttpWorkspacesService); } - m_FrontendService = std::make_unique(m_ContentRoot); + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); if (m_FrontendService) { @@ -314,18 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique(std::move(ObjCfg)); + m_ObjStoreService = std::make_unique(m_StatusService, std::move(ObjCfg)); m_Http->RegisterService(*m_ObjStoreService); } if (ServerOptions.BuildStoreConfig.Enabled) { - m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); m_Http->RegisterService(*m_BuildStoreService); } #if ZEN_WITH_VFS - m_VfsService = std::make_unique(); + m_VfsService = std::make_unique(m_StatusService); m_VfsService->AddService(Ref(m_ProjectStore)); m_VfsService->AddService(Ref(m_CacheStore)); m_Http->RegisterService(*m_VfsService); -- cgit v1.2.3 From 46a17320b25577a7f2a1f6636cb1f35414500422 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:23:28 -0600 Subject: Updating frontend archive --- src/zenserver/frontend/html.zip | Bin 154940 -> 156434 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index 9be6a8f79..ca44804b9 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From e244e2a07a0f3bf3b11d6d875d6c2c8bb169efc5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 15:46:05 +0200 Subject: added --find-max-block-count option to builds upload (#337) --- src/zen/cmds/builds_cmd.cpp | 16 +++++++++++++--- src/zen/cmds/builds_cmd.h | 3 ++- src/zenserver/projectstore/buildsremoteprojectstore.cpp | 2 +- src/zenutil/filebuildstorage.cpp | 9 ++++++++- src/zenutil/include/zenutil/buildstorage.h | 2 +- src/zenutil/include/zenutil/jupiter/jupitersession.h | 2 +- src/zenutil/jupiter/jupiterbuildstorage.cpp | 4 ++-- src/zenutil/jupiter/jupitersession.cpp | 8 +++++--- 8 files changed, 33 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1c24a3c5b..8e8fd480a 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2900,6 +2900,7 @@ namespace { const std::filesystem::path& Path, const std::filesystem::path& ZenFolderPath, const std::filesystem::path& ManifestPath, + const uint64_t FindBlockMaxCount, const uint8_t BlockReuseMinPercentLimit, bool AllowMultiparts, const CbObject& MetaData, @@ -2939,7 +2940,7 @@ namespace { FindBlocksStatistics FindBlocksStats; std::future PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task{ - [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + [&Storage, BuildId, FindBlockMaxCount, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { ZEN_TRACE_CPU("PrepareBuild"); PrepareBuildResult Result; @@ -2976,7 +2977,7 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount)); FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -7417,7 +7418,7 @@ namespace { std::vector AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); std::vector FoundBlocks = - ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, (uint64_t)-1)); for (const IoHash& BlockHash : BlockRawHashes) { @@ -8496,6 +8497,12 @@ BuildsCommand::BuildsCommand() ""); m_UploadOptions .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), ""); + m_UploadOptions.add_option("", + "", + "find-max-block-count", + "The maximum number of blocks we search for in the build context", + cxxopts::value(m_FindBlockMaxCount), + ""); m_UploadOptions.parse_positional({"local-path", "build-id"}); m_UploadOptions.positional_help("local-path build-id"); @@ -9221,6 +9228,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Path, ZenFolderPath, MakeSafeAbsolutePath(m_ManifestPath), + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -9489,6 +9497,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Path, ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -9677,6 +9686,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DownloadPath, ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData2, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index bc5cd235f..b40c3e08f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -89,7 +89,8 @@ private: std::string m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - bool m_PostUploadVerify = false; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; std::vector m_BuildPartNames; diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index fbb9bc344..a6583b722 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -308,7 +308,7 @@ public: { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId); + JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; if (Result.ErrorCode) diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 0fa05194f..7aa252e44 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,7 +442,7 @@ public: SimulateLatency(0, 0); } - virtual CbObject FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); @@ -451,6 +451,8 @@ public: auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; + uint64_t FoundCount = 0; + DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); CbObjectWriter Writer; @@ -469,6 +471,11 @@ public: CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); Writer.AddObject(BlockObject); + FoundCount++; + if (FoundCount == MaxBlockCount) + { + break; + } } } } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index f8c7c012c..b0665dbf8 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,7 +55,7 @@ public: std::function&& Receiver) = 0; virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index fda4a7bfe..417ed7384 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -152,7 +152,7 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); JupiterResult PutBuildPartStats(std::string_view Namespace, diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index b6d9e3990..f2d190408 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -289,13 +289,13 @@ public: } } - virtual CbObject FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId); + JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount); AddStatistic(FindResult); if (!FindResult.Success) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 2e4fe5258..fde86a478 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -758,10 +758,12 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace, } JupiterResult -JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId) +JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount) { - HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId), - HttpClient::Accept(ZenContentType::kCbObject)); + const std::string Parameters = MaxBlockCount == (uint64_t)-1 ? "" : fmt::format("?count={}", MaxBlockCount); + HttpClient::Response Response = + m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks{}", Namespace, BucketId, BuildId, Parameters), + HttpClient::Accept(ZenContentType::kCbObject)); return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv); } -- cgit v1.2.3 From 8e9a0b2d7a386dd390ab4b23c563b994e5f79f7a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 20:00:27 +0200 Subject: upload fail mac linux (#338) * fix macos/linux path resolving --- src/zen/cmds/builds_cmd.cpp | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8e8fd480a..ba2564fad 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -164,9 +164,9 @@ namespace { ); - std::filesystem::path MakeSafeAbsolutePath(const std::string Path) + std::filesystem::path MakeSafeAbsolutePath(std::filesystem::path Path) { - std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); + std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred(); #if ZEN_PLATFORM_WINDOWS && 1 const std::string_view Prefix = "\\\\?\\"; const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); @@ -180,6 +180,8 @@ namespace { return AbsolutePath; } + std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); } + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { std::error_code Ec; @@ -9061,8 +9063,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); CreateDirectories(ZenFolderPath); auto _ = MakeGuard([ZenFolderPath]() { if (CleanDirectory(ZenFolderPath, {})) @@ -9176,8 +9179,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); CreateDirectories(ZenFolderPath); auto _ = MakeGuard([ZenFolderPath]() { if (CleanDirectory(ZenFolderPath, {})) @@ -9227,7 +9231,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildPartName, Path, ZenFolderPath, - MakeSafeAbsolutePath(m_ManifestPath), + m_ManifestPath.empty() ? std::filesystem::path{} : MakeSafeAbsolutePath(m_ManifestPath), m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -9777,8 +9781,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); CreateDirectories(ZenFolderPath); auto _ = MakeGuard([ZenFolderPath]() { if (CleanDirectory(ZenFolderPath, {})) @@ -9828,8 +9833,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); CreateDirectories(ZenFolderPath); auto _ = MakeGuard([ZenFolderPath]() { if (CleanDirectory(ZenFolderPath, {})) -- cgit v1.2.3 From f535a53cdda32b2b4950e32901dcd333bea82c23 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 20:35:53 +0200 Subject: use oidctoken executable to generate auth (#336) - Feature: `zen builds` auth option `--oidctoken-exe-path` to let zen run the OidcToken executable to get and refresh authentication token --- src/zen/cmds/builds_cmd.cpp | 36 ++++++++- src/zen/cmds/builds_cmd.h | 2 + src/zencore/include/zencore/fmtutils.h | 14 ++++ src/zenhttp/httpclientauth.cpp | 108 +++++++++++++++++++++++++++ src/zenhttp/include/zenhttp/httpclient.h | 2 +- src/zenhttp/include/zenhttp/httpclientauth.h | 4 + src/zenstore/include/zenstore/gc.h | 13 ---- 7 files changed, 161 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ba2564fad..f6b7299cb 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8347,6 +8347,12 @@ BuildsCommand::BuildsCommand() "OAuth client secret", cxxopts::value(m_OAuthClientSecret)->default_value(""), ""); + Ops.add_option("auth", + "", + "oidctoken-exe-path", + "Path to OidcToken executable", + cxxopts::value(m_OidcTokenAuthExecutablePath)->default_value(""), + ""); }; auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { @@ -8750,6 +8756,27 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return {}; }; + auto FindOidcTokenExePath = [](const std::string& OidcTokenAuthExecutablePath) -> std::filesystem::path { + if (OidcTokenAuthExecutablePath.empty()) + { + const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path OidcTokenPath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + else + { + std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + return {}; + }; + if (!m_AccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); @@ -8776,15 +8803,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); } - else + else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) { - CreateAuthMgr(); - ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); + const std::string& CloudHost = m_OverrideHost.empty() ? m_Host : m_OverrideHost; + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, CloudHost); } if (!ClientSettings.AccessTokenProvider) { - ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials."); + CreateAuthMgr(); + ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); } }; diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index b40c3e08f..535d2b1d2 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -80,6 +80,8 @@ private: std::string m_OAuthClientId; std::string m_OAuthClientSecret; + std::string m_OidcTokenAuthExecutablePath; + std::string m_Verb; // list, upload, download cxxopts::Options m_ListOptions{"list", "List available builds"}; diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h index 8482157fb..10dfa5393 100644 --- a/src/zencore/include/zencore/fmtutils.h +++ b/src/zencore/include/zencore/fmtutils.h @@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +#include #include // Custom formatting for some zencore types @@ -102,3 +103,16 @@ struct fmt::formatter : fmt::formatter return fmt::formatter::format(a.ToView(), ctx); } }; + +template<> +struct fmt::formatter : formatter +{ + template + auto format(const std::chrono::system_clock::time_point& TimePoint, FormatContext& ctx) const + { + std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint); + char TimeString[std::size("yyyy-mm-ddThh:mm:ss")]; + std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time)); + return fmt::format_to(ctx.out(), "{}", TimeString); + } +}; diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 7fb3224f1..916b25bff 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -2,15 +2,25 @@ #include +#include #include +#include +#include +#include #include +#include + ZEN_THIRD_PARTY_INCLUDES_START #include #include #include ZEN_THIRD_PARTY_INCLUDES_END +#if ZEN_PLATFORM_WINDOWS +# define timegm _mkgmtime +#endif // ZEN_PLATFORM_WINDOWS + namespace zen { namespace httpclientauth { using namespace std::literals; @@ -76,4 +86,102 @@ namespace zen { namespace httpclientauth { return CreateFromOpenIdProvider(AuthManager, "Default"sv); } + static HttpClientAccessToken GetOidcTokenFromExe(const std::filesystem::path& OidcExecutablePath, + std::string_view CloudHost, + bool Unattended) + { + Stopwatch Timer; + + CreateProcOptions ProcOptions; + + const std::string AuthTokenPath = ".zen-auth-oidctoken"; + + auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); }); + + const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}", + OidcExecutablePath, + CloudHost, + AuthTokenPath, + Unattended ? "true"sv : "false"sv); + ZEN_DEBUG("Running: {}", ProcArgs); + ProcessHandle Proc; + Proc.Initialize(CreateProc(OidcExecutablePath, ProcArgs, ProcOptions)); + if (!Proc.IsValid()) + { + throw std::runtime_error(fmt::format("failed to launch '{}'", OidcExecutablePath)); + } + + int ExitCode = Proc.WaitExitCode(); + + auto EndTime = std::chrono::system_clock::now(); + + if (ExitCode == 0) + { + IoBuffer Body = IoBufferBuilder::MakeFromFile(AuthTokenPath); + std::string JsonText(reinterpret_cast(Body.GetData()), Body.GetSize()); + + std::string JsonError; + json11::Json Json = json11::Json::parse(JsonText, JsonError); + + if (JsonError.empty() == false) + { + ZEN_WARN("Unable to parse Oidcs json response from {}. Reason: '{}'", AuthTokenPath, JsonError); + return HttpClientAccessToken{}; + } + std::string Token = Json["Token"].string_value(); + std::string ExpiresAtUTCString = Json["ExpiresAtUtc"].string_value(); + ZEN_ASSERT(!ExpiresAtUTCString.empty()); + + int Year = 0; + int Month = 0; + int Day = 0; + int Hour = 0; + int Minute = 0; + int Second = 0; + int Millisecond = 0; + sscanf(ExpiresAtUTCString.c_str(), "%d-%d-%dT%d:%d:%d.%dZ", &Year, &Month, &Day, &Hour, &Minute, &Second, &Millisecond); + + std::tm Time = { + Second, + Minute, + Hour, + Day, + Month - 1, + Year - 1900, + }; + + time_t UTCTime = timegm(&Time); + HttpClientAccessToken::TimePoint ExpireTime = std::chrono::system_clock::from_time_t(UTCTime); + ExpireTime += std::chrono::microseconds(Millisecond); + + return HttpClientAccessToken{.Value = fmt::format("Bearer {}"sv, Token), .ExpireTime = ExpireTime}; + } + else + { + ZEN_WARN("Failed running {} to get auth token, error code {}", OidcExecutablePath, ExitCode); + } + return HttpClientAccessToken{}; + } + + std::optional> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, + std::string_view CloudHost) + { + HttpClientAccessToken InitialToken = GetOidcTokenFromExe(OidcExecutablePath, CloudHost, false); + if (InitialToken.IsValid()) + { + return [OidcExecutablePath = std::filesystem::path(OidcExecutablePath), + CloudHost = std::string(CloudHost), + InitialToken]() mutable { + if (InitialToken.IsValid()) + { + HttpClientAccessToken Result = InitialToken; + InitialToken = {}; + return Result; + } + return GetOidcTokenFromExe(OidcExecutablePath, CloudHost, true); + }; + } + return {}; + } + }} // namespace zen::httpclientauth diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 4d901bdb5..c991a71ea 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -34,7 +34,7 @@ struct HttpClientAccessToken using Clock = std::chrono::system_clock; using TimePoint = Clock::time_point; - static constexpr int64_t ExpireMarginInSeconds = 30; + static constexpr int64_t ExpireMarginInSeconds = 60 * 5; std::string Value; TimePoint ExpireTime; diff --git a/src/zenhttp/include/zenhttp/httpclientauth.h b/src/zenhttp/include/zenhttp/httpclientauth.h index aa07620ca..5b9b9d305 100644 --- a/src/zenhttp/include/zenhttp/httpclientauth.h +++ b/src/zenhttp/include/zenhttp/httpclientauth.h @@ -3,6 +3,7 @@ #pragma once #include +#include namespace zen { @@ -24,6 +25,9 @@ namespace httpclientauth { std::function CreateFromOpenIdProvider(AuthMgr& AuthManager, std::string_view OpenIdProvider); std::function CreateFromDefaultOpenIdProvider(AuthMgr& AuthManager); + + std::optional> CreateFromOidcTokenExecutable(const std::filesystem::path& OidcExecutablePath, + std::string_view CloudHost); } // namespace httpclientauth } // namespace zen diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 67aadef71..3223fba39 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -586,16 +586,3 @@ private: void gc_forcelink(); } // namespace zen - -template<> -struct fmt::formatter : formatter -{ - template - auto format(const zen::GcClock::TimePoint& TimePoint, FormatContext& ctx) const - { - std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint); - char TimeString[std::size("yyyy-mm-ddThh:mm:ss")]; - std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time)); - return fmt::format_to(ctx.out(), "{}", TimeString); - } -}; -- cgit v1.2.3 From 9e138d34eda99c57c1e55ab15b1c60f4757cd99f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 3 Apr 2025 14:28:15 +0200 Subject: `zen oplog-export`, `zen oplog-import` for `--url` (cloud) and `--builds` (builds) option now has `--oidctoken-exe-path` to let zen run the OidcToken executable to get and refresh authentication token (#340) --- src/zen/cmds/projectstore_cmd.cpp | 209 +++++++++------------ src/zen/cmds/projectstore_cmd.h | 2 + .../projectstore/buildsremoteprojectstore.cpp | 10 +- .../projectstore/buildsremoteprojectstore.h | 23 +-- .../projectstore/jupiterremoteprojectstore.cpp | 10 +- .../projectstore/jupiterremoteprojectstore.h | 23 +-- src/zenserver/projectstore/projectstore.cpp | 22 +++ 7 files changed, 159 insertions(+), 140 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 13c7c4b23..c73842b89 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -61,6 +61,75 @@ namespace { return AuthToken; } + std::filesystem::path FindOidcTokenExePath(std::string_view OidcTokenAuthExecutablePath) + { + if (OidcTokenAuthExecutablePath.empty()) + { + const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path OidcTokenPath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + OidcTokenPath = (std::filesystem::current_path() / OidcExecutableName).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + else + { + std::filesystem::path OidcTokenPath = std::filesystem::absolute(StringToPath(OidcTokenAuthExecutablePath)).make_preferred(); + if (IsFile(OidcTokenPath)) + { + return OidcTokenPath; + } + } + return {}; + }; + + void WriteAuthOptions(CbObjectWriter& Writer, + std::string_view JupiterOpenIdProvider, + std::string_view JupiterAccessToken, + std::string_view JupiterAccessTokenEnv, + std::string_view JupiterAccessTokenPath, + std::string_view OidcTokenAuthExecutablePath) + { + if (!JupiterOpenIdProvider.empty()) + { + Writer.AddString("openid-provider"sv, JupiterOpenIdProvider); + } + if (!JupiterAccessToken.empty()) + { + Writer.AddString("access-token"sv, JupiterAccessToken); + } + if (!JupiterAccessTokenPath.empty()) + { + std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(JupiterAccessTokenPath); + if (!ResolvedCloudAccessToken.empty()) + { + Writer.AddString("access-token"sv, ResolvedCloudAccessToken); + } + } + if (!JupiterAccessTokenEnv.empty()) + { + std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(JupiterAccessTokenEnv); + + if (!ResolvedCloudAccessTokenEnv.empty()) + { + Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); + } + else + { + Writer.AddString("access-token-env"sv, JupiterAccessTokenEnv); + } + } + if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) + { + Writer.AddString("oidc-exe-path"sv, OidcTokenExePath.generic_string()); + } + } + IoBuffer MakeCbObjectPayload(std::function WriteCB) { CbObjectWriter Writer; @@ -1160,35 +1229,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { Writer.AddString("basekey"sv, m_BaseCloudKey); } - if (!m_JupiterOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider); - } - if (!m_JupiterAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_JupiterAccessToken); - } - if (!m_JupiterAccessTokenPath.empty()) - { - std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath); - if (!ResolvedCloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessToken); - } - } - if (!m_JupiterAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv); - } - } + WriteAuthOptions(Writer, + m_JupiterOpenIdProvider, + m_JupiterAccessToken, + m_JupiterAccessTokenEnv, + m_JupiterAccessTokenPath, + m_OidcTokenAuthExecutablePath); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1219,35 +1265,12 @@ ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg Writer.AddString("namespace"sv, m_JupiterNamespace); Writer.AddString("bucket"sv, m_JupiterBucket); Writer.AddString("buildsid"sv, m_BuildsId); - if (!m_JupiterOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider); - } - if (!m_JupiterAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_JupiterAccessToken); - } - if (!m_JupiterAccessTokenPath.empty()) - { - std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath); - if (!ResolvedCloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessToken); - } - } - if (!m_JupiterAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv); - } - } + WriteAuthOptions(Writer, + m_JupiterOpenIdProvider, + m_JupiterAccessToken, + m_JupiterAccessTokenEnv, + m_JupiterAccessTokenPath, + m_OidcTokenAuthExecutablePath); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1586,35 +1609,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg Writer.AddString("namespace"sv, m_JupiterNamespace); Writer.AddString("bucket"sv, m_JupiterBucket); Writer.AddString("key"sv, m_CloudKey); - if (!m_JupiterOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider); - } - if (!m_JupiterAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_JupiterAccessToken); - } - if (!m_JupiterAccessTokenPath.empty()) - { - std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath); - if (!ResolvedCloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessToken); - } - } - if (!m_JupiterAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv); - } - } + WriteAuthOptions(Writer, + m_JupiterOpenIdProvider, + m_JupiterAccessToken, + m_JupiterAccessTokenEnv, + m_JupiterAccessTokenPath, + m_OidcTokenAuthExecutablePath); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); @@ -1631,35 +1631,12 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg Writer.AddString("namespace"sv, m_JupiterNamespace); Writer.AddString("bucket"sv, m_JupiterBucket); Writer.AddString("buildsid"sv, m_BuildsId); - if (!m_JupiterOpenIdProvider.empty()) - { - Writer.AddString("openid-provider"sv, m_JupiterOpenIdProvider); - } - if (!m_JupiterAccessToken.empty()) - { - Writer.AddString("access-token"sv, m_JupiterAccessToken); - } - if (!m_JupiterAccessTokenPath.empty()) - { - std::string ResolvedCloudAccessToken = ReadJupiterAccessTokenFromFile(m_JupiterAccessTokenPath); - if (!ResolvedCloudAccessToken.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessToken); - } - } - if (!m_JupiterAccessTokenEnv.empty()) - { - std::string ResolvedCloudAccessTokenEnv = GetEnvVariable(m_JupiterAccessTokenEnv); - - if (!ResolvedCloudAccessTokenEnv.empty()) - { - Writer.AddString("access-token"sv, ResolvedCloudAccessTokenEnv); - } - else - { - Writer.AddString("access-token-env"sv, m_JupiterAccessTokenEnv); - } - } + WriteAuthOptions(Writer, + m_JupiterOpenIdProvider, + m_JupiterAccessToken, + m_JupiterAccessTokenEnv, + m_JupiterAccessTokenPath, + m_OidcTokenAuthExecutablePath); if (m_JupiterAssumeHttp2) { Writer.AddBool("assumehttp2"sv, true); diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index e66e98414..0d24d8529 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -109,6 +109,7 @@ private: std::string m_JupiterAccessToken; std::string m_JupiterAccessTokenEnv; std::string m_JupiterAccessTokenPath; + std::string m_OidcTokenAuthExecutablePath; bool m_JupiterAssumeHttp2 = false; bool m_JupiterDisableTempBlocks = false; @@ -165,6 +166,7 @@ private: std::string m_JupiterAccessToken; std::string m_JupiterAccessTokenEnv; std::string m_JupiterAccessTokenPath; + std::string m_OidcTokenAuthExecutablePath; bool m_JupiterAssumeHttp2 = false; std::string m_CloudUrl; diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index a6583b722..2a04d5c40 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -494,7 +494,15 @@ CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, const std::file { TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); } - else + else if (!Options.OidcExePath.empty()) + { + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + { + TokenProvider = TokenProviderMaybe.value(); + } + } + + if (!TokenProvider) { TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.h b/src/zenserver/projectstore/buildsremoteprojectstore.h index 8b2c6c8c8..c52b13886 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.h +++ b/src/zenserver/projectstore/buildsremoteprojectstore.h @@ -10,17 +10,18 @@ class AuthMgr; struct BuildsRemoteStoreOptions : RemoteStoreOptions { - std::string Url; - std::string Namespace; - std::string Bucket; - Oid BuildId; - std::string OpenIdProvider; - std::string AccessToken; - AuthMgr& AuthManager; - bool ForceDisableBlocks = false; - bool ForceDisableTempBlocks = false; - bool AssumeHttp2 = false; - IoBuffer MetaData; + std::string Url; + std::string Namespace; + std::string Bucket; + Oid BuildId; + std::string OpenIdProvider; + std::string AccessToken; + AuthMgr& AuthManager; + std::filesystem::path OidcExePath; + bool ForceDisableBlocks = false; + bool ForceDisableTempBlocks = false; + bool AssumeHttp2 = false; + IoBuffer MetaData; }; std::shared_ptr CreateBuildsRemoteStore(const BuildsRemoteStoreOptions& Options, diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index e5839ad3b..20e6c28ac 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -371,7 +371,15 @@ CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, const std::fi { TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); } - else + else if (!Options.OidcExePath.empty()) + { + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, Url); TokenProviderMaybe) + { + TokenProvider = TokenProviderMaybe.value(); + } + } + + if (!TokenProvider) { TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); } diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.h b/src/zenserver/projectstore/jupiterremoteprojectstore.h index 27f3d9b73..8bf79d563 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.h +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.h @@ -10,17 +10,18 @@ class AuthMgr; struct JupiterRemoteStoreOptions : RemoteStoreOptions { - std::string Url; - std::string Namespace; - std::string Bucket; - IoHash Key; - IoHash OptionalBaseKey; - std::string OpenIdProvider; - std::string AccessToken; - AuthMgr& AuthManager; - bool ForceDisableBlocks = false; - bool ForceDisableTempBlocks = false; - bool AssumeHttp2 = false; + std::string Url; + std::string Namespace; + std::string Bucket; + IoHash Key; + IoHash OptionalBaseKey; + std::string OpenIdProvider; + std::string AccessToken; + AuthMgr& AuthManager; + std::filesystem::path OidcExePath; + bool ForceDisableBlocks = false; + bool ForceDisableTempBlocks = false; + bool AssumeHttp2 = false; }; std::shared_ptr CreateJupiterRemoteStore(const JupiterRemoteStoreOptions& Options, diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 1966eeef9..9aa800434 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -210,6 +210,16 @@ namespace { AccessToken = GetEnvVariable(AccessTokenEnvVariable); } } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (!IsFile(OidcExePathMaybe)) + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + OidcExePath = std::move(OidcExePathMaybe); + } + } std::string_view KeyParam = Cloud["key"sv].AsString(); if (KeyParam.empty()) { @@ -252,6 +262,7 @@ namespace { std::string(OpenIdProvider), AccessToken, AuthManager, + OidcExePath, ForceDisableBlocks, ForceDisableTempBlocks, AssumeHttp2}; @@ -307,6 +318,16 @@ namespace { AccessToken = GetEnvVariable(AccessTokenEnvVariable); } } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (!IsFile(OidcExePathMaybe)) + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + OidcExePath = std::move(OidcExePathMaybe); + } + } std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); if (BuildIdParam.empty()) { @@ -337,6 +358,7 @@ namespace { std::string(OpenIdProvider), AccessToken, AuthManager, + OidcExePath, ForceDisableBlocks, ForceDisableTempBlocks, AssumeHttp2, -- cgit v1.2.3 From 64cd6e328bee3a94bc0bf10441b4057f7be34d1c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 3 Apr 2025 14:28:51 +0200 Subject: build store save access times (#341) * save payload size in log for buildstore * read/write access times and manifest for buldstore * use retry when removing temporary files --- src/zen/cmds/builds_cmd.cpp | 10 +- src/zenstore/buildstore/buildstore.cpp | 283 +++++++++++++++++++-- src/zenstore/include/zenstore/accesstime.h | 12 +- .../include/zenstore/buildstore/buildstore.h | 39 ++- 4 files changed, 309 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f6b7299cb..d503ae45e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -4932,7 +4932,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - RemoveFile(CompressedChunkPath); + RemoveFileWithRetry(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -5761,7 +5761,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - RemoveFile(CompressedChunkPath); + RemoveFileWithRetry(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -6169,7 +6169,7 @@ namespace { throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } WritePartsComplete++; - RemoveFile(BlockChunkPath); + RemoveFileWithRetry(BlockChunkPath); if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -6328,7 +6328,7 @@ namespace { if (!BlockChunkPath.empty()) { - RemoveFile(BlockChunkPath); + RemoveFileWithRetry(BlockChunkPath); } WritePartsComplete++; @@ -6508,7 +6508,7 @@ namespace { if (!BlockChunkPath.empty()) { - RemoveFile(BlockChunkPath); + RemoveFileWithRetry(BlockChunkPath); } WritePartsComplete++; diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index f26901458..d6d727aa9 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -36,9 +37,18 @@ using namespace std::literals; namespace blobstore::impl { - const std::string BaseName = "builds"; - const char* IndexExtension = ".uidx"; - const char* LogExtension = ".slog"; + const std::string BaseName = "builds"; + const std::string ManifestExtension = ".cbo"; + const char* IndexExtension = ".uidx"; + const char* LogExtension = ".slog"; + const char* AccessTimeExtension = ".zacs"; + + const uint32_t ManifestVersion = (1 << 16) | (0 << 8) | (0); + + std::filesystem::path GetManifestPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + ManifestExtension); + } std::filesystem::path GetBlobIndexPath(const std::filesystem::path& RootDirectory) { @@ -56,10 +66,47 @@ namespace blobstore::impl { { return RootDirectory / (BaseName + "_meta" + LogExtension); } + + std::filesystem::path GetAccessTimesPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + AccessTimeExtension); + } + + struct AccessTimeRecord + { + IoHash Key; + std::uint32_t SecondsSinceEpoch = 0; + }; + + static_assert(sizeof(AccessTimeRecord) == 24); + +#pragma pack(push) +#pragma pack(1) + struct AccessTimesHeader + { + static constexpr uint32_t ExpectedMagic = 0x7363617a; // 'zacs'; + static constexpr uint32_t CurrentVersion = 1; + static constexpr uint64_t DataAlignment = 8; + + uint32_t Magic = ExpectedMagic; + uint32_t Version = CurrentVersion; + uint32_t AccessTimeCount = 0; + uint32_t Checksum = 0; + + static uint32_t ComputeChecksum(const AccessTimesHeader& Header) + { + return XXH32(&Header.Magic, sizeof(AccessTimesHeader) - sizeof(uint32_t), 0xC0C0'BABA); + } + }; +#pragma pack(pop) + + static_assert(sizeof(AccessTimesHeader) == 16); + } // namespace blobstore::impl BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) -: m_Config(Config) +: m_Log(logging::Get("builds")) +, m_Config(Config) , m_Gc(Gc) , m_LargeBlobStore(m_Gc) , m_SmallBlobStore(Gc) @@ -69,14 +116,57 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) ZEN_MEMSCOPE(GetBuildstoreTag()); try { - std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); - std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); - bool IsNew = !(IsFile(BlobLogPath) && IsFile(MetaLogPath)); + bool IsNew = true; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("{} build store at {} in {}", + IsNew ? "Initialized" : "Read", + m_Config.RootDirectory, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); + std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); + std::filesystem::path ManifestPath = blobstore::impl::GetManifestPath(Config.RootDirectory); + std::filesystem::path AccessTimesPath = blobstore::impl::GetAccessTimesPath(Config.RootDirectory); + if (IsFile(ManifestPath) && IsFile(BlobLogPath) && IsFile(MetaLogPath)) + { + IsNew = false; + } if (!IsNew) { - m_BlobLogFlushPosition = ReadPayloadLog(RwLock::ExclusiveLockScope(m_Lock), BlobLogPath, 0); - m_MetaLogFlushPosition = ReadMetadataLog(RwLock::ExclusiveLockScope(m_Lock), MetaLogPath, 0); + RwLock::ExclusiveLockScope Lock(m_Lock); + + CbObject ManifestReader = LoadCompactBinaryObject(ReadFile(ManifestPath).Flatten()); + Oid ManifestId = ManifestReader["id"].AsObjectId(); + uint32_t Version = ManifestReader["version"].AsUInt32(); + DateTime CreationDate = ManifestReader["createdAt"].AsDateTime(); + ZEN_UNUSED(CreationDate); + if (ManifestId == Oid::Zero || Version != blobstore::impl::ManifestVersion) + { + ZEN_WARN("Invalid manifest at {}, wiping state", ManifestPath); + IsNew = true; + } + else + { + m_BlobLogFlushPosition = ReadPayloadLog(Lock, BlobLogPath, 0); + m_MetaLogFlushPosition = ReadMetadataLog(Lock, MetaLogPath, 0); + if (IsFile(AccessTimesPath)) + { + ReadAccessTimes(Lock, AccessTimesPath); + } + } + } + + if (IsNew) + { + CleanDirectory(Config.RootDirectory, false); + CbObjectWriter ManifestWriter; + ManifestWriter.AddObjectId("id", Oid::NewOid()); + ManifestWriter.AddInteger("version", blobstore::impl::ManifestVersion); + ManifestWriter.AddDateTime("createdAt", DateTime::Now()); + TemporaryFile::SafeWriteFile(ManifestPath, ManifestWriter.Save().GetBuffer().AsIoBuffer()); } m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew); m_SmallBlobStore.Initialize(Config.RootDirectory, @@ -147,18 +237,19 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) } } + uint64_t PayloadSize = Payload.GetSize(); PayloadEntry Entry; if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize) { CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash); ZEN_UNUSED(Result); - Entry = {.Flags = PayloadEntry::kStandalone}; + Entry = PayloadEntry(PayloadEntry::kStandalone, PayloadSize); } else { CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash); ZEN_UNUSED(Result); - Entry = {.Flags = 0}; + Entry = PayloadEntry(0, PayloadSize); } m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); @@ -188,6 +279,7 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert({BlobHash, NewBlobIndex}); } + m_LastAccessTimeUpdateCount++; } IoBuffer @@ -204,7 +296,7 @@ BuildStore::GetBlob(const IoHash& BlobHash) if (Blob.Payload) { const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload]; - const bool IsStandalone = (Entry.Flags & PayloadEntry::kStandalone) != 0; + const bool IsStandalone = (Entry.GetFlags() & PayloadEntry::kStandalone) != 0; Lock.ReleaseNow(); IoBuffer Chunk; @@ -299,6 +391,7 @@ BuildStore::PutMetadatas(std::span BlobHashes, std::span BlobHashes, WorkerThreadPool* O ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; } ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount()); + m_LastAccessTimeUpdateCount++; } } } @@ -434,12 +528,23 @@ BuildStore::Flush() ZEN_TRACE_CPU("BuildStore::Flush"); try { + Stopwatch Timer; + const auto _ = MakeGuard( + [&] { ZEN_INFO("Flushed build store at {} in {}", m_Config.RootDirectory, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + m_LargeBlobStore.Flush(); m_SmallBlobStore.Flush(); m_MetadataBlockStore.Flush(false); m_PayloadlogFile.Flush(); m_MetadatalogFile.Flush(); + + if (uint64_t LastAccessTimeUpdateCount = m_LastAccessTimeUpdateCount.load(); LastAccessTimeUpdateCount > 0) + { + m_LastAccessTimeUpdateCount -= LastAccessTimeUpdateCount; + RwLock::ExclusiveLockScope UpdateLock(m_Lock); + WriteAccessTimes(UpdateLock, blobstore::impl::GetAccessTimesPath(m_Config.RootDirectory)); + } } catch (const std::exception& Ex) { @@ -447,6 +552,35 @@ BuildStore::Flush() } } +#if ZEN_WITH_TESTS +std::optional +BuildStore::GetLastAccessTime(const IoHash& Key) const +{ + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + const BlobEntry& Entry = m_BlobEntries[Index]; + return Entry.LastAccessTime; + } + return {}; +} + +bool +BuildStore::SetLastAccessTime(const IoHash& Key, const AccessTime& Time) +{ + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + BlobEntry& Entry = m_BlobEntries[Index]; + Entry.LastAccessTime = Time; + return true; + } + return false; +} +#endif // ZEN_WITH_TESTS + void BuildStore::CompactState() { @@ -540,7 +674,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys CasLog.Replay( [&](const PayloadDiskEntry& Record) { std::string InvalidEntryReason; - if (Record.Entry.Flags & PayloadEntry::kTombStone) + if (Record.Entry.GetFlags() & PayloadEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) @@ -702,6 +836,114 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy return LogEntryCount; } +void +BuildStore::ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath) +{ + ZEN_TRACE_CPU("BuildStore::ReadAccessTimes"); + + using namespace blobstore::impl; + + BasicFile AccessTimesFile; + AccessTimesFile.Open(AccessTimesPath, BasicFile::Mode::kRead); + uint64_t Size = AccessTimesFile.FileSize(); + if (Size >= sizeof(AccessTimesHeader)) + { + AccessTimesHeader Header; + uint64_t Offset = 0; + AccessTimesFile.Read(&Header, sizeof(Header), 0); + Offset += sizeof(AccessTimesHeader); + Offset = RoundUp(Offset, AccessTimesHeader::DataAlignment); + if ((Header.Magic == AccessTimesHeader::ExpectedMagic) && (Header.Version == AccessTimesHeader::CurrentVersion) && + (Header.Checksum == AccessTimesHeader::ComputeChecksum(Header))) + { + uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount; + if (AccessTimesFile.FileSize() >= Offset + RecordsSize) + { + std::vector AccessRecords(Header.AccessTimeCount); + AccessTimesFile.Read(AccessRecords.data(), RecordsSize, Offset); + for (const AccessTimeRecord& Record : AccessRecords) + { + const IoHash& Key = Record.Key; + const uint32_t SecondsSinceEpoch = Record.SecondsSinceEpoch; + if (auto It = m_BlobLookup.find(Key); It != m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + BlobEntry& Entry = m_BlobEntries[Index]; + Entry.LastAccessTime.SetSecondsSinceEpoch(SecondsSinceEpoch); + } + else + { + m_LastAccessTimeUpdateCount++; + } + } + } + else + { + m_LastAccessTimeUpdateCount++; + } + } + else + { + m_LastAccessTimeUpdateCount++; + } + } + else + { + m_LastAccessTimeUpdateCount++; + } +} + +void +BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath) +{ + ZEN_TRACE_CPU("BuildStore::WriteAccessTimes"); + + using namespace blobstore::impl; + + uint32_t Count = gsl::narrow(m_BlobLookup.size()); + AccessTimesHeader Header = {.AccessTimeCount = Count}; + Header.Checksum = AccessTimesHeader::ComputeChecksum(Header); + + TemporaryFile TempFile; + std::error_code Ec; + if (TempFile.CreateTemporary(AccessTimesPath.parent_path(), Ec); Ec) + { + throw std::runtime_error(fmt::format("Failed to create temporary file {} to write access times. Reason ({}) {}", + TempFile.GetPath(), + Ec.value(), + Ec.message())); + } + { + uint64_t Offset = 0; + TempFile.Write(&Header, sizeof(AccessTimesHeader), Offset); + Offset += sizeof(AccessTimesHeader); + Offset = RoundUp(Offset, AccessTimesHeader::DataAlignment); + + std::vector AccessRecords; + AccessRecords.reserve(Header.AccessTimeCount); + + for (auto It : m_BlobLookup) + { + const IoHash& Key = It.first; + const BlobIndex Index = It.second; + const BlobEntry& Entry = m_BlobEntries[Index]; + const uint32_t SecondsSinceEpoch = Entry.LastAccessTime.GetSecondsSinceEpoch(); + AccessRecords.emplace_back(AccessTimeRecord{.Key = Key, .SecondsSinceEpoch = SecondsSinceEpoch}); + } + uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount; + TempFile.Write(AccessRecords.data(), RecordsSize, Offset); + Offset += sizeof(AccessTimesHeader) * Header.AccessTimeCount; + } + if (TempFile.MoveTemporaryIntoPlace(AccessTimesPath, Ec); Ec) + { + throw std::runtime_error(fmt::format("Failed to move temporary file {} to {} when write access times. Reason ({}) {}", + TempFile.GetPath(), + AccessTimesPath, + Ec.value(), + Ec.message())); + } +} + bool BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason) { @@ -710,18 +952,18 @@ BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString()); return false; } - if (Entry.Entry.Flags & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) + if (Entry.Entry.GetFlags() & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) { - OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.Flags, Entry.BlobHash.ToHexString()); + OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.GetFlags(), Entry.BlobHash.ToHexString()); return false; } - if (Entry.Entry.Flags & PayloadEntry::kTombStone) + if (Entry.Entry.GetFlags() & PayloadEntry::kTombStone) { return true; } - if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0 || Entry.Entry.Reserved3 != 0) + if (Entry.Entry.GetSize() == 0 || Entry.Entry.GetSize() == 0x00ffffffffffffffu) { - OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + OutReason = fmt::format("Invalid size field {} for meta entry {}", Entry.Entry.GetSize(), Entry.BlobHash.ToHexString()); return false; } return true; @@ -869,6 +1111,8 @@ public: NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); + const auto __ = MakeGuard([&] { m_Store.Flush(); }); + if (!m_RemovedBlobs.empty()) { if (Ctx.Settings.CollectSmallObjects) @@ -1080,7 +1324,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) { RemovedPayloads.push_back( PayloadDiskEntry{.Entry = m_PayloadEntries[ReadBlobEntry.Payload], .BlobHash = ExpiredBlob}); - RemovedPayloads.back().Entry.Flags |= PayloadEntry::kTombStone; + RemovedPayloads.back().Entry.AddFlag(PayloadEntry::kTombStone); m_PayloadEntries[ReadBlobEntry.Payload] = {}; m_BlobEntries[ReadBlobIndex].Payload = {}; } @@ -1094,6 +1338,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) } m_BlobLookup.erase(It); + m_LastAccessTimeUpdateCount++; RemovedBlobs.push_back(ExpiredBlob); Stats.DeletedCount++; diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h index a28dc908b..e53937b52 100644 --- a/src/zenstore/include/zenstore/accesstime.h +++ b/src/zenstore/include/zenstore/accesstime.h @@ -11,10 +11,10 @@ namespace zen { // This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch struct AccessTime { - explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} + explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSecondsSinceEpoch(Tick)) {} AccessTime& operator=(GcClock::Tick Tick) noexcept { - SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); + SecondsSinceEpoch.store(ToSecondsSinceEpoch(Tick), std::memory_order_relaxed); return *this; } operator GcClock::Tick() const noexcept @@ -36,8 +36,14 @@ struct AccessTime return *this; } + void SetSecondsSinceEpoch(uint32_t InSecondsSinceEpoch) { SecondsSinceEpoch.store(InSecondsSinceEpoch); } + + uint32_t GetSecondsSinceEpoch() const { return SecondsSinceEpoch.load(); } + private: - static uint32_t ToSeconds(GcClock::Tick Tick) + AccessTime(uint32_t InSecondsSinceEpoch) noexcept : SecondsSinceEpoch(InSecondsSinceEpoch) {} + + static uint32_t ToSecondsSinceEpoch(GcClock::Tick Tick) { return gsl::narrow(std::chrono::duration_cast(GcClock::Duration(Tick)).count()); } diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index 302af5f9c..d88e682de 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -48,11 +48,19 @@ public: void Flush(); +#if ZEN_WITH_TESTS + std::optional GetLastAccessTime(const IoHash& Key) const; + bool SetLastAccessTime(const IoHash& Key, const AccessTime& Time); +#endif // ZEN_WITH_TESTS private: + LoggerRef Log() { return m_Log; } + void CompactState(); uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + void WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath); + void ReadAccessTimes(const RwLock::ExclusiveLockScope&, const std::filesystem::path& AccessTimesPath); //////// GcReferencer virtual std::string GetGcName(GcCtx& Ctx) override; @@ -67,22 +75,35 @@ private: #pragma pack(1) struct PayloadEntry { + PayloadEntry() {} + PayloadEntry(uint64_t Flags, uint64_t Size) + { + ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size); + ZEN_ASSERT((Flags & (kTombStone | kStandalone)) == Flags); + FlagsAndSize = (Size << 8) | Flags; + } static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value - uint8_t Flags = 0; - uint8_t Reserved1 = 0; - uint8_t Reserved2 = 0; - uint8_t Reserved3 = 0; + uint64_t FlagsAndSize = 0; + uint64_t GetSize() const { return FlagsAndSize >> 8; } + uint8_t GetFlags() const { return uint8_t(FlagsAndSize & 0xff); } + void AddFlag(uint8_t Flag) { FlagsAndSize |= Flag; } + void SetSize(uint64_t Size) + { + ZEN_ASSERT((Size & 0x00ffffffffffffffu) == Size); + FlagsAndSize = (Size << 8) | (FlagsAndSize & 0xff); + } + void SetFlags(uint8_t Flags) { FlagsAndSize = (FlagsAndSize & 0xffffffffffffff00u) | Flags; } }; - static_assert(sizeof(PayloadEntry) == 4); + static_assert(sizeof(PayloadEntry) == 8); struct PayloadDiskEntry { - PayloadEntry Entry; // 4 bytes + PayloadEntry Entry; // 8 bytes IoHash BlobHash; // 20 bytes }; - static_assert(sizeof(PayloadDiskEntry) == 24); + static_assert(sizeof(PayloadDiskEntry) == 28); struct MetadataEntry { @@ -154,10 +175,11 @@ private: }; static_assert(sizeof(BlobEntry) == 12); + LoggerRef m_Log; const BuildStoreConfig m_Config; GcManager& m_Gc; - RwLock m_Lock; + mutable RwLock m_Lock; std::vector m_PayloadEntries; std::vector m_MetadataEntries; @@ -175,6 +197,7 @@ private: uint64_t m_MetaLogFlushPosition = 0; std::unique_ptr m_TrackedCacheKeys; + std::atomic m_LastAccessTimeUpdateCount; friend class BuildStoreGcReferenceChecker; friend class BuildStoreGcReferencePruner; -- cgit v1.2.3 From 88ee30c9a4a734b83d9c6e51894f68f74a77d1ad Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Thu, 3 Apr 2025 13:41:57 -0600 Subject: Oplog search improvements - Case insensitive search - Allow search of 1 or 2 character strings - Reset table when doing a null search --- src/zenserver/frontend/html/indexer/indexer.js | 3 ++- src/zenserver/frontend/html/pages/oplog.js | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js index 4412e3a57..16b91e130 100644 --- a/src/zenserver/frontend/html/indexer/indexer.js +++ b/src/zenserver/frontend/html/indexer/indexer.js @@ -43,9 +43,10 @@ class Indexer *search(needle) { + var needleLwr = needle.toLowerCase(); for (const page of this._pages) for (const [_, name] of page) - if (name.indexOf(needle) >= 0) + if (name.toLowerCase().indexOf(needleLwr) >= 0) yield name; } diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js index f22c2a58f..bef5bacce 100644 --- a/src/zenserver/frontend/html/pages/oplog.js +++ b/src/zenserver/frontend/html/pages/oplog.js @@ -142,9 +142,12 @@ export class Page extends ZenPage async _search(needle) { - needle = needle.trim(); - if (needle.length < 3) + if (needle.length == 0) + { + this._build_table(this._index_start); return; + } + needle = needle.trim(); this._entry_table.clear(this._index_start); -- cgit v1.2.3 From 708735d15da5960ff3bccc1ca22351d70d54880f Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:14:57 -0600 Subject: Don't duplicate ID bytes when more than one pkg_data ID was getting extended and left shifted if we encountered multiple package data items in a single entry. So instead of the ID being 0x0c6500b7fb8dbe2e, it was 0x0C6500B7FB8DBE2E0C6500B7FB8DBE2E. When we went to look up an imported package by ID, it would not be found and the import would be presented as a blank string. Addressing this by making the first package data the only referenceable one. Second package datas are currently used for optional data blobs, and will not be imported or referenced. They are sidecar data. --- src/zenserver/frontend/html/indexer/worker.js | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js index 25c8d7671..dd0f8bb17 100644 --- a/src/zenserver/frontend/html/indexer/worker.js +++ b/src/zenserver/frontend/html/indexer/worker.js @@ -101,6 +101,7 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) id <<= 8n; id |= BigInt(pkg_id[i]); } + break; } if (bulk_data) -- cgit v1.2.3 From 8dd059cbfaaf49c8cfc30192ff999b8cd4f219c3 Mon Sep 17 00:00:00 2001 From: zousar Date: Thu, 3 Apr 2025 17:38:48 -0600 Subject: Bump db version Required to refresh db contents after ID fix. --- src/zenserver/frontend/html/indexer/cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/cache.js b/src/zenserver/frontend/html/indexer/cache.js index 390aa948d..b90194855 100644 --- a/src/zenserver/frontend/html/indexer/cache.js +++ b/src/zenserver/frontend/html/indexer/cache.js @@ -9,7 +9,7 @@ export class Cache { this._db_name = db_name; this._store_names = store_names; - this._version = 1; + this._version = 2; this._db = this._open(); } -- cgit v1.2.3 From 6d22f7e086646dcfb0d7f5e2c784acba53235c35 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 4 Apr 2025 13:07:42 +0200 Subject: blobstore size limit (#342) - Feature: zenserver option `--buildstore-disksizelimit` to set an soft upper limit for build storage data. Defaults to 1TB. --- src/zenserver/buildstore/httpbuildstore.cpp | 34 +++ src/zenserver/config.cpp | 7 + src/zenserver/config.h | 3 +- src/zenserver/zenserver.cpp | 7 +- src/zenstore/buildstore/buildstore.cpp | 247 ++++++++++++++++++++- .../include/zenstore/buildstore/buildstore.h | 21 +- 6 files changed, 302 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index c918f5683..75a333687 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -524,6 +524,40 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) } Cbo.EndObject(); + Cbo.BeginObject("size"); + { + BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); + + Cbo << "count" << StorageStats.EntryCount; + Cbo << "bytes" << StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes + StorageStats.MetadataByteCount; + Cbo.BeginObject("blobs"); + { + Cbo << "count" << (StorageStats.LargeBlobCount + StorageStats.SmallBlobCount); + Cbo << "bytes" << (StorageStats.LargeBlobBytes + StorageStats.SmallBlobBytes); + Cbo.BeginObject("large"); + { + Cbo << "count" << StorageStats.LargeBlobCount; + Cbo << "bytes" << StorageStats.LargeBlobBytes; + } + Cbo.EndObject(); // large + Cbo.BeginObject("small"); + { + Cbo << "count" << StorageStats.SmallBlobCount; + Cbo << "bytes" << StorageStats.SmallBlobBytes; + } + Cbo.EndObject(); // small + } + Cbo.EndObject(); // blobs + + Cbo.BeginObject("metadata"); + { + Cbo << "count" << StorageStats.MetadataCount; + Cbo << "bytes" << StorageStats.MetadataByteCount; + } + Cbo.EndObject(); // metadata + } + Cbo.EndObject(); // size + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 52f539dcd..31c104110 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -379,6 +379,7 @@ ParseConfigFile(const std::filesystem::path& Path, ////// buildsstore LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); @@ -1050,6 +1051,12 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Whether the builds store is enabled or not.", cxxopts::value(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit", + "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", + cxxopts::value(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), + ""); options.add_option("stats", "", diff --git a/src/zenserver/config.h b/src/zenserver/config.h index a87b6f8b3..6bd7aa357 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -138,7 +138,8 @@ struct ZenProjectStoreConfig struct ZenBuildStoreConfig { - bool Enabled = false; + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB }; struct ZenWorkspacesConfig diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index c7cb2ba6e..3f2f01d5a 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -266,9 +266,10 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen if (ServerOptions.BuildStoreConfig.Enabled) { - BuildStoreConfig ObjCfg; - ObjCfg.RootDirectory = m_DataRoot / "builds"; - m_BuildStore = std::make_unique(std::move(ObjCfg), m_GcManager); + BuildStoreConfig BuildsCfg; + BuildsCfg.RootDirectory = m_DataRoot / "builds"; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; + m_BuildStore = std::make_unique(std::move(BuildsCfg), m_GcManager); } if (ServerOptions.StructuredCacheConfig.Enabled) diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index d6d727aa9..cf518c06f 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -193,10 +193,12 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) m_Gc.AddGcReferencer(*this); m_Gc.AddGcReferenceLocker(*this); + m_Gc.AddGcStorage(this); } catch (const std::exception& Ex) { ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what()); + m_Gc.RemoveGcStorage(this); m_Gc.RemoveGcReferenceLocker(*this); m_Gc.RemoveGcReferencer(*this); } @@ -207,6 +209,7 @@ BuildStore::~BuildStore() try { ZEN_TRACE_CPU("BuildStore::~BuildStore"); + m_Gc.RemoveGcStorage(this); m_Gc.RemoveGcReferenceLocker(*this); m_Gc.RemoveGcReferencer(*this); Flush(); @@ -552,6 +555,44 @@ BuildStore::Flush() } } +BuildStore::StorageStats +BuildStore::GetStorageStats() const +{ + StorageStats Result; + { + RwLock::SharedLockScope _(m_Lock); + Result.EntryCount = m_BlobLookup.size(); + + for (auto LookupIt : m_BlobLookup) + { + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + if (ReadBlobEntry.Payload) + { + const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload]; + uint64_t Size = Payload.GetSize(); + if ((Payload.GetFlags() & PayloadEntry::kStandalone) != 0) + { + Result.LargeBlobCount++; + Result.LargeBlobBytes += Size; + } + else + { + Result.SmallBlobCount++; + Result.SmallBlobBytes += Size; + } + } + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata]; + Result.MetadataCount++; + Result.MetadataByteCount += Metadata.Location.Size; + } + } + } + return Result; +} + #if ZEN_WITH_TESTS std::optional BuildStore::GetLastAccessTime(const IoHash& Key) const @@ -1273,24 +1314,103 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) } }); - const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count(); + const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count(); + std::vector ExpiredBlobs; + tsl::robin_set SizeDroppedBlobs; - std::vector ExpiredBlobs; { - RwLock::SharedLockScope __(m_Lock); - for (const auto& It : m_BlobLookup) + struct SizeInfo { - const BlobIndex ReadBlobIndex = It.second; - const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + const IoHash Key; + uint32_t SecondsSinceEpoch = 0; + uint64_t BlobSize = 0; + }; + + bool DiskSizeExceeded = false; + const uint64_t CurrentDiskSize = + m_LargeBlobStore.StorageSize().DiskSize + m_SmallBlobStore.StorageSize().DiskSize + m_MetadataBlockStore.TotalSize(); + if (CurrentDiskSize > m_Config.MaxDiskSpaceLimit) + { + DiskSizeExceeded = true; + } + + uint64_t ExpiredDataSize = 0; + + std::vector NonExpiredBlobSizeInfos; + + { + RwLock::SharedLockScope __(m_Lock); + if (DiskSizeExceeded) + { + NonExpiredBlobSizeInfos.reserve(m_BlobLookup.size()); + } + for (const auto& It : m_BlobLookup) + { + const BlobIndex ReadBlobIndex = It.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + uint64_t Size = 0; + if (ReadBlobEntry.Payload) + { + const PayloadEntry& Payload = m_PayloadEntries[ReadBlobEntry.Payload]; + Size += Payload.GetSize(); + } + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& Metadata = m_MetadataEntries[ReadBlobEntry.Metadata]; + Size += Metadata.Location.Size; + } + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + if (AccessTick < ExpireTicks) + { + ExpiredBlobs.push_back(It.first); + ExpiredDataSize += ExpiredDataSize; + } + else if (DiskSizeExceeded) + { + NonExpiredBlobSizeInfos.emplace_back(SizeInfo{.Key = It.first, + .SecondsSinceEpoch = ReadBlobEntry.LastAccessTime.GetSecondsSinceEpoch(), + .BlobSize = Size}); + } + } + Stats.CheckedCount += m_BlobLookup.size(); + Stats.FoundCount += ExpiredBlobs.size(); + } - const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; - if (AccessTick < ExpireTicks) + if (DiskSizeExceeded) + { + const uint64_t NewSizeLimit = + m_Config.MaxDiskSpaceLimit - + (m_Config.MaxDiskSpaceLimit >> 4); // Remove a bit more than just below the limit so we have some space to grow + if ((CurrentDiskSize - ExpiredDataSize) > NewSizeLimit) { - ExpiredBlobs.push_back(It.first); + std::vector NonExpiredOrder; + NonExpiredOrder.resize(NonExpiredBlobSizeInfos.size()); + for (size_t Index = 0; Index < NonExpiredOrder.size(); Index++) + { + NonExpiredOrder[Index] = Index; + } + std::sort(NonExpiredOrder.begin(), NonExpiredOrder.end(), [&NonExpiredBlobSizeInfos](const size_t Lhs, const size_t Rhs) { + const SizeInfo& LhsInfo = NonExpiredBlobSizeInfos[Lhs]; + const SizeInfo& RhsInfo = NonExpiredBlobSizeInfos[Rhs]; + return LhsInfo.SecondsSinceEpoch < RhsInfo.SecondsSinceEpoch; + }); + + auto It = NonExpiredOrder.begin(); + while (It != NonExpiredOrder.end()) + { + const SizeInfo& Info = NonExpiredBlobSizeInfos[*It]; + if ((CurrentDiskSize - ExpiredDataSize) < NewSizeLimit) + { + break; + } + ExpiredDataSize += Info.BlobSize; + ExpiredBlobs.push_back(Info.Key); + SizeDroppedBlobs.insert(Info.Key); + It++; + } } } - Stats.CheckedCount += m_BlobLookup.size(); - Stats.FoundCount += ExpiredBlobs.size(); } std::vector RemovedBlobs; @@ -1318,7 +1438,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; - if (AccessTick < ExpireTicks) + if (SizeDroppedBlobs.contains(ExpiredBlob) || (AccessTick < ExpireTicks)) { if (ReadBlobEntry.Payload) { @@ -1388,6 +1508,21 @@ BuildStore::LockState(GcCtx& Ctx) return Locks; } +void +BuildStore::ScrubStorage(ScrubContext& ScrubCtx) +{ + ZEN_UNUSED(ScrubCtx); + // TODO +} + +GcStorageSize +BuildStore::StorageSize() const +{ + GcStorageSize Result; + Result.DiskSize = m_MetadataBlockStore.TotalSize(); + return Result; +} + /* ___________ __ \__ ___/___ _______/ |_ ______ @@ -1731,6 +1866,94 @@ TEST_CASE("BuildStore.GC") } } +TEST_CASE("BuildStore.SizeLimit") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config = {.MaxDiskSpaceLimit = 1024u * 1024u}; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector CompressedBlobsHashes; + std::vector BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 64; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(65537 + I * 7); + CompressedBuffer CompressedBlob = + CompressedBuffer::Compress(SharedBuffer(std::move(Blob)), OodleCompressor::Mermaid, OodleCompressionLevel::None); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + + { + for (size_t I = 0; I < 64; I++) + { + const IoHash& Key = CompressedBlobsHashes[I]; + GcClock::Tick AccessTick = (GcClock::Now() + std::chrono::minutes(I)).time_since_epoch().count(); + + Store.SetLastAccessTime(Key, AccessTime(AccessTick)); + } + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .Verbose = true}); + + uint32_t DeletedBlobs = 0; + + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + if (!Blob) + { + DeletedBlobs++; + } + else + { + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash); + } + } + CHECK(DeletedBlobs == 50); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + if (I < DeletedBlobs) + { + CHECK(!MetadataPayload); + } + else + { + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + } + } +} + void buildstore_forcelink() { diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index d88e682de..adf48dc26 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -24,9 +24,10 @@ struct BuildStoreConfig uint32_t SmallBlobBlockStoreAlignement = 16; uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024; uint32_t MetadataBlockStoreAlignement = 8; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB }; -class BuildStore : public GcReferencer, public GcReferenceLocker //, public GcStorage +class BuildStore : public GcReferencer, public GcReferenceLocker, public GcStorage { public: explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc); @@ -48,10 +49,24 @@ public: void Flush(); + struct StorageStats + { + uint64_t EntryCount = 0; + uint64_t LargeBlobCount = 0; + uint64_t LargeBlobBytes = 0; + uint64_t SmallBlobCount = 0; + uint64_t SmallBlobBytes = 0; + uint64_t MetadataCount = 0; + uint64_t MetadataByteCount = 0; + }; + + StorageStats GetStorageStats() const; + #if ZEN_WITH_TESTS std::optional GetLastAccessTime(const IoHash& Key) const; bool SetLastAccessTime(const IoHash& Key, const AccessTime& Time); #endif // ZEN_WITH_TESTS + private: LoggerRef Log() { return m_Log; } @@ -71,6 +86,10 @@ private: //////// GcReferenceLocker virtual std::vector LockState(GcCtx& Ctx) override; + //////// GcStorage + virtual void ScrubStorage(ScrubContext& ScrubCtx) override; + virtual GcStorageSize StorageSize() const override; + #pragma pack(push) #pragma pack(1) struct PayloadEntry -- cgit v1.2.3 From d142c80beb9cc5c43192ba978198280906c5bd39 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 4 Apr 2025 15:23:50 +0200 Subject: progress bar improvements (#346) * hide ETA until at least 5% is complete * dynamically adjust progres output length --- src/zen/zen.cpp | 100 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 6f831349b..50ee43341 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -58,7 +58,8 @@ ZEN_THIRD_PARTY_INCLUDES_END #include -#ifndef ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +# include # include #endif @@ -329,29 +330,92 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) { size_t ProgressBarSize = 20; - size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; - uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; - uint64_t ETAMS = (Completed > 0) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0; - std::string ETA = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; - - std::string Output = fmt::format("\r{} {:#3}%: |{}{}|: {}{}{}", - NewState.Task, - PercentDone, - std::string(ProgressBarCount, '#'), - std::string(ProgressBarSize - ProgressBarCount, ' '), - NiceTimeSpanMs(ElapsedTimeMS), - ETA, - NewState.Details.empty() ? "" : fmt::format(". {}", NewState.Details)); + size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; + uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; + uint64_t ETAMS = (PercentDone > 5) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0; + +#if ZEN_PLATFORM_WINDOWS + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + CONSOLE_SCREEN_BUFFER_INFO csbi; + GetConsoleScreenBufferInfo(hStdOut, &csbi); + uint32_t ConsoleColumns = (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); +#else + struct winsize w; + ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); + uint32_t ConsoleColumns = (uint32_t)w.ws_col; +#endif + + std::string_view TaskString = NewState.Task; + + const std::string PercentString = fmt::format("{:#3}%", PercentDone); + + const std::string ProgressBarString = + fmt::format(": |{}{}|", std::string(ProgressBarCount, '#'), std::string(ProgressBarSize - ProgressBarCount, ' ')); + + const std::string ElapsedString = fmt::format(": {}", NiceTimeSpanMs(ElapsedTimeMS)); + + const std::string ETAString = (ETAMS > 0) ? fmt::format(" ETA {}", NiceTimeSpanMs(ETAMS)) : ""; + + const std::string DetailsString = (!NewState.Details.empty()) ? fmt::format(". {}", NewState.Details) : ""; + + ExtendableStringBuilder<256> OutputBuilder; + + OutputBuilder << "\r" << TaskString << PercentString; + if (OutputBuilder.Size() + 1 < ConsoleColumns) + { + size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); + bool ElapsedFits = RemainingSpace >= ElapsedString.length(); + RemainingSpace -= ElapsedString.length(); + bool ETAFits = ElapsedFits && RemainingSpace >= ETAString.length(); + RemainingSpace -= ETAString.length(); + bool DetailsFits = ETAFits && RemainingSpace >= DetailsString.length(); + RemainingSpace -= DetailsString.length(); + bool ProgressBarFits = DetailsFits && RemainingSpace >= ProgressBarString.length(); + RemainingSpace -= ProgressBarString.length(); + + if (ProgressBarFits) + { + OutputBuilder << ProgressBarString; + } + if (ElapsedFits) + { + OutputBuilder << ElapsedString; + } + if (ETAFits) + { + OutputBuilder << ETAString; + } + if (DetailsFits) + { + OutputBuilder << DetailsString; + } + } + + std::string_view Output = OutputBuilder.ToView(); std::string::size_type EraseLength = m_LastOutputLength > Output.length() ? (m_LastOutputLength - Output.length()) : 0; - ExtendableStringBuilder<128> LineToPrint; - LineToPrint << Output << std::string(EraseLength, ' '); + ExtendableStringBuilder<256> LineToPrint; + + if (Output.length() + EraseLength >= ConsoleColumns) + { + if (m_LastOutputLength > 0) + { + LineToPrint << "\n"; + } + LineToPrint << Output.substr(1); + DoLinebreak = true; + } + else + { + LineToPrint << Output << std::string(EraseLength, ' '); + } + if (DoLinebreak) + { LineToPrint << "\n"; + } #if ZEN_PLATFORM_WINDOWS - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - if (m_StdoutIsTty) { WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0); -- cgit v1.2.3 From 716f48c9d2aab6d624840332785064271e5fccbe Mon Sep 17 00:00:00 2001 From: zousar Date: Fri, 4 Apr 2025 07:53:35 -0600 Subject: Alternate fix by explicitly initializing pkg_id --- src/zenserver/frontend/html/indexer/worker.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js index dd0f8bb17..b4d547ef9 100644 --- a/src/zenserver/frontend/html/indexer/worker.js +++ b/src/zenserver/frontend/html/indexer/worker.js @@ -81,7 +81,7 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) var raw_size = 0; for (const item of pkg_data.as_array()) { - var found = 0, pkg_id; + var found = 0, pkg_id = undefined; for (const field of item.as_object()) { if (!id && field.is_named("id")) pkg_id = field.as_value(); @@ -101,7 +101,6 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) id <<= 8n; id |= BigInt(pkg_id[i]); } - break; } if (bulk_data) -- cgit v1.2.3 From b57274715ae1d47f8e59d3c098f37393815ba031 Mon Sep 17 00:00:00 2001 From: zousar Date: Fri, 4 Apr 2025 07:59:21 -0600 Subject: xmake updatefrontend --- src/zenserver/frontend/html.zip | Bin 156434 -> 156557 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index ca44804b9..b23b2efab 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From abd29e8c81594244e476d9def5333fc726597fb0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 4 Apr 2025 19:16:00 +0200 Subject: "unlimited" line length when using plain progress (#347) --- src/zen/zen.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 50ee43341..5ce0a89ec 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -334,16 +334,28 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; uint64_t ETAMS = (PercentDone > 5) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0; + uint32_t ConsoleColumns = 1024; + +#if ZEN_PLATFORM_WINDOWS + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); +#endif + + if (!m_PlainProgress) + { #if ZEN_PLATFORM_WINDOWS - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); - CONSOLE_SCREEN_BUFFER_INFO csbi; - GetConsoleScreenBufferInfo(hStdOut, &csbi); - uint32_t ConsoleColumns = (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) + { + ConsoleColumns = (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); + } #else - struct winsize w; - ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); - uint32_t ConsoleColumns = (uint32_t)w.ws_col; + struct winsize w; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) + { + ConsoleColumns = (uint32_t)w.ws_col; + } #endif + } std::string_view TaskString = NewState.Task; -- cgit v1.2.3 From fe6fb23a53bee3ffdb47b285a9446ea0ac4831fd Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 7 Apr 2025 10:00:08 +0200 Subject: improved layout of end of run stats output (#348) --- src/zen/cmds/builds_cmd.cpp | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d503ae45e..2f4a2921e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3829,7 +3829,7 @@ namespace { const uint64_t DownloadedCount = ValidateDownloadStats.DownloadedChunkCount + ValidateDownloadStats.DownloadedBlockCount; const uint64_t DownloadedByteCount = ValidateDownloadStats.DownloadedChunkByteCount + ValidateDownloadStats.DownloadedBlockByteCount; - ValidateInfo = fmt::format("\n Verified: {} ({}) {}B/sec in {}", + ValidateInfo = fmt::format("\n Verified: {:>8} ({}), {}B/sec, {}", DownloadedCount, NiceBytes(DownloadedByteCount), NiceNum(GetBytesPerSecond(ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)), @@ -3837,14 +3837,14 @@ namespace { } ZEN_CONSOLE( - "Uploaded part {} ('{}') to build {} in {} \n" - " Scanned files: {} ({}) {}B/sec in {}\n" - " New data: {} ({:.1f}%)\n" - " New blocks: {} ({}) {}B/sec\n" - " New chunks: {} ({} -> {}) {}B/sec\n" - " Uploaded: {} ({}) {}bits/sec\n" - " Blocks: {} ({})\n" - " Chunks: {} ({}){}" + "Uploaded part {} ('{}') to build {}, {}\n" + " Scanned files: {:>8} ({}), {}B/sec, {}\n" + " New data: {:>8} ({}) {:.1f}%\n" + " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n" + " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n" + " Uploaded: {:>8} ({}), {}bits/sec, {}\n" + " Blocks: {:>8} ({})\n" + " Chunks: {:>8} ({}){}" "{}", BuildPartId, BuildPartName, @@ -3856,21 +3856,26 @@ namespace { NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)), NiceTimeSpanMs(ChunkingStats.ElapsedWallTimeUS / 1000), + FindBlocksStats.NewBlocksChunkCount + LooseChunksStats.CompressedChunkCount, NiceBytes(FindBlocksStats.NewBlocksChunkByteCount + LooseChunksStats.CompressedChunkBytes), DeltaByteCountPercent, GenerateBlocksStats.GeneratedBlockCount.load(), + NiceBytes(FindBlocksStats.NewBlocksChunkByteCount), NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()), NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS, GenerateBlocksStats.GeneratedBlockByteCount)), + NiceTimeSpanMs(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000), LooseChunksStats.CompressedChunkCount.load(), NiceBytes(LooseChunksStats.ChunkByteCount), NiceBytes(LooseChunksStats.CompressedChunkBytes.load()), NiceNum(GetBytesPerSecond(LooseChunksStats.CompressChunksElapsedWallTimeUS, LooseChunksStats.ChunkByteCount)), + NiceTimeSpanMs(LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000), - NiceBytes(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load()), + UploadStats.BlockCount.load() + UploadStats.ChunkCount.load(), NiceBytes(UploadStats.BlocksBytes + UploadStats.ChunksBytes), NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, (UploadStats.ChunksBytes + UploadStats.BlocksBytes) * 8)), + NiceTimeSpanMs(UploadStats.ElapsedWallTimeUS / 1000), UploadStats.BlockCount.load(), NiceBytes(UploadStats.BlocksBytes.load()), -- cgit v1.2.3 From 61123ee1ac750059aa50136c9c7b0e521133dd87 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 7 Apr 2025 11:08:44 +0200 Subject: save global download info file for scavenging (#349) * save global download info file for scavenging * don't let test code write to official state folder --- src/zen/cmds/builds_cmd.cpp | 294 ++++++++++++++++++++++++++++---------------- 1 file changed, 189 insertions(+), 105 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2f4a2921e..ebcbb59b7 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1292,10 +1292,10 @@ namespace { } } - CbObject CreateStateObject(const Oid& BuildId, - std::vector> AllBuildParts, - std::span PartContents, - const FolderContent& LocalFolderState) + CbObject CreateStateObject(const Oid& BuildId, + const std::vector>& AllBuildParts, + std::span PartContents, + const FolderContent& LocalFolderState) { CbObjectWriter CurrentStateWriter; CurrentStateWriter.BeginArray("builds"sv); @@ -1334,6 +1334,51 @@ namespace { return CurrentStateWriter.Save(); } + void AddDownloadedPath(const std::filesystem::path& SystemRootDir, + const Oid& BuildId, + const std::vector>& BuildParts, + const std::filesystem::path& StateFilePath, + const std::filesystem::path& LocalPath) + { + ZEN_ASSERT(!SystemRootDir.empty()); + ZEN_ASSERT(!StateFilePath.empty()); + ZEN_ASSERT(!LocalPath.empty()); + const std::u8string LocalPathString = LocalPath.generic_u8string(); + IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); + std::filesystem::path WritePath = SystemRootDir / "builds" / "downloads" / (PathHash.ToHexString() + ".json"); + CreateDirectories(WritePath.parent_path()); + CbObjectWriter Writer; + Writer.AddString("path", (const char*)LocalPath.u8string().c_str()); + Writer.AddString("statePath", (const char*)StateFilePath.u8string().c_str()); + Writer.AddDateTime("date", DateTime::Now()); + Writer.BeginArray("builds"sv); + { + Writer.BeginObject(); + { + Writer.AddObjectId("buildId", BuildId); + Writer.BeginArray("parts"); + for (const auto& It : BuildParts) + { + Writer.BeginObject(); + { + Writer.AddObjectId("partId", It.first); + Writer.AddString("partName", It.second); + } + Writer.EndObject(); + } + Writer.EndArray(); // parts + } + Writer.EndObject(); + } + Writer.EndArray(); // builds + + CbObject Payload = Writer.Save(); + ExtendableStringBuilder<512> SB; + CompactBinaryToJson(Payload.GetView(), SB); + MemoryView JsonPayload(SB.Data(), SB.Size()); + TemporaryFile::SafeWriteFile(WritePath, JsonPayload); + } + class BufferedOpenFile { public: @@ -7882,6 +7927,7 @@ namespace { std::span BuildPartNames, const std::filesystem::path& Path, const std::filesystem::path& ZenFolderPath, + const std::filesystem::path SystemRootDir, bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, @@ -8004,6 +8050,8 @@ namespace { CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + + AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); } else { @@ -8055,6 +8103,8 @@ namespace { TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + AddDownloadedPath(SystemRootDir, BuildId, AllBuildParts, ZenStateFilePath(ZenFolderPath), Path); + #if 0 ExtendableStringBuilder<1024> SB; CompactBinaryToJson(StateObject, SB); @@ -8285,9 +8335,11 @@ BuildsCommand::BuildsCommand() { m_Options.add_options()("h,help", "Print help"); - auto AddAuthOptions = [this](cxxopts::Options& Ops) { + auto AddSystemOptions = [this](cxxopts::Options& Ops) { Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + }; + auto AddAuthOptions = [this](cxxopts::Options& Ops) { // Direct access token (may expire) Ops.add_option("auth-token", "", @@ -8428,6 +8480,7 @@ BuildsCommand::BuildsCommand() m_Options.positional_help("verb"); // list + AddSystemOptions(m_ListOptions); AddCloudOptions(m_ListOptions); AddFileOptions(m_ListOptions); AddOutputOptions(m_ListOptions); @@ -8449,6 +8502,7 @@ BuildsCommand::BuildsCommand() m_ListOptions.positional_help("query-path result-path"); // upload + AddSystemOptions(m_UploadOptions); AddCloudOptions(m_UploadOptions); AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); @@ -8521,6 +8575,7 @@ BuildsCommand::BuildsCommand() m_UploadOptions.positional_help("local-path build-id"); // download + AddSystemOptions(m_DownloadOptions); AddCloudOptions(m_DownloadOptions); AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); @@ -8585,6 +8640,7 @@ BuildsCommand::BuildsCommand() m_DiffOptions.parse_positional({"local-path", "compare-path"}); m_DiffOptions.positional_help("local-path compare-path"); + AddSystemOptions(m_TestOptions); AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); @@ -8607,6 +8663,7 @@ BuildsCommand::BuildsCommand() m_TestOptions.parse_positional({"local-path"}); m_TestOptions.positional_help("local-path"); + AddSystemOptions(m_FetchBlobOptions); AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); @@ -8618,6 +8675,7 @@ BuildsCommand::BuildsCommand() m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"}); m_FetchBlobOptions.positional_help("build-id blob-hash"); + AddSystemOptions(m_ValidateBuildPartOptions); AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); @@ -8640,6 +8698,7 @@ BuildsCommand::BuildsCommand() m_ValidateBuildPartOptions.parse_positional({"build-id", "build-part-id"}); m_ValidateBuildPartOptions.positional_help("build-id build-part-id"); + AddSystemOptions(m_MultiTestDownloadOptions); AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); @@ -8684,6 +8743,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } + std::filesystem::path SystemRootDir; + auto ParseSystemOptions = [&]() { + SystemRootDir = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); + }; + auto ParseStorageOptions = [&]() { if (!m_OverrideHost.empty() || !m_Host.empty()) { @@ -8710,10 +8774,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .RetryCount = 2}; auto CreateAuthMgr = [&]() { + ZEN_ASSERT(!SystemRootDir.empty()); if (!Auth) { - std::filesystem::path DataRoot = - m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; @@ -8726,7 +8789,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Warning: Using default encryption initialization vector"); } - AuthConfig AuthMgrConfig = {.RootDirectory = DataRoot / "auth", + AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth", .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; if (!AuthMgrConfig.EncryptionKey.IsValid()) @@ -8742,6 +8805,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseAuthOptions = [&]() { + ParseSystemOptions(); if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) { CreateAuthMgr(); @@ -9300,6 +9364,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ParseSystemOptions(); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -9362,6 +9428,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildPartNames, Path, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, @@ -9407,8 +9474,109 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return AbortFlag ? 11 : 0; } + if (SubOption == &m_FetchBlobOptions) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + if (m_BlobHash.empty()) + { + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + } + + IoHash BlobHash; + if (!IoHash::TryParse(m_BlobHash, BlobHash)) + { + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + } + + const Oid BuildId = Oid::FromHexString(m_BuildId); + + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); + if (AbortFlag) + { + return 11; + } + ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", + BlobHash, + CompressedSize, + DecompressedSize); + return 0; + } + + if (SubOption == &m_ValidateBuildPartOptions) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + + if (m_BuildId.empty()) + { + throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); + } + Oid BuildId = Oid::TryFromHexString(m_BuildId); + if (BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); + } + + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + } + + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + + Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); + + ValidateStatistics ValidateStats; + DownloadStatistics DownloadStats; + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); + + return AbortFlag ? 13 : 0; + } + if (SubOption == &m_MultiTestDownloadOptions) { + SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(SystemRootDir); + CleanDirectory(SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -9448,6 +9616,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, Path, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -9466,6 +9635,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { + SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(SystemRootDir); + CleanDirectory(SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -9490,7 +9664,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CleanDirectory(StoragePath, {}); m_StoragePath = StoragePath.generic_string(); } - auto _ = MakeGuard([&]() { + auto __ = MakeGuard([&]() { if (m_OverrideHost.empty() && StoragePath.empty()) { DeleteDirectories(StoragePath); @@ -9554,6 +9728,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, true, @@ -9576,6 +9751,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -9693,6 +9869,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -9743,6 +9920,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -9761,6 +9939,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -9779,6 +9958,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, DownloadPath, ZenFolderPath, + SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -9792,102 +9972,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } - - if (SubOption == &m_FetchBlobOptions) - { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BlobHash.empty()) - { - throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); - } - - IoHash BlobHash; - if (!IoHash::TryParse(m_BlobHash, BlobHash)) - { - throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); - } - - const Oid BuildId = Oid::FromHexString(m_BuildId); - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); - } - }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); - - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); - if (AbortFlag) - { - return 11; - } - ZEN_CONSOLE("Blob '{}' has a compressed size {} and a decompressed size of {} bytes", - BlobHash, - CompressedSize, - DecompressedSize); - return 0; - } - - if (SubOption == &m_ValidateBuildPartOptions) - { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } - - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) - { - std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); - } - }); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); - - Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); - - ValidateStatistics ValidateStats; - DownloadStatistics DownloadStats; - ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); - - return AbortFlag ? 13 : 0; - } } catch (const ParallellWorkException& Ex) { -- cgit v1.2.3 From cf99543cff63bb95d7b86a98d03c94857ef90ea4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 7 Apr 2025 14:05:57 +0200 Subject: fixed host resolving if both host and and override-host (url) was given (#350) --- src/zen/cmds/builds_cmd.cpp | 57 +++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ebcbb59b7..7014a6f15 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8928,6 +8928,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::string_view JsonResponse = ServerInfoResponse.AsText(); CbObject ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject(); + auto TestHostEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair { + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + return {true, ""}; + } + return {false, TestResponse.ErrorMessage("")}; + }; + if (m_OverrideHost.empty()) { CbArrayView ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView(); @@ -8945,23 +8962,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { const bool AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false); std::string_view Name = ServerEndpointObject["name"sv].AsString(); - - HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", - .ConnectTimeout = std::chrono::milliseconds{1000}, - .Timeout = std::chrono::milliseconds{2000}, - .AssumeHttp2 = AssumeHttp2, - .AllowResume = true, - .RetryCount = 0}; - - HttpClient TestHttpClient(BaseUrl, TestClientSettings); - HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); - if (TestResponse.IsSuccess()) + if (auto TestResult = TestHostEndpoint(BaseUrl, AssumeHttp2); TestResult.first) { CloudHost = BaseUrl; m_AssumeHttp2 = AssumeHttp2; BuildStorageName = Name; break; } + else + { + ZEN_DEBUG("Unable to reach host {}. Reason: {}", BaseUrl, TestResult.second); + } } } if (CloudHost.empty()) @@ -8970,8 +8981,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host)); } } + else if (auto TestResult = TestHostEndpoint(m_OverrideHost, m_AssumeHttp2); TestResult.first) + { + CloudHost = m_OverrideHost; + } + else + { + throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.second)); + } - auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool { + auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair { HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{1000}, .Timeout = std::chrono::milliseconds{2000}, @@ -8982,9 +9001,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); if (TestResponse.IsSuccess()) { - return true; + return {true, ""}; } - return false; + return {false, TestResponse.ErrorMessage("")}; }; if (m_ZenCacheHost.empty()) @@ -9001,7 +9020,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const bool AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false); std::string_view Name = CacheEndpointObject["name"sv].AsString(); - if (TestCacheEndpoint(BaseUrl, AssumeHttp2)) + if (auto TestResult = TestCacheEndpoint(BaseUrl, AssumeHttp2); TestResult.first) { m_ZenCacheHost = BaseUrl; CacheAssumeHttp2 = AssumeHttp2; @@ -9020,7 +9039,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); - if (TestCacheEndpoint(ZenServerLocalHostUrl, false)) + if (auto TestResult = TestCacheEndpoint(ZenServerLocalHostUrl, false); TestResult.first) { m_ZenCacheHost = ZenServerLocalHostUrl; CacheAssumeHttp2 = false; @@ -9035,7 +9054,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } } - else if (TestCacheEndpoint(m_ZenCacheHost, false)) + else if (auto TestResult = TestCacheEndpoint(m_ZenCacheHost, false); TestResult.first) { std::string::size_type HostnameStart = 0; std::string::size_type HostnameLength = std::string::npos; @@ -9049,6 +9068,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength); } + else + { + ZEN_WARN("Unable to reach cache host {}. Reason: {}", m_ZenCacheHost, TestResult.second); + } } } else -- cgit v1.2.3 From b904e8db99f1bfcd33708d6f1f5b2c37c3273ff8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 7 Apr 2025 22:08:44 +0200 Subject: tweaked verbose output (#351) --- src/zen/cmds/builds_cmd.cpp | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 7014a6f15..0d2601bd8 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1910,10 +1910,6 @@ namespace { uint64_t CompressedSize; uint64_t DecompressedSize; ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); - ZEN_CONSOLE_VERBOSE("Chunk attachment {} ({} -> {}) is valid", - ChunkHash, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); ValidateStats.VerifiedAttachmentCount++; ValidateStats.VerifiedByteCount += DecompressedSize; if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) @@ -1965,10 +1961,6 @@ namespace { uint64_t CompressedSize; uint64_t DecompressedSize; ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); - ZEN_CONSOLE_VERBOSE("Chunk block {} ({} -> {}) is valid", - BlockAttachment, - NiceBytes(CompressedSize), - NiceBytes(DecompressedSize)); ValidateStats.VerifiedAttachmentCount++; ValidateStats.VerifiedByteCount += DecompressedSize; if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) @@ -2258,12 +2250,14 @@ namespace { FilteredGeneratedBytesPerSecond.Start(); // TODO: Convert ScheduleWork body to function + Stopwatch GenerateTimer; CompressedBuffer CompressedBlock = GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); - ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks", + ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}", OutBlocks.BlockDescriptions[BlockIndex].BlockHash, NiceBytes(CompressedBlock.GetCompressedSize()), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); { @@ -2657,6 +2651,7 @@ namespace { FilteredGenerateBlockBytesPerSecond.Start(); + Stopwatch GenerateTimer; CompositeBuffer Payload; if (NewBlocks.BlockHeaders[BlockIndex]) { @@ -2687,14 +2682,16 @@ namespace { { FilteredGenerateBlockBytesPerSecond.Stop(); } + ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}", + NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(NewBlocks.BlockSizes[BlockIndex]), + NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); if (!AbortFlag) { AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); } - ZEN_CONSOLE_VERBOSE("Regenerated block {} ({}) containing {} chunks", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(NewBlocks.BlockSizes[BlockIndex]), - NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); } }, Work.DefaultErrorFunction()); @@ -2713,12 +2710,14 @@ namespace { ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); FilteredCompressedBytesPerSecond.Start(); + Stopwatch CompressTimer; CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); - ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", + ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), - NiceBytes(Payload.GetSize())); + NiceBytes(Payload.GetSize()), + NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; UploadStats.ReadFromDiskBytes += ChunkRawSize; if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) @@ -2857,10 +2856,8 @@ namespace { } else if (FoundAttachmentCount > 0) { - ZEN_CONSOLE_VERBOSE("Skipping block {}. {} attachments found, usage level: {}%", - KnownBlock.BlockHash, - FoundAttachmentCount, - ReusePercent); + // ZEN_CONSOLE_VERBOSE("Skipping block {}. {} attachments found, usage level: {}%", KnownBlock.BlockHash, + // FoundAttachmentCount, ReusePercent); FindBlocksStats.RejectedBlockCount++; FindBlocksStats.RejectedChunkCount += FoundAttachmentCount; FindBlocksStats.RejectedByteCount += ReuseSize; @@ -2920,7 +2917,7 @@ namespace { } else { - ZEN_CONSOLE_VERBOSE("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent); + // ZEN_CONSOLE_VERBOSE("Skipping block {}. filtered usage level: {}%", KnownBlock.BlockHash, ReusePercent); FindBlocksStats.RejectedBlockCount++; FindBlocksStats.RejectedChunkCount += FoundChunkIndexes.size(); FindBlocksStats.RejectedByteCount += AdjustedReuseSize; @@ -3561,8 +3558,6 @@ namespace { auto UploadAttachments = [&](std::span RawHashes) { if (!AbortFlag) { - ZEN_CONSOLE_VERBOSE("Uploading attachments: {}", FormatArray(RawHashes, "\n "sv)); - UploadStatistics TempUploadStats; LooseChunksStatistics TempLooseChunksStats; -- cgit v1.2.3 From 1ca32ca4718dad5bf1e2f381fe93b47d8159807b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 8 Apr 2025 18:57:25 +0200 Subject: scavenge builds (#352) - Improvement: `zen builds` now scavenges previous download locations for data to reduce download size, enabled by default, disable with `--enable-scavenge=false` - Bugfix: Failing to rename a file during download sometimes reported an error when it succeeded when retrying --- src/zen/cmds/builds_cmd.cpp | 851 ++++++++++++++++++++++++++++++++--------- src/zen/cmds/builds_cmd.h | 1 + src/zenutil/chunkedcontent.cpp | 4 +- 3 files changed, 669 insertions(+), 187 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 0d2601bd8..b33ec659d 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -189,6 +189,7 @@ namespace { for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); + Ec.clear(); RenameFile(SourcePath, TargetPath, Ec); } if (Ec) @@ -197,6 +198,23 @@ namespace { } } + bool IsFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + bool Result = IsFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + Result = IsFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) { std::error_code Ec; @@ -204,7 +222,7 @@ namespace { for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); - if (!IsFile(Path)) + if (!IsFileWithRetry(Path)) { return false; } @@ -225,7 +243,7 @@ namespace { for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); - if (!IsFile(Path)) + if (!IsFileWithRetry(Path)) { return; } @@ -833,11 +851,23 @@ namespace { uint64_t CacheSequenceHashesCount = 0; uint64_t CacheSequenceHashesByteCount = 0; + uint64_t CacheScanElapsedWallTimeUs = 0; + uint32_t LocalPathsMatchingSequencesCount = 0; uint64_t LocalPathsMatchingSequencesByteCount = 0; uint64_t LocalChunkMatchingRemoteCount = 0; uint64_t LocalChunkMatchingRemoteByteCount = 0; + + uint64_t LocalScanElapsedWallTimeUs = 0; + + uint32_t ScavengedPathsMatchingSequencesCount = 0; + uint64_t ScavengedPathsMatchingSequencesByteCount = 0; + + uint64_t ScavengedChunkMatchingRemoteCount = 0; + uint64_t ScavengedChunkMatchingRemoteByteCount = 0; + + uint64_t ScavengeElapsedWallTimeUs = 0; }; struct DownloadStatistics @@ -1295,9 +1325,11 @@ namespace { CbObject CreateStateObject(const Oid& BuildId, const std::vector>& AllBuildParts, std::span PartContents, - const FolderContent& LocalFolderState) + const FolderContent& LocalFolderState, + const std::filesystem::path& LocalPath) { CbObjectWriter CurrentStateWriter; + CurrentStateWriter.AddString("path", (const char*)LocalPath.u8string().c_str()); CurrentStateWriter.BeginArray("builds"sv); { CurrentStateWriter.BeginObject(); @@ -1379,6 +1411,64 @@ namespace { TemporaryFile::SafeWriteFile(WritePath, JsonPayload); } + struct ScavengeSource + { + std::filesystem::path StateFilePath; + std::filesystem::path Path; + }; + + std::vector GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir) + { + std::vector Result; + DirectoryContent Content; + GetDirectoryContent(SystemRootDir / "builds" / "downloads", DirectoryContentFlags::IncludeFiles, Content); + for (const std::filesystem::path& EntryPath : Content.Files) + { + bool DeleteEntry = false; + IoHash EntryPathHash; + if (IoHash::TryParse(EntryPath.stem().string(), EntryPathHash)) + { + // Read state and verify that it is valid + IoBuffer MetaDataJson = ReadFile(EntryPath).Flatten(); + std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (JsonError.empty()) + { + std::filesystem::path StateFilePath = DownloadInfo["statePath"].AsU8String(); + if (IsFile(StateFilePath)) + { + std::filesystem::path Path = DownloadInfo["path"].AsU8String(); + if (IsDir(Path)) + { + Result.push_back({.StateFilePath = std::move(StateFilePath), .Path = std::move(Path)}); + } + else + { + DeleteEntry = true; + } + } + else + { + DeleteEntry = true; + } + } + else + { + ZEN_WARN("Invalid download state file at {}. '{}'", EntryPath, JsonError); + DeleteEntry = true; + } + } + + if (DeleteEntry) + { + std::error_code DummyEc; + std::filesystem::remove(EntryPath, DummyEc); + } + } + return Result; + } + class BufferedOpenFile { public: @@ -4842,6 +4932,7 @@ namespace { IoBuffer&& CompressedPart, DiskStatistics& DiskStats) { + ZEN_TRACE_CPU("WriteCompressedChunk"); auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; @@ -5000,7 +5091,143 @@ namespace { Work.DefaultErrorFunction()); }; - void UpdateFolder(StorageInstance& Storage, + bool ReadStateFile(const std::filesystem::path& StateFilePath, + FolderContent& OutLocalFolderState, + ChunkedFolderContent& OutLocalContent) + { + ZEN_TRACE_CPU("ReadStateFile"); + bool HasLocalState = false; + try + { + CbObject CurrentStateObject = LoadCompactBinaryObject(StateFilePath).Object; + if (CurrentStateObject) + { + Oid CurrentBuildId; + std::vector SavedBuildPartIds; + std::vector SavedBuildPartsNames; + std::vector SavedPartContents; + if (ReadStateObject(CurrentStateObject, + CurrentBuildId, + SavedBuildPartIds, + SavedBuildPartsNames, + SavedPartContents, + OutLocalFolderState)) + { + if (!SavedPartContents.empty()) + { + if (SavedPartContents.size() == 1) + { + OutLocalContent = std::move(SavedPartContents[0]); + } + else + { + OutLocalContent = + MergeChunkedFolderContents(SavedPartContents[0], + std::span(SavedPartContents).subspan(1)); + } + HasLocalState = true; + } + } + } + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what()); + } + return HasLocalState; + } + + FolderContent GetValidFolderContent(GetFolderContentStatistics& LocalFolderScanStats, + const std::filesystem::path& Path, + std::span PathsToCheck) + { + ZEN_TRACE_CPU("GetValidFolderContent"); + FolderContent Result; + const uint32_t PathCount = gsl::narrow(PathsToCheck.size()); + + Result.Paths.resize(PathCount); + Result.RawSizes.resize(PathCount); + Result.Attributes.resize(PathCount); + Result.ModificationTicks.resize(PathCount); + + { + Stopwatch Timer; + auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + ProgressBar ProgressBar(UsePlainProgress); + + ParallellWork Work(AbortFlag); + std::atomic CompletedPathCount = 0; + uint32_t PathIndex = 0; + + while (PathIndex < PathCount) + { + uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); + Work.ScheduleWork( + GetIOWorkerPool(), + [PathIndex, PathRangeCount, &PathsToCheck, &Path, &Result, &CompletedPathCount, &LocalFolderScanStats]( + std::atomic&) { + for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) + { + const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; + std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); + if (TryGetFileProperties(LocalFilePath, + Result.RawSizes[PathRangeIndex], + Result.ModificationTicks[PathRangeIndex], + Result.Attributes[PathRangeIndex])) + { + Result.Paths[PathRangeIndex] = std::move(FilePath); + LocalFolderScanStats.FoundFileCount++; + LocalFolderScanStats.FoundFileByteCount += Result.RawSizes[PathRangeIndex]; + LocalFolderScanStats.AcceptedFileCount++; + LocalFolderScanStats.AcceptedFileByteCount += Result.RawSizes[PathRangeIndex]; + } + CompletedPathCount++; + } + }, + Work.DefaultErrorFunction()); + PathIndex += PathRangeCount; + } + Work.Wait(200, [&](bool, ptrdiff_t) { + // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} checked, {} found", + CompletedPathCount.load(), + PathCount, + LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount.load()}, + false); + }); + ProgressBar.Finish(); + } + + uint32_t WritePathIndex = 0; + for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++) + { + if (!Result.Paths[ReadPathIndex].empty()) + { + if (WritePathIndex < ReadPathIndex) + { + Result.Paths[WritePathIndex] = std::move(Result.Paths[ReadPathIndex]); + Result.RawSizes[WritePathIndex] = Result.RawSizes[ReadPathIndex]; + Result.Attributes[WritePathIndex] = Result.Attributes[ReadPathIndex]; + Result.ModificationTicks[WritePathIndex] = Result.ModificationTicks[ReadPathIndex]; + } + WritePathIndex++; + } + } + + Result.Paths.resize(WritePathIndex); + Result.RawSizes.resize(WritePathIndex); + Result.Attributes.resize(WritePathIndex); + Result.ModificationTicks.resize(WritePathIndex); + return Result; + } + + void UpdateFolder(const std::filesystem::path& SystemRootDir, + StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, const std::filesystem::path& ZenFolderPath, @@ -5013,6 +5240,7 @@ namespace { bool AllowPartialBlockRequests, bool WipeTargetFolder, bool PrimeCacheOnly, + bool EnableScavenging, FolderContent& OutLocalFolderState, DiskStatistics& DiskStats, CacheMappingStatistics& CacheMappingStats, @@ -5046,6 +5274,8 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); + Stopwatch CacheTimer; + DirectoryContent CacheDirContent; GetDirectoryContent(CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, @@ -5091,6 +5321,7 @@ namespace { } RemoveFileWithRetry(CacheDirContent.Files[Index]); } + CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); } tsl::robin_map CachedBlocksFound; @@ -5098,6 +5329,8 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); + Stopwatch CacheTimer; + tsl::robin_map AllBlockSizes; AllBlockSizes.reserve(BlockDescriptions.size()); for (uint32_t BlockIndex = 0; BlockIndex < BlockDescriptions.size(); BlockIndex++) @@ -5137,54 +5370,62 @@ namespace { } RemoveFileWithRetry(BlockDirContent.Files[Index]); } + + CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); } - std::vector LocalPathIndexesMatchingSequenceIndexes; + std::vector LocalPathIndexesMatchingSequenceIndexes; + tsl::robin_map SequenceIndexesLeftToFindToRemoteIndex; if (!PrimeCacheOnly) { // Pick up all whole files we can use from current local state - ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); + ZEN_TRACE_CPU("UpdateFolder_GetLocalSequences"); + + Stopwatch LocalTimer; + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) { - const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const IoHash& RemoteSequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); + const uint64_t RemoteRawSize = RemoteContent.RawSizes[RemotePathIndex]; if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash); CacheSequenceIt != CachedSequenceHashesFound.end()) { - // const uint32_t RemoteSequenceIndex = CacheSequenceIt->second; - // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); - // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) { - // const uint32_t RemoteChunkIndex = CacheChunkIt->second; - // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); - // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + ZEN_CONSOLE_VERBOSE("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { - const uint32_t LocalSequenceIndex = It->second; - const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); - uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + const uint32_t LocalSequenceIndex = It->second; + const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + ZEN_ASSERT_SLOW(IsFile(LocalFilePath)); LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; - CacheMappingStats.LocalPathsMatchingSequencesByteCount += RawSize; + CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize; + ZEN_CONSOLE_VERBOSE("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize)); } else { // We must write the sequence const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + SequenceIndexesLeftToFindToRemoteIndex.insert({RemoteSequenceRawHash, RemoteSequenceIndex}); } } + + CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); } else { @@ -5195,10 +5436,138 @@ namespace { SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; } } + + std::vector ScavengedContents; + std::vector ScavengedLookups; + std::vector ScavengedPaths; + + struct ScavengeCopyOperation + { + uint32_t ScavengedContentIndex = (uint32_t)-1; + uint32_t ScavengedPathIndex = (uint32_t)-1; + uint32_t RemoteSequenceIndex = (uint32_t)-1; + uint64_t RawSize = (uint32_t)-1; + }; + + std::vector ScavengeCopyOperations; + uint64_t ScavengedPathsCount = 0; + + if (!PrimeCacheOnly && EnableScavenging) + { + ZEN_TRACE_CPU("UpdateFolder_GetScavengedSequences"); + + Stopwatch ScavengeTimer; + + if (!SequenceIndexesLeftToFindToRemoteIndex.empty()) + { + std::vector ScavengeSources = GetDownloadedStatePaths(SystemRootDir); + auto EraseIt = std::remove_if(ScavengeSources.begin(), ScavengeSources.end(), [&Path](const ScavengeSource& Source) { + return Source.Path == Path; + }); + ScavengeSources.erase(EraseIt, ScavengeSources.end()); + + const size_t ScavengePathCount = ScavengeSources.size(); + + ScavengedContents.resize(ScavengePathCount); + ScavengedLookups.resize(ScavengePathCount); + ScavengedPaths.resize(ScavengePathCount); + for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++) + { + const ScavengeSource& Source = ScavengeSources[ScavengeIndex]; + + ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengeIndex]; + std::filesystem::path& ScavengePath = ScavengedPaths[ScavengeIndex]; + FolderContent LocalFolderState; + if (ReadStateFile(Source.StateFilePath, LocalFolderState, ScavengedLocalContent)) + { + GetFolderContentStatistics ScavengedFolderScanStats; + + FolderContent ValidFolderContent = + GetValidFolderContent(ScavengedFolderScanStats, Source.Path, LocalFolderState.Paths); + + if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent)) + { + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths); + + // If the files are modified since the state was saved we ignore the files since we don't want to incur the + // cost of scanning/hashing scavenged files + DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end()); + if (!DeletedPaths.empty()) + { + ScavengedLocalContent = DeletePathsFromChunkedContent(ScavengedLocalContent, DeletedPaths); + } + } + + if (!ScavengedLocalContent.Paths.empty()) + { + ScavengePath = Source.Path; + } + } + } + + for (uint32_t ScavengedContentIndex = 0; + ScavengedContentIndex < ScavengedContents.size() && (!SequenceIndexesLeftToFindToRemoteIndex.empty()); + ScavengedContentIndex++) + { + const std::filesystem::path& ScavengePath = ScavengedPaths[ScavengedContentIndex]; + if (!ScavengePath.empty()) + { + const ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengedContentIndex]; + ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; + ScavengedLookup = BuildChunkedContentLookup(ScavengedLocalContent); + + for (uint32_t ScavengedSequenceIndex = 0; + ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); + ScavengedSequenceIndex++) + { + const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex]; + if (auto It = SequenceIndexesLeftToFindToRemoteIndex.find(SequenceRawHash); + It != SequenceIndexesLeftToFindToRemoteIndex.end()) + { + const uint32_t RemoteSequenceIndex = It->second; + const uint64_t RawSize = + RemoteContent.RawSizes[RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]]; + ZEN_ASSERT(RawSize > 0); + + const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex]; + ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred())); + + ScavengeCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex, + .ScavengedPathIndex = ScavengedPathIndex, + .RemoteSequenceIndex = RemoteSequenceIndex, + .RawSize = RawSize}); + + SequenceIndexesLeftToFindToRemoteIndex.erase(SequenceRawHash); + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0; + + CacheMappingStats.ScavengedPathsMatchingSequencesCount++; + CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize; + } + } + ScavengedPathsCount++; + } + } + } + CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); + } + + uint32_t RemainingChunkCount = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) + { + uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + if (ChunkWriteCount > 0) + { + RemainingChunkCount++; + } + } + // Pick up all chunks in current local state + // TODO: Rename to LocalStateCopyData struct CacheCopyData { - uint32_t LocalSequenceIndex = (uint32_t)-1; + uint32_t ScavengeSourceIndex = (uint32_t)-1; + uint32_t SourceSequenceIndex = (uint32_t)-1; std::vector TargetChunkLocationPtrs; struct ChunkTarget { @@ -5216,7 +5585,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); - for (uint32_t LocalSequenceIndex = 0; LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size(); + Stopwatch LocalTimer; + + for (uint32_t LocalSequenceIndex = 0; + LocalSequenceIndex < LocalContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); LocalSequenceIndex++) { const IoHash& LocalSequenceRawHash = LocalContent.ChunkedContent.SequenceRawHashes[LocalSequenceIndex]; @@ -5254,7 +5626,8 @@ namespace { { RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); CacheCopyDatas.push_back( - CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, + CacheCopyData{.ScavengeSourceIndex = (uint32_t)-1, + .SourceSequenceIndex = LocalSequenceIndex, .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector{Target}}); } @@ -5270,13 +5643,15 @@ namespace { { RawHashToCacheCopyDataIndex.insert_or_assign(LocalSequenceRawHash, CacheCopyDatas.size()); CacheCopyDatas.push_back( - CacheCopyData{.LocalSequenceIndex = LocalSequenceIndex, + CacheCopyData{.ScavengeSourceIndex = (uint32_t)-1, + .SourceSequenceIndex = LocalSequenceIndex, .TargetChunkLocationPtrs = ChunkTargetPtrs, .ChunkTargets = std::vector{Target}}); } CacheMappingStats.LocalChunkMatchingRemoteCount++; CacheMappingStats.LocalChunkMatchingRemoteByteCount += LocalChunkRawSize; RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + RemainingChunkCount--; } } } @@ -5284,26 +5659,130 @@ namespace { } } } + CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs(); + } + + if (!PrimeCacheOnly) + { + ZEN_TRACE_CPU("UpdateFolder_GetScavengeChunks"); + + Stopwatch ScavengeTimer; + + for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0); + ScavengedContentIndex++) + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex]; + // const std::filesystem::path& ScavengedPath = ScavengedPaths[ScavengedContentIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; + + for (uint32_t ScavengedSequenceIndex = 0; + ScavengedSequenceIndex < ScavengedContent.ChunkedContent.SequenceRawHashes.size() && (RemainingChunkCount > 0); + ScavengedSequenceIndex++) + { + const IoHash& ScavengedSequenceRawHash = ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex]; + const uint32_t ScavengedOrderOffset = ScavengedLookup.SequenceIndexChunkOrderOffset[ScavengedSequenceIndex]; + + { + uint64_t SourceOffset = 0; + const uint32_t ScavengedChunkCount = ScavengedContent.ChunkedContent.ChunkCounts[ScavengedSequenceIndex]; + for (uint32_t ScavengedOrderIndex = 0; ScavengedOrderIndex < ScavengedChunkCount; ScavengedOrderIndex++) + { + const uint32_t ScavengedChunkIndex = + ScavengedContent.ChunkedContent.ChunkOrders[ScavengedOrderOffset + ScavengedOrderIndex]; + const IoHash& ScavengedChunkHash = ScavengedContent.ChunkedContent.ChunkHashes[ScavengedChunkIndex]; + const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex]; + + if (auto RemoteChunkIt = RemoteLookup.ChunkHashToChunkIndex.find(ScavengedChunkHash); + RemoteChunkIt != RemoteLookup.ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = RemoteChunkIt->second; + if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + { + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + + if (!ChunkTargetPtrs.empty()) + { + CacheCopyData::ChunkTarget Target = { + .TargetChunkLocationCount = gsl::narrow(ChunkTargetPtrs.size()), + .RemoteChunkIndex = RemoteChunkIndex, + .CacheFileOffset = SourceOffset}; + if (auto CopySourceIt = RawHashToCacheCopyDataIndex.find(ScavengedSequenceRawHash); + CopySourceIt != RawHashToCacheCopyDataIndex.end()) + { + CacheCopyData& Data = CacheCopyDatas[CopySourceIt->second]; + if (Data.TargetChunkLocationPtrs.size() > 1024) + { + RawHashToCacheCopyDataIndex.insert_or_assign(ScavengedSequenceRawHash, + CacheCopyDatas.size()); + CacheCopyDatas.push_back( + CacheCopyData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengedSequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + else + { + Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(), + ChunkTargetPtrs.begin(), + ChunkTargetPtrs.end()); + Data.ChunkTargets.push_back(Target); + } + } + else + { + RawHashToCacheCopyDataIndex.insert_or_assign(ScavengedSequenceRawHash, CacheCopyDatas.size()); + CacheCopyDatas.push_back( + CacheCopyData{.ScavengeSourceIndex = ScavengedContentIndex, + .SourceSequenceIndex = ScavengedSequenceIndex, + .TargetChunkLocationPtrs = ChunkTargetPtrs, + .ChunkTargets = std::vector{Target}}); + } + CacheMappingStats.ScavengedChunkMatchingRemoteCount++; + CacheMappingStats.ScavengedChunkMatchingRemoteByteCount += ScavengedChunkRawSize; + RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true; + RemainingChunkCount--; + } + } + } + SourceOffset += ScavengedChunkRawSize; + } + } + } + } + CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs(); } if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) { - ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks.", + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}", CachedSequenceHashesFound.size(), NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), CachedChunkHashesFound.size(), NiceBytes(CacheMappingStats.CacheChunkByteCount), CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount)); + NiceBytes(CacheMappingStats.CacheBlocksByteCount), + NiceTimeSpanMs(CacheMappingStats.CacheScanElapsedWallTimeUs / 1000)); } if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) { - ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks", + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}", LocalPathIndexesMatchingSequenceIndexes.size(), NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.LocalScanElapsedWallTimeUs / 1000)); + } + if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Scavenge of {} paths found {} ({}) chunk sequences, {} ({}) chunks in {}", + ScavengedPathsCount, + CacheMappingStats.ScavengedPathsMatchingSequencesCount, + NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), + CacheMappingStats.ScavengedChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.ScavengedChunkMatchingRemoteByteCount), + NiceTimeSpanMs(CacheMappingStats.ScavengeElapsedWallTimeUs / 1000)); } uint64_t BytesToWrite = 0; @@ -5321,6 +5800,11 @@ namespace { } } + for (const ScavengeCopyOperation& ScavengeCopyOp : ScavengeCopyOperations) + { + BytesToWrite += ScavengeCopyOp.RawSize; + } + uint64_t TotalRequestCount = 0; uint64_t TotalPartWriteCount = 0; std::atomic WritePartsComplete = 0; @@ -5347,6 +5831,7 @@ namespace { std::vector LooseChunkHashWorks; TotalPartWriteCount += CacheCopyDatas.size(); + TotalPartWriteCount += ScavengeCopyOperations.size(); for (const IoHash ChunkHash : LooseChunkHashes) { @@ -5694,6 +6179,53 @@ namespace { } } } + + for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < ScavengeCopyOperations.size(); ScavengeOpIndex++) + { + if (AbortFlag) + { + break; + } + if (!PrimeCacheOnly) + { + Work.ScheduleWork( + WritePool, + [&, ScavengeOpIndex](std::atomic&) mutable { + if (!AbortFlag) + { + const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; + const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; + + const std::filesystem::path ScavengedFilePath = + (ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred(); + ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); + + const IoHash& RemoteSequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; + const std::filesystem::path TempFilePath = + GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + + CopyFile(ScavengedFilePath, TempFilePath, {.EnableClone = false}); + + DiskStats.WriteCount++; + DiskStats.WriteByteCount += ScavengeOp.RawSize; + + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + RenameFile(TempFilePath, CacheFilePath); + + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + } + for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) { if (AbortFlag) @@ -6011,9 +6543,25 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); FilteredWrittenBytesPerSecond.Start(); - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.LocalSequenceIndex]; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + + std::filesystem::path SourceFilePath; + + if (CopyData.ScavengeSourceIndex == (uint32_t)-1) + { + const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + } + else + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; + const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; + const uint32_t ScavengedPathIndex = + ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); + } + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); uint64_t CacheLocalFileBytesRead = 0; @@ -6071,7 +6619,7 @@ namespace { tsl::robin_set ChunkIndexesWritten; - BufferedOpenFile SourceFile(LocalFilePath, DiskStats); + BufferedOpenFile SourceFile(SourceFilePath, DiskStats); WriteFileCache OpenFileCache(DiskStats); for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) { @@ -6155,9 +6703,7 @@ namespace { CompletedChunkSequences, Work, WritePool); - ZEN_CONSOLE_VERBOSE("Copied {} from {}", - NiceBytes(CacheLocalFileBytesRead), - LocalContent.Paths[LocalPathIndex]); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); } WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) @@ -6781,7 +7327,7 @@ namespace { const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); + ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); RenameFileWithRetry(LocalFilePath, CacheFilePath); CachedCount++; @@ -6942,7 +7488,7 @@ namespace { std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - if (IsFile(TargetFilePath)) + if (IsFileWithRetry(TargetFilePath)) { SetFileReadOnlyWithRetry(TargetFilePath, false); } @@ -6979,11 +7525,11 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); } else { - if (IsFile(FirstTargetFilePath)) + if (IsFileWithRetry(FirstTargetFilePath)) { SetFileReadOnlyWithRetry(FirstTargetFilePath, false); } @@ -6999,7 +7545,7 @@ namespace { const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); @@ -7010,7 +7556,7 @@ namespace { ZEN_TRACE_CPU("Rename"); const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); @@ -7043,12 +7589,12 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(IsFile(TargetFilePath)); + ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); } else { ZEN_TRACE_CPU("Copy"); - if (IsFile(TargetFilePath)) + if (IsFileWithRetry(TargetFilePath)) { SetFileReadOnlyWithRetry(TargetFilePath, false); } @@ -7057,7 +7603,7 @@ namespace { CreateDirectories(TargetFilePath.parent_path()); } - ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; @@ -7620,7 +8166,7 @@ namespace { ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, + const std::filesystem::path& StateFilePath, ChunkingController& ChunkController, const ChunkedFolderContent& ReferenceContent, FolderContent& OutLocalFolderContent) @@ -7628,50 +8174,12 @@ namespace { FolderContent LocalFolderState; ChunkedFolderContent LocalContent; - bool HasLocalState = false; - if (IsFile(ZenStateFilePath(ZenFolderPath))) + Stopwatch ReadStateTimer; + const bool HasLocalState = IsFile(StateFilePath) && ReadStateFile(StateFilePath, LocalFolderState, LocalContent); + if (HasLocalState) { - try - { - Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; - if (CurrentStateObject) - { - Oid CurrentBuildId; - std::vector SavedBuildPartIds; - std::vector SavedBuildPartsNames; - std::vector SavedPartContents; - if (ReadStateObject(CurrentStateObject, - CurrentBuildId, - SavedBuildPartIds, - SavedBuildPartsNames, - SavedPartContents, - LocalFolderState)) - { - if (!SavedPartContents.empty()) - { - if (SavedPartContents.size() == 1) - { - LocalContent = std::move(SavedPartContents[0]); - } - else - { - LocalContent = - MergeChunkedFolderContents(SavedPartContents[0], - std::span(SavedPartContents).subspan(1)); - } - HasLocalState = true; - } - } - } - ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - } - catch (const std::exception& Ex) - { - ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); - } + ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); } - { const uint32_t LocalPathCount = gsl::narrow(ReferenceContent.Paths.size()); const uint32_t RemotePathCount = gsl::narrow(LocalFolderState.Paths.size()); @@ -7696,92 +8204,7 @@ namespace { } } - const uint32_t PathCount = gsl::narrow(PathsToCheck.size()); - - OutLocalFolderContent.Paths.resize(PathCount); - OutLocalFolderContent.RawSizes.resize(PathCount); - OutLocalFolderContent.Attributes.resize(PathCount); - OutLocalFolderContent.ModificationTicks.resize(PathCount); - - { - Stopwatch Timer; - auto _ = - MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); - - ProgressBar ProgressBar(UsePlainProgress); - - ParallellWork Work(AbortFlag); - std::atomic CompletedPathCount = 0; - uint32_t PathIndex = 0; - - while (PathIndex < PathCount) - { - uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); - Work.ScheduleWork( - GetIOWorkerPool(), - [PathIndex, - PathRangeCount, - &PathsToCheck, - &Path, - &OutLocalFolderContent, - &CompletedPathCount, - &LocalFolderScanStats](std::atomic&) { - for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) - { - const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; - std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (TryGetFileProperties(LocalFilePath, - OutLocalFolderContent.RawSizes[PathRangeIndex], - OutLocalFolderContent.ModificationTicks[PathRangeIndex], - OutLocalFolderContent.Attributes[PathRangeIndex])) - { - OutLocalFolderContent.Paths[PathRangeIndex] = std::move(FilePath); - LocalFolderScanStats.FoundFileCount++; - LocalFolderScanStats.FoundFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; - LocalFolderScanStats.AcceptedFileCount++; - LocalFolderScanStats.AcceptedFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; - } - CompletedPathCount++; - } - }, - Work.DefaultErrorFunction()); - PathIndex += PathRangeCount; - } - Work.Wait(200, [&](bool, ptrdiff_t) { - // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} checked, {} found", - CompletedPathCount.load(), - PathCount, - LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount.load()}, - false); - }); - ProgressBar.Finish(); - } - - uint32_t WritePathIndex = 0; - for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++) - { - if (!OutLocalFolderContent.Paths[ReadPathIndex].empty()) - { - if (WritePathIndex < ReadPathIndex) - { - OutLocalFolderContent.Paths[WritePathIndex] = std::move(OutLocalFolderContent.Paths[ReadPathIndex]); - OutLocalFolderContent.RawSizes[WritePathIndex] = OutLocalFolderContent.RawSizes[ReadPathIndex]; - OutLocalFolderContent.Attributes[WritePathIndex] = OutLocalFolderContent.Attributes[ReadPathIndex]; - OutLocalFolderContent.ModificationTicks[WritePathIndex] = OutLocalFolderContent.ModificationTicks[ReadPathIndex]; - } - WritePathIndex++; - } - } - - OutLocalFolderContent.Paths.resize(WritePathIndex); - OutLocalFolderContent.RawSizes.resize(WritePathIndex); - OutLocalFolderContent.Attributes.resize(WritePathIndex); - OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); + OutLocalFolderContent = GetValidFolderContent(LocalFolderScanStats, Path, PathsToCheck); } bool ScanContent = true; @@ -7927,7 +8350,8 @@ namespace { bool AllowPartialBlockRequests, bool WipeTargetFolder, bool PostDownloadVerify, - bool PrimeCacheOnly) + bool PrimeCacheOnly, + bool EnableScavenging) { ZEN_TRACE_CPU("DownloadFolder"); @@ -7977,7 +8401,7 @@ namespace { LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, - ZenFolderPath, + ZenStateFilePath(ZenFolderPath), *ChunkController, RemoteContent, LocalFolderContent); @@ -8041,7 +8465,7 @@ namespace { NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent); + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent, Path); CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); @@ -8065,7 +8489,8 @@ namespace { RebuildFolderStateStatistics RebuildFolderStateStats; VerifyFolderStatistics VerifyFolderStats; - UpdateFolder(Storage, + UpdateFolder(SystemRootDir, + Storage, BuildId, Path, ZenFolderPath, @@ -8078,6 +8503,7 @@ namespace { AllowPartialBlockRequests, WipeTargetFolder, PrimeCacheOnly, + EnableScavenging, LocalFolderState, DiskStats, CacheMappingStats, @@ -8092,7 +8518,7 @@ namespace { VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState, Path); CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); @@ -8618,6 +9044,12 @@ BuildsCommand::BuildsCommand() m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), ""); + m_DownloadOptions.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + ""); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); m_DownloadOptions.positional_help("local-path build-id build-part-name"); @@ -8655,6 +9087,12 @@ BuildsCommand::BuildsCommand() "Allow request for partial chunk blocks. Defaults to true.", cxxopts::value(m_AllowPartialBlockRequests), ""); + m_TestOptions.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + ""); m_TestOptions.parse_positional({"local-path"}); m_TestOptions.positional_help("local-path"); @@ -8702,6 +9140,12 @@ BuildsCommand::BuildsCommand() m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); + m_MultiTestDownloadOptions.add_option("", + "", + "enable-scavenge", + "Enable scavenging of data from previouse download locations", + cxxopts::value(m_EnableScavenging), + ""); m_MultiTestDownloadOptions.parse_positional({"local-path"}); m_MultiTestDownloadOptions.positional_help("local-path"); } @@ -9451,7 +9895,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, m_PostDownloadVerify, - m_PrimeCacheOnly); + m_PrimeCacheOnly, + m_EnableScavenging); if (false) { @@ -9639,7 +10084,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); @@ -9692,7 +10138,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath2 = Path.parent_path() / (m_BuildPartName + "_test2"); + + auto ___ = MakeGuard([DownloadPath, DownloadPath2]() { + CleanDirectory(DownloadPath, true); + DeleteDirectories(DownloadPath); + CleanDirectory(DownloadPath2, true); + DeleteDirectories(DownloadPath2); + }); + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); @@ -9751,7 +10206,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, true, true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -9774,7 +10230,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, false, true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -9892,7 +10349,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, false, true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -9943,7 +10401,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, false, true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -9962,7 +10421,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, false, true, - false); + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -9981,7 +10441,28 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowPartialBlockRequests, false, true, - false); + false, + m_EnableScavenging); + if (AbortFlag) + { + ZEN_CONSOLE("Re-download failed."); + return 11; + } + + ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath2); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath2, + ZenFolderPath, + SystemRootDir, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false, + m_EnableScavenging); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 535d2b1d2..7e1e7d0ca 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -98,6 +98,7 @@ private: std::vector m_BuildPartNames; std::vector m_BuildPartIds; bool m_PostDownloadVerify = false; + bool m_EnableScavenging = true; cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; std::string m_DiffPath; diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 32ae2d94a..17b348f8d 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -305,7 +305,7 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector& OutP } FolderContent -GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) +GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPaths) { ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); @@ -342,7 +342,7 @@ GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vecto } else { - OutDeletedPathIndexes.push_back(Old.Paths[OldPathIndex]); + OutDeletedPaths.push_back(Old.Paths[OldPathIndex]); } } return Result; -- cgit v1.2.3 From d03a6c7824c0cf7c8925cbfd61008e305872d895 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 10 Apr 2025 09:42:32 +0200 Subject: multpart download crash (#353) * fix lambda capture during multipart-download --- src/zen/cmds/builds_cmd.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b33ec659d..2562d8244 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -6437,7 +6437,8 @@ namespace { Work, NetworkPool, DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( + IoBuffer&& Payload) mutable { DownloadStats.RequestsCompleteCount++; if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { -- cgit v1.2.3 From 81fb756a872817c625aef2b19c6b05a77a514587 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 10 Apr 2025 20:07:49 +0200 Subject: filesystem retry fixes (#354) * add more forgiving retries with filesystem * fall back to FindFirstFile if access prevents us from using GetFileAttributes * only validate hash if we have a complete payload in http client * changelog --- src/zen/cmds/builds_cmd.cpp | 51 +++++++++++++--------- src/zencore/filesystem.cpp | 101 +++++++++++++++++++++++++++++--------------- src/zenhttp/httpclient.cpp | 10 ++--- 3 files changed, 103 insertions(+), 59 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2562d8244..f7f9e3abb 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -182,65 +182,74 @@ namespace { std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); } - void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) + bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; - RenameFile(SourcePath, TargetPath, Ec); + bool Result = IsFile(Path, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); Ec.clear(); - RenameFile(SourcePath, TargetPath, Ec); + Result = IsFile(Path, Ec); } if (Ec) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Check path '{}' is file failed with: {} ({})", Path, Ec.message(), Ec.value())); } + return Result; } - bool IsFileWithRetry(const std::filesystem::path& Path) + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) { std::error_code Ec; - bool Result = IsFile(Path, Ec); + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); for (size_t Retries = 0; Ec && Retries < 3; Retries++) { Sleep(100 + int(Retries * 50)); + if (!IsFileWithRetry(Path)) + { + return false; + } Ec.clear(); - Result = IsFile(Path, Ec); + Result = SetFileReadOnly(Path, ReadOnly, Ec); } if (Ec) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error( + std::error_code(Ec.value(), std::system_category()), + fmt::format("Failed {} read only flag for '{}' failed with: {}", ReadOnly ? "setting" : "clearing", Path, Ec.message())); } return Result; } - bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { std::error_code Ec; - bool Result = SetFileReadOnly(Path, ReadOnly, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + RenameFile(SourcePath, TargetPath, Ec); + for (size_t Retries = 0; Ec && Retries < 10; Retries++) { - Sleep(100 + int(Retries * 50)); - if (!IsFileWithRetry(Path)) + ZEN_ASSERT_SLOW(IsFile(SourcePath)); + if (Retries > 5) { - return false; + ZEN_CONSOLE("Unable to overwrite file {} ({}: {}), retrying...", TargetPath, Ec.value(), Ec.message()); } + Sleep(50 + int(Retries * 150)); Ec.clear(); - Result = SetFileReadOnly(Path, ReadOnly, Ec); + RenameFile(SourcePath, TargetPath, Ec); } if (Ec) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Rename from '{}' to '{}' failed with: {}", SourcePath, TargetPath, Ec.message())); } - return Result; } void RemoveFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; RemoveFile(Path, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + for (size_t Retries = 0; Ec && Retries < 6; Retries++) { Sleep(100 + int(Retries * 50)); if (!IsFileWithRetry(Path)) @@ -252,7 +261,8 @@ namespace { } if (Ec) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Removing file '{}' failed with: {}", Path, Ec.message())); } } @@ -272,7 +282,8 @@ namespace { } if (Ec) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(std::error_code(Ec.value(), std::system_category()), + fmt::format("Removing directory '{}' failed with: {}", Path, Ec.message())); } } diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 4ec563ba3..8ee21d9ab 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -251,6 +251,51 @@ MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); } +#if ZEN_PLATFORM_WINDOWS + +static DWORD +WinGetFileAttributes(const std::filesystem::path& Path, std::error_code& Ec) +{ + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + case ERROR_ACCESS_DENIED: + { + WIN32_FIND_DATA FindData; + HANDLE FindHandle = ::FindFirstFile(Path.native().c_str(), &FindData); + if (FindHandle == INVALID_HANDLE_VALUE) + { + DWORD LastFindError = GetLastError(); + if (LastFindError != ERROR_FILE_NOT_FOUND) + { + Ec = MakeErrorCode(LastError); + } + } + else + { + CloseHandle(FindHandle); + Attributes = FindData.dwFileAttributes; + } + } + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + } + return Attributes; +} + +#endif // ZEN_PLATFORM_WINDOWS + bool RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec) { @@ -286,7 +331,12 @@ RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFile { if (ForceRemoveReadOnlyFiles) { - DWORD FileAttributes = ::GetFileAttributes(NativePath); + DWORD FileAttributes = WinGetFileAttributes(NativePath, Ec); + if (Ec) + { + return false; + } + if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) { ::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false)); @@ -1597,21 +1647,13 @@ bool IsFile(const std::filesystem::path& Path, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + DWORD Attributes = WinGetFileAttributes(Path, Ec); + if (Ec) + { + return false; + } if (Attributes == INVALID_FILE_ATTRIBUTES) { - DWORD LastError = GetLastError(); - switch (LastError) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_BAD_NETPATH: - case ERROR_INVALID_DRIVE: - break; - default: - Ec = MakeErrorCode(LastError); - break; - } return false; } return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; @@ -1651,21 +1693,13 @@ bool IsDir(const std::filesystem::path& Path, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + DWORD Attributes = WinGetFileAttributes(Path, Ec); + if (Ec) + { + return false; + } if (Attributes == INVALID_FILE_ATTRIBUTES) { - DWORD LastError = GetLastError(); - switch (LastError) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - case ERROR_BAD_NETPATH: - case ERROR_INVALID_DRIVE: - break; - default: - Ec = MakeErrorCode(LastError); - break; - } return false; } return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY; @@ -2492,13 +2526,7 @@ PickDefaultSystemRootDirectory() uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec) { - DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); - if (Attributes == INVALID_FILE_ATTRIBUTES) - { - Ec = MakeErrorCodeFromLastError(); - return 0; - } - return (uint32_t)Attributes; + return WinGetFileAttributes(Filename, Ec); } uint32_t @@ -2594,6 +2622,11 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error { return false; } + if (CurrentAttributes == INVALID_FILE_ATTRIBUTES) + { + Ec = MakeErrorCode(ERROR_FILE_NOT_FOUND); + return false; + } uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); if (CurrentAttributes != NewAttributes) { diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index f3baf37ce..763f3262a 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -410,6 +410,11 @@ ValidatePayload(cpr::Response& Response, std::unique_ptrsecond == "application/x-ue-comp") -- cgit v1.2.3 From afda5c6345f4c0b274ae8084bfbd371169791c57 Mon Sep 17 00:00:00 2001 From: zousar Date: Fri, 11 Apr 2025 00:25:20 -0600 Subject: Avoid signed overflow using BigInt Bias for use of BigInt when consuming integer fields in compact binary to avoid values showing up as negative due to overflow on the Number type. --- src/zenserver/frontend/html/indexer/worker.js | 4 ++-- src/zenserver/frontend/html/pages/entry.js | 5 ++++- src/zenserver/frontend/html/util/compactbinary.js | 4 ++-- src/zenserver/frontend/html/util/friendly.js | 14 +++++++------- src/zenserver/frontend/html/util/widgets.js | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/worker.js b/src/zenserver/frontend/html/indexer/worker.js index b4d547ef9..69ee234fa 100644 --- a/src/zenserver/frontend/html/indexer/worker.js +++ b/src/zenserver/frontend/html/indexer/worker.js @@ -77,8 +77,8 @@ async function map_id_to_key(project_id, oplog, start, end, page_size, stride) continue; var id = 0n; - var size = 0; - var raw_size = 0; + var size = 0n; + var raw_size = 0n; for (const item of pkg_data.as_array()) { var found = 0, pkg_id = undefined; diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index f127cb0a3..41fc47218 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -210,8 +210,11 @@ export class Page extends ZenPage _display_unsupported(section, entry) { + const replacer = (key, value) => + typeof value === "bigint" ? { $bigint: value.toString() } : value; + const object = entry.to_js_object(); - const text = JSON.stringify(object, null, " "); + const text = JSON.stringify(object, replacer, " "); section.tag("pre").text(text); } diff --git a/src/zenserver/frontend/html/util/compactbinary.js b/src/zenserver/frontend/html/util/compactbinary.js index 366ec6aff..90e4249f6 100644 --- a/src/zenserver/frontend/html/util/compactbinary.js +++ b/src/zenserver/frontend/html/util/compactbinary.js @@ -284,7 +284,7 @@ CbFieldView.prototype.as_array = function() } //////////////////////////////////////////////////////////////////////////////// -CbFieldView.prototype.as_value = function(int_type=Number) +CbFieldView.prototype.as_value = function(int_type=BigInt) { switch (CbFieldTypeOps.get_type(this.get_type())) { @@ -388,8 +388,8 @@ CbObjectView.prototype.to_js_object = function() } if (node.is_string()) return node.as_value(); - if (node.is_float()) return node.as_value(); if (node.is_integer()) return node.as_value(); + if (node.is_float()) return node.as_value(); var ret = node.as_value(); if (ret instanceof Uint8Array) diff --git a/src/zenserver/frontend/html/util/friendly.js b/src/zenserver/frontend/html/util/friendly.js index b27721964..a15252faf 100644 --- a/src/zenserver/frontend/html/util/friendly.js +++ b/src/zenserver/frontend/html/util/friendly.js @@ -7,17 +7,17 @@ export class Friendly { static sep(value, prec=0) { - return (+value).toLocaleString("en", { + return (+Number(value)).toLocaleString("en", { style: "decimal", minimumFractionDigits : prec, maximumFractionDigits : prec, }); } - static k(x, p=0) { return Friendly.sep((x + 999) / Math.pow(10, 3)|0, p) + "K"; } - static m(x, p=1) { return Friendly.sep( x / Math.pow(10, 6), p) + "M"; } - static g(x, p=2) { return Friendly.sep( x / Math.pow(10, 9), p) + "G"; } - static kib(x, p=0) { return Friendly.sep((x + 1023) / (1 << 10)|0, p) + " KiB"; } - static mib(x, p=1) { return Friendly.sep( x / (1 << 20), p) + " MiB"; } - static gib(x, p=2) { return Friendly.sep( x / (1 << 30), p) + " GiB"; } + static k(x, p=0) { return Friendly.sep((BigInt(x) + 999n) / BigInt(Math.pow(10, 3))|0n, p) + "K"; } + static m(x, p=1) { return Friendly.sep( BigInt(x) / BigInt(Math.pow(10, 6)), p) + "M"; } + static g(x, p=2) { return Friendly.sep( BigInt(x) / BigInt(Math.pow(10, 9)), p) + "G"; } + static kib(x, p=0) { return Friendly.sep((BigInt(x) + 1023n) / (1n << 10n)|0n, p) + " KiB"; } + static mib(x, p=1) { return Friendly.sep( BigInt(x) / (1n << 20n), p) + " MiB"; } + static gib(x, p=2) { return Friendly.sep( BigInt(x) / (1n << 30n), p) + " GiB"; } } diff --git a/src/zenserver/frontend/html/util/widgets.js b/src/zenserver/frontend/html/util/widgets.js index d4f9875cd..32a3f4d28 100644 --- a/src/zenserver/frontend/html/util/widgets.js +++ b/src/zenserver/frontend/html/util/widgets.js @@ -173,7 +173,7 @@ export class PropTable extends Table continue; } - if (friendly && typeof value == "number") + if (friendly && ((typeof value == "number") || (typeof value == "bigint"))) { if (key.indexOf("memory") >= 0) value = Friendly.kib(value); else if (key.indexOf("disk") >= 0) value = Friendly.kib(value); -- cgit v1.2.3 From 1bf45ed4d820be75ec0c677976463b09d50d4ac5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Apr 2025 16:07:47 +0200 Subject: use FindClose (#357) * use FindClose --- src/zencore/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8ee21d9ab..ad796cb4a 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -281,7 +281,7 @@ WinGetFileAttributes(const std::filesystem::path& Path, std::error_code& Ec) } else { - CloseHandle(FindHandle); + FindClose(FindHandle); Attributes = FindData.dwFileAttributes; } } -- cgit v1.2.3 From c0fd6871d57a1e211472abb92e10df0585872792 Mon Sep 17 00:00:00 2001 From: zousar Date: Fri, 11 Apr 2025 08:45:31 -0600 Subject: xmake updatefrontend --- src/zenserver/frontend/html.zip | Bin 156557 -> 156798 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index b23b2efab..c28404da6 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From dec3f27c488a1dda8a2f1133361e2fda9315e0d2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Apr 2025 19:22:21 +0200 Subject: fix race condition in multipart download (#358) --- src/zen/cmds/builds_cmd.cpp | 26 +++++++-------- src/zenutil/filebuildstorage.cpp | 26 +++++++++------ src/zenutil/include/zenutil/buildstorage.h | 12 +++---- .../include/zenutil/jupiter/jupitersession.h | 15 +++++---- src/zenutil/jupiter/jupiterbuildstorage.cpp | 20 ++++++++---- src/zenutil/jupiter/jupitersession.cpp | 38 ++++++++++++++-------- 6 files changed, 81 insertions(+), 56 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f7f9e3abb..cdcd79f58 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1843,25 +1843,25 @@ namespace { BuildId, ChunkHash, PreferredMultipartChunkSize, - [Workload, &DownloadStats, OnDownloadComplete = std::move(OnDownloadComplete)](uint64_t Offset, - const IoBuffer& Chunk, - uint64_t BytesRemaining) { + [Workload, &DownloadStats](uint64_t Offset, const IoBuffer& Chunk) { DownloadStats.DownloadedChunkByteCount += Chunk.GetSize(); if (!AbortFlag.load()) { ZEN_TRACE_CPU("DownloadLargeBlob_Save"); Workload->TempFile.Write(Chunk.GetView(), Offset); - if (Chunk.GetSize() == BytesRemaining) - { - DownloadStats.DownloadedChunkCount++; - uint64_t PayloadSize = Workload->TempFile.FileSize(); - void* FileHandle = Workload->TempFile.Detach(); - ZEN_ASSERT(FileHandle != nullptr); - IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true); - Payload.SetDeleteOnClose(true); - OnDownloadComplete(std::move(Payload)); - } + } + }, + [Workload, &DownloadStats, OnDownloadComplete = std::move(OnDownloadComplete)]() { + DownloadStats.DownloadedChunkCount++; + if (!AbortFlag.load()) + { + uint64_t PayloadSize = Workload->TempFile.FileSize(); + void* FileHandle = Workload->TempFile.Detach(); + ZEN_ASSERT(FileHandle != nullptr); + IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true); + Payload.SetDeleteOnClose(true); + OnDownloadComplete(std::move(Payload)); } }); if (!WorkItems.empty()) diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 7aa252e44..f335a03a3 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -369,11 +369,11 @@ public: return IoBuffer{}; } - virtual std::vector> GetLargeBuildBlob( - const Oid& BuildId, - const IoHash& RawHash, - uint64_t ChunkSize, - std::function&& Receiver) override + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete) override { ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob"); ZEN_UNUSED(BuildId); @@ -387,16 +387,18 @@ public: { struct WorkloadData { - std::atomic BytesRemaining; - BasicFile BlobFile; - std::function Receiver; + std::atomic BytesRemaining; + BasicFile BlobFile; + std::function OnReceive; + std::function OnComplete; }; std::shared_ptr Workload(std::make_shared()); Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead); const uint64_t BlobSize = Workload->BlobFile.FileSize(); - Workload->Receiver = std::move(Receiver); + Workload->OnReceive = std::move(OnReceive); + Workload->OnComplete = std::move(OnComplete); Workload->BytesRemaining = BlobSize; std::vector> WorkItems; @@ -410,8 +412,12 @@ public: IoBuffer PartPayload(Size); Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset); m_Stats.TotalBytesRead += PartPayload.GetSize(); + Workload->OnReceive(Offset, PartPayload); uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size); - Workload->Receiver(Offset, PartPayload, ByteRemaning); + if (ByteRemaning == Size) + { + Workload->OnComplete(); + } SimulateLatency(Size, PartPayload.GetSize()); }); diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index b0665dbf8..05e3ca22d 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -47,12 +47,12 @@ public: virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset = 0, - uint64_t RangeBytes = (uint64_t)-1) = 0; - virtual std::vector> GetLargeBuildBlob( - const Oid& BuildId, - const IoHash& RawHash, - uint64_t ChunkSize, - std::function&& Receiver) = 0; + uint64_t RangeBytes = (uint64_t)-1) = 0; + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete) = 0; virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 417ed7384..c2886ca4c 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -135,13 +135,14 @@ public: uint64_t PayloadSize, std::function&& Transmitter, std::vector>& OutWorkItems); - JupiterResult GetMultipartBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const IoHash& Hash, - uint64_t ChunkSize, - std::function&& Receiver, - std::vector>& OutWorkItems); + JupiterResult GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete, + std::vector>& OutWorkItems); JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index f2d190408..24e062c7b 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -235,19 +235,25 @@ public: return std::move(GetBuildBlobResult.Response); } - virtual std::vector> GetLargeBuildBlob( - const Oid& BuildId, - const IoHash& RawHash, - uint64_t ChunkSize, - std::function&& Receiver) override + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete) override { ZEN_TRACE_CPU("Jupiter::GetLargeBuildBlob"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); std::vector> WorkItems; - JupiterResult GetMultipartBlobResult = - m_Session.GetMultipartBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, ChunkSize, std::move(Receiver), WorkItems); + JupiterResult GetMultipartBlobResult = m_Session.GetMultipartBuildBlob(m_Namespace, + m_Bucket, + BuildId, + RawHash, + ChunkSize, + std::move(OnReceive), + std::move(OnComplete), + WorkItems); AddStatistic(GetMultipartBlobResult); if (!GetMultipartBlobResult.Success) diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index fde86a478..1f71c29b7 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -626,13 +626,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, } JupiterResult -JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const IoHash& Hash, - uint64_t ChunkSize, - std::function&& Receiver, - std::vector>& OutWorkItems) +JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete, + std::vector>& OutWorkItems) { std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); HttpClient::Response Response = @@ -649,18 +650,20 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespa uint64_t TotalSize = TotalSizeMaybe.value(); uint64_t PayloadSize = Response.ResponsePayload.GetSize(); - Receiver(0, Response.ResponsePayload, TotalSize); + OnReceive(0, Response.ResponsePayload); if (TotalSize > PayloadSize) { struct WorkloadData { - std::function Receiver; - std::atomic BytesRemaining; + std::function OnReceive; + std::function OnComplete; + std::atomic BytesRemaining; }; std::shared_ptr Workload(std::make_shared()); - Workload->Receiver = std::move(Receiver); + Workload->OnReceive = std::move(OnReceive); + Workload->OnComplete = std::move(OnComplete); Workload->BytesRemaining = TotalSize - PayloadSize; uint64_t Offset = PayloadSize; @@ -676,19 +679,28 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespa HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); if (Response.IsSuccess()) { + Workload->OnReceive(Offset, Response.ResponsePayload); uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize()); - Workload->Receiver(Offset, Response.ResponsePayload, ByteRemaning); + if (ByteRemaning == Response.ResponsePayload.GetSize()) + { + Workload->OnComplete(); + } } return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); }); Offset += PartSize; } } + else + { + OnComplete(); + } return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); } } } - Receiver(0, Response.ResponsePayload, Response.ResponsePayload.GetSize()); + OnReceive(0, Response.ResponsePayload); + OnComplete(); } return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); } -- cgit v1.2.3 From 9cdfac225f0dc986ce449cbe8ef72faa39025971 Mon Sep 17 00:00:00 2001 From: zousar Date: Mon, 14 Apr 2025 21:51:39 -0600 Subject: Add a list-container subcommand to zen builds command --- src/zen/cmds/builds_cmd.cpp | 71 +++++++++++++++++++++- src/zen/cmds/builds_cmd.h | 6 +- src/zenutil/filebuildstorage.cpp | 22 ++++++- src/zenutil/include/zenutil/buildstorage.h | 1 + .../include/zenutil/jupiter/jupitersession.h | 2 + src/zenutil/jupiter/jupiterbuildstorage.cpp | 52 ++++++++++++++++ src/zenutil/jupiter/jupitersession.cpp | 15 +++++ 7 files changed, 165 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cdcd79f58..2d24db956 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8906,12 +8906,28 @@ BuildsCommand::BuildsCommand() m_Options.add_option("", "v", "verb", - "Verb for build - list, upload, download, diff, fetch-blob, validate-part", + "Verb for build - list-container, list, upload, download, diff, fetch-blob, validate-part", cxxopts::value(m_Verb), ""); m_Options.parse_positional({"verb"}); m_Options.positional_help("verb"); + // list-container + AddSystemOptions(m_ListContainerOptions); + AddCloudOptions(m_ListContainerOptions); + AddFileOptions(m_ListContainerOptions); + AddOutputOptions(m_ListContainerOptions); + AddZenFolderOptions(m_ListContainerOptions); + m_ListContainerOptions.add_options()("h,help", "Print help"); + m_ListContainerOptions.add_option("", + "", + "result-path", + "Path to json or compactbinary to write query result to", + cxxopts::value(m_ListResultPath), + ""); + m_ListContainerOptions.parse_positional({"query-path", "result-path"}); + m_ListContainerOptions.positional_help("query-path result-path"); + // list AddSystemOptions(m_ListOptions); AddCloudOptions(m_ListOptions); @@ -9206,7 +9222,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help())); } - if (m_Namespace.empty() || m_Bucket.empty()) + if (ToLower(m_Verb) != "list-container" && (m_Namespace.empty() || m_Bucket.empty())) { throw zen::OptionParseException( fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); @@ -9596,6 +9612,57 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) try { + if (SubOption == &m_ListContainerOptions) + { + if (!m_ListResultPath.empty()) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + } + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() + ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName + : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + + CbObject Response = Storage.BuildStorage->ListContainers(); + ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); + if (m_ListResultPath.empty()) + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + ZEN_CONSOLE("{}", SB.ToView()); + } + else + { + std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); + if (ToLower(ListResultPath.extension().string()) == ".cbo") + { + MemoryView ResponseView = Response.GetView(); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); + } + else + { + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(Response.GetView(), SB); + WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + } + } + + return 0; + } + if (SubOption == &m_ListOptions) { if (!m_ListResultPath.empty()) diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 7e1e7d0ca..3da1922c9 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -84,6 +84,9 @@ private: std::string m_Verb; // list, upload, download + cxxopts::Options m_ListContainerOptions{"list-container", "List available build containers"}; + std::string m_ListContainerResultPath; + cxxopts::Options m_ListOptions{"list", "List available builds"}; std::string m_ListQueryPath; std::string m_ListResultPath; @@ -114,7 +117,8 @@ private: cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; std::vector m_BuildIds; - cxxopts::Options* m_SubCommands[8] = {&m_ListOptions, + cxxopts::Options* m_SubCommands[9] = {&m_ListContainerOptions, + &m_ListOptions, &m_UploadOptions, &m_DownloadOptions, &m_DiffOptions, diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f335a03a3..b77eb1d11 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -35,6 +35,26 @@ public: virtual ~FileBuildStorage() {} + virtual CbObject ListContainers() override + { + ZEN_TRACE_CPU("FileBuildStorage::ListContainers"); + + SimulateLatency(0, 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + CbObjectWriter Writer; + Writer.BeginArray("results"); + { + } + Writer.EndArray(); // results + Writer.Save(); + SimulateLatency(Writer.GetSaveSize(), 0); + return Writer.Save(); + } + virtual CbObject ListBuilds(CbObject Query) override { ZEN_TRACE_CPU("FileBuildStorage::ListBuilds"); @@ -66,7 +86,7 @@ public: } } } - Writer.EndArray(); // builds + Writer.EndArray(); // results Writer.Save(); SimulateLatency(Writer.GetSaveSize(), 0); return Writer.Save(); diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 05e3ca22d..0785acd62 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -25,6 +25,7 @@ public: virtual ~BuildStorage() {} + virtual CbObject ListContainers() = 0; virtual CbObject ListBuilds(CbObject Query) = 0; virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; virtual CbObject GetBuild(const Oid& BuildId) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index c2886ca4c..32bfd50f4 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -102,6 +102,8 @@ public: std::vector Filter(std::string_view Namespace, std::string_view BucketId, const std::vector& ChunkHashes); + JupiterResult ListBuildNamespaces(); + JupiterResult ListBuildBuckets(std::string_view Namespace); JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 24e062c7b..b2fabb43f 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -35,6 +35,58 @@ public: } virtual ~JupiterBuildStorage() {} + virtual CbObject ListContainers() override + { + ZEN_TRACE_CPU("Jupiter::ListContainers"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + JupiterResult ListResult = m_Session.ListBuildNamespaces(); + AddStatistic(ListResult); + if (!ListResult.Success) + { + throw std::runtime_error(fmt::format("Failed listing containers: {} ({})", ListResult.Reason, ListResult.ErrorCode)); + } + CbObject NamespaceResponse = PayloadToCbObject("Failed listing containers"sv, ListResult.Response); + + CbObjectWriter Response; + Response.BeginArray("results"sv); + for (CbFieldView NamespaceField : NamespaceResponse["namespaces"]) + { + std::string_view Namespace = NamespaceField.AsString(); + if (!Namespace.empty()) + { + Response.BeginObject(); + Response.AddString("name", Namespace); + + JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace); + AddStatistic(BucketsResult); + if (!BucketsResult.Success) + { + throw std::runtime_error( + fmt::format("Failed listing containers: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode)); + } + CbObject BucketResponse = PayloadToCbObject("Failed listing containers"sv, BucketsResult.Response); + + Response.BeginArray("items"); + for (CbFieldView BucketField : BucketResponse["buckets"]) + { + std::string_view Bucket = BucketField.AsString(); + if (!Bucket.empty()) + { + Response.AddString(Bucket); + } + } + Response.EndArray(); + + Response.EndObject(); + } + } + Response.EndArray(); + + return Response.Save(); + } + virtual CbObject ListBuilds(CbObject Query) override { ZEN_TRACE_CPU("Jupiter::ListBuilds"); diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 1f71c29b7..d3076d36b 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -356,6 +356,21 @@ JupiterSession::CacheTypeExists(std::string_view Namespace, std::string_view Typ return Result; } +JupiterResult +JupiterSession::ListBuildNamespaces() +{ + HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds"), {HttpClient::Accept(ZenContentType::kJSON)}); + return detail::ConvertResponse(Response, "JupiterSession::ListBuildNamespaces"sv); +} + +JupiterResult +JupiterSession::ListBuildBuckets(std::string_view Namespace) +{ + HttpClient::Response Response = + m_HttpClient.Get(fmt::format("/api/v2/builds/{}", Namespace), {HttpClient::Accept(ZenContentType::kJSON)}); + return detail::ConvertResponse(Response, "JupiterSession::ListBuildBuckets"sv); +} + JupiterResult JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload) { -- cgit v1.2.3 From 2b0c494dece513477c94444c94250cc19a117873 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 15 Apr 2025 00:23:59 -0600 Subject: Fix list-container positional args to remove query-path --- src/zen/cmds/builds_cmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2d24db956..40cd8fa74 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8925,8 +8925,8 @@ BuildsCommand::BuildsCommand() "Path to json or compactbinary to write query result to", cxxopts::value(m_ListResultPath), ""); - m_ListContainerOptions.parse_positional({"query-path", "result-path"}); - m_ListContainerOptions.positional_help("query-path result-path"); + m_ListContainerOptions.parse_positional({"result-path"}); + m_ListContainerOptions.positional_help("result-path"); // list AddSystemOptions(m_ListOptions); -- cgit v1.2.3 From 0f968eb47c9f4860779bcc769f09e1d4287969bc Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 15 Apr 2025 13:59:51 -0600 Subject: Fix for BigInt conversion bug --- src/zenserver/frontend/html/indexer/indexer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/indexer/indexer.js b/src/zenserver/frontend/html/indexer/indexer.js index 16b91e130..688bc71b0 100644 --- a/src/zenserver/frontend/html/indexer/indexer.js +++ b/src/zenserver/frontend/html/indexer/indexer.js @@ -61,7 +61,7 @@ class Indexer { for (const page of this._pages) for (const [_, name, size, raw_size] of page) - yield [name, size|0, raw_size|0]; + yield [name, size|0n, raw_size|0n]; } } -- cgit v1.2.3 From 7dc91e3a9c830c173e26f1eee84a908f6244157f Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 15 Apr 2025 14:33:00 -0600 Subject: xmake updatefrontend --- src/zenserver/frontend/html.zip | Bin 156798 -> 156800 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index c28404da6..d1a3cfc14 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From 311040be0b5220db7d526f95fa42fa4e9dd13518 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 15 Apr 2025 23:42:32 -0600 Subject: Make metadata presentation more generic --- src/zenserver/frontend/html/pages/entry.js | 65 +++++++++++++++++++----------- 1 file changed, 42 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 41fc47218..54fb11c18 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -79,10 +79,32 @@ export class Page extends ZenPage async _build_meta(section, entry) { var tree = {} - const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); - if (cookart != null) + + for (const field of entry) { - tree["cook"] = { CookPackageArtifacts: cookart}; + var visibleKey = undefined; + const name = field.get_name(); + if (name == "CookPackageArtifacts") + { + visibleKey = name; + } + else if (name.startsWith("meta.")) + { + visibleKey = name.slice(5); + } + + if (visibleKey != undefined) + { + var found_value = field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + tree[visibleKey] = ret; + } + } + } if (Object.keys(tree).length == 0) @@ -90,28 +112,25 @@ export class Page extends ZenPage const sub_section = section.add_section("meta"); - for (const cat_name in tree) + const table = sub_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + for (const key in tree) { - const cat_section = sub_section.add_section(cat_name); - const table = cat_section.add_widget( - Table, - ["name", "actions"], Table.Flag_PackRight + const row = table.add_row(key); + const value = tree[key]; + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); - Object.entries(tree[cat_name]).forEach(([key, value]) => - { - const row = table.add_row(key); - - const project = this.get_param("project"); - const oplog = this.get_param("oplog"); - const link = row.get_cell(0).link( - "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") - ); - - const action_tb = new Toolbar(row.get_cell(-1), true); - action_tb.left().add("copy-hash").on_click(async (v) => { - await navigator.clipboard.writeText(v); - }, value); - }); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); } } -- cgit v1.2.3 From b1a8b3337c552fa4694769f51b9a58e2bc1910ae Mon Sep 17 00:00:00 2001 From: zousar Date: Wed, 16 Apr 2025 16:00:03 -0600 Subject: xmake updatefrontend --- src/zenserver/frontend/html.zip | Bin 156800 -> 157061 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index d1a3cfc14..eadfcf84c 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From 732a1cb1e78abbabaa0d926e9b1e58a36538dc1b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 22 Apr 2025 16:28:08 +0200 Subject: add cxxopts overload for parsing file paths from command line (#362) --- src/zen/cmds/admin_cmd.cpp | 19 +- src/zen/cmds/admin_cmd.h | 8 +- src/zen/cmds/builds_cmd.cpp | 603 +++++++++++------------ src/zen/cmds/builds_cmd.h | 50 +- src/zen/cmds/cache_cmd.cpp | 22 +- src/zen/cmds/cache_cmd.h | 18 +- src/zen/cmds/copy_cmd.cpp | 7 +- src/zen/cmds/copy_cmd.h | 10 +- src/zen/cmds/dedup_cmd.h | 4 +- src/zen/cmds/status_cmd.cpp | 9 +- src/zen/cmds/status_cmd.h | 6 +- src/zen/cmds/up_cmd.cpp | 41 +- src/zen/cmds/up_cmd.h | 28 +- src/zen/cmds/workspaces_cmd.cpp | 55 +-- src/zen/cmds/workspaces_cmd.h | 28 +- src/zen/zen.h | 5 +- src/zencore/filesystem.cpp | 16 - src/zencore/include/zencore/filesystem.h | 2 - src/zencore/include/zencore/process.h | 3 - src/zencore/process.cpp | 142 ------ src/zenserver/config.cpp | 38 +- src/zenserver/config/luaconfig.cpp | 23 +- src/zenserver/config/luaconfig.h | 5 +- src/zenutil/commandlineoptions.cpp | 213 ++++++++ src/zenutil/include/zenutil/commandlineoptions.h | 28 ++ src/zenutil/zenutil.cpp | 2 + 26 files changed, 682 insertions(+), 703 deletions(-) create mode 100644 src/zenutil/commandlineoptions.cpp create mode 100644 src/zenutil/include/zenutil/commandlineoptions.h (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 573639c2d..a7cfa6a4e 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -714,29 +714,26 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw OptionParseException("data path must be given"); } - std::filesystem::path DataPath = StringToPath(m_DataPath); - std::filesystem::path TargetPath = StringToPath(m_TargetPath); - - if (!IsDir(DataPath)) + if (!IsDir(m_DataPath)) { throw OptionParseException("data path must exist"); } - if (TargetPath.empty()) + if (m_TargetPath.empty()) { throw OptionParseException("target path must be given"); } - std::filesystem::path RootManifestPath = DataPath / "root_manifest"; - std::filesystem::path TargetRootManifestPath = TargetPath / "root_manifest"; + std::filesystem::path RootManifestPath = m_DataPath / "root_manifest"; + std::filesystem::path TargetRootManifestPath = m_TargetPath / "root_manifest"; if (!TryCopy(RootManifestPath, TargetRootManifestPath)) { throw OptionParseException("data path is invalid, missing root_manifest"); } - std::filesystem::path CachePath = DataPath / "cache"; - std::filesystem::path TargetCachePath = TargetPath / "cache"; + std::filesystem::path CachePath = m_DataPath / "cache"; + std::filesystem::path TargetCachePath = m_TargetPath / "cache"; // Copy cache state DirectoryContent CacheDirectoryContent; @@ -781,8 +778,8 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path CasPath = DataPath / "cas"; - std::filesystem::path TargetCasPath = TargetPath / "cas"; + std::filesystem::path CasPath = m_DataPath / "cas"; + std::filesystem::path TargetCasPath = m_TargetPath / "cas"; { std::filesystem::path UCasRootPath = CasPath / ".ucas_root"; diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 8b6d3e258..c593b2cac 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -155,10 +155,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; - std::string m_DataPath; - std::string m_TargetPath; - bool m_SkipLogs = false; + cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + std::filesystem::path m_DataPath; + std::filesystem::path m_TargetPath; + bool m_SkipLogs = false; }; } // namespace zen diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index cdcd79f58..6607fd6d2 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -164,24 +164,6 @@ namespace { ); - std::filesystem::path MakeSafeAbsolutePath(std::filesystem::path Path) - { - std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred(); -#if ZEN_PLATFORM_WINDOWS && 1 - const std::string_view Prefix = "\\\\?\\"; - const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); - std::u8string PathString = AbsolutePath.u8string(); - if (!PathString.empty() && !PathString.starts_with(PrefixU8)) - { - PathString.insert(0, PrefixU8); - return std::filesystem::path(PathString); - } -#endif - return AbsolutePath; - } - - std::filesystem::path MakeSafeAbsolutePath(const std::string PathString) { return MakeSafeAbsolutePath(StringToPath(PathString)); } - bool IsFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; @@ -3169,8 +3151,9 @@ namespace { auto ParseManifest = [](const std::filesystem::path& Path, const std::filesystem::path& ManifestPath) -> std::vector { std::vector AssetPaths; - std::filesystem::path AbsoluteManifestPath = ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath; - IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); + std::filesystem::path AbsoluteManifestPath = + MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : Path / ManifestPath); + IoBuffer ManifestContent = ReadFile(AbsoluteManifestPath).Flatten(); std::string_view ManifestString((const char*)ManifestContent.GetView().GetData(), ManifestContent.GetSize()); std::string_view::size_type Offset = 0; while (Offset < ManifestContent.GetSize()) @@ -8769,7 +8752,7 @@ BuildsCommand::BuildsCommand() m_Options.add_options()("h,help", "Print help"); auto AddSystemOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); }; auto AddAuthOptions = [this](cxxopts::Options& Ops) { @@ -9196,7 +9179,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path SystemRootDir; auto ParseSystemOptions = [&]() { - SystemRootDir = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); + if (m_SystemRootDir.empty()) + { + m_SystemRootDir = PickDefaultSystemRootDirectory(); + } + MakeSafeAbsolutePath(m_SystemRootDir); }; auto ParseStorageOptions = [&]() { @@ -9216,6 +9203,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); } + MakeSafeAbsolutePath(m_StoragePath); }; std::unique_ptr Auth; @@ -9225,7 +9213,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .RetryCount = 2}; auto CreateAuthMgr = [&]() { - ZEN_ASSERT(!SystemRootDir.empty()); + ZEN_ASSERT(!m_SystemRootDir.empty()); if (!Auth) { if (m_EncryptionKey.empty()) @@ -9240,7 +9228,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Warning: Using default encryption initialization vector"); } - AuthConfig AuthMgrConfig = {.RootDirectory = SystemRootDir / "auth", + AuthConfig AuthMgrConfig = {.RootDirectory = m_SystemRootDir / "auth", .EncryptionKey = AesKey256Bit::FromString(m_EncryptionKey), .EncryptionIV = AesIV128Bit::FromString(m_EncryptionIV)}; if (!AuthMgrConfig.EncryptionKey.IsValid()) @@ -9303,7 +9291,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath)); + MakeSafeAbsolutePath(m_AccessTokenPath); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -9545,10 +9534,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_StoragePath.empty()) { - std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - StorageDescription = fmt::format("folder {}", StoragePath); - Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); + StorageDescription = fmt::format("folder {}", m_StoragePath); + Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem()); } else { @@ -9592,12 +9580,146 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return Result; }; + auto ParsePath = [&]() { + if (m_Path.empty()) + { + throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); + } + MakeSafeAbsolutePath(m_Path); + }; + + auto ParseDiffPath = [&]() { + if (m_DiffPath.empty()) + { + throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); + } + MakeSafeAbsolutePath(m_DiffPath); + }; + + auto ParseBlobHash = [&]() -> IoHash { + if (m_BlobHash.empty()) + { + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + } + + IoHash BlobHash; + if (!IoHash::TryParse(m_BlobHash, BlobHash)) + { + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + } + + return BlobHash; + }; + + auto ParseBuildId = [&]() -> Oid { + if (m_BuildId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + } + else + { + return BuildId; + } + }; + + auto ParseBuildPartId = [&]() -> Oid { + if (m_BuildPartId.length() != Oid::StringLength) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } + else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + } + else + { + return BuildPartId; + } + }; + + auto ParseBuildPartIds = [&]() -> std::vector { + std::vector BuildPartIds; + for (const std::string& BuildPartId : m_BuildPartIds) + { + BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); + if (BuildPartIds.back() == Oid::Zero) + { + throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + } + } + return BuildPartIds; + }; + + auto ParseBuildMetadata = [&]() -> CbObject { + if (m_CreateBuild) + { + if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) + { + throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); + } + + if (!m_BuildMetadataPath.empty()) + { + MakeSafeAbsolutePath(m_BuildMetadataPath); + IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); + std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); + std::string JsonError; + CbObject MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); + } + return MetaData; + } + if (!m_BuildMetadata.empty()) + { + CbObjectWriter MetaDataWriter(1024); + ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { + size_t SplitPos = Pair.find('='); + if (SplitPos == std::string::npos || SplitPos == 0) + { + throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); + } + MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); + return true; + }); + return MetaDataWriter.Save(); + } + } + else + { + if (!m_BuildMetadataPath.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + if (!m_BuildMetadata.empty()) + { + throw zen::OptionParseException( + fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + } + } + return {}; + }; + BoostWorkerThreads = m_BoostWorkerThreads; try { if (SubOption == &m_ListOptions) { + MakeSafeAbsolutePath(m_ListQueryPath); + MakeSafeAbsolutePath(m_ListResultPath); + if (!m_ListResultPath.empty()) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); @@ -9612,14 +9734,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath); - if (ToLower(ListQueryPath.extension().string()) == ".cbo") + if (ToLower(m_ListQueryPath.extension().string()) == ".cbo") { - QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); + QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_ListQueryPath)); } else { - IoBuffer MetaDataJson = ReadFile(ListQueryPath).Flatten(); + IoBuffer MetaDataJson = ReadFile(m_ListQueryPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; QueryObject = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); @@ -9634,19 +9755,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePath(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -9658,17 +9782,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); - if (ToLower(ListResultPath.extension().string()) == ".cbo") + if (ToLower(m_ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, ResponseView.GetData(), ResponseView.GetSize())); } else { ExtendableStringBuilder<1024> SB; CompactBinaryToJson(Response.GetView(), SB); - WriteFile(ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(m_ListResultPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); } } @@ -9679,130 +9802,53 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); - } - - if (m_CreateBuild) - { - if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) - { - throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); - } - if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) - { - throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); - } - } - else - { - if (!m_BuildMetadataPath.empty()) - { - throw zen::OptionParseException( - fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); - } - if (!m_BuildMetadata.empty()) - { - throw zen::OptionParseException( - fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); - } - } - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + ParsePath(); if (m_BuildPartName.empty()) { - m_BuildPartName = Path.filename().string(); + m_BuildPartName = m_Path.filename().string(); } - const bool GeneratedBuildId = m_BuildId.empty(); - if (GeneratedBuildId) - { - m_BuildId = Oid::NewOid().ToString(); - } - else if (m_BuildId.length() != Oid::StringLength) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } - else if (Oid::FromHexString(m_BuildId) == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); - } - - const bool GeneratedBuildPartId = m_BuildPartId.empty(); - if (GeneratedBuildPartId) - { - m_BuildPartId = Oid::NewOid().ToString(); - } - else if (m_BuildPartId.length() != Oid::StringLength) + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); + if (m_BuildId.empty()) { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + m_BuildId = BuildId.ToString(); } - else if (Oid::FromHexString(m_BuildPartId) == Oid::Zero) + const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); + if (m_BuildPartId.empty()) { - throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + m_BuildPartId = BuildPartId.ToString(); } - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePath(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); - CbObject MetaData; - if (m_CreateBuild) - { - if (!m_BuildMetadataPath.empty()) - { - std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath); - IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); - std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); - std::string JsonError; - MetaData = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - if (!JsonError.empty()) - { - throw std::runtime_error( - fmt::format("build metadata file '{}' is malformed. Reason: '{}'", m_BuildMetadataPath, JsonError)); - } - } - if (!m_BuildMetadata.empty()) - { - CbObjectWriter MetaDataWriter(1024); - ForEachStrTok(m_BuildMetadata, ';', [&](std::string_view Pair) { - size_t SplitPos = Pair.find('='); - if (SplitPos == std::string::npos || SplitPos == 0) - { - throw std::runtime_error(fmt::format("build metadata key-value pair '{}' is malformed", Pair)); - } - MetaDataWriter.AddString(Pair.substr(0, SplitPos), Pair.substr(SplitPos + 1)); - return true; - }); - MetaData = MetaDataWriter.Save(); - } - } + CbObject MetaData = ParseBuildMetadata(); UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, - Path, - ZenFolderPath, - m_ManifestPath.empty() ? std::filesystem::path{} : MakeSafeAbsolutePath(m_ManifestPath), + m_Path, + m_ZenFolderPath, + m_ManifestPath, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -9838,26 +9884,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - ParseSystemOptions(); + ParsePath(); - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } - - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } + const Oid BuildId = ParseBuildId(); if (m_PostDownloadVerify && m_PrimeCacheOnly) { @@ -9875,34 +9904,26 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); } - std::vector BuildPartIds; - for (const std::string& BuildPartId : m_BuildPartIds) + std::vector BuildPartIds = ParseBuildPartIds(); + + if (m_ZenFolderPath.empty()) { - BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); - if (BuildPartIds.back() == Oid::Zero) - { - throw zen::OptionParseException( - fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); - } + m_ZenFolderPath = m_Path / ZenFolderName; } - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + MakeSafeAbsolutePath(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, - Path, - ZenFolderPath, - SystemRootDir, + m_Path, + m_ZenFolderPath, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, @@ -9910,79 +9931,43 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_PrimeCacheOnly, m_EnableScavenging); - if (false) - { - ZEN_CONSOLE( - "{}:\n" - "Read: {}\n" - "Write: {}\n" - "Requests: {}\n" - "Avg Request Time: {}\n" - "Avg I/O Time: {}", - Storage.StorageName, - NiceBytes(StorageStats.TotalBytesRead.load()), - NiceBytes(StorageStats.TotalBytesWritten.load()), - StorageStats.TotalRequestCount.load(), - StorageStats.TotalExecutionTimeUs.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalExecutionTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0, - StorageStats.TotalRequestCount.load() > 0 - ? NiceTimeSpanMs(StorageStats.TotalRequestTimeUs.load() / 1000 / StorageStats.TotalRequestCount.load()) - : 0); - } - return AbortFlag ? 11 : 0; } if (SubOption == &m_DiffOptions) { - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } - if (m_DiffPath.empty()) - { - throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); - } - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath); - DiffFolders(Path, DiffPath, m_OnlyChunked); + ParsePath(); + ParseDiffPath(); + + DiffFolders(m_Path, m_DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } if (SubOption == &m_FetchBlobOptions) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BlobHash.empty()) - { - throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); - } - - IoHash BlobHash; - if (!IoHash::TryParse(m_BlobHash, BlobHash)) - { - throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); - } + IoHash BlobHash = ParseBlobHash(); const Oid BuildId = Oid::FromHexString(m_BuildId); - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePath(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); uint64_t CompressedSize; uint64_t DecompressedSize; @@ -10002,41 +9987,34 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BuildId.empty()) - { - throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); - } - Oid BuildId = Oid::TryFromHexString(m_BuildId); - if (BuildId == Oid::Zero) - { - throw zen::OptionParseException(fmt::format("build-id is invalid\n{}", m_DownloadOptions.help())); - } + Oid BuildId = ParseBuildId(); if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = m_ZenFolderPath.empty() - ? MakeSafeAbsolutePath(std::filesystem::current_path()) / ZenFolderName - : MakeSafeAbsolutePath(m_ZenFolderPath); - CreateDirectories(ZenFolderPath); - auto _ = MakeGuard([ZenFolderPath]() { - if (CleanDirectory(ZenFolderPath, {})) + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; + } + MakeSafeAbsolutePath(m_ZenFolderPath); + + CreateDirectories(m_ZenFolderPath); + auto _ = MakeGuard([this]() { + if (CleanDirectory(m_ZenFolderPath, {})) { std::error_code DummyEc; - RemoveDir(ZenFolderPath, DummyEc); + RemoveDir(m_ZenFolderPath, DummyEc); } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); - Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); ValidateStatistics ValidateStats; DownloadStatistics DownloadStats; @@ -10047,35 +10025,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_MultiTestDownloadOptions) { - SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(SystemRootDir); - CleanDirectory(SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); + + ParsePath(); - if (m_Path.empty()) + if (m_ZenFolderPath.empty()) { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); + m_ZenFolderPath = m_Path / ZenFolderName; } - - // m_StoragePath = "D:\\buildstorage"; - // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; - // std::vector BuildIdStrings{"07d3942f0e7f4ca1b13b0587", - // "07d394eed89d769f2254e75d", - // "07d3953f22fa3f8000fa6f0a", - // "07d3959df47ed1f42ddbe44c", - // "07d395fa7803d50804f14417", - // "07d3964f919d577a321a1fdd", - // "07d396a6ce875004e16b9528"}; - - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + MakeSafeAbsolutePath(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); - - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -10089,9 +10055,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, {}, {}, - Path, - ZenFolderPath, - SystemRootDir, + m_Path, + m_ZenFolderPath, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -10111,47 +10077,41 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { - SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); - CreateDirectories(SystemRootDir); - CleanDirectory(SystemRootDir, {}); - auto _ = MakeGuard([&]() { DeleteDirectories(SystemRootDir); }); - - if (m_Path.empty()) - { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); - } + m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); + CreateDirectories(m_SystemRootDir); + CleanDirectory(m_SystemRootDir, {}); + auto _ = MakeGuard([&]() { DeleteDirectories(m_SystemRootDir); }); - std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + ParsePath(); m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = Path.filename().string(); + m_BuildPartName = m_Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; const Oid BuildId = Oid::FromHexString(m_BuildId); const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - - if (m_OverrideHost.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && m_StoragePath.empty()) { - StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); - CreateDirectories(StoragePath); - CleanDirectory(StoragePath, {}); - m_StoragePath = StoragePath.generic_string(); + m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); + CreateDirectories(m_StoragePath); + CleanDirectory(m_StoragePath, {}); + m_StoragePath = m_StoragePath.generic_string(); } + auto __ = MakeGuard([&]() { - if (m_OverrideHost.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && m_StoragePath.empty()) { - DeleteDirectories(StoragePath); + DeleteDirectories(m_StoragePath); } }); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); - const std::filesystem::path DownloadPath2 = Path.parent_path() / (m_BuildPartName + "_test2"); + const std::filesystem::path DownloadPath = m_Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path DownloadPath2 = m_Path.parent_path() / (m_BuildPartName + "_test2"); auto ___ = MakeGuard([DownloadPath, DownloadPath2]() { CleanDirectory(DownloadPath, true); @@ -10160,10 +10120,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DeleteDirectories(DownloadPath2); }); - const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePath(m_ZenFolderPath); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -10190,8 +10153,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildId, BuildPartId, m_BuildPartName, - Path, - ZenFolderPath, + m_Path, + m_ZenFolderPath, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10212,8 +10175,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, true, @@ -10236,8 +10199,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10355,8 +10318,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10386,7 +10349,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, - ZenFolderPath, + m_ZenFolderPath, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10407,8 +10370,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10427,8 +10390,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10447,8 +10410,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, - ZenFolderPath, - SystemRootDir, + DownloadPath / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -10467,8 +10430,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath2, - ZenFolderPath, - SystemRootDir, + DownloadPath2 / ZenFolderName, + m_SystemRootDir, m_AllowMultiparts, m_AllowPartialBlockRequests, false, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 7e1e7d0ca..cd2ae1f08 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -25,13 +25,13 @@ public: private: cxxopts::Options m_Options{Name, Description}; - std::string m_SystemRootDir; + std::filesystem::path m_SystemRootDir; bool m_PlainProgress = false; bool m_Verbose = false; bool m_BoostWorkerThreads = false; - std::string m_ZenFolderPath; + std::filesystem::path m_ZenFolderPath; // cloud builds std::string m_OverrideHost; @@ -41,29 +41,29 @@ private: std::string m_Bucket; // file storage - std::string m_StoragePath; - bool m_WriteMetadataAsJson = false; + std::filesystem::path m_StoragePath; + bool m_WriteMetadataAsJson = false; // cache std::string m_ZenCacheHost; bool m_PrimeCacheOnly = false; - std::string m_BuildId; - bool m_CreateBuild = false; - std::string m_BuildMetadataPath; - std::string m_BuildMetadata; - std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path - std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName - bool m_Clean = false; - uint8_t m_BlockReuseMinPercentLimit = 85; - bool m_AllowMultiparts = true; - bool m_AllowPartialBlockRequests = true; - std::string m_ManifestPath; + std::string m_BuildId; + bool m_CreateBuild = false; + std::filesystem::path m_BuildMetadataPath; + std::string m_BuildMetadata; + std::string m_BuildPartName; // Defaults to name of leaf folder in m_Path + std::string m_BuildPartId; // Defaults to a generated id when creating part, looked up when downloading using m_BuildPartName + bool m_Clean = false; + uint8_t m_BlockReuseMinPercentLimit = 85; + bool m_AllowMultiparts = true; + bool m_AllowPartialBlockRequests = true; + std::string m_ManifestPath; // Not a std::filesystem::path since it can be relative to m_Path // Direct access token (may expire) - std::string m_AccessToken; - std::string m_AccessTokenEnv; - std::string m_AccessTokenPath; + std::string m_AccessToken; + std::string m_AccessTokenEnv; + std::filesystem::path m_AccessTokenPath; // Auth manager token encryption std::string m_EncryptionKey; // 256 bit AES encryption key @@ -84,11 +84,11 @@ private: std::string m_Verb; // list, upload, download - cxxopts::Options m_ListOptions{"list", "List available builds"}; - std::string m_ListQueryPath; - std::string m_ListResultPath; + cxxopts::Options m_ListOptions{"list", "List available builds"}; + std::filesystem::path m_ListQueryPath; + std::filesystem::path m_ListResultPath; - std::string m_Path; + std::filesystem::path m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; uint64_t m_FindBlockMaxCount = 10000; @@ -100,9 +100,9 @@ private: bool m_PostDownloadVerify = false; bool m_EnableScavenging = true; - cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; - std::string m_DiffPath; - bool m_OnlyChunked = false; + cxxopts::Options m_DiffOptions{"diff", "Compare two local folders"}; + std::filesystem::path m_DiffPath; + bool m_OnlyChunked = false; cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; std::string m_BlobHash; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 185edc35d..4412eaf34 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -625,22 +625,20 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) HttpClient Http(m_HostName); - std::filesystem::path TargetPath; if (!m_OutputPath.empty()) { - TargetPath = std::filesystem::path(m_OutputPath); - if (IsDir(TargetPath)) + if (IsDir(m_OutputPath)) { - TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); + m_OutputPath = m_OutputPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } else { - CreateDirectories(TargetPath.parent_path()); + CreateDirectories(m_OutputPath.parent_path()); } } - if (TargetPath.empty()) + if (m_OutputPath.empty()) { - TargetPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); + m_OutputPath = (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } std::string Url = fmt::format("/z$/{}/{}/{}", m_Namespace, m_Bucket, m_ValueKey); @@ -670,17 +668,17 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - WriteFile(TargetPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length())); - ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), TargetPath, ToString(ChunkData.GetContentType())); + WriteFile(m_OutputPath, IoBuffer(IoBuffer::Wrap, StringData.data(), StringData.length())); + ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(StringData.length()), m_OutputPath, ToString(ChunkData.GetContentType())); } } else { - if (!MoveToFile(TargetPath, ChunkData)) + if (!MoveToFile(m_OutputPath, ChunkData)) { - WriteFile(TargetPath, ChunkData); + WriteFile(m_OutputPath, ChunkData); } - ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), TargetPath, ToString(ChunkData.GetContentType())); + ZEN_CONSOLE("Wrote {} to '{}' ({})", NiceBytes(ChunkData.GetSize()), m_OutputPath, ToString(ChunkData.GetContentType())); } } else diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 73702cada..b8a319359 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -108,15 +108,15 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_Namespace; - std::string m_Bucket; - std::string m_ValueKey; - std::string m_AttachmentHash; - std::string m_OutputPath; - bool m_AsText = false; - bool m_Decompress = true; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_Namespace; + std::string m_Bucket; + std::string m_ValueKey; + std::string m_AttachmentHash; + std::filesystem::path m_OutputPath; + bool m_AsText = false; + bool m_Decompress = true; }; } // namespace zen diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index 53e80c896..e86b6964c 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -42,11 +42,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_CopyTarget.empty()) throw std::runtime_error("No target specified"); - std::filesystem::path FromPath; - std::filesystem::path ToPath; - - FromPath = m_CopySource; - ToPath = m_CopyTarget; + std::filesystem::path FromPath = m_CopySource; + std::filesystem::path ToPath = m_CopyTarget; std::error_code Ec; std::filesystem::path FromCanonical = std::filesystem::canonical(FromPath, Ec); diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h index 876aff3f5..e9735c159 100644 --- a/src/zen/cmds/copy_cmd.h +++ b/src/zen/cmds/copy_cmd.h @@ -19,11 +19,11 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"copy", "Copy files efficiently"}; - std::string m_CopySource; - std::string m_CopyTarget; - bool m_NoClone = false; - bool m_MustClone = false; + cxxopts::Options m_Options{"copy", "Copy files efficiently"}; + std::filesystem::path m_CopySource; + std::filesystem::path m_CopyTarget; + bool m_NoClone = false; + bool m_MustClone = false; }; } // namespace zen diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index c4f0068e4..2721be2b9 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -21,8 +21,8 @@ public: private: cxxopts::Options m_Options{"dedup", "Deduplicate files"}; std::vector m_Positional; - std::string m_DedupSource; - std::string m_DedupTarget; + std::filesystem::path m_DedupSource; + std::filesystem::path m_DedupTarget; size_t m_SizeThreshold = 1024 * 1024; }; diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 88c0b22a2..2b507e43d 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -32,17 +32,16 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) uint16_t EffectivePort = 0; if (!m_DataDir.empty()) { - std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } EffectivePort = Info.EffectiveListenPort; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index 00ad0e758..46bda9ee6 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -20,9 +20,9 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; - uint16_t m_Port = 0; - std::string m_DataDir; + cxxopts::Options m_Options{"status", "Show zen status"}; + uint16_t m_Port = 0; + std::filesystem::path m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index aacc115a0..f3bf2f66d 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -77,15 +77,13 @@ UpCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); - - if (ProgramBaseDir.empty()) + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = zen::GetRunningExecutablePath(); - ProgramBaseDir = ExePath.parent_path(); + m_ProgramBaseDir = ExePath.parent_path(); } ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ProgramBaseDir); + ServerEnvironment.Initialize(m_ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); std::string ServerArguments = GlobalOptions.PassthroughCommandLine; if ((m_Port != 0) && (ServerArguments.find("--port"sv) == std::string::npos)) @@ -155,20 +153,18 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Sweep(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - std::filesystem::path DataDir = StringToPath(m_DataDir); - - if (!DataDir.empty()) + if (!m_DataDir.empty()) { - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -218,27 +214,24 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Instance.Initialize(); ZenServerState::ZenServerEntry* Entry = Instance.Lookup(m_Port); - std::filesystem::path ProgramBaseDir = StringToPath(m_ProgramBaseDir); - if (ProgramBaseDir.empty()) + if (m_ProgramBaseDir.empty()) { std::filesystem::path ExePath = GetRunningExecutablePath(); - ProgramBaseDir = ExePath.parent_path(); + m_ProgramBaseDir = ExePath.parent_path(); } - std::filesystem::path DataDir = StringToPath(m_DataDir); - - if (!DataDir.empty()) + if (!m_DataDir.empty()) { - if (!IsFile(DataDir / ".lock")) + if (!IsFile(m_DataDir / ".lock")) { - ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); + ZEN_CONSOLE("lock file does not exist in directory '{}'", m_DataDir); return 1; } - LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(DataDir / ".lock"))); + LockFileInfo Info = ReadLockFilePayload(LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(m_DataDir / ".lock"))); std::string Reason; if (!ValidateLockFileInfo(Info, Reason)) { - ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", DataDir, Reason); + ZEN_CONSOLE("lock file in directory '{}' is not valid. Reason: '{}'", m_DataDir, Reason); return 1; } Entry = Instance.LookupByEffectivePort(Info.EffectiveListenPort); @@ -251,7 +244,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) try { ZenServerEnvironment ServerEnvironment; - ServerEnvironment.Initialize(ProgramBaseDir); + ServerEnvironment.Initialize(m_ProgramBaseDir); ZenServerInstance Server(ServerEnvironment); Server.AttachToRunningServer(EntryPort); @@ -316,7 +309,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_ForceTerminate) { // Try to find the running executable by path name - std::filesystem::path ServerExePath = ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec) { diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index 32d8ddab3..c9af16749 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -18,11 +18,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; - uint16_t m_Port = 0; - bool m_ShowConsole = false; - bool m_ShowLog = false; - std::string m_ProgramBaseDir; + cxxopts::Options m_Options{"up", "Bring up zen service"}; + uint16_t m_Port = 0; + bool m_ShowConsole = false; + bool m_ShowLog = false; + std::filesystem::path m_ProgramBaseDir; }; class AttachCommand : public ZenCmdBase @@ -35,10 +35,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; - uint16_t m_Port = 0; - int m_OwnerPid = 0; - std::string m_DataDir; + cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + uint16_t m_Port = 0; + int m_OwnerPid = 0; + std::filesystem::path m_DataDir; }; class DownCommand : public ZenCmdBase @@ -51,11 +51,11 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; - uint16_t m_Port = 0; - bool m_ForceTerminate = false; - std::string m_ProgramBaseDir; - std::string m_DataDir; + cxxopts::Options m_Options{"down", "Bring down zen service"}; + uint16_t m_Port = 0; + bool m_ForceTerminate = false; + std::filesystem::path m_ProgramBaseDir; + std::filesystem::path m_DataDir; }; } // namespace zen diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 5f3f8f7ca..2ddd0c73a 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -139,18 +139,16 @@ WorkspaceCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_HostName = ResolveTargetHostSpec(m_HostName); - std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); - - if (SystemRootDir.empty()) + if (m_SystemRootDir.empty()) { - SystemRootDir = PickDefaultSystemRootDirectory(); - if (SystemRootDir.empty()) + m_SystemRootDir = PickDefaultSystemRootDirectory(); + if (m_SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } - std::filesystem::path StatePath = SystemRootDir / "workspaces"; + std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -392,18 +390,20 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** m_HostName = ResolveTargetHostSpec(m_HostName); - std::filesystem::path SystemRootDir = StringToPath(m_SystemRootDir); - - if (SystemRootDir.empty()) + if (m_SystemRootDir.empty()) { - SystemRootDir = PickDefaultSystemRootDirectory(); - if (SystemRootDir.empty()) + m_SystemRootDir = PickDefaultSystemRootDirectory(); + if (m_SystemRootDir.empty()) { throw zen::OptionParseException("unable to resolve system root directory"); } } + else + { + MakeSafeAbsolutePath(m_SystemRootDir); + } - std::filesystem::path StatePath = SystemRootDir / "workspaces"; + std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; if (!ParseOptions(*SubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) { @@ -412,8 +412,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (SubOption == &m_CreateOptions) { - std::filesystem::path WorkspaceRoot = StringToPath(m_WorkspaceRoot); - if (WorkspaceRoot.empty()) + if (m_WorkspaceRoot.empty()) { if (m_WorkspaceId.empty()) { @@ -432,15 +431,15 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ZEN_CONSOLE("Workspace {} does not exist", m_WorkspaceId); return 0; } - WorkspaceRoot = WorkspaceConfig.RootPath; + m_WorkspaceRoot = WorkspaceConfig.RootPath; } else { - RemoveTrailingPathSeparator(WorkspaceRoot); + RemoveTrailingPathSeparator(m_WorkspaceRoot); if (m_WorkspaceId.empty()) { - m_WorkspaceId = Workspaces::PathToId(WorkspaceRoot).ToString(); - ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, WorkspaceRoot); + m_WorkspaceId = Workspaces::PathToId(m_WorkspaceRoot).ToString(); + ZEN_CONSOLE("Using generated workspace id {} from path '{}'", m_WorkspaceId, m_WorkspaceRoot); } else { @@ -449,25 +448,23 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** throw zen::OptionParseException(fmt::format("workspace id '{}' is invalid", m_WorkspaceId)); } } - if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = WorkspaceRoot})) + if (Workspaces::AddWorkspace(Log(), StatePath, {.Id = Oid::FromHexString(m_WorkspaceId), .RootPath = m_WorkspaceRoot})) { - ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, WorkspaceRoot); + ZEN_CONSOLE("Created workspace {} using root path '{}'", m_WorkspaceId, m_WorkspaceRoot); } else { - ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, WorkspaceRoot); + ZEN_CONSOLE("Using existing workspace {} with root path '{}'", m_WorkspaceId, m_WorkspaceRoot); } } - std::filesystem::path SharePath = StringToPath(m_SharePath); - - RemoveLeadingPathSeparator(SharePath); - RemoveTrailingPathSeparator(SharePath); + RemoveLeadingPathSeparator(m_SharePath); + RemoveTrailingPathSeparator(m_SharePath); if (m_ShareId.empty()) { - m_ShareId = Workspaces::PathToId(SharePath).ToString(); - ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, SharePath); + m_ShareId = Workspaces::PathToId(m_SharePath).ToString(); + ZEN_CONSOLE("Using generated share id {}, for path '{}'", m_ShareId, m_SharePath); } if (Oid::TryFromHexString(m_ShareId) == Oid::Zero) @@ -476,8 +473,8 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } if (Workspaces::AddWorkspaceShare(Log(), - WorkspaceRoot, - {.Id = Oid::FromHexString(m_ShareId), .SharePath = SharePath, .Alias = m_Alias})) + m_WorkspaceRoot, + {.Id = Oid::FromHexString(m_ShareId), .SharePath = m_SharePath, .Alias = m_Alias})) { if (!m_HostName.empty()) { diff --git a/src/zen/cmds/workspaces_cmd.h b/src/zen/cmds/workspaces_cmd.h index 86452e25e..d85d8f7d8 100644 --- a/src/zen/cmds/workspaces_cmd.h +++ b/src/zen/cmds/workspaces_cmd.h @@ -21,9 +21,9 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_SystemRootDir; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::filesystem::path m_SystemRootDir; std::string m_Verb; // create, info, remove @@ -53,17 +53,17 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{Name, Description}; - std::string m_HostName; - std::string m_SystemRootDir; - std::string m_WorkspaceId; - std::string m_WorkspaceRoot; - std::string m_Verb; // create, info, remove - std::string m_ShareId; - std::string m_Alias; - - cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; - std::string m_SharePath; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::filesystem::path m_SystemRootDir; + std::string m_WorkspaceId; + std::filesystem::path m_WorkspaceRoot; + std::string m_Verb; // create, info, remove + std::string m_ShareId; + std::string m_Alias; + + cxxopts::Options m_CreateOptions{"create", "Create a workspace share"}; + std::filesystem::path m_SharePath; bool m_Refresh = false; diff --git a/src/zen/zen.h b/src/zen/zen.h index 6765101db..dd0fd44b3 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -5,10 +5,7 @@ #include #include #include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END +#include namespace cpr { class Response; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index ad796cb4a..018330d9b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2670,22 +2670,6 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) return Result; } -std::filesystem::path -StringToPath(const std::string_view& Path) -{ - std::string_view UnquotedPath = Path; - - if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') - { - UnquotedPath = Path.substr(1, Path.length() - 2); - } - if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) - { - UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); - } - return std::filesystem::path(UnquotedPath).make_preferred(); -} - ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 66deffa6f..1bc3943df 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -372,8 +372,6 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -std::filesystem::path StringToPath(const std::string_view& Path); - ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 0c5931ba0..d1394cd9a 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -100,9 +100,6 @@ int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); -std::vector ParseCommandLine(std::string_view CommandLine); -std::vector StripCommandlineQuotes(std::vector& InOutArgs); - void process_forcelink(); // internal } // namespace zen diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 2fe5b8948..48efc3f85 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -1047,118 +1047,6 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand #endif // ZEN_PLATFORM_LINUX } -std::vector -ParseCommandLine(std::string_view CommandLine) -{ - auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { - if (Pos == CommandLine.length()) - { - return true; - } - if (CommandLine[Pos] == ' ') - { - return true; - } - return false; - }; - - bool IsParsingArg = false; - bool IsInQuote = false; - - std::string::size_type Pos = 0; - std::string::size_type ArgStart = 0; - std::vector Args; - while (Pos < CommandLine.length()) - { - if (IsInQuote) - { - if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); - Pos++; - IsInQuote = false; - IsParsingArg = false; - } - else - { - Pos++; - } - } - else if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - if (CommandLine[Pos] == ' ') - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); - Pos++; - IsParsingArg = false; - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - Pos++; - } - else - { - Pos++; - } - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else if (CommandLine[Pos] != ' ') - { - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else - { - Pos++; - } - } - if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - Args.push_back(std::string(CommandLine.substr(ArgStart))); - } - - return Args; -} - -std::vector -StripCommandlineQuotes(std::vector& InOutArgs) -{ - std::vector RawArgs; - RawArgs.reserve(InOutArgs.size()); - for (std::string& Arg : InOutArgs) - { - std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); - while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) - { - Arg.erase(EscapedQuotePos, 1); - EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); - } - - if (Arg.starts_with("\"")) - { - if (Arg.find('"', 1) == Arg.length() - 1) - { - if (Arg.find(' ', 1) == std::string::npos) - { - Arg = Arg.substr(1, Arg.length() - 2); - } - } - } - RawArgs.push_back(const_cast(Arg.c_str())); - } - return RawArgs; -} - #if ZEN_WITH_TESTS void @@ -1235,36 +1123,6 @@ TEST_CASE("BuildArgV") } } -TEST_CASE("CommandLine") -{ - std::vector v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); - CHECK_EQ(v1[0], "c:\\my\\exe.exe"); - CHECK_EQ(v1[1], "\"quoted arg\""); - CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); - - std::vector v2 = ParseCommandLine( - "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " - "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " - "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " - "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); - - std::vector v2Stripped = StripCommandlineQuotes(v2); - CHECK_EQ(v2Stripped[0], std::string("--tracehost")); - CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); - CHECK_EQ(v2Stripped[2], std::string("builds")); - CHECK_EQ(v2Stripped[3], std::string("download")); - CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); - CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); - CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); - CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); - CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\"")); - CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\"")); - CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\"")); - CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); - CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); - CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); -} - TEST_SUITE_END(/* core.process */); #endif diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 31c104110..fecc1043a 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -15,6 +15,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -178,27 +179,6 @@ ParseBucketConfigs(std::span Buckets) return Cfg; } -static std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - class CachePolicyOption : public LuaConfig::OptionValue { public: @@ -324,7 +304,7 @@ public: std::string Name = Bucket.value().get_or("name", std::string("Default")); std::string Directory = Bucket.value().get_or("directory", std::string()); - Value.Buckets.push_back({.Name = std::move(Name), .Directory = LuaConfig::MakeSafePath(Directory)}); + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); } } } @@ -525,7 +505,7 @@ ParseConfigFile(const std::filesystem::path& Path, if (!OutputConfigFile.empty()) { - std::filesystem::path WritePath(MakeSafePath(OutputConfigFile)); + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); zen::ExtendableStringBuilder<512> ConfigStringBuilder; LuaOptions.Print(ConfigStringBuilder, CmdLineResult); zen::BasicFile Output; @@ -1110,12 +1090,12 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) } logging::RefreshLogLevels(); - ServerOptions.SystemRootDir = MakeSafePath(SystemRootDir); - ServerOptions.DataDir = MakeSafePath(DataDir); - ServerOptions.BaseSnapshotDir = MakeSafePath(BaseSnapshotDir); - ServerOptions.ContentDir = MakeSafePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafePath(AbsLogFile); - ServerOptions.ConfigFile = MakeSafePath(ConfigFile); + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); if (!BaseSnapshotDir.empty()) diff --git a/src/zenserver/config/luaconfig.cpp b/src/zenserver/config/luaconfig.cpp index f742fa34a..2c54de29e 100644 --- a/src/zenserver/config/luaconfig.cpp +++ b/src/zenserver/config/luaconfig.cpp @@ -4,27 +4,6 @@ namespace zen::LuaConfig { -std::string -MakeSafePath(const std::string_view Path) -{ -#if ZEN_PLATFORM_WINDOWS - if (Path.empty()) - { - return std::string(Path); - } - - std::string FixedPath(Path); - std::replace(FixedPath.begin(), FixedPath.end(), '/', '\\'); - if (!FixedPath.starts_with("\\\\?\\")) - { - FixedPath.insert(0, "\\\\?\\"); - } - return FixedPath; -#else - return std::string(Path); -#endif -}; - void EscapeBackslash(std::string& InOutString) { @@ -101,7 +80,7 @@ FilePathOption::Parse(sol::object Object) std::string Str = Object.as(); if (!Str.empty()) { - Value = MakeSafePath(Str); + Value = MakeSafeAbsolutePath(Str); } } diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h index 76b3088a3..ce7013a9a 100644 --- a/src/zenserver/config/luaconfig.h +++ b/src/zenserver/config/luaconfig.h @@ -4,10 +4,10 @@ #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include -#include #include ZEN_THIRD_PARTY_INCLUDES_END @@ -20,8 +20,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen::LuaConfig { -std::string MakeSafePath(const std::string_view Path); -void EscapeBackslash(std::string& InOutString); +void EscapeBackslash(std::string& InOutString); class OptionValue { diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp new file mode 100644 index 000000000..0dffa42f0 --- /dev/null +++ b/src/zenutil/commandlineoptions.cpp @@ -0,0 +1,213 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#if ZEN_WITH_TESTS +# include +#endif // ZEN_WITH_TESTS + +void +cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value) +{ + value = zen::StringToPath(text); +} + +namespace zen { + +std::vector +ParseCommandLine(std::string_view CommandLine) +{ + auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { + if (Pos == CommandLine.length()) + { + return true; + } + if (CommandLine[Pos] == ' ') + { + return true; + } + return false; + }; + + bool IsParsingArg = false; + bool IsInQuote = false; + + std::string::size_type Pos = 0; + std::string::size_type ArgStart = 0; + std::vector Args; + while (Pos < CommandLine.length()) + { + if (IsInQuote) + { + if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); + Pos++; + IsInQuote = false; + IsParsingArg = false; + } + else + { + Pos++; + } + } + else if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + if (CommandLine[Pos] == ' ') + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); + Pos++; + IsParsingArg = false; + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + Pos++; + } + else + { + Pos++; + } + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else if (CommandLine[Pos] != ' ') + { + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else + { + Pos++; + } + } + if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + Args.push_back(std::string(CommandLine.substr(ArgStart))); + } + + return Args; +} + +std::vector +StripCommandlineQuotes(std::vector& InOutArgs) +{ + std::vector RawArgs; + RawArgs.reserve(InOutArgs.size()); + for (std::string& Arg : InOutArgs) + { + std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); + while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) + { + Arg.erase(EscapedQuotePos, 1); + EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); + } + + if (Arg.starts_with("\"")) + { + if (Arg.find('"', 1) == Arg.length() - 1) + { + if (Arg.find(' ', 1) == std::string::npos) + { + Arg = Arg.substr(1, Arg.length() - 2); + } + } + } + RawArgs.push_back(const_cast(Arg.c_str())); + } + return RawArgs; +} + +void +MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path) +{ + if (!Path.empty()) + { + std::filesystem::path AbsolutePath = std::filesystem::absolute(Path).make_preferred(); +#if ZEN_PLATFORM_WINDOWS + const std::string_view Prefix = "\\\\?\\"; + const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); + std::u8string PathString = AbsolutePath.u8string(); + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) + { + PathString.insert(0, PrefixU8); + Path = PathString; + } +#endif // ZEN_PLATFORM_WINDOWS + } +} + +std::filesystem::path +MakeSafeAbsolutePath(const std::filesystem::path& Path) +{ + std::filesystem::path Tmp(Path); + MakeSafeAbsolutePathÍnPlace(Tmp); + return Tmp; +} + +std::filesystem::path +StringToPath(const std::string_view& Path) +{ + std::string_view UnquotedPath = Path; + + if (UnquotedPath.length() > 2 && UnquotedPath.front() == '\"' && UnquotedPath.back() == '\"') + { + UnquotedPath = UnquotedPath.substr(1, UnquotedPath.length() - 2); + } + + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) + { + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); + } + + return std::filesystem::path(UnquotedPath).make_preferred(); +} + +#if ZEN_WITH_TESTS + +void +commandlineoptions_forcelink() +{ +} + +TEST_CASE("CommandLine") +{ + std::vector v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); + CHECK_EQ(v1[0], "c:\\my\\exe.exe"); + CHECK_EQ(v1[1], "\"quoted arg\""); + CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); + + std::vector v2 = ParseCommandLine( + "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " + "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " + "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " + "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); + + std::vector v2Stripped = StripCommandlineQuotes(v2); + CHECK_EQ(v2Stripped[0], std::string("--tracehost")); + CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); + CHECK_EQ(v2Stripped[2], std::string("builds")); + CHECK_EQ(v2Stripped[3], std::string("download")); + CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); + CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); + CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); + CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); + CHECK_EQ(v2Stripped[8], std::string("--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\"")); + CHECK_EQ(v2Stripped[9], std::string("\"D:\\Dev\\Spaced Folder\\Target\"")); + CHECK_EQ(v2Stripped[10], std::string("--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\"")); + CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); + CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); + CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); +} + +#endif +} // namespace zen diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h new file mode 100644 index 000000000..3afbac854 --- /dev/null +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START + +namespace cxxopts::values { +// We declare this specialization before including cxxopts to make it stick +void parse_value(const std::string& text, std::filesystem::path& value); +} // namespace cxxopts::values + +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); +void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); +std::filesystem::path StringToPath(const std::string_view& Path); + +void commandlineoptions_forcelink(); // internal + +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index 19eb63ce9..aff9156f4 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -7,6 +7,7 @@ # include # include # include +# include namespace zen { @@ -17,6 +18,7 @@ zenutil_forcelinktests() cache::rpcrecord_forcelink(); cacherequests_forcelink(); chunkedfile_forcelink(); + commandlineoptions_forcelink(); } } // namespace zen -- cgit v1.2.3 From ea2918767afa3c00c8dfa7dd7d9960a80a460eb9 Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Wed, 9 Apr 2025 15:35:37 +0200 Subject: Added config, versioning and logging for plugins --- .../transport-sdk/include/transportplugin.h | 42 ++++- src/transports/winsock/winsock.cpp | 16 +- src/zenhttp/httpserver.cpp | 177 ++++++++++----------- src/zenhttp/include/zenhttp/httpserver.h | 15 +- src/zenhttp/transports/dlltransport.cpp | 81 ++++++++-- src/zenserver/config.cpp | 72 +++++++++ src/zenserver/config.h | 1 + 7 files changed, 295 insertions(+), 109 deletions(-) (limited to 'src') diff --git a/src/transports/transport-sdk/include/transportplugin.h b/src/transports/transport-sdk/include/transportplugin.h index 4347868e6..a78a758bc 100644 --- a/src/transports/transport-sdk/include/transportplugin.h +++ b/src/transports/transport-sdk/include/transportplugin.h @@ -17,10 +17,14 @@ namespace zen { +// Current API version, value will be incremented to represent breaking changes +static const uint32_t kTransportApiVersion = 1; + class TransportConnection; class TransportPlugin; class TransportServerConnection; class TransportServer; +class TransportLogger; /************************************************************************* @@ -60,6 +64,29 @@ public: virtual TransportServerConnection* CreateConnectionHandler(TransportConnection* Connection) = 0; }; +/** Logger interface + + There will be one instance of this provided by the system to the transport plugin + + The plugin can use this to log messages back to zen server + + */ +class TransportLogger +{ +public: + enum class LogLevel : uint32_t + { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Err = 4, + Critical = 5, + }; + + virtual void LogMessage(LogLevel Level, const char* Message) = 0; +}; + /************************************************************************* The following interfaces are to be implemented by transport plugins. @@ -116,7 +143,18 @@ public: extern "C" { - DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(); + /** Provide information about plugin version + + Fills out API version (kTransportApiVersion) plugin was built against. + Fills out plugin own version ever increasing version number, + a copy of plugin with higher version will be used. + */ + DLL_TRANSPORT_API void GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion); + + // Return nullptr if requested api version mismatches api version plugin was built against + DLL_TRANSPORT_API zen::TransportPlugin* CreateTransportPlugin(zen::TransportLogger* Logger); } -typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(); +typedef void (*PfnGetTransportPluginVersion)(uint32_t* OutApiVersion, uint32_t* OutPluginVersion); + +typedef zen::TransportPlugin* (*PfnCreateTransportPlugin)(zen::TransportLogger* Logger); diff --git a/src/transports/winsock/winsock.cpp b/src/transports/winsock/winsock.cpp index 1c3ee909a..f98984726 100644 --- a/src/transports/winsock/winsock.cpp +++ b/src/transports/winsock/winsock.cpp @@ -364,8 +364,22 @@ WinsockTransportPlugin::IsAvailable() ////////////////////////////////////////////////////////////////////////// +void +GetTransportPluginVersion(uint32_t* OutApiVersion, uint32_t* OutPluginVersion) +{ + if (OutApiVersion != nullptr) + { + *OutApiVersion = kTransportApiVersion; + } + + if (OutPluginVersion != nullptr) + { + *OutPluginVersion = 1; + } +} + TransportPlugin* -CreateTransportPlugin() +CreateTransportPlugin([[maybe_unused]] TransportLogger* Logger) { return new WinsockTransportPlugin; } diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 27a09f339..d12f89a92 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -787,120 +787,115 @@ HttpRpcHandler::AddRpc(std::string_view RpcId, std::function -CreateHttpServerClass(HttpServerClass Class, const HttpServerConfig& Config) +CreateHttpServerClass(const std::string_view ServerClass, const HttpServerConfig& Config) { - switch (Class) + if (ServerClass == "asio"sv) { - default: - case HttpServerClass::kHttpAsio: - ZEN_INFO("using asio HTTP server implementation"); - return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); - - case HttpServerClass::kHttpMulti: - { - ZEN_INFO("using multi HTTP server implementation"); - Ref Server{new HttpMultiServer()}; - - // This is hardcoded for now, but should be configurable in the future - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpSys, Config)); - Server->AddServer(CreateHttpServerClass(HttpServerClass::kHttpPlugin, Config)); + ZEN_INFO("using asio HTTP server implementation") + return CreateHttpAsioServer(Config.ForceLoopback, Config.ThreadCount); + } +#if ZEN_WITH_HTTPSYS + else if (ServerClass == "httpsys"sv) + { + ZEN_INFO("using http.sys server implementation") + return Ref(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, + .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, + .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, + .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, + .IsDedicatedServer = Config.IsDedicatedServer, + .ForceLoopback = Config.ForceLoopback})); + } +#endif + else if (ServerClass == "null"sv) + { + ZEN_INFO("using null HTTP server implementation") + return Ref(new HttpNullServer); + } + else + { + ZEN_WARN("unknown HTTP server implementation '{}', falling back to default", ServerClass) - return Server; - } +#if ZEN_WITH_HTTPSYS + return CreateHttpServerClass("httpsys"sv, Config); +#else + return CreateHttpServerClass("asio"sv, Config); +#endif + } +} #if ZEN_WITH_PLUGINS - case HttpServerClass::kHttpPlugin: - { - ZEN_INFO("using plugin HTTP server implementation"); - Ref Server{CreateHttpPluginServer()}; - - // This is hardcoded for now, but should be configurable in the future +Ref +CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) +{ + const std::string& PluginName = PluginConfig.PluginName; -# if 0 - Ref WinsockPlugin{CreateSocketTransportPlugin()}; - WinsockPlugin->Configure("port", "8558"); - Server->AddPlugin(WinsockPlugin); -# endif + ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName) + Ref Server{CreateHttpPluginServer()}; + if (PluginName.starts_with("builtin:"sv)) + { # if 0 - Ref AsioPlugin{CreateAsioTransportPlugin()}; - AsioPlugin->Configure("port", "8558"); - Server->AddPlugin(AsioPlugin); -# endif - -# if 1 - Ref DllPlugin{CreateDllTransportPlugin()}; - DllPlugin->LoadDll("winsock"); - DllPlugin->ConfigureDll("winsock", "port", "8558"); - Server->AddPlugin(DllPlugin); -# endif + Ref Plugin = {}; + if (PluginName == "builtin:winsock"sv) + { + Plugin = CreateSocketTransportPlugin(); + } + else if (PluginName == "builtin:asio"sv) + { + Plugin = CreateAsioTransportPlugin(); + } - return Server; + if (!Plugin.IsNull()) + { + for (const std::pair& Option : PluginConfig.PluginOptions) + { + Plugin->Configure(Option.first.c_str(), Option.second.c_str()); } -#endif - -#if ZEN_WITH_HTTPSYS - case HttpServerClass::kHttpSys: - ZEN_INFO("using http.sys server implementation"); - return Ref(CreateHttpSysServer({.ThreadCount = Config.ThreadCount, - .AsyncWorkThreadCount = Config.HttpSys.AsyncWorkThreadCount, - .IsAsyncResponseEnabled = Config.HttpSys.IsAsyncResponseEnabled, - .IsRequestLoggingEnabled = Config.HttpSys.IsRequestLoggingEnabled, - .IsDedicatedServer = Config.IsDedicatedServer, - .ForceLoopback = Config.ForceLoopback})); -#endif - - case HttpServerClass::kHttpNull: - ZEN_INFO("using null HTTP server implementation"); - return Ref(new HttpNullServer); + Server->AddPlugin(Plugin); + } +# endif } + else + { + Ref DllPlugin{CreateDllTransportPlugin()}; + DllPlugin->LoadDll(PluginName); + for (const std::pair& Option : PluginConfig.PluginOptions) + { + DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); + } + Server->AddPlugin(DllPlugin); + } + + return Server; } +#endif Ref CreateHttpServer(const HttpServerConfig& Config) { using namespace std::literals; - HttpServerClass Class = HttpServerClass::kHttpNull; - -#if ZEN_WITH_HTTPSYS - Class = HttpServerClass::kHttpSys; -#else - Class = HttpServerClass::kHttpAsio; -#endif - - if (Config.ServerClass == "asio"sv) - { - Class = HttpServerClass::kHttpAsio; - } - else if (Config.ServerClass == "httpsys"sv) - { - Class = HttpServerClass::kHttpSys; - } - else if (Config.ServerClass == "plugin"sv) - { - Class = HttpServerClass::kHttpPlugin; - } - else if (Config.ServerClass == "null"sv) +#if ZEN_WITH_PLUGINS + if (Config.PluginConfigs.empty()) { - Class = HttpServerClass::kHttpNull; + return CreateHttpServerClass(Config.ServerClass, Config); } - else if (Config.ServerClass == "multi"sv) + else { - Class = HttpServerClass::kHttpMulti; - } + Ref Server{new HttpMultiServer()}; + Server->AddServer(CreateHttpServerClass(Config.ServerClass, Config)); - return CreateHttpServerClass(Class, Config); + for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs) + { + Server->AddServer(CreateHttpServerPlugin(PluginConfig)); + } + + return Server; + } +#else + return CreateHttpServerClass(Config.ServerClass, Config); +#endif } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 217455dba..03e547bf3 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -184,12 +184,19 @@ public: virtual void Close() = 0; }; +struct HttpServerPluginConfig +{ + std::string PluginName; + std::vector> PluginOptions; +}; + struct HttpServerConfig { - bool IsDedicatedServer = false; // Should be set to true for shared servers - std::string ServerClass; // Choice of HTTP server implementation - bool ForceLoopback = false; - unsigned int ThreadCount = 0; + bool IsDedicatedServer = false; // Should be set to true for shared servers + std::string ServerClass; // Choice of HTTP server implementation + std::vector PluginConfigs; + bool ForceLoopback = false; + unsigned int ThreadCount = 0; struct { diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index e09e62ec5..e9336f1be 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -21,18 +21,31 @@ namespace zen { ////////////////////////////////////////////////////////////////////////// +class DllTransportLogger : public TransportLogger, public RefCounted +{ +public: + DllTransportLogger(std::string_view PluginName); + virtual ~DllTransportLogger() = default; + + void LogMessage(LogLevel Level, const char* Message) override; + +private: + std::string m_PluginName; +}; + struct LoadedDll { std::string Name; std::filesystem::path LoadedFromPath; + DllTransportLogger* Logger; Ref Plugin; }; class DllTransportPluginImpl : public DllTransportPlugin, RefCounted { public: - DllTransportPluginImpl(); - ~DllTransportPluginImpl(); + DllTransportPluginImpl() = default; + ~DllTransportPluginImpl() = default; virtual uint32_t AddRef() const override; virtual uint32_t Release() const override; @@ -51,12 +64,27 @@ private: std::vector m_Transports; }; -DllTransportPluginImpl::DllTransportPluginImpl() +DllTransportLogger::DllTransportLogger(std::string_view PluginName) : m_PluginName(PluginName) { } -DllTransportPluginImpl::~DllTransportPluginImpl() +void +DllTransportLogger::LogMessage(LogLevel PluginLogLevel, const char* Message) { + logging::level::LogLevel Level; + // clang-format off + switch (PluginLogLevel) + { + case LogLevel::Trace: Level = logging::level::Trace; break; + case LogLevel::Debug: Level = logging::level::Debug; break; + case LogLevel::Info: Level = logging::level::Info; break; + case LogLevel::Warn: Level = logging::level::Warn; break; + case LogLevel::Err: Level = logging::level::Err; break; + case LogLevel::Critical: Level = logging::level::Critical; break; + default: Level = logging::level::Off; break; + } + // clang-format on + ZEN_LOG(Log(), Level, "[{}] {}", m_PluginName, Message) } uint32_t @@ -109,6 +137,7 @@ DllTransportPluginImpl::Shutdown() try { Transport.Plugin->Shutdown(); + Transport.Logger->Release(); } catch (const std::exception&) { @@ -148,8 +177,15 @@ DllTransportPluginImpl::LoadDll(std::string_view Name) { RwLock::ExclusiveLockScope _(m_Lock); - ExtendableStringBuilder<128> DllPath; - DllPath << Name << ".dll"; + ExtendableStringBuilder<1024> DllPath; + DllPath << Name; + if (!Name.ends_with(".dll")) + { + DllPath << ".dll"; + } + + std::string FileName = std::filesystem::path(DllPath.c_str()).filename().replace_extension().string(); + HMODULE DllHandle = LoadLibraryA(DllPath.c_str()); if (!DllHandle) @@ -159,24 +195,47 @@ DllTransportPluginImpl::LoadDll(std::string_view Name) throw std::system_error(Ec, fmt::format("failed to load transport DLL from '{}'", DllPath)); } - TransportPlugin* CreateTransportPlugin(); + PfnGetTransportPluginVersion GetVersion = (PfnGetTransportPluginVersion)GetProcAddress(DllHandle, "GetTransportPluginVersion"); + PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin"); - PfnCreateTransportPlugin CreatePlugin = (PfnCreateTransportPlugin)GetProcAddress(DllHandle, "CreateTransportPlugin"); + uint32_t APIVersion = 0; + uint32_t PluginVersion = 0; - if (!CreatePlugin) + if (GetVersion) + { + GetVersion(&APIVersion, &PluginVersion); + } + + const bool bValidApiVersion = APIVersion == kTransportApiVersion; + + if (!GetVersion || !CreatePlugin || !bValidApiVersion) { std::error_code Ec = MakeErrorCodeFromLastError(); FreeLibrary(DllHandle); - throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath)); + if (GetVersion && !bValidApiVersion) + { + throw std::system_error( + Ec, + fmt::format("invalid API version '{}' detected in transport DLL loaded from '{}', supported API version '{}'", + APIVersion, + DllPath, + kTransportApiVersion)); + } + else + { + throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath)); + } } LoadedDll NewDll; NewDll.Name = Name; NewDll.LoadedFromPath = DllPath.c_str(); - NewDll.Plugin = CreatePlugin(); + NewDll.Logger = new DllTransportLogger(FileName); + NewDll.Logger->AddRef(); + NewDll.Plugin = CreatePlugin(NewDll.Logger); m_Transports.emplace_back(std::move(NewDll)); } diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index fecc1043a..9a12719c0 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -22,6 +22,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include #include #include +#include #include ZEN_THIRD_PARTY_INCLUDES_END @@ -349,6 +350,7 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "quiet"sv); @@ -514,6 +516,68 @@ ParseConfigFile(const std::filesystem::path& Path, } } +void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + throw std::runtime_error(fmt::format("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", + Path, + PluginInfo.dump())); + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + throw std::runtime_error( + fmt::format("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump())); + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + void ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) { @@ -546,6 +610,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) std::string ContentDir; std::string AbsLogFile; std::string ConfigFile; + std::string PluginsConfigFile; std::string OutputConfigFile; std::string BaseSnapshotDir; @@ -573,6 +638,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "Exit immediately after initialization is complete", cxxopts::value(ServerOptions.IsPowerCycle)); options.add_options()("config", "Path to Lua config file", cxxopts::value(ConfigFile)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value(PluginsConfigFile)); options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", @@ -1096,6 +1162,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); if (!BaseSnapshotDir.empty()) @@ -1129,6 +1196,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ParseConfigFile(ServerOptions.DataDir / "zen_cfg.lua", ServerOptions, Result, OutputConfigFile); } + if (!ServerOptions.PluginsConfigFile.empty()) + { + ParsePluginsConfigFile(ServerOptions.PluginsConfigFile, ServerOptions, ServerOptions.BasePort); + } + ValidateOptions(ServerOptions); } catch (const zen::OptionParseException& e) diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 6bd7aa357..bd277cd88 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -165,6 +165,7 @@ struct ZenServerOptions std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) std::filesystem::path AbsLogFile; // Absolute path to main log file std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) std::string ChildId; // Id assigned by parent process (used for lifetime management) std::string LogId; // Id for tagging log output -- cgit v1.2.3 From 675fb3bed15b88ec50db1a38443e0520a69b9313 Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Tue, 22 Apr 2025 17:03:54 +0200 Subject: review fixes --- src/zenhttp/transports/dlltransport.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index e9336f1be..dd05f6f0c 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -37,7 +37,7 @@ struct LoadedDll { std::string Name; std::filesystem::path LoadedFromPath; - DllTransportLogger* Logger; + DllTransportLogger* Logger = nullptr; Ref Plugin; }; -- cgit v1.2.3 From 6646a12bbffd349ad234a4265e339797e815493b Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 15:57:02 -0600 Subject: Replace container with namespaces --- src/zen/cmds/builds_cmd.cpp | 28 +++++++++++++-------- src/zen/cmds/builds_cmd.h | 7 +++--- src/zenutil/filebuildstorage.cpp | 5 ++-- src/zenutil/include/zenutil/buildstorage.h | 2 +- src/zenutil/jupiter/jupiterbuildstorage.cpp | 39 ++++++++++++++++------------- 5 files changed, 46 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 40cd8fa74..bdf349888 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8913,20 +8913,26 @@ BuildsCommand::BuildsCommand() m_Options.positional_help("verb"); // list-container - AddSystemOptions(m_ListContainerOptions); - AddCloudOptions(m_ListContainerOptions); - AddFileOptions(m_ListContainerOptions); - AddOutputOptions(m_ListContainerOptions); - AddZenFolderOptions(m_ListContainerOptions); - m_ListContainerOptions.add_options()("h,help", "Print help"); - m_ListContainerOptions.add_option("", + AddSystemOptions(m_ListNamespacesOptions); + AddCloudOptions(m_ListNamespacesOptions); + AddFileOptions(m_ListNamespacesOptions); + AddOutputOptions(m_ListNamespacesOptions); + AddZenFolderOptions(m_ListNamespacesOptions); + m_ListNamespacesOptions.add_options()("h,help", "Print help"); + m_ListNamespacesOptions.add_option("", + "", + "recursive", + "Enable fetch of buckets within namespaces also", + cxxopts::value(m_ListNamespacesRecursive), + ""); + m_ListNamespacesOptions.add_option("", "", "result-path", "Path to json or compactbinary to write query result to", cxxopts::value(m_ListResultPath), ""); - m_ListContainerOptions.parse_positional({"result-path"}); - m_ListContainerOptions.positional_help("result-path"); + m_ListNamespacesOptions.parse_positional({"result-path"}); + m_ListNamespacesOptions.positional_help("result-path"); // list AddSystemOptions(m_ListOptions); @@ -9612,7 +9618,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) try { - if (SubOption == &m_ListContainerOptions) + if (SubOption == &m_ListNamespacesOptions) { if (!m_ListResultPath.empty()) { @@ -9636,7 +9642,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); - CbObject Response = Storage.BuildStorage->ListContainers(); + CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); if (m_ListResultPath.empty()) { diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 3da1922c9..f44e758ae 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -84,8 +84,9 @@ private: std::string m_Verb; // list, upload, download - cxxopts::Options m_ListContainerOptions{"list-container", "List available build containers"}; - std::string m_ListContainerResultPath; + cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"}; + std::string m_ListNamespacesResultPath; + bool m_ListNamespacesRecursive = false; cxxopts::Options m_ListOptions{"list", "List available builds"}; std::string m_ListQueryPath; @@ -117,7 +118,7 @@ private: cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; std::vector m_BuildIds; - cxxopts::Options* m_SubCommands[9] = {&m_ListContainerOptions, + cxxopts::Options* m_SubCommands[9] = {&m_ListNamespacesOptions, &m_ListOptions, &m_UploadOptions, &m_DownloadOptions, diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index b77eb1d11..badfb4840 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -35,9 +35,10 @@ public: virtual ~FileBuildStorage() {} - virtual CbObject ListContainers() override + virtual CbObject ListNamespaces(bool bRecursive) override { - ZEN_TRACE_CPU("FileBuildStorage::ListContainers"); + ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces"); + ZEN_UNUSED(bRecursive); SimulateLatency(0, 0); diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 0785acd62..5422c837c 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -25,7 +25,7 @@ public: virtual ~BuildStorage() {} - virtual CbObject ListContainers() = 0; + virtual CbObject ListNamespaces(bool bRecursive = false) = 0; virtual CbObject ListBuilds(CbObject Query) = 0; virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; virtual CbObject GetBuild(const Oid& BuildId) = 0; diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index b2fabb43f..6ada72e1e 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -35,9 +35,9 @@ public: } virtual ~JupiterBuildStorage() {} - virtual CbObject ListContainers() override + virtual CbObject ListNamespaces(bool bRecursive) override { - ZEN_TRACE_CPU("Jupiter::ListContainers"); + ZEN_TRACE_CPU("Jupiter::ListNamespaces"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); @@ -45,9 +45,9 @@ public: AddStatistic(ListResult); if (!ListResult.Success) { - throw std::runtime_error(fmt::format("Failed listing containers: {} ({})", ListResult.Reason, ListResult.ErrorCode)); + throw std::runtime_error(fmt::format("Failed listing namespaces: {} ({})", ListResult.Reason, ListResult.ErrorCode)); } - CbObject NamespaceResponse = PayloadToCbObject("Failed listing containers"sv, ListResult.Response); + CbObject NamespaceResponse = PayloadToCbObject("Failed listing namespaces"sv, ListResult.Response); CbObjectWriter Response; Response.BeginArray("results"sv); @@ -59,25 +59,28 @@ public: Response.BeginObject(); Response.AddString("name", Namespace); - JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace); - AddStatistic(BucketsResult); - if (!BucketsResult.Success) + if (bRecursive) { - throw std::runtime_error( - fmt::format("Failed listing containers: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode)); - } - CbObject BucketResponse = PayloadToCbObject("Failed listing containers"sv, BucketsResult.Response); + JupiterResult BucketsResult = m_Session.ListBuildBuckets(Namespace); + AddStatistic(BucketsResult); + if (!BucketsResult.Success) + { + throw std::runtime_error( + fmt::format("Failed listing namespaces: {} ({})", BucketsResult.Reason, BucketsResult.ErrorCode)); + } + CbObject BucketResponse = PayloadToCbObject("Failed listing namespaces"sv, BucketsResult.Response); - Response.BeginArray("items"); - for (CbFieldView BucketField : BucketResponse["buckets"]) - { - std::string_view Bucket = BucketField.AsString(); - if (!Bucket.empty()) + Response.BeginArray("items"); + for (CbFieldView BucketField : BucketResponse["buckets"]) { - Response.AddString(Bucket); + std::string_view Bucket = BucketField.AsString(); + if (!Bucket.empty()) + { + Response.AddString(Bucket); + } } + Response.EndArray(); } - Response.EndArray(); Response.EndObject(); } -- cgit v1.2.3 From 45e81401ecf98b035bb3b022af0e2de9f5cffccf Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 16:22:02 -0600 Subject: Replacing list-container with list-namespaces --- src/zen/cmds/builds_cmd.cpp | 6 +++--- src/zen/cmds/builds_cmd.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index bdf349888..571050654 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8906,13 +8906,13 @@ BuildsCommand::BuildsCommand() m_Options.add_option("", "v", "verb", - "Verb for build - list-container, list, upload, download, diff, fetch-blob, validate-part", + "Verb for build - list-namespaces, list, upload, download, diff, fetch-blob, validate-part", cxxopts::value(m_Verb), ""); m_Options.parse_positional({"verb"}); m_Options.positional_help("verb"); - // list-container + // list-namespaces AddSystemOptions(m_ListNamespacesOptions); AddCloudOptions(m_ListNamespacesOptions); AddFileOptions(m_ListNamespacesOptions); @@ -9228,7 +9228,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help())); } - if (ToLower(m_Verb) != "list-container" && (m_Namespace.empty() || m_Bucket.empty())) + if (ToLower(m_Verb) != "list-namespaces" && (m_Namespace.empty() || m_Bucket.empty())) { throw zen::OptionParseException( fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index f44e758ae..6489a7564 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -14,7 +14,7 @@ class BuildsCommand : public CacheStoreCommand { public: static constexpr char Name[] = "builds"; - static constexpr char Description[] = "Manage builds - list, upload, download, diff"; + static constexpr char Description[] = "Manage builds - list, list-namespaces, upload, download, diff"; BuildsCommand(); ~BuildsCommand(); -- cgit v1.2.3 From 4f70ce781de168f78397c8ce03924cf9a3fb8a40 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 16:23:24 -0600 Subject: xmake precommit --- src/zen/cmds/builds_cmd.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 571050654..bc9fa51ac 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8926,11 +8926,11 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_ListNamespacesRecursive), ""); m_ListNamespacesOptions.add_option("", - "", - "result-path", - "Path to json or compactbinary to write query result to", - cxxopts::value(m_ListResultPath), - ""); + "", + "result-path", + "Path to json or compactbinary to write query result to", + cxxopts::value(m_ListResultPath), + ""); m_ListNamespacesOptions.parse_positional({"result-path"}); m_ListNamespacesOptions.positional_help("result-path"); -- cgit v1.2.3 From d892428a92a1cc13a01abe714e90a9c2d3eaeb4c Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 16:27:46 -0600 Subject: Remove list-namespaces from owning command description --- src/zen/cmds/builds_cmd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 6489a7564..f44e758ae 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -14,7 +14,7 @@ class BuildsCommand : public CacheStoreCommand { public: static constexpr char Name[] = "builds"; - static constexpr char Description[] = "Manage builds - list, list-namespaces, upload, download, diff"; + static constexpr char Description[] = "Manage builds - list, upload, download, diff"; BuildsCommand(); ~BuildsCommand(); -- cgit v1.2.3 From cf6c57b9ffb5eca184c4913c3fca9fabed90e8ba Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 16:32:27 -0600 Subject: Use SubOption instead of Verb to control flow --- src/zen/cmds/builds_cmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index bc9fa51ac..624bb2270 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9228,7 +9228,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help())); } - if (ToLower(m_Verb) != "list-namespaces" && (m_Namespace.empty() || m_Bucket.empty())) + if (SubOption != &m_ListNamespacesOptions && (m_Namespace.empty() || m_Bucket.empty())) { throw zen::OptionParseException( fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); -- cgit v1.2.3 From 2d6e62b7914c96e8156c4b47a5d8a3e2364a728f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 23 Apr 2025 09:57:51 +0200 Subject: =?UTF-8?q?make=20sure=20to=20call=20MakeSafeAbsolutePath=C3=8DnPl?= =?UTF-8?q?ace=20where=20appropriate=20(#363)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/zen/cmds/builds_cmd.cpp | 30 +++++++++++++++--------------- src/zen/cmds/workspaces_cmd.cpp | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 6607fd6d2..43e0ed689 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9183,7 +9183,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePath(m_SystemRootDir); + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); }; auto ParseStorageOptions = [&]() { @@ -9203,7 +9203,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); } - MakeSafeAbsolutePath(m_StoragePath); + MakeSafeAbsolutePathÍnPlace(m_StoragePath); }; std::unique_ptr Auth; @@ -9291,7 +9291,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - MakeSafeAbsolutePath(m_AccessTokenPath); + MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath); std::string ResolvedAccessToken = ReadAccessTokenFromFile(m_AccessTokenPath); if (!ResolvedAccessToken.empty()) { @@ -9585,7 +9585,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); } - MakeSafeAbsolutePath(m_Path); + MakeSafeAbsolutePathÍnPlace(m_Path); }; auto ParseDiffPath = [&]() { @@ -9593,7 +9593,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - MakeSafeAbsolutePath(m_DiffPath); + MakeSafeAbsolutePathÍnPlace(m_DiffPath); }; auto ParseBlobHash = [&]() -> IoHash { @@ -9668,7 +9668,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildMetadataPath.empty()) { - MakeSafeAbsolutePath(m_BuildMetadataPath); + MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -9717,8 +9717,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (SubOption == &m_ListOptions) { - MakeSafeAbsolutePath(m_ListQueryPath); - MakeSafeAbsolutePath(m_ListResultPath); + MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); + MakeSafeAbsolutePathÍnPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -9759,7 +9759,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { @@ -9827,7 +9827,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { @@ -9910,7 +9910,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -9956,7 +9956,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { @@ -10001,7 +10001,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { @@ -10036,7 +10036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10124,7 +10124,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePath(m_ZenFolderPath); + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 2ddd0c73a..773734f12 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -400,7 +400,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } else { - MakeSafeAbsolutePath(m_SystemRootDir); + MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); } std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; -- cgit v1.2.3 From 199553c86444dcd9c4cff91165938dbd9a5b4bbb Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Wed, 23 Apr 2025 10:54:22 +0200 Subject: Make plugin loading errors non fatal (#364) make plugin loading errors non fatal --- src/zenhttp/httpserver.cpp | 48 ++++++++++++++++++++++----------- src/zenhttp/transports/dlltransport.cpp | 25 ++++++++--------- src/zenhttp/transports/dlltransport.h | 2 +- 3 files changed, 46 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index d12f89a92..764f2a2a7 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -831,7 +831,6 @@ CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) const std::string& PluginName = PluginConfig.PluginName; ZEN_INFO("using '{}' plugin HTTP server implementation", PluginName) - Ref Server{CreateHttpPluginServer()}; if (PluginName.starts_with("builtin:"sv)) { @@ -845,28 +844,41 @@ CreateHttpServerPlugin(const HttpServerPluginConfig& PluginConfig) { Plugin = CreateAsioTransportPlugin(); } + else + { + ZEN_WARN("Unknown builtin plugin '{}'", PluginName) + return {}; + } + + ZEN_ASSERT(!Plugin.IsNull()); - if (!Plugin.IsNull()) + for (const std::pair& Option : PluginConfig.PluginOptions) { - for (const std::pair& Option : PluginConfig.PluginOptions) - { - Plugin->Configure(Option.first.c_str(), Option.second.c_str()); - } - Server->AddPlugin(Plugin); + Plugin->Configure(Option.first.c_str(), Option.second.c_str()); } + + Ref Server{CreateHttpPluginServer()}; + Server->AddPlugin(Plugin); + return Server; +# else + ZEN_WARN("Builtin plugin '{}' is not supported", PluginName) + return {}; # endif } - else + + Ref DllPlugin{CreateDllTransportPlugin()}; + if (!DllPlugin->LoadDll(PluginName)) { - Ref DllPlugin{CreateDllTransportPlugin()}; - DllPlugin->LoadDll(PluginName); - for (const std::pair& Option : PluginConfig.PluginOptions) - { - DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); - } - Server->AddPlugin(DllPlugin); + return {}; + } + + for (const std::pair& Option : PluginConfig.PluginOptions) + { + DllPlugin->ConfigureDll(PluginName, Option.first.c_str(), Option.second.c_str()); } + Ref Server{CreateHttpPluginServer()}; + Server->AddPlugin(DllPlugin); return Server; } #endif @@ -888,7 +900,11 @@ CreateHttpServer(const HttpServerConfig& Config) for (const HttpServerPluginConfig& PluginConfig : Config.PluginConfigs) { - Server->AddServer(CreateHttpServerPlugin(PluginConfig)); + Ref PluginServer = CreateHttpServerPlugin(PluginConfig); + if (!PluginServer.IsNull()) + { + Server->AddServer(PluginServer); + } } return Server; diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index dd05f6f0c..fb3dd23b5 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -55,7 +55,7 @@ public: virtual const char* GetDebugName() override; virtual bool IsAvailable() override; - virtual void LoadDll(std::string_view Name) override; + virtual bool LoadDll(std::string_view Name) override; virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) override; private: @@ -172,7 +172,7 @@ DllTransportPluginImpl::ConfigureDll(std::string_view Name, const char* OptionTa } } -void +bool DllTransportPluginImpl::LoadDll(std::string_view Name) { RwLock::ExclusiveLockScope _(m_Lock); @@ -190,9 +190,8 @@ DllTransportPluginImpl::LoadDll(std::string_view Name) if (!DllHandle) { - std::error_code Ec = MakeErrorCodeFromLastError(); - - throw std::system_error(Ec, fmt::format("failed to load transport DLL from '{}'", DllPath)); + ZEN_WARN("Failed to load transport DLL from '{}' due to '{}'", DllPath, GetLastErrorAsString()) + return false; } PfnGetTransportPluginVersion GetVersion = (PfnGetTransportPluginVersion)GetProcAddress(DllHandle, "GetTransportPluginVersion"); @@ -216,17 +215,18 @@ DllTransportPluginImpl::LoadDll(std::string_view Name) if (GetVersion && !bValidApiVersion) { - throw std::system_error( - Ec, - fmt::format("invalid API version '{}' detected in transport DLL loaded from '{}', supported API version '{}'", - APIVersion, - DllPath, - kTransportApiVersion)); + ZEN_WARN("Failed to load transport DLL from '{}' due to invalid API version {}, supported API version is {}", + DllPath, + APIVersion, + kTransportApiVersion) } else { - throw std::system_error(Ec, fmt::format("API mismatch detected in transport DLL loaded from '{}'", DllPath)); + ZEN_WARN("Failed to load transport DLL from '{}' due to not finding GetTransportPluginVersion or CreateTransportPlugin", + DllPath) } + + return false; } LoadedDll NewDll; @@ -238,6 +238,7 @@ DllTransportPluginImpl::LoadDll(std::string_view Name) NewDll.Plugin = CreatePlugin(NewDll.Logger); m_Transports.emplace_back(std::move(NewDll)); + return true; } DllTransportPlugin* diff --git a/src/zenhttp/transports/dlltransport.h b/src/zenhttp/transports/dlltransport.h index 9346a10ce..c49f888da 100644 --- a/src/zenhttp/transports/dlltransport.h +++ b/src/zenhttp/transports/dlltransport.h @@ -15,7 +15,7 @@ namespace zen { class DllTransportPlugin : public TransportPlugin { public: - virtual void LoadDll(std::string_view Name) = 0; + virtual bool LoadDll(std::string_view Name) = 0; virtual void ConfigureDll(std::string_view Name, const char* OptionTag, const char* OptionValue) = 0; }; -- cgit v1.2.3 From 37d3a1b76ef256431f3883853fd625dc33b7a991 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 23 Apr 2025 14:14:27 +0200 Subject: parse system dir for builds (#365) * make sure we always parse system options for zen builds command * make MakeSafeAbsolutePath nodiscard --- src/zen/cmds/builds_cmd.cpp | 2 +- src/zenutil/include/zenutil/commandlineoptions.h | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 43e0ed689..b113ce6d1 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9185,6 +9185,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); }; + ParseSystemOptions(); auto ParseStorageOptions = [&]() { if (!m_OverrideHost.empty() || !m_Host.empty()) @@ -9244,7 +9245,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseAuthOptions = [&]() { - ParseSystemOptions(); if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) { CreateAuthMgr(); diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h index 3afbac854..b7581f6cd 100644 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -17,11 +17,11 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -std::vector ParseCommandLine(std::string_view CommandLine); -std::vector StripCommandlineQuotes(std::vector& InOutArgs); -void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); -std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); -std::filesystem::path StringToPath(const std::string_view& Path); +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); +void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); +std::filesystem::path StringToPath(const std::string_view& Path); void commandlineoptions_forcelink(); // internal -- cgit v1.2.3 From 2ec01ef9c6e07b77afa33fe3db6335f231c78006 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 23 Apr 2025 17:16:16 +0200 Subject: zen wipe command (#366) - Feature: New `zen wipe` command for fast cleaning of directories, it will not remove the directory itself, only the content - `--directory` - path to directory to wipe, if the directory does not exist or is empty, no action will be taken - `--keep-readonly` - skip removal of read-only files found in directory, defaults to `true`, set to `false` to remove read-only files - `--quiet` - reduce output to console, defaults to `false` - `--dryrun` - simulate the wipe without removing anything, defaults to `false` - `--yes` - skips prompt to confirm wipe of directory - `--plain-progress` - show progress using plain output - `--verbose` - enable verbose console output - `--boost-workers` - increase the number of worker threads, may cause computer to be less responsive, defaults to `false` --- src/zen/cmds/builds_cmd.cpp | 2 +- src/zen/cmds/wipe_cmd.cpp | 575 ++++++++++++++++++++++++++++++++++++++++++++ src/zen/cmds/wipe_cmd.h | 36 +++ src/zen/zen.cpp | 3 + 4 files changed, 615 insertions(+), 1 deletion(-) create mode 100644 src/zen/cmds/wipe_cmd.cpp create mode 100644 src/zen/cmds/wipe_cmd.h (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b113ce6d1..6c97645cd 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8873,7 +8873,7 @@ BuildsCommand::BuildsCommand() Ops.add_option("", "", "boost-workers", - "Increase the number of worker threads - may cause computer to less responsive", + "Increase the number of worker threads - may cause computer to be less responsive", cxxopts::value(m_BoostWorkerThreads), ""); }; diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp new file mode 100644 index 000000000..2b4e9ab3c --- /dev/null +++ b/src/zen/cmds/wipe_cmd.cpp @@ -0,0 +1,575 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "wipe_cmd.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +# include +# include +# include +#endif + +namespace zen { + +namespace { + static std::atomic AbortFlag = false; + static bool IsVerbose = false; + static bool Quiet = false; + static bool UsePlainProgress = false; + const bool SingleThreaded = false; + bool BoostWorkerThreads = true; + + WorkerThreadPool& GetIOWorkerPool() + { + return SingleThreaded ? GetSyncWorkerPool() + : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst) + : GetMediumWorkerPool(EWorkloadType::Burst); + } + +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ + } + + static void SignalCallbackHandler(int SigNum) + { + if (SigNum == SIGINT) + { + AbortFlag = true; + } +#if ZEN_PLATFORM_WINDOWS + if (SigNum == SIGBREAK) + { + AbortFlag = true; + } +#endif // ZEN_PLATFORM_WINDOWS + } + + bool IsReadOnly(uint32_t Attributes) + { +#if ZEN_PLATFORM_WINDOWS + return IsFileAttributeReadOnly(Attributes); +#else + return IsFileModeReadOnly(Attributes); +#endif + } + + bool IsFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + bool Result = IsFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + Result = IsFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFileWithRetry(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFileWithRetry(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool CleanDirectory(const std::filesystem::path& Path, + std::span ExcludeDirectories, + bool RemoveReadonly, + bool Dryrun) + { + ZEN_TRACE_CPU("CleanDirectory"); + Stopwatch Timer; + + ProgressBar Progress(UsePlainProgress); + + std::atomic CleanWipe = true; + std::atomic DiscoveredItemCount = 0; + std::atomic DeletedItemCount = 0; + std::atomic DeletedByteCount = 0; + std::atomic FailedDeleteCount = 0; + + std::vector SubdirectoriesToDelete; + tsl::robin_map SubdirectoriesToDeleteLookup; + tsl::robin_set SubdirectoriesToKeep; + RwLock SubdirectoriesLock; + + auto AddFoundDirectory = [&](std::filesystem::path Directory, bool Keep) -> bool { + bool Added = false; + if (Keep) + { + bool IsLeaf = true; + while (Directory != Path) + { + const std::string DirectoryString = Directory.generic_string(); + IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length()); + RwLock::ExclusiveLockScope _(SubdirectoriesLock); + if (auto It = SubdirectoriesToKeep.find(DirectoryNameHash); It == SubdirectoriesToKeep.end()) + { + SubdirectoriesToKeep.insert(DirectoryNameHash); + if (IsLeaf) + { + Added = true; + } + } + else + { + break; + } + Directory = Directory.parent_path(); + IsLeaf = false; + } + } + else + { + bool IsLeaf = true; + while (Directory != Path) + { + const std::string DirectoryString = Directory.generic_string(); + IoHash DirectoryNameHash = IoHash::HashBuffer(DirectoryString.data(), DirectoryString.length()); + RwLock::ExclusiveLockScope _(SubdirectoriesLock); + if (SubdirectoriesToKeep.contains(DirectoryNameHash)) + { + break; + } + if (auto It = SubdirectoriesToDeleteLookup.find(DirectoryNameHash); It == SubdirectoriesToDeleteLookup.end()) + { + SubdirectoriesToDeleteLookup.insert({DirectoryNameHash, SubdirectoriesToDelete.size()}); + SubdirectoriesToDelete.push_back(Directory); + if (IsLeaf) + { + Added = true; + } + } + else + { + break; + } + Directory = Directory.parent_path(); + IsLeaf = false; + } + } + return Added; + }; + + ParallellWork Work(AbortFlag); + + struct AsyncVisitor : public GetDirectoryContentVisitor + { + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic& InCleanWipe, + std::atomic& InDiscoveredItemCount, + std::atomic& InDeletedItemCount, + std::atomic& InDeletedByteCount, + std::atomic& InFailedDeleteCount, + std::span InExcludeDirectories, + bool InRemoveReadonly, + bool InDryrun, + const std::function& InAddFoundDirectoryFunc) + : Path(InPath) + , CleanWipe(InCleanWipe) + , DiscoveredItemCount(InDiscoveredItemCount) + , DeletedItemCount(InDeletedItemCount) + , DeletedByteCount(InDeletedByteCount) + , FailedDeleteCount(InFailedDeleteCount) + , ExcludeDirectories(InExcludeDirectories) + , RemoveReadonly(InRemoveReadonly) + , Dryrun(InDryrun) + , AddFoundDirectoryFunc(InAddFoundDirectoryFunc) + { + } + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override + { + ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); + if (!AbortFlag) + { + if (!RelativeRoot.empty()) + { + DiscoveredItemCount++; + } + if (!Content.FileNames.empty()) + { + DiscoveredItemCount += Content.FileNames.size(); + + const std::string RelativeRootString = RelativeRoot.generic_string(); + bool RemoveContent = true; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (RelativeRootString.starts_with(ExcludeDirectory)) + { + if (RelativeRootString.length() > ExcludeDirectory.length()) + { + const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; + if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || + MaybePathDelimiter == std::filesystem::path::preferred_separator) + { + RemoveContent = false; + break; + } + } + else + { + RemoveContent = false; + break; + } + } + } + + const std::filesystem::path ParentPath = Path / RelativeRoot; + bool KeepDirectory = RelativeRoot.empty(); + + if (RemoveContent) + { + ZEN_TRACE_CPU("DeleteFiles"); + uint64_t RemovedCount = 0; + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (ParentPath / FileName).make_preferred(); + try + { + const uint32_t Attributes = Content.FileAttributes[FileIndex]; + const bool IsReadonly = IsReadOnly(Attributes); + bool RemoveFile = false; + if (IsReadonly) + { + if (RemoveReadonly) + { + if (!Dryrun) + { + SetFileReadOnlyWithRetry(FilePath, false); + } + RemoveFile = true; + } + } + else + { + RemoveFile = true; + } + + if (RemoveFile) + { + if (!Dryrun) + { + RemoveFileWithRetry(FilePath); + } + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + RemovedCount++; + ZEN_CONSOLE_VERBOSE("Removed file {}", FilePath); + } + else + { + ZEN_CONSOLE_VERBOSE("Skipped readonly file {}", FilePath); + KeepDirectory = true; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + FailedDeleteCount++; + CleanWipe = false; + KeepDirectory = true; + } + } + ZEN_CONSOLE_VERBOSE("Removed {} files in {}", RemovedCount, ParentPath); + } + else + { + ZEN_CONSOLE_VERBOSE("Skipped removal of {} files in {}", Content.FileNames.size(), ParentPath); + } + bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory); + if (Added) + { + ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath); + } + } + } + } + const std::filesystem::path& Path; + std::atomic& CleanWipe; + std::atomic& DiscoveredItemCount; + std::atomic& DeletedItemCount; + std::atomic& DeletedByteCount; + std::atomic& FailedDeleteCount; + std::span ExcludeDirectories; + const bool RemoveReadonly; + const bool Dryrun; + std::function AddFoundDirectoryFunc; + } Visitor(Path, + CleanWipe, + DiscoveredItemCount, + DeletedItemCount, + DeletedByteCount, + FailedDeleteCount, + ExcludeDirectories, + RemoveReadonly, + Dryrun, + AddFoundDirectory); + + uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + GetDirectoryContent(Path, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | + DirectoryContentFlags::IncludeFileSizes | DirectoryContentFlags::IncludeAttributes, + Visitor, + GetIOWorkerPool(), + Work.PendingWork()); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + if (Quiet) + { + return; + } + ZEN_UNUSED(IsAborted, PendingWork); + LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Removing files ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); + }); + + std::vector DirectoriesToDelete; + DirectoriesToDelete.reserve(SubdirectoriesToDelete.size()); + for (auto It : SubdirectoriesToDeleteLookup) + { + const IoHash& DirHash = It.first; + if (auto KeepIt = SubdirectoriesToKeep.find(DirHash); KeepIt == SubdirectoriesToKeep.end()) + { + DirectoriesToDelete.emplace_back(std::move(SubdirectoriesToDelete[It.second])); + } + } + + std::sort(DirectoriesToDelete.begin(), + DirectoriesToDelete.end(), + [](const std::filesystem::path& Lhs, const std::filesystem::path& Rhs) { + return Lhs.string().length() > Rhs.string().length(); + }); + + for (size_t SubDirectoryIndex = 0; SubDirectoryIndex < DirectoriesToDelete.size(); SubDirectoryIndex++) + { + ZEN_TRACE_CPU("DeleteDirs"); + const std::filesystem::path& DirectoryToDelete = DirectoriesToDelete[SubDirectoryIndex]; + try + { + if (!Dryrun) + { + RemoveDirWithRetry(DirectoryToDelete); + } + ZEN_CONSOLE_VERBOSE("Removed directory {}", DirectoryToDelete); + DeletedItemCount++; + } + catch (const std::exception& Ex) + { + if (!Quiet) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + } + CleanWipe = false; + FailedDeleteCount++; + } + + uint64_t NowMs = Timer.GetElapsedTimeMs(); + if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + { + LastUpdateTimeMs = NowMs; + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Removing folders", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = DirectoriesToDelete.size(), + .RemainingCount = DirectoriesToDelete.size() - SubDirectoryIndex}, + false); + } + } + + Progress.Finish(); + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!Quiet) + { + ZEN_CONSOLE("Wiped folder '{}' {} ({}) ({} failed) in {}", + Path, + DeletedItemCount.load(), + NiceBytes(DeletedByteCount.load()), + FailedDeleteCount.load(), + NiceTimeSpanMs(ElapsedTimeMs)); + } + if (FailedDeleteCount.load() > 0) + { + throw std::runtime_error(fmt::format("Failed to delete {} files/directories in '{}'", FailedDeleteCount.load(), Path)); + } + return CleanWipe; + } +} // namespace + +WipeCommand::WipeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "d", "directory", "Directory to wipe", cxxopts::value(m_Directory), ""); + m_Options.add_option("", "r", "keep-readonly", "Leave read-only files", cxxopts::value(m_KeepReadOnlyFiles), ""); + m_Options.add_option("", "q", "quiet", "Reduce output to console", cxxopts::value(m_Quiet), ""); + m_Options.add_option("", "y", "yes", "Don't query for confirmation", cxxopts::value(m_Yes), ""); + m_Options.add_option("", "", "dryrun", "Do a dry run without deleting anything", cxxopts::value(m_Dryrun), ""); + m_Options.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); + m_Options.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); + m_Options.add_option("", + "", + "boost-workers", + "Increase the number of worker threads - may cause computer to be less responsive", + cxxopts::value(m_BoostWorkerThreads), + ""); + + m_Options.parse_positional({"directory"}); +} + +WipeCommand::~WipeCommand() = default; + +int +WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + signal(SIGINT, SignalCallbackHandler); +#if ZEN_PLATFORM_WINDOWS + signal(SIGBREAK, SignalCallbackHandler); +#endif // ZEN_PLATFORM_WINDOWS + + if (!ZenCmdBase::ParseOptions(argc, argv)) + { + return 0; + } + + Quiet = m_Quiet; + IsVerbose = m_Verbose; + UsePlainProgress = IsVerbose || m_PlainProgress; + BoostWorkerThreads = m_BoostWorkerThreads; + + MakeSafeAbsolutePathÍnPlace(m_Directory); + + if (!IsDir(m_Directory)) + { + return 0; + } + + while (!m_Yes) + { + const std::string Prompt = fmt::format("Do you want to wipe directory '{}'? (yes/no) ", m_Directory); + printf("%s", Prompt.c_str()); + std::string Reponse; + std::getline(std::cin, Reponse); + Reponse = ToLower(Reponse); + if (Reponse == "y" || Reponse == "yes") + { + m_Yes = true; + } + else if (Reponse == "n" || Reponse == "no") + { + return 0; + } + } + + try + { + CleanDirectory(m_Directory, {}, !m_KeepReadOnlyFiles, m_Dryrun); + } + catch (std::exception& Ex) + { + if (!m_Quiet) + { + ZEN_ERROR("{}", Ex.what()); + } + return 3; + } + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/wipe_cmd.h b/src/zen/cmds/wipe_cmd.h new file mode 100644 index 000000000..0e910bb81 --- /dev/null +++ b/src/zen/cmds/wipe_cmd.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** Wipe directories + */ +class WipeCommand : public ZenCmdBase +{ +public: + static constexpr char Name[] = "wipe"; + static constexpr char Description[] = "Wipe the contents of a directory"; + + WipeCommand(); + ~WipeCommand(); + + virtual cxxopts::Options& Options() override { return m_Options; } + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::filesystem::path m_Directory; + bool m_KeepReadOnlyFiles = true; + bool m_Quiet = false; + bool m_Yes = false; + bool m_PlainProgress = false; + bool m_Verbose = false; + bool m_Dryrun = false; + bool m_BoostWorkerThreads = false; +}; + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 5ce0a89ec..e442f8a4b 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -23,6 +23,7 @@ #include "cmds/up_cmd.h" #include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" +#include "cmds/wipe_cmd.h" #include "cmds/workspaces_cmd.h" #include @@ -557,6 +558,7 @@ main(int argc, char** argv) UpCommand UpCmd; VersionCommand VersionCmd; VfsCommand VfsCmd; + WipeCommand WipeCmd; WorkspaceCommand WorkspaceCmd; WorkspaceShareCommand WorkspaceShareCmd; @@ -613,6 +615,7 @@ main(int argc, char** argv) {"version", &VersionCmd, "Get zen server version"}, {"vfs", &VfsCmd, "Manage virtual file system"}, {"flush", &FlushCmd, "Flush storage"}, + {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, {WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description}, {WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description}, // clang-format on -- cgit v1.2.3 From 66a6f6726dc5ff495f3000e745d6e13b9a989dd4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 24 Apr 2025 09:54:08 +0200 Subject: predicate to enable compiling with later EASTL version (#367) --- src/zenstore/cache/cachedisklayer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index e4d962b56..4f72a711a 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -3572,7 +3572,7 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer() } template -struct equal_to_2 : public eastl::binary_function +struct equal_to_2 { constexpr bool operator()(const T& a, const U& b) const { return a == b; } -- cgit v1.2.3 From 1301b12c57206df23886b004bcfbc21cac5e953a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Apr 2025 12:52:00 +0200 Subject: add retry on internal error / bad gateway (#370) * do http client retry on internal error and bad gateway --- src/zenhttp/httpclient.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 763f3262a..ca1b820c9 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -365,9 +365,10 @@ ShouldRetry(const cpr::Response& Response) { case cpr::ErrorCode::OK: break; - case cpr::ErrorCode::OPERATION_TIMEDOUT: + case cpr::ErrorCode::INTERNAL_ERROR: case cpr::ErrorCode::NETWORK_RECEIVE_ERROR: case cpr::ErrorCode::NETWORK_SEND_FAILURE: + case cpr::ErrorCode::OPERATION_TIMEDOUT: return true; default: return false; @@ -377,6 +378,7 @@ ShouldRetry(const cpr::Response& Response) case HttpResponseCode::RequestTimeout: case HttpResponseCode::TooManyRequests: case HttpResponseCode::InternalServerError: + case HttpResponseCode::BadGateway: case HttpResponseCode::ServiceUnavailable: case HttpResponseCode::GatewayTimeout: return true; -- cgit v1.2.3 From 70f7fdfc433811c92ec47a32665b5a6a676d51e8 Mon Sep 17 00:00:00 2001 From: Dmytro Ivanov Date: Thu, 24 Apr 2025 13:09:08 +0200 Subject: close servers in multi server (#371) --- src/zenhttp/servers/httpmulti.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 2a6a90d2e..f4dc1e15b 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -103,6 +103,10 @@ HttpMultiServer::RequestExit() void HttpMultiServer::Close() { + for (auto& Server : m_Servers) + { + Server->Close(); + } } void -- cgit v1.2.3 From a6f5506c2be0d30db37a2b8a51d0bd221e984fbc Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Apr 2025 14:26:23 +0200 Subject: fix buildstore disksizelimit lua config name (#372) --- src/zenserver/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 9a12719c0..e81e8eb54 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -361,7 +361,7 @@ ParseConfigFile(const std::filesystem::path& Path, ////// buildsstore LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); - LuaOptions.AddOption("buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); -- cgit v1.2.3 From c0003b3e259dafbef71144cdb5353fd531946db4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Apr 2025 15:33:41 +0200 Subject: use state file if available when doing builds diff command (#369) * use state file if available when doing builds diff command * remove dead code --- src/zen/cmds/builds_cmd.cpp | 143 +++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 81 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 6c97645cd..855d7012f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -672,75 +672,6 @@ namespace { return CacheFolderPath / RawHash.ToHexString(); } - ChunkedFolderContent ScanAndChunkFolder( - GetFolderContentStatistics& GetFolderContentStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - std::function&& IsAcceptedFolder, - std::function&& IsAcceptedFile, - ChunkingController& ChunkController) - { - ZEN_TRACE_CPU("ScanAndChunkFolder"); - - FolderContent Content = GetFolderContent( - GetFolderContentStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, - [](bool, std::ptrdiff_t) {}, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - ProgressBar ProgressBar(UsePlainProgress); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent FolderContent = ChunkFolderContent( - ChunkingStats, - GetIOWorkerPool(), - Path, - Content, - ChunkController, - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - GetFolderContentStats.AcceptedFileCount.load(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(GetFolderContentStats.FoundFileByteCount), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = GetFolderContentStats.AcceptedFileByteCount, - .RemainingCount = GetFolderContentStats.AcceptedFileByteCount - ChunkingStats.BytesHashed.load()}, - false); - }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - - ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", - ChunkingStats.FilesProcessed.load(), - NiceBytes(ChunkingStats.BytesHashed.load()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load()), - Path, - NiceTimeSpanMs((GetFolderContentStats.ElapsedWallTimeUS + ChunkingStats.ElapsedWallTimeUS) / 1000), - NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); - return FolderContent; - }; - struct DiskStatistics { std::atomic OpenReadCount = 0; @@ -8158,13 +8089,13 @@ namespace { return RemoteContent; } - ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, - ChunkingStatistics& ChunkingStats, - const std::filesystem::path& Path, - const std::filesystem::path& StateFilePath, - ChunkingController& ChunkController, - const ChunkedFolderContent& ReferenceContent, - FolderContent& OutLocalFolderContent) + ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + const std::filesystem::path& StateFilePath, + ChunkingController& ChunkController, + std::span ReferencePaths, + FolderContent& OutLocalFolderContent) { FolderContent LocalFolderState; ChunkedFolderContent LocalContent; @@ -8176,7 +8107,7 @@ namespace { ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); } { - const uint32_t LocalPathCount = gsl::narrow(ReferenceContent.Paths.size()); + const uint32_t LocalPathCount = gsl::narrow(ReferencePaths.size()); const uint32_t RemotePathCount = gsl::narrow(LocalFolderState.Paths.size()); std::vector PathsToCheck; @@ -8191,7 +8122,7 @@ namespace { PathsToCheck.push_back(LocalPath); } - for (const std::filesystem::path& RemotePath : ReferenceContent.Paths) + for (const std::filesystem::path& RemotePath : ReferencePaths) { if (FileSet.insert(RemotePath.generic_string()).second) { @@ -8334,6 +8265,56 @@ namespace { return LocalContent; } + ChunkedFolderContent ScanAndChunkFolder( + GetFolderContentStatistics& GetFolderContentStats, + ChunkingStatistics& ChunkingStats, + const std::filesystem::path& Path, + std::function&& IsAcceptedFolder, + std::function&& IsAcceptedFile, + ChunkingController& ChunkController) + { + Stopwatch Timer; + + ZEN_TRACE_CPU("ScanAndChunkFolder"); + + FolderContent Content = GetFolderContent( + GetFolderContentStats, + Path, + std::move(IsAcceptedFolder), + std::move(IsAcceptedFile), + GetIOWorkerPool(), + UsePlainProgress ? 5000 : 200, + [](bool, std::ptrdiff_t) {}, + AbortFlag); + if (AbortFlag) + { + return {}; + } + + FolderContent _; + ChunkedFolderContent Result = GetLocalContent(GetFolderContentStats, + ChunkingStats, + Path, + ZenStateFilePath(Path / ZenFolderName), + ChunkController, + Content.Paths, + _); + + const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0)); + const uint64_t ChunkedRawSize = + std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0)); + + ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", + Result.Paths.size(), + NiceBytes(TotalRawSize), + Result.ChunkedContent.ChunkHashes.size(), + NiceBytes(ChunkedRawSize), + Path, + NiceTimeSpanMs(Timer.GetElapsedTimeMs()), + NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed))); + return Result; + }; + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector& BuildPartIds, @@ -8398,7 +8379,7 @@ namespace { Path, ZenStateFilePath(ZenFolderPath), *ChunkController, - RemoteContent, + RemoteContent.Paths, LocalFolderContent); } } @@ -8695,7 +8676,7 @@ namespace { double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0; - ZEN_CONSOLE("{} ({}) files removed, {} ({}) files added, {} ({} {:.1f}%) files kept", + ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept", RemovedHashes.size(), NiceBytes(RemovedSize), AddedHashes.size(), @@ -8730,7 +8711,7 @@ namespace { double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0; double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0; - ZEN_CONSOLE("Found {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", + ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.", FoundChunkCount, NiceBytes(FoundChunkSize), FoundPercent, -- cgit v1.2.3 From 0a3d04458199a1bfe2c60fc7ce174dbea20713a6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 24 Apr 2025 15:45:34 +0200 Subject: limit retries on buildpart finalize (#374) * limit retries on buildpart finalize --- src/zen/cmds/builds_cmd.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 855d7012f..f50f6205b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2475,7 +2475,7 @@ namespace { BlockIndexes.push_back(It->second); TotalBlocksSize += NewBlocks.BlockSizes[It->second]; } - if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) + else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end()) { const uint32_t ChunkIndex = ChunkIndexIt->second; if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex); @@ -2485,6 +2485,11 @@ namespace { TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; } } + else + { + throw std::runtime_error( + fmt::format("Can not upload requested build blob {} as it was not generated by this upload", RawHash)); + } } uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize; @@ -3644,7 +3649,8 @@ namespace { UploadAttachments(PutBuildPartResult.second); } - while (!AbortFlag) + uint32_t FinalizeBuildPartRetryCount = 5; + while (!AbortFlag && (FinalizeBuildPartRetryCount--) > 0) { Stopwatch FinalizeBuildPartTimer; std::vector Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); -- cgit v1.2.3 From a1c40f05af26f374a5d1c5a7b5993b09d3c2fda8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 25 Apr 2025 11:06:22 +0200 Subject: fix race in rpcrecorder (#375) --- src/zenutil/cache/rpcrecording.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src') diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 380c182b2..46e80f6b7 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -366,6 +366,7 @@ private: }; std::unique_ptr m_WriterThread; + std::atomic_bool m_IsWriterReady{false}; std::atomic_bool m_IsActive{false}; std::atomic_int64_t m_PendingRequests{0}; RwLock m_RequestQueueLock; @@ -658,6 +659,8 @@ RecordedRequestsWriter::BeginWrite(const std::filesystem::path& BasePath) m_IsActive = true; m_WriterThread.reset(new std::thread(&RecordedRequestsWriter::WriterThreadMain, this)); + + m_IsWriterReady.wait(false); } void @@ -707,6 +710,9 @@ RecordedRequestsWriter::WriterThreadMain() SetCurrentThreadName("rpc_writer"); EnsureCurrentSegment(); + m_IsWriterReady.store(true); + m_IsWriterReady.notify_all(); + while (m_IsActive) { m_PendingRequests.wait(0); -- cgit v1.2.3 From d0d065d70428a3ef699a1d125f493d79683b3295 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 25 Apr 2025 12:33:47 +0200 Subject: replace local equal_to_2 with eastl impl (#368) --- src/zenstore/cache/cachedisklayer.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 4f72a711a..da5038984 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -3571,20 +3571,6 @@ ZenCacheDiskLayer::~ZenCacheDiskLayer() } } -template -struct equal_to_2 -{ - constexpr bool operator()(const T& a, const U& b) const { return a == b; } - - template, eastl::remove_const_t>>> - constexpr bool operator()(const U& b, const T& a) const - { - return b == a; - } -}; - ZenCacheDiskLayer::CacheBucket* ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) { @@ -3592,7 +3578,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) { RwLock::SharedLockScope SharedLock(m_Lock); - if (auto It = m_Buckets.find_as(InBucket, std::hash(), equal_to_2()); + if (auto It = m_Buckets.find_as(InBucket, std::hash(), eastl::equal_to_2()); It != m_Buckets.end()) { return It->second.get(); @@ -3604,7 +3590,7 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) std::unique_ptr Bucket(std::make_unique(m_Gc, m_TotalMemCachedSize, InBucket, m_Configuration.BucketConfig)); RwLock::ExclusiveLockScope Lock(m_Lock); - if (auto It = m_Buckets.find_as(InBucket, std::hash(), equal_to_2()); + if (auto It = m_Buckets.find_as(InBucket, std::hash(), eastl::equal_to_2()); It != m_Buckets.end()) { return It->second.get(); -- cgit v1.2.3 From 8d927c40cf10361870c1b42384f4f00d6cc2b10c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 2 May 2025 14:15:55 +0200 Subject: cbobject validation (#377) * validate incoming CbObject to cache when receiving a package * validate records when fetched from store in cache before parsing them --- src/zenstore/cache/cacherpc.cpp | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 97e26a38d..bf78dae86 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -220,6 +220,11 @@ CacheRpcHandler::HandleRpcRequest(const CacheRequestContext& Context, ZEN_WARN("Content format not supported, expected package message format"); return RpcResponseCode::BadRequest; } + if (CbValidateError Error = ValidateCompactBinary(Object.GetView(), CbValidateMode::Default); Error != CbValidateError::None) + { + ZEN_WARN("Content format is corrupt, compact binary format validation failed. Reason: '{}'", ToString(Error)); + return RpcResponseCode::BadRequest; + } } if (!UriNamespace.empty()) @@ -558,6 +563,13 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb { FoundLocalInvalid = true; } + else if (CbValidateError Error = ValidateCompactBinary(Request.RecordCacheValue.GetView(), CbValidateMode::Default); + Error != CbValidateError::None) + { + ZEN_WARN("HandleRpcGetCacheRecords stored record is corrupt, compact binary format validation failed. Reason: '{}'", + ToString(Error)); + FoundLocalInvalid = true; + } else { Request.RecordObject = CbObjectView(Request.RecordCacheValue.GetData()); @@ -1563,18 +1575,27 @@ CacheRpcHandler::GetLocalCacheRecords(const CacheRequestContext& Context, Record.ValuesRead = true; if (Record.CacheValue && Record.CacheValue.GetContentType() == ZenContentType::kCbObject) { - CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData()); - CbArrayView ValuesArray = RecordObject["Values"sv].AsArrayView(); - Record.Values.reserve(ValuesArray.Num()); - for (CbFieldView ValueField : ValuesArray) + if (CbValidateError Error = ValidateCompactBinary(Record.CacheValue.GetView(), CbValidateMode::Default); + Error != CbValidateError::None) + { + ZEN_WARN("GetLocalCacheRecords stored record for is corrupt, compact binary format validation failed. Reason: '{}'", + ToString(Error)); + } + else { - CbObjectView ValueObject = ValueField.AsObjectView(); - Oid ValueId = ValueObject["Id"sv].AsObjectId(); - CbFieldView RawHashField = ValueObject["RawHash"sv]; - IoHash RawHash = RawHashField.AsBinaryAttachment(); - if (ValueId && !RawHashField.HasError()) + CbObjectView RecordObject = CbObjectView(Record.CacheValue.GetData()); + CbArrayView ValuesArray = RecordObject["Values"sv].AsArrayView(); + Record.Values.reserve(ValuesArray.Num()); + for (CbFieldView ValueField : ValuesArray) { - Record.Values.push_back({ValueId, RawHash, ValueObject["RawSize"sv].AsUInt64()}); + CbObjectView ValueObject = ValueField.AsObjectView(); + Oid ValueId = ValueObject["Id"sv].AsObjectId(); + CbFieldView RawHashField = ValueObject["RawHash"sv]; + IoHash RawHash = RawHashField.AsBinaryAttachment(); + if (ValueId && !RawHashField.HasError()) + { + Record.Values.push_back({ValueId, RawHash, ValueObject["RawSize"sv].AsUInt64()}); + } } } } -- cgit v1.2.3 From a9c83b3c1299692923e2c16b696f5b9e211f5737 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 2 May 2025 15:33:58 +0200 Subject: iterate chunks crash fix (#376) * Bugfix: Add explicit lambda capture in CasContainer::IterateChunks to avoid accessing state data references --- src/zenserver/projectstore/projectstore.cpp | 2 +- src/zenstore/blockstore.cpp | 31 ++- src/zenstore/buildstore/buildstore.cpp | 76 +++++--- src/zenstore/cache/cacherpc.cpp | 2 +- src/zenstore/compactcas.cpp | 281 ++++++++++++++++++++++++---- 5 files changed, 315 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 9aa800434..e91e6ac51 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -2185,7 +2185,7 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); if (!Payload) { - ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[ChunkIndex], FilePath); + ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath); } if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 7cc09be15..c58080e6a 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -735,6 +735,8 @@ BlockStore::IterateBlock(std::span ChunkLocations, return true; } + ZEN_ASSERT(ChunkLocations.size() >= InChunkIndexes.size()); + if (LargeSizeLimit == 0) { LargeSizeLimit = DefaultIterateSmallChunkWindowSize; @@ -746,7 +748,10 @@ BlockStore::IterateBlock(std::span ChunkLocations, IterateSmallChunkWindowSize = Min((LargeSizeLimit + IterateSmallChunkMaxGapSize) * ChunkLocations.size(), IterateSmallChunkWindowSize); - uint32_t BlockIndex = ChunkLocations[InChunkIndexes[0]].BlockIndex; + const size_t FirstLocationIndex = InChunkIndexes[0]; + ZEN_ASSERT(FirstLocationIndex < ChunkLocations.size()); + + const uint32_t BlockIndex = ChunkLocations[FirstLocationIndex].BlockIndex; std::vector ChunkIndexes(InChunkIndexes.begin(), InChunkIndexes.end()); std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&](size_t IndexA, size_t IndexB) -> bool { return ChunkLocations[IndexA].Offset < ChunkLocations[IndexB].Offset; @@ -756,8 +761,9 @@ BlockStore::IterateBlock(std::span ChunkLocations, IterateSmallChunkWindowSize, IterateSmallChunkMaxGapSize, &ChunkLocations](uint64_t BlockFileSize, std::span ChunkIndexes, size_t StartIndexOffset) -> size_t { - size_t ChunkCount = 0; - size_t StartIndex = ChunkIndexes[StartIndexOffset]; + size_t ChunkCount = 0; + size_t StartIndex = ChunkIndexes[StartIndexOffset]; + ZEN_ASSERT(StartIndex < ChunkLocations.size()); const BlockStoreLocation& StartLocation = ChunkLocations[StartIndex]; uint64_t StartOffset = StartLocation.Offset; uint64_t LastEnd = StartOffset + StartLocation.Size; @@ -810,22 +816,26 @@ BlockStore::IterateBlock(std::span ChunkLocations, ZEN_ASSERT(BlockFile); InsertLock.ReleaseNow(); + const size_t BlockSize = BlockFile->FileSize(); + IoBuffer ReadBuffer; void* BufferBase = nullptr; size_t LocationIndexOffset = 0; while (LocationIndexOffset < ChunkIndexes.size()) { - size_t ChunkIndex = ChunkIndexes[LocationIndexOffset]; + size_t ChunkIndex = ChunkIndexes[LocationIndexOffset]; + ZEN_ASSERT(ChunkIndex < ChunkLocations.size()); const BlockStoreLocation& FirstLocation = ChunkLocations[ChunkIndex]; + ZEN_ASSERT(FirstLocation.BlockIndex == BlockIndex); - const size_t BlockSize = BlockFile->FileSize(); const size_t RangeCount = GetNextRange(BlockSize, ChunkIndexes, LocationIndexOffset); if (RangeCount > 1) { - size_t LastChunkIndex = ChunkIndexes[LocationIndexOffset + RangeCount - 1]; - const BlockStoreLocation& LastLocation = ChunkLocations[LastChunkIndex]; - uint64_t Size = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset; + size_t LastChunkIndex = ChunkIndexes[LocationIndexOffset + RangeCount - 1]; + ZEN_ASSERT(LastChunkIndex < ChunkLocations.size()); + const BlockStoreLocation& LastLocation = ChunkLocations[LastChunkIndex]; + uint64_t Size = LastLocation.Offset + LastLocation.Size - FirstLocation.Offset; if (ReadBuffer.GetSize() < Size) { ReadBuffer = IoBuffer(Min(Size * 2, IterateSmallChunkWindowSize)); @@ -834,8 +844,9 @@ BlockStore::IterateBlock(std::span ChunkLocations, BlockFile->Read(BufferBase, Size, FirstLocation.Offset); for (size_t RangeIndex = 0; RangeIndex < RangeCount; ++RangeIndex) { - size_t NextChunkIndex = ChunkIndexes[LocationIndexOffset + RangeIndex]; - const BlockStoreLocation& ChunkLocation = ChunkLocations[NextChunkIndex]; + size_t NextChunkIndex = ChunkIndexes[LocationIndexOffset + RangeIndex]; + ZEN_ASSERT(NextChunkIndex < ChunkLocations.size()); + const BlockStoreLocation& ChunkLocation = ChunkLocations[NextChunkIndex]; if (ChunkLocation.Size == 0 || ((ChunkLocation.Offset + ChunkLocation.Size) > BlockSize)) { ZEN_LOG_SCOPE("chunk [{},{}] out of bounds (block #{} file size = {})", diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index cf518c06f..b4891a742 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -442,7 +442,10 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O } } - auto DoOneBlock = [&](std::span ChunkIndexes) { + auto DoOneBlock = [this](std::span MetaLocations, + std::span MetaLocationResultIndexes, + std::span ChunkIndexes, + std::vector& Result) { if (ChunkIndexes.size() < 4) { for (size_t ChunkIndex : ChunkIndexes) @@ -459,7 +462,7 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O return m_MetadataBlockStore.IterateBlock( MetaLocations, ChunkIndexes, - [&](size_t ChunkIndex, const void* Data, uint64_t Size) { + [&MetaLocationResultIndexes, &Result](size_t ChunkIndex, const void* Data, uint64_t Size) { if (Data != nullptr) { size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; @@ -479,38 +482,51 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O { Latch WorkLatch(1); - m_MetadataBlockStore.IterateChunks(MetaLocations, [&](uint32_t BlockIndex, std::span ChunkIndexes) -> bool { - ZEN_UNUSED(BlockIndex); - if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) - { - return DoOneBlock(ChunkIndexes); - } - else - { - ZEN_ASSERT(OptionalWorkerPool != nullptr); - WorkLatch.AddCount(1); - try + m_MetadataBlockStore.IterateChunks( + MetaLocations, + [this, OptionalWorkerPool, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock, &WorkLatch]( + uint32_t BlockIndex, + std::span ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) { - OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector(ChunkIndexes.begin(), ChunkIndexes.end())]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - try - { - DoOneBlock(ChunkIndexes); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); - } - }); + return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); } - catch (const std::exception& Ex) + else { - WorkLatch.CountDown(); - ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + ZEN_ASSERT(OptionalWorkerPool != nullptr); + std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + WorkLatch.AddCount(1); + try + { + OptionalWorkerPool->ScheduleWork([this, + &Result, + &MetaLocations, + &MetaLocationResultIndexes, + DoOneBlock, + &WorkLatch, + ChunkIndexes = std::move(TmpChunkIndexes)]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + try + { + DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + } + catch (const std::exception& Ex) + { + WorkLatch.CountDown(); + ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", + ChunkIndexes.size(), + Ex.what()); + } + return true; } - return true; - } - }); + }); WorkLatch.CountDown(); WorkLatch.Wait(); diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index bf78dae86..de4b0a37c 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -653,7 +653,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb { m_CidStore.IterateChunks( CidHashes, - [this, &Request, ValueCount, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool { + [this, &Request, &RequestValueIndexes](size_t Index, const IoBuffer& Payload) -> bool { try { const size_t ValueIndex = RequestValueIndexes[Index]; diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 184251da7..90e77e48a 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -360,7 +360,11 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas return true; } - auto DoOneBlock = [&](std::span ChunkIndexes) { + auto DoOneBlock = [this](const std::function& AsyncCallback, + uint64_t LargeSizeLimit, + std::span FoundChunkIndexes, + std::span FoundChunkLocations, + std::span ChunkIndexes) { if (ChunkIndexes.size() < 4) { for (size_t ChunkIndex : ChunkIndexes) @@ -376,7 +380,7 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas return m_BlockStore.IterateBlock( FoundChunkLocations, ChunkIndexes, - [&](size_t ChunkIndex, const void* Data, uint64_t Size) { + [AsyncCallback, FoundChunkIndexes, LargeSizeLimit](size_t ChunkIndex, const void* Data, uint64_t Size) { if (Data == nullptr) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer()); @@ -391,39 +395,59 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas Latch WorkLatch(1); std::atomic_bool AsyncContinue = true; - bool Continue = m_BlockStore.IterateChunks(FoundChunkLocations, [&](uint32_t BlockIndex, std::span ChunkIndexes) { - if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) - { - WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector(ChunkIndexes.begin(), ChunkIndexes.end())]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - if (!AsyncContinue) - { - return; - } - try - { - bool Continue = DoOneBlock(ChunkIndexes); - if (!Continue) - { - AsyncContinue.store(false); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", - m_RootDirectory, - BlockIndex, - Ex.what()); - } - }); - return AsyncContinue.load(); - } - else - { - return DoOneBlock(ChunkIndexes); - } - }); + bool Continue = m_BlockStore.IterateChunks( + FoundChunkLocations, + [this, + &AsyncContinue, + &WorkLatch, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + &FoundChunkIndexes, + &FoundChunkLocations, + OptionalWorkerPool](uint32_t BlockIndex, std::span ChunkIndexes) { + if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) + { + std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + WorkLatch.AddCount(1); + OptionalWorkerPool->ScheduleWork([this, + &AsyncContinue, + &WorkLatch, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + BlockIndex, + &FoundChunkIndexes, + &FoundChunkLocations, + ChunkIndexes = std::move(TmpChunkIndexes)]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + if (!AsyncContinue) + { + return; + } + try + { + bool Continue = DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + if (!Continue) + { + AsyncContinue.store(false); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", + m_RootDirectory, + BlockIndex, + Ex.what()); + } + }); + return AsyncContinue.load(); + } + else + { + return DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + } + }); WorkLatch.CountDown(); WorkLatch.Wait(); return AsyncContinue.load() && Continue; @@ -1573,6 +1597,193 @@ TEST_CASE("compactcas.threadedinsert") } } +TEST_CASE("compactcas.iteratechunks") +{ + std::atomic WorkCompleted = 0; + WorkerThreadPool ThreadPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "put"); + + const uint64_t kChunkSize = 1048 + 395; + const size_t kChunkCount = 63840; + + for (uint32_t N = 0; N < 4; N++) + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + ScopedTemporaryDirectory TempDir; + Cas.Initialize(TempDir.Path(), "test", 65536 * 128, 8, true); + + CHECK(Cas.IterateChunks( + {}, + [](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index, Payload); + return true; + }, + &ThreadPool, + 2048u)); + + uint64_t ExpectedSize = 0; + + std::vector Hashes; + Hashes.reserve(kChunkCount); + + { + Latch WorkLatch(1); + tsl::robin_set ChunkHashesLookup; + ChunkHashesLookup.reserve(kChunkCount); + RwLock InsertLock; + for (size_t Offset = 0; Offset < kChunkCount;) + { + size_t BatchCount = Min(kChunkCount - Offset, 512u); + WorkLatch.AddCount(1); + ThreadPool.ScheduleWork( + [N, &WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + + std::vector BatchBlobs; + std::vector BatchHashes; + BatchBlobs.reserve(BatchCount); + BatchHashes.reserve(BatchCount); + + while (BatchBlobs.size() < BatchCount) + { + IoBuffer Chunk = CreateRandomBlob( + N + kChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377)); + IoHash Hash = IoHash::HashBuffer(Chunk); + { + RwLock::ExclusiveLockScope __(InsertLock); + if (ChunkHashesLookup.contains(Hash)) + { + continue; + } + ChunkHashesLookup.insert(Hash); + ExpectedSize += Chunk.Size(); + } + + BatchBlobs.emplace_back(std::move(Chunk)); + BatchHashes.push_back(Hash); + } + + Cas.InsertChunks(BatchBlobs, BatchHashes); + { + RwLock::ExclusiveLockScope __(InsertLock); + Hashes.insert(Hashes.end(), BatchHashes.begin(), BatchHashes.end()); + } + }); + Offset += BatchCount; + } + WorkLatch.CountDown(); + WorkLatch.Wait(); + } + + WorkerThreadPool BatchWorkerPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "fetch"); + { + std::vector> FetchedFlags(Hashes.size()); + std::atomic FetchedSize = 0; + CHECK(Cas.IterateChunks( + Hashes, + [&Hashes, &FetchedFlags, &FetchedSize](size_t Index, const IoBuffer& Payload) { + CHECK(FetchedFlags[Index].load() == false); + FetchedFlags[Index].store(true); + const IoHash& Hash = Hashes[Index]; + CHECK(Hash == IoHash::HashBuffer(Payload)); + FetchedSize += Payload.GetSize(); + return true; + }, + &BatchWorkerPool, + 2048u)); + for (const auto& Flag : FetchedFlags) + { + CHECK(Flag.load()); + } + CHECK(FetchedSize == ExpectedSize); + } + + Latch WorkLatch(1); + for (size_t I = 0; I < 2; I++) + { + WorkLatch.AddCount(1); + ThreadPool.ScheduleWork([&Cas, &Hashes, &BatchWorkerPool, &WorkLatch, I]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + std::vector PartialHashes; + PartialHashes.reserve(Hashes.size() / 4); + for (size_t Index = 0; Index < Hashes.size(); Index++) + { + size_t TestIndex = Index + I; + if ((TestIndex % 7 == 1) || (TestIndex % 13 == 1) || (TestIndex % 17 == 1)) + { + PartialHashes.push_back(Hashes[Index]); + } + } + std::reverse(PartialHashes.begin(), PartialHashes.end()); + + std::vector NoFoundHashes; + std::vector NoFindIndexes; + + NoFoundHashes.reserve(9); + for (size_t J = 0; J < 9; J++) + { + std::string Data = fmt::format("oh no, we don't exist {}", J + 1); + NoFoundHashes.push_back(IoHash::HashBuffer(Data.data(), Data.length())); + } + + NoFindIndexes.reserve(9); + + // Sprinkle in chunks that are not found! + auto It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 0, NoFoundHashes[0]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 0 + 1, NoFoundHashes[1]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 1, NoFoundHashes[2]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 1 + 1, NoFoundHashes[3]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 2, NoFoundHashes[4]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 3, NoFoundHashes[5]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 3 + 1, NoFoundHashes[6]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.begin() + (PartialHashes.size() / 4) * 4, NoFoundHashes[7]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + It = PartialHashes.insert(PartialHashes.end(), NoFoundHashes[8]); + NoFindIndexes.push_back(std::distance(PartialHashes.begin(), It)); + + std::vector> FoundFlags(PartialHashes.size() + NoFoundHashes.size()); + std::vector> FetchedCounts(PartialHashes.size() + NoFoundHashes.size()); + + CHECK(Cas.IterateChunks( + PartialHashes, + [&PartialHashes, &FoundFlags, &FetchedCounts, &NoFindIndexes](size_t Index, const IoBuffer& Payload) { + CHECK_EQ(NoFindIndexes.end(), std::find(NoFindIndexes.begin(), NoFindIndexes.end(), Index)); + uint32_t PreviousCount = FetchedCounts[Index].fetch_add(1); + CHECK(PreviousCount == 0); + FoundFlags[Index] = !!Payload; + const IoHash& Hash = PartialHashes[Index]; + CHECK(Hash == IoHash::HashBuffer(Payload)); + return true; + }, + &BatchWorkerPool, + 2048u)); + + for (size_t FoundIndex = 0; FoundIndex < PartialHashes.size(); FoundIndex++) + { + CHECK(FetchedCounts[FoundIndex].load() <= 1); + if (std::find(NoFindIndexes.begin(), NoFindIndexes.end(), FoundIndex) == NoFindIndexes.end()) + { + CHECK(FoundFlags[FoundIndex]); + } + else + { + CHECK(!FoundFlags[FoundIndex]); + } + } + }); + } + WorkLatch.CountDown(); + WorkLatch.Wait(); + } +} + #endif void -- cgit v1.2.3 From f74d56510a19d693f9febb1a1686c2382a924dfb Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 5 May 2025 14:37:48 +0200 Subject: silence Out Of Disk errors to sentry (#378) * block writing GC state/info if disk is full * fix if/else on error while writing gc state --- src/zenstore/gc.cpp | 104 ++++++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 48 deletions(-) (limited to 'src') diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index ac4dda83f..56d0e8db6 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -2538,7 +2538,11 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, ZEN_INFO("GCV2: {}", SB.ToView()); - AppendGCLog(GcId, GcStartTime, Settings, Result); + CheckDiskSpace(); + if (!m_AreDiskWritesBlocked.load()) + { + AppendGCLog(GcId, GcStartTime, Settings, Result); + } if (SkipCid) { @@ -2580,65 +2584,69 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, m_LastFullGCDiff = Diff; } - for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++) + CheckDiskSpace(); + if (!m_AreDiskWritesBlocked.load()) { - if (RetryCount > 0) + for (uint32_t RetryCount = 0; RetryCount < 3; RetryCount++) { - ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount); - Sleep(250); - } - try - { - const fs::path Path = m_Config.RootDirectory / "gc_state"; - ZEN_DEBUG("saving scheduler state to '{}'", Path); - CbObjectWriter SchedulerState; - SchedulerState << "LastGcTime"sv << static_cast(m_LastGcTime.time_since_epoch().count()); - SchedulerState << "LastGcExpireTime"sv << static_cast(m_LastGcExpireTime.time_since_epoch().count()); - SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex; - - SaveCompactBinaryObject(Path, SchedulerState.Save()); if (RetryCount > 0) { - ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1); - } - break; - } - catch (const std::system_error& SystemError) - { - if (IsOOM(SystemError.code())) - { - ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what()); + ZEN_INFO("Writing GC state failed {} time(s), pausing and trying again", RetryCount); + Sleep(250); } - else if (IsOOD(SystemError.code())) + try { - ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what()); + const fs::path Path = m_Config.RootDirectory / "gc_state"; + ZEN_DEBUG("saving scheduler state to '{}'", Path); + CbObjectWriter SchedulerState; + SchedulerState << "LastGcTime"sv << static_cast(m_LastGcTime.time_since_epoch().count()); + SchedulerState << "LastGcExpireTime"sv << static_cast(m_LastGcExpireTime.time_since_epoch().count()); + SchedulerState << "AttachmentPassIndex"sv << m_AttachmentPassIndex; + + SaveCompactBinaryObject(Path, SchedulerState.Save()); + if (RetryCount > 0) + { + ZEN_INFO("Writing GC state succeeded after {} attempts", RetryCount + 1); + } + break; } - if (RetryCount == 0) + catch (const std::system_error& SystemError) { - ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})", - SystemError.what(), - SystemError.code().value()); - } - else - { - ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})", - SystemError.what(), - SystemError.code().value()); + if (IsOOM(SystemError.code())) + { + ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", SystemError.what()); + } + else if (IsOOD(SystemError.code())) + { + ZEN_WARN("writing gc scheduler state ran out of disk space: '{}'", SystemError.what()); + } + else if (RetryCount == 0) + { + ZEN_ERROR("writing gc scheduler state failed with system error exception: '{}' ({})", + SystemError.what(), + SystemError.code().value()); + } + else + { + ZEN_WARN("writing gc scheduler state failed with system error exception: '{}' ({})", + SystemError.what(), + SystemError.code().value()); + } } - } - catch (const std::bad_alloc& BadAlloc) - { - ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what()); - } - catch (const std::exception& Ex) - { - if (RetryCount == 0) + catch (const std::bad_alloc& BadAlloc) { - ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what()); + ZEN_WARN("writing gc scheduler state ran out of memory: '{}'", BadAlloc.what()); } - else + catch (const std::exception& Ex) { - ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what()); + if (RetryCount == 0) + { + ZEN_ERROR("writing gc scheduler state failed with: '{}'", Ex.what()); + } + else + { + ZEN_WARN("writing gc scheduler state failed with: '{}'", Ex.what()); + } } } } -- cgit v1.2.3 From fd5b8fde6bd838b17e6100f01d466bcbf01f91fe Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 5 May 2025 18:57:15 +0200 Subject: UE style formatted progress output (#380) * add UE style @progress style progress --- src/zen/cmds/builds_cmd.cpp | 206 ++++++++++++++++++++++++++++++++------ src/zen/cmds/builds_cmd.h | 1 + src/zen/cmds/projectstore_cmd.cpp | 6 +- src/zen/cmds/wipe_cmd.cpp | 26 ++++- src/zen/zen.cpp | 181 ++++++++++++++++++++++++--------- src/zen/zen.h | 25 +++-- 6 files changed, 350 insertions(+), 95 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 15c635594..134cde0f8 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -145,8 +145,24 @@ namespace { const std::vector DefaultExcludeFolders({UnsyncFolderName, ZenFolderName, UGSFolderName, LegacyZenTempFolderName}); const std::vector DefaultExcludeExtensions({}); - static bool IsVerbose = false; - static bool UsePlainProgress = false; + static bool IsVerbose = false; + static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; + + uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) + { + switch (InMode) + { + case ProgressBar::Mode::Plain: + return 5000; + case ProgressBar::Mode::Pretty: + return 200; + case ProgressBar::Mode::Log: + return 2000; + default: + ZEN_ASSERT(false); + return 0; + } + } #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ @@ -333,7 +349,7 @@ namespace { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(UsePlainProgress); + ProgressBar Progress(ProgressMode, "Clean Folder"); std::atomic CleanWipe = true; std::atomic DiscoveredItemCount = 0; @@ -453,7 +469,7 @@ namespace { uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); @@ -492,7 +508,7 @@ namespace { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) { LastUpdateTimeMs = NowMs; @@ -1814,15 +1830,34 @@ namespace { ValidateStatistics& ValidateStats, DownloadStatistics& DownloadStats) { + ZEN_TRACE_CPU("ValidateBuildPart"); + + ProgressBar::SetLogOperationName(ProgressMode, "Validate Part"); + + enum TaskSteps : uint32_t + { + FetchBuild, + FetchBuildPart, + ValidateBlobs, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + Stopwatch Timer; - auto _ = MakeGuard([&]() { + auto _ = MakeGuard([&]() { ZEN_CONSOLE("Validated build part {}/{} ('{}') in {}", BuildId, BuildPartId, BuildPartName, NiceTimeSpanMs(Timer.GetElapsedTimeMs())); }); - CbObject Build = Storage.GetBuild(BuildId); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuild, TaskSteps::StepCount); + + CbObject Build = Storage.GetBuild(BuildId); if (!BuildPartName.empty()) { BuildPartId = Build["parts"sv].AsObjectView()[BuildPartName].AsObjectId(); @@ -1837,6 +1872,9 @@ namespace { { PreferredMultipartChunkSize = ChunkSize; } + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::FetchBuildPart, TaskSteps::StepCount); + CbObject BuildPart = Storage.GetBuildPart(BuildId, BuildPartId); ValidateStats.BuildPartSize = BuildPart.GetSize(); ZEN_CONSOLE("Validating build part {}/{} ({})", BuildId, BuildPartId, NiceBytes(BuildPart.GetSize())); @@ -1876,7 +1914,9 @@ namespace { } }); - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::ValidateBlobs, TaskSteps::StepCount); + + ProgressBar ProgressBar(ProgressMode, "Validate Blobs"); uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size(); FilteredRate FilteredDownloadedBytesPerSecond; @@ -1990,7 +2030,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t DownloadedAttachmentCount = DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount; @@ -2020,6 +2060,8 @@ namespace { ProgressBar.Finish(); ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } void ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content, @@ -2226,7 +2268,7 @@ namespace { const std::size_t NewBlockCount = NewBlockChunks.size(); if (NewBlockCount > 0) { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Generate Blocks"); OutBlocks.BlockDescriptions.resize(NewBlockCount); OutBlocks.BlockSizes.resize(NewBlockCount); @@ -2389,7 +2431,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); @@ -2439,7 +2481,7 @@ namespace { { ZEN_TRACE_CPU("UploadPartBlobs"); { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Upload Blobs"); WorkerThreadPool& ReadChunkPool = GetIOWorkerPool(); WorkerThreadPool& UploadChunkPool = GetNetworkPool(); @@ -2752,7 +2794,7 @@ namespace { Work.DefaultErrorFunction()); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); @@ -2971,6 +3013,23 @@ namespace { bool IgnoreExistingBlocks, bool PostUploadVerify) { + ZEN_TRACE_CPU("UploadFolder"); + + ProgressBar::SetLogOperationName(ProgressMode, "Upload Folder"); + + enum TaskSteps : uint32_t + { + PrepareBuild, + CalculateDelta, + Upload, + Validate, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + Stopwatch ProcessTimer; const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); @@ -2986,6 +3045,8 @@ namespace { CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::PrepareBuild, TaskSteps::StepCount); + std::uint64_t TotalRawSize = 0; CbObject ChunkerParameters; @@ -3157,7 +3218,7 @@ namespace { return true; }, GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, @@ -3212,7 +3273,7 @@ namespace { TotalRawSize = std::accumulate(Content.RawSizes.begin(), Content.RawSizes.end(), std::uint64_t(0)); { - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( @@ -3221,7 +3282,7 @@ namespace { Path, Content, *ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -3277,6 +3338,8 @@ namespace { PrepBuildResult.KnownBlocks.size(), NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs))); + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CalculateDelta, TaskSteps::StepCount); + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PrepBuildResult.PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; Stopwatch BlockArrangeTimer; @@ -3386,6 +3449,8 @@ namespace { UploadStatistics UploadStats; GeneratedBlocks NewBlocks; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Upload, TaskSteps::StepCount); + if (!NewBlockChunks.empty()) { Stopwatch GenerateBuildBlocksTimer; @@ -3710,6 +3775,7 @@ namespace { DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Validate, TaskSteps::StepCount); ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } @@ -3961,6 +4027,8 @@ namespace { {"uploadedBytesPerSec", double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))}, {"elapsedTimeSec", double(ProcessTimer.GetElapsedTimeMs() / 1000.0)}}); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } void VerifyFolder(const ChunkedFolderContent& Content, @@ -3972,7 +4040,7 @@ namespace { Stopwatch Timer; - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Verify Files"); WorkerThreadPool& VerifyPool = GetIOWorkerPool(); @@ -4114,7 +4182,7 @@ namespace { }); } - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", VerifyFolderStats.FilesVerified.load(), @@ -5085,7 +5153,7 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); - ProgressBar ProgressBar(UsePlainProgress); + ProgressBar ProgressBar(ProgressMode, "Check Files"); ParallellWork Work(AbortFlag); std::atomic CompletedPathCount = 0; @@ -5751,7 +5819,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar WriteProgressBar(UsePlainProgress); + ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); ParallellWork Work(AbortFlag); struct LooseChunkHashWorkData @@ -7054,7 +7122,7 @@ namespace { { ZEN_TRACE_CPU("WriteChunks_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + @@ -7241,7 +7309,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar CacheLocalProgressBar(UsePlainProgress); + ProgressBar CacheLocalProgressBar(ProgressMode, "Cache Local Data"); ParallellWork Work(AbortFlag); for (uint32_t LocalPathIndex : FilesToCache) @@ -7272,7 +7340,7 @@ namespace { { ZEN_TRACE_CPU("CacheLocal_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t WorkTotal = FilesToCache.size(); const uint64_t WorkComplete = CachedCount.load(); @@ -7329,7 +7397,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar RebuildProgressBar(UsePlainProgress); + ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); ParallellWork Work(AbortFlag); OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); @@ -7566,7 +7634,7 @@ namespace { { ZEN_TRACE_CPU("FinalizeTree_Wait"); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); @@ -8164,7 +8232,7 @@ namespace { { ByteCountToScan += RawSize; } - ProgressBar ProgressBar(false); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( @@ -8173,7 +8241,7 @@ namespace { Path, UpdatedContent, ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -8232,7 +8300,7 @@ namespace { { ByteCountToScan += RawSize; } - ProgressBar ProgressBar(false); + ProgressBar ProgressBar(ProgressMode, "Scan Files"); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( @@ -8241,7 +8309,7 @@ namespace { Path, OutLocalFolderContent, ChunkController, - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", @@ -8289,7 +8357,7 @@ namespace { std::move(IsAcceptedFolder), std::move(IsAcceptedFile), GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, + GetUpdateDelayMS(ProgressMode), [](bool, std::ptrdiff_t) {}, AbortFlag); if (AbortFlag) @@ -8337,10 +8405,27 @@ namespace { { ZEN_TRACE_CPU("DownloadFolder"); + ProgressBar::SetLogOperationName(ProgressMode, "Download Folder"); + + enum TaskSteps : uint32_t + { + CheckState, + CompareState, + Download, + Verify, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); Stopwatch DownloadTimer; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckState, TaskSteps::StepCount); + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); @@ -8360,6 +8445,8 @@ namespace { std::vector BlockDescriptions; std::vector LooseChunkHashes; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CompareState, TaskSteps::StepCount); + ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); @@ -8471,6 +8558,8 @@ namespace { RebuildFolderStateStatistics RebuildFolderStateStats; VerifyFolderStatistics VerifyFolderStats; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); + UpdateFolder(SystemRootDir, Storage, BuildId, @@ -8497,6 +8586,8 @@ namespace { { if (!PrimeCacheOnly) { + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Verify, TaskSteps::StepCount); + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); Stopwatch WriteStateTimer; @@ -8564,6 +8655,9 @@ namespace { }); } } + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); + if (CleanDirectory(ZenTempFolder, {})) { RemoveDirWithRetry(ZenTempFolder); @@ -8572,6 +8666,22 @@ namespace { void DiffFolders(const std::filesystem::path& BasePath, const std::filesystem::path& ComparePath, bool OnlyChunked) { + ZEN_TRACE_CPU("DiffFolders"); + + ProgressBar::SetLogOperationName(ProgressMode, "Diff Folders"); + + enum TaskSteps : uint32_t + { + CheckBase, + CheckCompare, + Diff, + Cleanup, + StepCount + }; + + auto EndProgress = + MakeGuard([&]() { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::StepCount, TaskSteps::StepCount); }); + ChunkedFolderContent BaseFolderContent; ChunkedFolderContent CompareFolderContent; @@ -8614,6 +8724,8 @@ namespace { return true; }; + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckBase, TaskSteps::StepCount); + GetFolderContentStatistics BaseGetFolderContentStats; ChunkingStatistics BaseChunkingStats; BaseFolderContent = ScanAndChunkFolder(BaseGetFolderContentStats, @@ -8627,6 +8739,8 @@ namespace { return; } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::CheckCompare, TaskSteps::StepCount); + GetFolderContentStatistics CompareGetFolderContentStats; ChunkingStatistics CompareChunkingStats; CompareFolderContent = ScanAndChunkFolder(CompareGetFolderContentStats, @@ -8642,6 +8756,8 @@ namespace { } } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Diff, TaskSteps::StepCount); + std::vector AddedHashes; std::vector RemovedHashes; uint64_t RemovedSize = 0; @@ -8728,6 +8844,8 @@ namespace { NewChunkCount, NiceBytes(NewChunkSize), NewPercent); + + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } } // namespace @@ -8852,7 +8970,18 @@ BuildsCommand::BuildsCommand() }; auto AddOutputOptions = [this](cxxopts::Options& Ops) { - Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); + Ops.add_option("output", + "", + "plain-progress", + "Show progress using plain output", + cxxopts::value(m_PlainProgress), + ""); + Ops.add_option("output", + "", + "log-progress", + "Write @progress style progress to output", + cxxopts::value(m_LogProgress), + ""); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); }; @@ -9335,8 +9464,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseOutputOptions = [&]() { - IsVerbose = m_Verbose; - UsePlainProgress = IsVerbose || m_PlainProgress; + IsVerbose = m_Verbose; + if (m_LogProgress) + { + ProgressMode = ProgressBar::Mode::Log; + } + else if (m_Verbose || m_PlainProgress) + { + ProgressMode = ProgressBar::Mode::Plain; + } + else + { + ProgressMode = ProgressBar::Mode::Pretty; + } }; ParseOutputOptions(); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 01d510e1b..9f6c1f6ec 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -28,6 +28,7 @@ private: std::filesystem::path m_SystemRootDir; bool m_PlainProgress = false; + bool m_LogProgress = false; bool m_Verbose = false; bool m_BoostWorkerThreads = false; diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index c73842b89..540c4c9ac 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -168,7 +168,7 @@ namespace { throw std::runtime_error(fmt::format("invalid job id returned, received '{}'", JobIdText)); } - ProgressBar ProgressBar(PlainProgress); + ProgressBar ProgressBar(PlainProgress ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty, ""sv); auto OuputMessages = [&](CbObjectView StatusObject) { CbArrayView Messages = StatusObject["Messages"sv].AsArrayView(); @@ -2091,7 +2091,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { std::unique_ptr EmitProgressBar; { - ProgressBar ParseProgressBar(false); + ProgressBar ParseProgressBar(ProgressBar::Mode::Pretty, ""); CbArrayView Entries = ResponseObject["entries"sv].AsArrayView(); uint64_t Remaining = Entries.Num(); for (auto EntryIter : Entries) @@ -2110,7 +2110,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg } if (!EmitProgressBar) { - EmitProgressBar = std::make_unique(false); + EmitProgressBar = std::make_unique(ProgressBar::Mode::Pretty, ""sv); WriteStopWatch.Reset(); } diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 2b4e9ab3c..9bf0c6f9e 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -35,10 +35,26 @@ namespace { static std::atomic AbortFlag = false; static bool IsVerbose = false; static bool Quiet = false; - static bool UsePlainProgress = false; + static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; const bool SingleThreaded = false; bool BoostWorkerThreads = true; + uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) + { + switch (InMode) + { + case ProgressBar::Mode::Plain: + return 5000; + case ProgressBar::Mode::Pretty: + return 200; + case ProgressBar::Mode::Log: + return 2000; + default: + ZEN_ASSERT(false); + return 0; + } + } + WorkerThreadPool& GetIOWorkerPool() { return SingleThreaded ? GetSyncWorkerPool() @@ -161,7 +177,7 @@ namespace { ZEN_TRACE_CPU("CleanDirectory"); Stopwatch Timer; - ProgressBar Progress(UsePlainProgress); + ProgressBar Progress(ProgressMode, "Clean Folder"); std::atomic CleanWipe = true; std::atomic DiscoveredItemCount = 0; @@ -396,7 +412,7 @@ namespace { GetIOWorkerPool(), Work.PendingWork()); - Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, ptrdiff_t PendingWork) { if (Quiet) { return; @@ -455,7 +471,7 @@ namespace { } uint64_t NowMs = Timer.GetElapsedTimeMs(); - if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + if ((NowMs - LastUpdateTimeMs) >= GetUpdateDelayMS(ProgressMode)) { LastUpdateTimeMs = NowMs; @@ -529,7 +545,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Quiet = m_Quiet; IsVerbose = m_Verbose; - UsePlainProgress = IsVerbose || m_PlainProgress; + ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; MakeSafeAbsolutePathÍnPlace(m_Directory); diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index e442f8a4b..399c43990 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -269,25 +269,120 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy); } +#if ZEN_PLATFORM_WINDOWS +static HANDLE +GetConsoleHandle() +{ + static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + return hStdOut; +} +#endif + static bool -IsStdoutTty() +CheckStdoutTty() { #if ZEN_PLATFORM_WINDOWS - static HANDLE hStdOut = ::GetStdHandle(STD_OUTPUT_HANDLE); - DWORD dwMode = 0; - static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); + HANDLE hStdOut = GetConsoleHandle(); + DWORD dwMode = 0; + static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); return IsConsole; #else return isatty(fileno(stdout)); #endif } -ProgressBar::ProgressBar(bool PlainProgress, bool ShowDetails) -: m_StdoutIsTty(IsStdoutTty()) -, m_PlainProgress(PlainProgress || !m_StdoutIsTty) -, m_ShowDetails(ShowDetails) +static bool +IsStdoutTty() +{ + static bool StdoutIsTty = CheckStdoutTty(); + return StdoutIsTty; +} + +static void +OutputToConsoleRaw(const char* String, size_t Length) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); +#endif + +#if ZEN_PLATFORM_WINDOWS + if (IsStdoutTty()) + { + WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); + } + else + { + ::WriteFile(hStdOut, (LPCVOID)String, (DWORD)Length, 0, 0); + } +#else + fwrite(String, 1, Length, stdout); +#endif +} + +static void +OutputToConsoleRaw(const std::string& String) +{ + OutputToConsoleRaw(String.c_str(), String.length()); +} + +static void +OutputToConsoleRaw(const StringBuilderBase& SB) +{ + OutputToConsoleRaw(SB.c_str(), SB.Size()); +} + +static uint32_t +GetConsoleColumns() +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdOut = GetConsoleHandle(); + CONSOLE_SCREEN_BUFFER_INFO csbi; + if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) + { + return (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); + } +#else + struct winsize w; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) + { + return (uint32_t)w.ws_col; + } +#endif + return 1024; +} + +void +ProgressBar::SetLogOperationName(Mode InMode, std::string_view Name) +{ + if (InMode == Mode::Log) + { + std::string String = fmt::format("@progress {}\n", Name); + OutputToConsoleRaw(String); + } +} + +void +ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount) +{ + if (InMode == Mode::Log) + { + const size_t PercentDone = StepCount > 0u ? gsl::narrow((100 * StepIndex) / StepCount) : 0u; + + std::string String = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(String); + } +} + +ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) +: m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) , m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000) +, m_SubTask(InSubTask) { + if (!m_SubTask.empty() && InMode == Mode::Log) + { + std::string String = fmt::format("@progress push {}\n", m_SubTask); + OutputToConsoleRaw(String); + } } ProgressBar::~ProgressBar() @@ -295,6 +390,11 @@ ProgressBar::~ProgressBar() try { ForceLinebreak(); + if (!m_SubTask.empty() && m_Mode == Mode::Log) + { + const std::string String("@progress pop\n"); + OutputToConsoleRaw(String); + } } catch (const std::exception& Ex) { @@ -319,15 +419,16 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) m_LastUpdateMS = ElapsedTimeMS; - size_t PercentDone = + const size_t PercentDone = NewState.TotalCount > 0u ? gsl::narrow((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; - if (m_PlainProgress) + if (m_Mode == Mode::Plain) { - std::string Details = (m_ShowDetails && !NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - ZEN_CONSOLE("{} {}% ({}){}", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); + const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + const std::string Output = fmt::format("{} {}% ({}){}\n", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); + OutputToConsoleRaw(Output); } - else + else if (m_Mode == Mode::Pretty) { size_t ProgressBarSize = 20; @@ -335,28 +436,7 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; uint64_t ETAMS = (PercentDone > 5) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0; - uint32_t ConsoleColumns = 1024; - -#if ZEN_PLATFORM_WINDOWS - static HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); -#endif - - if (!m_PlainProgress) - { -#if ZEN_PLATFORM_WINDOWS - CONSOLE_SCREEN_BUFFER_INFO csbi; - if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) - { - ConsoleColumns = (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); - } -#else - struct winsize w; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) - { - ConsoleColumns = (uint32_t)w.ws_col; - } -#endif - } + uint32_t ConsoleColumns = GetConsoleColumns(); std::string_view TaskString = NewState.Task; @@ -428,21 +508,30 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) LineToPrint << "\n"; } -#if ZEN_PLATFORM_WINDOWS - if (m_StdoutIsTty) + OutputToConsoleRaw(LineToPrint); + + m_LastOutputLength = DoLinebreak ? 0 : Output.length(); + m_State = NewState; + } + else if (m_Mode == Mode::Log) + { + if (m_State.Task != NewState.Task || + m_State.Details != NewState.Details) // TODO: Should we output just because details change? Will this spam the log collector? { - WriteConsoleA(hStdOut, LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0); + const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; + const std::string Message = fmt::format("@progress {} ({}){}\n", NewState.Task, NiceTimeSpanMs(ElapsedTimeMS), Details); + OutputToConsoleRaw(Message); } - else + + const size_t OldPercentDone = + m_State.TotalCount > 0u ? gsl::narrow((100 * (m_State.TotalCount - m_State.RemainingCount)) / m_State.TotalCount) : 0u; + + if (OldPercentDone != PercentDone) { - ::WriteFile(hStdOut, (LPCVOID)LineToPrint.c_str(), (DWORD)LineToPrint.Size(), 0, 0); + const std::string Progress = fmt::format("@progress {}%\n", PercentDone); + OutputToConsoleRaw(Progress); } -#else - fwrite(LineToPrint.c_str(), 1, LineToPrint.Size(), stdout); -#endif - - m_LastOutputLength = DoLinebreak ? 0 : Output.length(); - m_State = NewState; + m_State = NewState; } } diff --git a/src/zen/zen.h b/src/zen/zen.h index dd0fd44b3..9ca228e37 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -81,7 +81,17 @@ public: uint64_t RemainingCount = 0; }; - explicit ProgressBar(bool PlainProgress, bool ShowDetails = true); + enum class Mode + { + Plain, + Pretty, + Log + }; + + static void SetLogOperationName(Mode InMode, std::string_view Name); + static void SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t StepCount); + + explicit ProgressBar(Mode InMode, std::string_view InSubTask); ~ProgressBar(); void UpdateState(const State& NewState, bool DoLinebreak); @@ -91,13 +101,12 @@ public: bool HasActiveTask() const; private: - const bool m_StdoutIsTty = true; - const bool m_PlainProgress; - const bool m_ShowDetails; - Stopwatch m_SW; - uint64_t m_LastUpdateMS; - State m_State; - size_t m_LastOutputLength = 0; + const Mode m_Mode; + Stopwatch m_SW; + uint64_t m_LastUpdateMS; + State m_State; + const std::string m_SubTask; + size_t m_LastOutputLength = 0; }; } // namespace zen -- cgit v1.2.3 From 2ee2eb1cd22ab5c9eab7ec4de0a3abcced50f7f2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 5 May 2025 19:11:01 +0200 Subject: make OOD and OOM in gc non critical (#381) * oom and ood exceptions in GC are now treated as warnings instead of errors --- src/zenstore/gc.cpp | 245 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 218 insertions(+), 27 deletions(-) (limited to 'src') diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 56d0e8db6..5bd34fc37 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -757,6 +757,23 @@ GcManager::CollectGarbage(const GcSettings& Settings) StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats); } } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); @@ -832,6 +849,29 @@ GcManager::CollectGarbage(const GcSettings& Settings) ReferencePruners.insert_or_assign(Index, std::move(ReferencePruner)); } } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed creating reference pruners for {}. Reason: '{}'", + ReferenceStore->GetGcName(Ctx), + Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'", + ReferenceStore->GetGcName(Ctx), + Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'", + ReferenceStore->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed creating reference pruners for {}. Reason: '{}'", @@ -890,36 +930,65 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_MEMSCOPE(GetGcTag()); auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - // The Referencer will create a reference checker that guarantees that the references do not change - // as long as it lives - std::vector Checkers; - try + if (!CheckGCCancel()) { + // The Referencer will create a reference checker that guarantees that the references do not + // change as long as it lives + std::vector Checkers; + auto __ = MakeGuard([&Checkers]() { + while (!Checkers.empty()) + { + delete Checkers.back(); + Checkers.pop_back(); + } + }); + try { - SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS = - std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - Checkers = Referencer->CreateReferenceCheckers(Ctx); + { + SCOPED_TIMER(Stats->second.CreateReferenceCheckersMS = + std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + Checkers = Referencer->CreateReferenceCheckers(Ctx); + } + if (!Checkers.empty()) + { + RwLock::ExclusiveLockScope __(ReferenceCheckersLock); + for (auto& Checker : Checkers) + { + ReferenceCheckers.insert_or_assign(std::unique_ptr(Checker), + Index); + Checker = nullptr; + } + } } - if (!Checkers.empty()) + catch (const std::system_error& Ex) { - RwLock::ExclusiveLockScope __(ReferenceCheckersLock); - for (auto& Checker : Checkers) + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed creating reference checkers for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + } + else { - ReferenceCheckers.insert_or_assign(std::unique_ptr(Checker), Index); - Checker = nullptr; + ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); } + SetCancelGC(true); } - } - catch (const std::exception& Ex) - { - ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'", - Referencer->GetGcName(Ctx), - Ex.what()); - SetCancelGC(true); - while (!Checkers.empty()) + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); + } + catch (const std::exception& Ex) { - delete Checkers.back(); - Checkers.pop_back(); + ZEN_ERROR("GCV2: Failed creating reference checkers for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); } } }); @@ -975,6 +1044,13 @@ GcManager::CollectGarbage(const GcSettings& Settings) auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); std::vector Validators; + auto __ = MakeGuard([&Validators]() { + while (!Validators.empty()) + { + delete Validators.back(); + Validators.pop_back(); + } + }); try { { @@ -995,17 +1071,35 @@ GcManager::CollectGarbage(const GcSettings& Settings) } } } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed creating reference validators for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'", + Referencer->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed creating reference validators for {}. Reason: '{}'", Referencer->GetGcName(Ctx), Ex.what()); SetCancelGC(true); - while (!Validators.empty()) - { - delete Validators.back(); - Validators.pop_back(); - } } }); } @@ -1054,6 +1148,23 @@ GcManager::CollectGarbage(const GcSettings& Settings) SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); Checker->PreCache(Ctx); } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); @@ -1142,6 +1253,29 @@ GcManager::CollectGarbage(const GcSettings& Settings) std::chrono::milliseconds(Timer.GetElapsedTimeMs());); Checker->UpdateLockedState(Ctx); } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'", + Checker->GetGcName(Ctx), + Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'", + Checker->GetGcName(Ctx), + Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_WARN("GCV2: Failed Updating locked state for {}. Reason: '{}'", + Checker->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed Updating locked state for {}. Reason: '{}'", @@ -1231,6 +1365,29 @@ GcManager::CollectGarbage(const GcSettings& Settings) StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->CompactStoreStats); } } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed removing unused data for {}. Reason: '{}'", + Pruner->GetGcName(Ctx), + Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'", + Pruner->GetGcName(Ctx), + Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'", + Pruner->GetGcName(Ctx), + Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed removing unused data for {}. Reason: '{}'", @@ -1294,6 +1451,23 @@ GcManager::CollectGarbage(const GcSettings& Settings) SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); Compactor->CompactStore(Ctx, Stats, ClaimDiskReserve); } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed compacting store {}. Reason: '{}'", Compactor->GetGcName(Ctx), Ex.what()); @@ -1335,6 +1509,23 @@ GcManager::CollectGarbage(const GcSettings& Settings) SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); ReferenceValidator->Validate(Ctx, Stats); } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) + { + ZEN_WARN("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what()); + } + else + { + ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what()); + } + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } catch (const std::exception& Ex) { ZEN_ERROR("GCV2: Failed validating referencer {}. Reason: '{}'", ReferenceValidator->GetGcName(Ctx), Ex.what()); -- cgit v1.2.3 From 2a727175c7eaf369593a4ac5039ecd7f3da5aa66 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 5 May 2025 19:27:42 +0200 Subject: builds allow redirect option (#379) * add --allow-redirect to zen builds upload/download --- src/zen/cmds/builds_cmd.cpp | 17 ++++- src/zen/cmds/builds_cmd.h | 3 +- .../projectstore/buildsremoteprojectstore.cpp | 29 ++++----- .../projectstore/jupiterremoteprojectstore.cpp | 27 ++++---- src/zenserver/upstream/upstreamcache.cpp | 15 ++--- .../include/zenutil/jupiter/jupiterbuildstorage.h | 1 + .../include/zenutil/jupiter/jupitersession.h | 3 +- src/zenutil/jupiter/jupiterbuildstorage.cpp | 6 +- src/zenutil/jupiter/jupitersession.cpp | 73 ++++++++++++++++------ 9 files changed, 107 insertions(+), 67 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 134cde0f8..c4a1344d4 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -8953,6 +8953,12 @@ BuildsCommand::BuildsCommand() Ops.add_option("cloud build", "", "namespace", "Builds Storage namespace", cxxopts::value(m_Namespace), ""); Ops.add_option("cloud build", "", "bucket", "Builds Storage bucket", cxxopts::value(m_Bucket), ""); + Ops.add_option("cloud build", + "", + "allow-redirect", + "Allow redirect of requests", + cxxopts::value(m_AllowRedirect), + ""); }; auto AddFileOptions = [this](cxxopts::Options& Ops) { @@ -9677,9 +9683,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Result.BuildStorageHttp->GetSessionId(), m_Namespace, m_Bucket); - Result.BuildStorage = - CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); - Result.StorageName = BuildStorageName; + Result.BuildStorage = CreateJupiterBuildStorage(Log(), + *Result.BuildStorageHttp, + StorageStats, + m_Namespace, + m_Bucket, + m_AllowRedirect, + TempPath / "storage"); + Result.StorageName = BuildStorageName; } else if (!m_StoragePath.empty()) { diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 9f6c1f6ec..de2d6fffc 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -37,7 +37,8 @@ private: // cloud builds std::string m_OverrideHost; std::string m_Host; - bool m_AssumeHttp2 = false; + bool m_AssumeHttp2 = false; + bool m_AllowRedirect = false; std::string m_Namespace; std::string m_Bucket; diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index 2a04d5c40..ab96ae92d 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -35,16 +35,10 @@ public: , m_BuildId(BuildId) , m_MetaData(MetaData) , m_TempFilePath(TempFilePath) + , m_EnableBlocks(!ForceDisableBlocks) + , m_UseTempBlocks(!ForceDisableTempBlocks) { m_MetaData.MakeOwned(); - if (ForceDisableBlocks) - { - m_EnableBlocks = false; - } - if (ForceDisableTempBlocks) - { - m_UseTempBlocks = false; - } } virtual RemoteStoreInfo GetInfo() const override @@ -71,7 +65,7 @@ public: { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); IoBuffer Payload = m_MetaData; Payload.SetContentType(ZenContentType::kCbObject); @@ -95,7 +89,7 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); PutBuildPartResult PutResult = Session.PutBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, OplogContainerPartName, Payload); AddStats(PutResult); @@ -120,7 +114,7 @@ public: ChunkBlockDescription&& Block) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult PutResult = Session.PutBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); @@ -183,7 +177,7 @@ public: ZEN_UNUSED(RawHash); ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); FinalizeBuildPartResult FinalizeRefResult = Session.FinalizeBuildPart(m_Namespace, m_Bucket, m_BuildId, m_OplogBuildPartId, RawHash); AddStats(FinalizeRefResult); @@ -222,7 +216,7 @@ public: { ZEN_ASSERT(m_OplogBuildPartId == Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult GetBuildResult = Session.GetBuild(m_Namespace, m_Bucket, m_BuildId); AddStats(GetBuildResult); LoadContainerResult Result{ConvertResult(GetBuildResult)}; @@ -307,7 +301,7 @@ public: virtual GetKnownBlocksResult GetKnownBlocks() override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; @@ -356,7 +350,7 @@ public: virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult GetResult = Session.GetBuildBlob(m_Namespace, m_Bucket, m_BuildId, RawHash, m_TempFilePath); AddStats(GetResult); @@ -452,8 +446,9 @@ private: IoBuffer m_MetaData; Oid m_OplogBuildPartId = Oid::Zero; std::filesystem::path m_TempFilePath; - bool m_EnableBlocks = true; - bool m_UseTempBlocks = true; + const bool m_EnableBlocks = true; + const bool m_UseTempBlocks = true; + const bool m_AllowRedirect = false; std::atomic_uint64_t m_SentBytes = {}; std::atomic_uint64_t m_ReceivedBytes = {}; diff --git a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp index 20e6c28ac..3728babb5 100644 --- a/src/zenserver/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenserver/projectstore/jupiterremoteprojectstore.cpp @@ -31,15 +31,9 @@ public: , m_Key(Key) , m_OptionalBaseKey(OptionalBaseKey) , m_TempFilePath(TempFilePath) + , m_EnableBlocks(!ForceDisableBlocks) + , m_UseTempBlocks(!ForceDisableTempBlocks) { - if (ForceDisableBlocks) - { - m_EnableBlocks = false; - } - if (ForceDisableTempBlocks) - { - m_UseTempBlocks = false; - } } virtual RemoteStoreInfo GetInfo() const override @@ -75,7 +69,7 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) override { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); PutRefResult PutResult = Session.PutRef(m_Namespace, m_Bucket, m_Key, Payload, ZenContentType::kCbObject); AddStats(PutResult); @@ -94,7 +88,7 @@ public: virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult PutResult = Session.PutCompressedBlob(m_Namespace, RawHash, Payload); AddStats(PutResult); @@ -127,7 +121,7 @@ public: virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) override { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); FinalizeRefResult FinalizeRefResult = Session.FinalizeRef(m_Namespace, m_Bucket, m_Key, RawHash); AddStats(FinalizeRefResult); @@ -164,7 +158,7 @@ public: {.ErrorCode = static_cast(HttpResponseCode::NoContent), .ElapsedSeconds = LoadResult.ElapsedSeconds}}; } - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterExistsResult ExistsResult = Session.CompressedBlobExists(m_Namespace, std::set(BlockHashes.begin(), BlockHashes.end())); AddStats(ExistsResult); @@ -203,7 +197,7 @@ public: virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); AddStats(GetResult); @@ -239,7 +233,7 @@ public: private: LoadContainerResult LoadContainer(const IoHash& Key) { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult GetResult = Session.GetRef(m_Namespace, m_Bucket, Key, ZenContentType::kCbObject); AddStats(GetResult); if (GetResult.ErrorCode || !GetResult.Success) @@ -329,8 +323,9 @@ private: const IoHash m_Key; const IoHash m_OptionalBaseKey; std::filesystem::path m_TempFilePath; - bool m_EnableBlocks = true; - bool m_UseTempBlocks = true; + const bool m_EnableBlocks = true; + const bool m_UseTempBlocks = true; + const bool m_AllowRedirect = false; std::atomic_uint64_t m_SentBytes = {}; std::atomic_uint64_t m_ReceivedBytes = {}; diff --git a/src/zenserver/upstream/upstreamcache.cpp b/src/zenserver/upstream/upstreamcache.cpp index e438a840a..744b861dd 100644 --- a/src/zenserver/upstream/upstreamcache.cpp +++ b/src/zenserver/upstream/upstreamcache.cpp @@ -134,7 +134,7 @@ namespace detail { return {.State = UpstreamEndpointState::kOk}; } - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); const JupiterResult Result = Session.Authenticate(); if (Result.Success) @@ -181,7 +181,7 @@ namespace detail { try { - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); JupiterResult Result; std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); @@ -301,7 +301,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords"); - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheKeyRequest* Request : Requests) @@ -365,7 +365,7 @@ namespace detail { try { - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); const JupiterResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); @@ -398,7 +398,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks"); - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheChunkRequest* RequestPtr : CacheChunkRequests) @@ -453,7 +453,7 @@ namespace detail { { ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues"); - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); GetUpstreamCacheResult Result; for (CacheValueRequest* RequestPtr : CacheValueRequests) @@ -533,7 +533,7 @@ namespace detail { try { - JupiterSession Session(m_Client->Logger(), m_Client->Client()); + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); if (CacheRecord.Type == ZenContentType::kBinary) { @@ -756,6 +756,7 @@ namespace detail { UpstreamStatus m_Status; UpstreamEndpointStats m_Stats; RefPtr m_Client; + const bool m_AllowRedirect = false; }; class ZenUpstreamEndpoint final : public UpstreamEndpoint diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h index 89fc70140..bbf070993 100644 --- a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h +++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h @@ -13,5 +13,6 @@ std::unique_ptr CreateJupiterBuildStorage(LoggerRef InLog, BuildStorage::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, + bool AllowRedirect, const std::filesystem::path& TempFolderPath); } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 32bfd50f4..b79790f25 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -65,7 +65,7 @@ struct FinalizeBuildPartResult : JupiterResult class JupiterSession { public: - JupiterSession(LoggerRef InLog, HttpClient& InHttpClient); + JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect); ~JupiterSession(); JupiterResult Authenticate(); @@ -173,6 +173,7 @@ private: LoggerRef m_Log; HttpClient& m_HttpClient; + const bool m_AllowRedirect = false; }; } // namespace zen diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 6ada72e1e..53cad78da 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -25,8 +25,9 @@ public: Statistics& Stats, std::string_view Namespace, std::string_view Bucket, + bool AllowRedirect, const std::filesystem::path& TempFolderPath) - : m_Session(InLog, InHttpClient) + : m_Session(InLog, InHttpClient, AllowRedirect) , m_Stats(Stats) , m_Namespace(Namespace) , m_Bucket(Bucket) @@ -462,11 +463,12 @@ CreateJupiterBuildStorage(LoggerRef InLog, BuildStorage::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, + bool AllowRedirect, const std::filesystem::path& TempFolderPath) { ZEN_TRACE_CPU("CreateJupiterBuildStorage"); - return std::make_unique(InLog, InHttpClient, Stats, Namespace, Bucket, TempFolderPath); + return std::make_unique(InLog, InHttpClient, Stats, Namespace, Bucket, AllowRedirect, TempFolderPath); } } // namespace zen diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index d3076d36b..de138f994 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -48,7 +49,10 @@ namespace detail { } } // namespace detail -JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient) : m_Log(InLog), m_HttpClient(InHttpClient) +JupiterSession::JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect) +: m_Log(InLog) +, m_HttpClient(InHttpClient) +, m_AllowRedirect(AllowRedirect) { } @@ -542,12 +546,14 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, bool& OutIsComplete) -> JupiterResult { const MultipartUploadResponse::Part& Part = Workload->PartDescription.Parts[PartIndex]; IoBuffer PartPayload = Workload->Transmitter(Part.FirstByte, Part.LastByte - Part.FirstByte); - std::string MultipartUploadResponseRequestString = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}", - Namespace, - BucketId, - BuildId, - Hash.ToHexString(), - Part.QueryString); + std::string MultipartUploadResponseRequestString = + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + Part.QueryString, + m_AllowRedirect ? "true"sv : "false"sv); // ZEN_INFO("PUT: {}", MultipartUploadResponseRequestString); HttpClient::Response MultipartUploadResponse = m_HttpClient.Put(MultipartUploadResponseRequestString, PartPayload); if (!MultipartUploadResponse.IsSuccess()) @@ -605,12 +611,13 @@ JupiterSession::PutMultipartBuildBlob(std::string_view Namespace, IoBuffer RetryPartPayload = Workload->Transmitter(RetryPart.FirstByte, RetryPart.LastByte - RetryPart.FirstByte - 1); std::string RetryMultipartUploadResponseRequestString = - fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}", + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}/uploadMultipart{}&supportsRedirect={}", Namespace, BucketId, BuildId, Hash.ToHexString(), - RetryPart.QueryString); + RetryPart.QueryString, + m_AllowRedirect ? "true"sv : "false"sv); MultipartUploadResponse = m_HttpClient.Put(RetryMultipartUploadResponseRequestString, RetryPartPayload); TotalUploadedBytes = MultipartUploadResponse.UploadedBytes; @@ -650,7 +657,12 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, std::function&& OnComplete, std::vector>& OutWorkItems) { - std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); + std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + m_AllowRedirect ? "true"sv : "false"sv); HttpClient::Response Response = m_HttpClient.Get(RequestUrl, HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", 0, ChunkSize - 1)}})); if (Response.IsSuccess()) @@ -687,11 +699,15 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, uint64_t PartSize = Min(ChunkSize, TotalSize - Offset); OutWorkItems.emplace_back( [this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult { - std::string RequestUrl = - fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()); - HttpClient::Response Response = m_HttpClient.Get( - RequestUrl, - HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); + std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + m_AllowRedirect ? "true"sv : "false"sv); + HttpClient::Response Response = m_HttpClient.Get( + RequestUrl, + HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); if (Response.IsSuccess()) { Workload->OnReceive(Offset, Response.ResponsePayload); @@ -734,11 +750,28 @@ JupiterSession::GetBuildBlob(std::string_view Namespace, { Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", Offset, Offset + Size - 1)}); } - HttpClient::Response Response = - m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}", Namespace, BucketId, BuildId, Hash.ToHexString()), - TempFolderPath, - Headers); - + HttpClient::Response Response = m_HttpClient.Download(fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + m_AllowRedirect ? "true"sv : "false"sv), + TempFolderPath, + Headers); + if (Response.IsSuccess()) + { + // If we get a redirect to S3 or a non-Jupiter endpoint the content type will not be correct, validate it and set it + if (m_AllowRedirect && (Response.ResponsePayload.GetContentType() == HttpContentType::kBinary)) + { + IoHash ValidateRawHash; + uint64_t ValidateRawSize = 0; + ZEN_ASSERT_SLOW(CompressedBuffer::ValidateCompressedHeader(Response.ResponsePayload, ValidateRawHash, ValidateRawSize)); + ZEN_ASSERT_SLOW(ValidateRawHash == Hash); + ZEN_ASSERT_SLOW(ValidateRawSize > 0); + ZEN_UNUSED(ValidateRawHash, ValidateRawSize); + Response.ResponsePayload.SetContentType(ZenContentType::kCompressedBinary); + } + } return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv); } -- cgit v1.2.3 From 6d9ff7e404a22ed1cc7e529cfa77ef7d593d9547 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 6 May 2025 16:50:57 +0200 Subject: add sentry for zen command (#373) * refactor sentry integration and add to zen command line tool * move add_ldflags("-framework Security") --- src/zen/xmake.lua | 4 +- src/zen/zen.cpp | 35 +++ src/zencore/include/zencore/sentryintegration.h | 50 ++++ src/zencore/sentryintegration.cpp | 327 ++++++++++++++++++++++++ src/zencore/xmake.lua | 21 ++ src/zenserver/main.cpp | 2 +- src/zenserver/sentryintegration.cpp | 324 ----------------------- src/zenserver/sentryintegration.h | 50 ---- src/zenserver/xmake.lua | 15 -- src/zenserver/zenserver.cpp | 3 +- 10 files changed, 438 insertions(+), 393 deletions(-) create mode 100644 src/zencore/include/zencore/sentryintegration.h create mode 100644 src/zencore/sentryintegration.cpp delete mode 100644 src/zenserver/sentryintegration.cpp delete mode 100644 src/zenserver/sentryintegration.h (limited to 'src') diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua index 78b2a3c2b..bf595a21d 100644 --- a/src/zen/xmake.lua +++ b/src/zen/xmake.lua @@ -18,13 +18,15 @@ target("zen") add_ldflags("/subsystem:console,5.02") add_ldflags("/LTCG") add_links("crypt32", "wldap32", "Ws2_32") + + add_links("dbghelp", "winhttp", "version") -- for Sentry end if is_plat("macosx") then add_ldflags("-framework CoreFoundation") + add_ldflags("-framework Foundation") add_ldflags("-framework Security") add_ldflags("-framework SystemConfiguration") - add_syslinks("bsm") end add_packages("vcpkg::cpr", "vcpkg::cxxopts", "vcpkg::mimalloc", "vcpkg::fmt") diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 399c43990..7c21035c0 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -581,6 +582,8 @@ ProgressBar::HasActiveTask() const int main(int argc, char** argv) { + zen::SetCurrentThreadName("main"); + std::vector Args; #if ZEN_PLATFORM_WINDOWS LPWSTR RawCommandLine = GetCommandLine(); @@ -838,6 +841,38 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE Options.parse_positional({"command"}); +#if ZEN_USE_SENTRY + bool NoSentry = false; + bool SentryAllowPII = false; + Options.add_options()("no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false")); + Options.add_options()("sentry-allow-personal-info", + "Allow personally identifiable information in sentry crash reports", + cxxopts::value(SentryAllowPII)->default_value("false")); +#endif + +#if ZEN_USE_SENTRY + SentryIntegration Sentry; + + if (NoSentry == false) + { + std::string SentryDatabasePath = (GetRunningExecutablePath().parent_path() / ".sentry-native").string(); + + ExtendableStringBuilder<512> SB; + for (int i = 0; i < argc; ++i) + { + if (i) + { + SB.Append(' '); + } + + SB.Append(argv[i]); + } + + Sentry.Initialize(SentryDatabasePath, {}, SentryAllowPII, SB.ToString()); + + SentryIntegration::ClearCaches(); + } +#endif const bool IsNullInvoke = (argc == 1); // If no arguments are passed we want to print usage information diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h new file mode 100644 index 000000000..40e22af4e --- /dev/null +++ b/src/zencore/include/zencore/sentryintegration.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#if !defined(ZEN_USE_SENTRY) +# define ZEN_USE_SENTRY 1 +#endif + +#if ZEN_USE_SENTRY + +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace sentry { + +struct SentryAssertImpl; + +} // namespace sentry + +namespace zen { + +class SentryIntegration +{ +public: + SentryIntegration(); + ~SentryIntegration(); + + void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine); + void LogStartupInformation(); + static void ClearCaches(); + +private: + int m_SentryErrorCode = 0; + bool m_IsInitialized = false; + bool m_AllowPII = false; + std::unique_ptr m_SentryAssert; + std::string m_SentryUserName; + std::string m_SentryHostName; + std::string m_SentryId; + std::shared_ptr m_SentryLogger; +}; + +} // namespace zen +#endif diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp new file mode 100644 index 000000000..d08fb7f1d --- /dev/null +++ b/src/zencore/sentryintegration.cpp @@ -0,0 +1,327 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include + +#include +#include + +#if ZEN_PLATFORM_LINUX +# include +#endif + +#if ZEN_PLATFORM_MAC +# include +#endif + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_USE_SENTRY +# define SENTRY_BUILD_STATIC 1 +ZEN_THIRD_PARTY_INCLUDES_START +# include +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace sentry { + +struct SentryAssertImpl : zen::AssertImpl +{ + virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION + OnAssert(const char* Filename, int LineNumber, const char* FunctionName, const char* Msg, zen::CallstackFrames* Callstack) override; +}; + +class sentry_sink final : public spdlog::sinks::base_sink +{ +public: + sentry_sink(); + ~sentry_sink(); + +protected: + void sink_it_(const spdlog::details::log_msg& msg) override; + void flush_() override; +}; + +////////////////////////////////////////////////////////////////////////// + +static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_INFO, + SENTRY_LEVEL_WARNING, + SENTRY_LEVEL_ERROR, + SENTRY_LEVEL_FATAL, + SENTRY_LEVEL_DEBUG}; + +sentry_sink::sentry_sink() +{ +} +sentry_sink::~sentry_sink() +{ +} + +void +sentry_sink::sink_it_(const spdlog::details::log_msg& msg) +{ + if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) + { + return; + } + try + { + std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); + sentry_value_t event = sentry_value_new_message_event( + /* level */ MapToSentryLevel[msg.level], + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } + catch (const std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw + char TmpBuffer[256]; + size_t MaxCopy = zen::Min(msg.payload.size(), size_t(255)); + memcpy(TmpBuffer, msg.payload.data(), MaxCopy); + TmpBuffer[MaxCopy] = '\0'; + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ TmpBuffer); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } +} +void +sentry_sink::flush_() +{ +} + +void +SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionName, const char* Msg, zen::CallstackFrames* Callstack) +{ + // Sentry will provide its own callstack + ZEN_UNUSED(Callstack); + try + { + std::string Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg); + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } + catch (const std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the Msg raw + sentry_value_t event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ Msg); + sentry_event_value_add_stacktrace(event, NULL, 0); + sentry_capture_event(event); + } +} + +} // namespace sentry + +namespace zen { + +# if ZEN_USE_SENTRY +static void +SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) +{ + char LogMessageBuffer[160]; + std::string LogMessage; + const char* MessagePtr = LogMessageBuffer; + + int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); + + if (n >= int(sizeof LogMessageBuffer)) + { + LogMessage.resize(n + 1); + + n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args); + + MessagePtr = LogMessage.c_str(); + } + + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + break; + } +} +# endif + +SentryIntegration::SentryIntegration() +{ +} + +SentryIntegration::~SentryIntegration() +{ + if (m_IsInitialized && m_SentryErrorCode == 0) + { + logging::SetErrorLog(""); + m_SentryAssert.reset(); + sentry_close(); + } +} + +void +SentryIntegration::Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + bool AllowPII, + const std::string& CommandLine) +{ + m_AllowPII = AllowPII; + + if (SentryDatabasePath.starts_with("\\\\?\\")) + { + SentryDatabasePath = SentryDatabasePath.substr(4); + } + sentry_options_t* SentryOptions = sentry_options_new(); + sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); + sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); + sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + if (!SentryAttachmentsPath.empty()) + { + if (SentryAttachmentsPath.starts_with("\\\\?\\")) + { + SentryAttachmentsPath = SentryAttachmentsPath.substr(4); + } + sentry_options_add_attachment(SentryOptions, SentryAttachmentsPath.c_str()); + } + sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); + + // sentry_options_set_debug(SentryOptions, 1); + + m_SentryErrorCode = sentry_init(SentryOptions); + + if (m_SentryErrorCode == 0) + { + sentry_value_t SentryUserObject = sentry_value_new_object(); + + if (m_AllowPII) + { +# if ZEN_PLATFORM_WINDOWS + CHAR Buffer[511 + 1]; + DWORD BufferLength = sizeof(Buffer) / sizeof(CHAR); + BOOL OK = GetUserNameA(Buffer, &BufferLength); + if (OK && BufferLength) + { + m_SentryUserName = std::string(Buffer, BufferLength - 1); + } + BufferLength = sizeof(Buffer) / sizeof(CHAR); + OK = GetComputerNameA(Buffer, &BufferLength); + if (OK && BufferLength) + { + m_SentryHostName = std::string(Buffer, BufferLength); + } + else + { + m_SentryHostName = "unknown"; + } +# endif // ZEN_PLATFORM_WINDOWS + +# if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC) + uid_t uid = geteuid(); + struct passwd* pw = getpwuid(uid); + if (pw) + { + m_SentryUserName = std::string(pw->pw_name); + } + else + { + m_SentryUserName = "unknown"; + } + char HostNameBuffer[1023 + 1]; + int err = gethostname(HostNameBuffer, sizeof(HostNameBuffer)); + if (err == 0) + { + m_SentryHostName = std::string(HostNameBuffer); + } + else + { + m_SentryHostName = "unknown"; + } +# endif + m_SentryId = fmt::format("{}@{}", m_SentryUserName, m_SentryHostName); + sentry_value_set_by_key(SentryUserObject, "id", sentry_value_new_string(m_SentryId.c_str())); + sentry_value_set_by_key(SentryUserObject, "username", sentry_value_new_string(m_SentryUserName.c_str())); + sentry_value_set_by_key(SentryUserObject, "ip_address", sentry_value_new_string("{{auto}}")); + } + + sentry_value_set_by_key(SentryUserObject, "cmd", sentry_value_new_string(CommandLine.c_str())); + + const std::string SessionId(GetSessionIdString()); + sentry_value_set_by_key(SentryUserObject, "session", sentry_value_new_string(SessionId.c_str())); + + sentry_set_user(SentryUserObject); + + m_SentryLogger = spdlog::create("sentry"); + logging::SetErrorLog("sentry"); + + m_SentryAssert = std::make_unique(); + } + + m_IsInitialized = true; +} + +void +SentryIntegration::LogStartupInformation() +{ + if (m_IsInitialized) + { + if (m_SentryErrorCode == 0) + { + if (m_AllowPII) + { + ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); + } + else + { + ZEN_INFO("sentry initialized with anonymous reports"); + } + } + else + { + ZEN_WARN( + "sentry_init returned failure! (error code: {}) note that sentry expects crashpad_handler to exist alongside the running " + "executable", + m_SentryErrorCode); + } + } +} + +void +SentryIntegration::ClearCaches() +{ + sentry_clear_modulecache(); +} + +} // namespace zen +#endif diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 13611a2e9..b3a33e052 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -64,6 +64,27 @@ target('zencore') {public=true} ) + if has_config("zensentry") then + add_packages("vcpkg::sentry-native") + + if is_plat("windows") then + add_links("dbghelp", "winhttp", "version") -- for Sentry + end + + if is_plat("linux") then + -- As sentry_native uses symbols from breakpad_client, the latter must + -- be specified after the former with GCC-like toolchains. xmake however + -- is unaware of this and simply globs files from vcpkg's output. The + -- line below forces breakpad_client to be to the right of sentry_native + add_syslinks("breakpad_client") + end + + if is_plat("macosx") then + add_syslinks("bsm") + end + + end + if is_plat("linux") then add_syslinks("rt") end diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 78ddd39a0..0f647cd5c 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include "config.h" #include "diag/logging.h" -#include "sentryintegration.h" #if ZEN_PLATFORM_WINDOWS # include diff --git a/src/zenserver/sentryintegration.cpp b/src/zenserver/sentryintegration.cpp deleted file mode 100644 index 7996f25bb..000000000 --- a/src/zenserver/sentryintegration.cpp +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "sentryintegration.h" - -#include -#include -#include -#include - -#include -#include - -#if ZEN_PLATFORM_LINUX -# include -#endif - -#if ZEN_PLATFORM_MAC -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - -#if ZEN_USE_SENTRY -# define SENTRY_BUILD_STATIC 1 -ZEN_THIRD_PARTY_INCLUDES_START -# include -# include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace sentry { - -struct SentryAssertImpl : zen::AssertImpl -{ - virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION - OnAssert(const char* Filename, int LineNumber, const char* FunctionName, const char* Msg, zen::CallstackFrames* Callstack) override; -}; - -class sentry_sink final : public spdlog::sinks::base_sink -{ -public: - sentry_sink(); - ~sentry_sink(); - -protected: - void sink_it_(const spdlog::details::log_msg& msg) override; - void flush_() override; -}; - -////////////////////////////////////////////////////////////////////////// - -static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_INFO, - SENTRY_LEVEL_WARNING, - SENTRY_LEVEL_ERROR, - SENTRY_LEVEL_FATAL, - SENTRY_LEVEL_DEBUG}; - -sentry_sink::sentry_sink() -{ -} -sentry_sink::~sentry_sink() -{ -} - -void -sentry_sink::sink_it_(const spdlog::details::log_msg& msg) -{ - if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) - { - return; - } - try - { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (const std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw - char TmpBuffer[256]; - size_t MaxCopy = zen::Min(msg.payload.size(), size_t(255)); - memcpy(TmpBuffer, msg.payload.data(), MaxCopy); - TmpBuffer[MaxCopy] = '\0'; - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ TmpBuffer); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} -void -sentry_sink::flush_() -{ -} - -void -SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionName, const char* Msg, zen::CallstackFrames* Callstack) -{ - // Sentry will provide its own callstack - ZEN_UNUSED(Callstack); - try - { - std::string Message = fmt::format("ASSERT {}:({}) [{}]\n\"{}\"", Filename, LineNumber, FunctionName, Msg); - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (const std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the Msg raw - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ Msg); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} - -} // namespace sentry - -namespace zen { - -# if ZEN_USE_SENTRY -static void -SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) -{ - char LogMessageBuffer[160]; - std::string LogMessage; - const char* MessagePtr = LogMessageBuffer; - - int n = vsnprintf(LogMessageBuffer, sizeof LogMessageBuffer, Message, Args); - - if (n >= int(sizeof LogMessageBuffer)) - { - LogMessage.resize(n + 1); - - n = vsnprintf(LogMessage.data(), LogMessage.size(), Message, Args); - - MessagePtr = LogMessage.c_str(); - } - - switch (Level) - { - case SENTRY_LEVEL_DEBUG: - ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_INFO: - ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_WARNING: - ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_ERROR: - ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); - break; - - case SENTRY_LEVEL_FATAL: - ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); - break; - } -} -# endif - -SentryIntegration::SentryIntegration() -{ -} - -SentryIntegration::~SentryIntegration() -{ - if (m_IsInitialized && m_SentryErrorCode == 0) - { - logging::SetErrorLog(""); - m_SentryAssert.reset(); - sentry_close(); - } -} - -void -SentryIntegration::Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - bool AllowPII, - const std::string& CommandLine) -{ - m_AllowPII = AllowPII; - - if (SentryDatabasePath.starts_with("\\\\?\\")) - { - SentryDatabasePath = SentryDatabasePath.substr(4); - } - sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); - sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); - sentry_options_set_logger(SentryOptions, SentryLogFunction, this); - if (SentryAttachmentsPath.starts_with("\\\\?\\")) - { - SentryAttachmentsPath = SentryAttachmentsPath.substr(4); - } - sentry_options_add_attachment(SentryOptions, SentryAttachmentsPath.c_str()); - sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - - // sentry_options_set_debug(SentryOptions, 1); - - m_SentryErrorCode = sentry_init(SentryOptions); - - if (m_SentryErrorCode == 0) - { - sentry_value_t SentryUserObject = sentry_value_new_object(); - - if (m_AllowPII) - { -# if ZEN_PLATFORM_WINDOWS - CHAR Buffer[511 + 1]; - DWORD BufferLength = sizeof(Buffer) / sizeof(CHAR); - BOOL OK = GetUserNameA(Buffer, &BufferLength); - if (OK && BufferLength) - { - m_SentryUserName = std::string(Buffer, BufferLength - 1); - } - BufferLength = sizeof(Buffer) / sizeof(CHAR); - OK = GetComputerNameA(Buffer, &BufferLength); - if (OK && BufferLength) - { - m_SentryHostName = std::string(Buffer, BufferLength); - } - else - { - m_SentryHostName = "unknown"; - } -# endif // ZEN_PLATFORM_WINDOWS - -# if (ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC) - uid_t uid = geteuid(); - struct passwd* pw = getpwuid(uid); - if (pw) - { - m_SentryUserName = std::string(pw->pw_name); - } - else - { - m_SentryUserName = "unknown"; - } - char HostNameBuffer[1023 + 1]; - int err = gethostname(HostNameBuffer, sizeof(HostNameBuffer)); - if (err == 0) - { - m_SentryHostName = std::string(HostNameBuffer); - } - else - { - m_SentryHostName = "unknown"; - } -# endif - m_SentryId = fmt::format("{}@{}", m_SentryUserName, m_SentryHostName); - sentry_value_set_by_key(SentryUserObject, "id", sentry_value_new_string(m_SentryId.c_str())); - sentry_value_set_by_key(SentryUserObject, "username", sentry_value_new_string(m_SentryUserName.c_str())); - sentry_value_set_by_key(SentryUserObject, "ip_address", sentry_value_new_string("{{auto}}")); - } - - sentry_value_set_by_key(SentryUserObject, "cmd", sentry_value_new_string(CommandLine.c_str())); - - const std::string SessionId(GetSessionIdString()); - sentry_value_set_by_key(SentryUserObject, "session", sentry_value_new_string(SessionId.c_str())); - - sentry_set_user(SentryUserObject); - - m_SentryLogger = spdlog::create("sentry"); - logging::SetErrorLog("sentry"); - - m_SentryAssert = std::make_unique(); - } - - m_IsInitialized = true; -} - -void -SentryIntegration::LogStartupInformation() -{ - if (m_IsInitialized) - { - if (m_SentryErrorCode == 0) - { - if (m_AllowPII) - { - ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); - } - else - { - ZEN_INFO("sentry initialized with anonymous reports"); - } - } - else - { - ZEN_WARN( - "sentry_init returned failure! (error code: {}) note that sentry expects crashpad_handler to exist alongside the running " - "executable", - m_SentryErrorCode); - } - } -} - -void -SentryIntegration::ClearCaches() -{ - sentry_clear_modulecache(); -} - -} // namespace zen -#endif diff --git a/src/zenserver/sentryintegration.h b/src/zenserver/sentryintegration.h deleted file mode 100644 index 40e22af4e..000000000 --- a/src/zenserver/sentryintegration.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include - -#if !defined(ZEN_USE_SENTRY) -# define ZEN_USE_SENTRY 1 -#endif - -#if ZEN_USE_SENTRY - -# include - -ZEN_THIRD_PARTY_INCLUDES_START -# include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace sentry { - -struct SentryAssertImpl; - -} // namespace sentry - -namespace zen { - -class SentryIntegration -{ -public: - SentryIntegration(); - ~SentryIntegration(); - - void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine); - void LogStartupInformation(); - static void ClearCaches(); - -private: - int m_SentryErrorCode = 0; - bool m_IsInitialized = false; - bool m_AllowPII = false; - std::unique_ptr m_SentryAssert; - std::string m_SentryUserName; - std::string m_SentryHostName; - std::string m_SentryId; - std::shared_ptr m_SentryLogger; -}; - -} // namespace zen -#endif diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index a3d7aa124..470fbd24e 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -28,8 +28,6 @@ target("zenserver") add_cxxflags("/bigobj") add_links("delayimp", "projectedfslib") add_ldflags("/delayload:ProjectedFSLib.dll") - - add_links("dbghelp", "winhttp", "version") -- for Sentry else remove_files("windows/**") end @@ -41,7 +39,6 @@ target("zenserver") add_ldflags("-framework Foundation") add_ldflags("-framework Security") add_ldflags("-framework SystemConfiguration") - add_syslinks("bsm") end add_options("compute") @@ -57,18 +54,6 @@ target("zenserver") "vcpkg::sol2" ) - if has_config("zensentry") then - add_packages("vcpkg::sentry-native") - end - - if is_plat("linux") then - -- As sentry_native uses symbols from breakpad_client, the latter must - -- be specified after the former with GCC-like toolchains. xmake however - -- is unaware of this and simply globs files from vcpkg's output. The - -- line below forces breakpad_client to be to the right of sentry_native - add_syslinks("breakpad_client") - end - -- to work around some unfortunate Ctrl-C behaviour on Linux/Mac due to -- our use of setsid() at startup we pass in `--no-detach` to zenserver -- ensure that it recieves signals when the user requests termination diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 3f2f01d5a..366944ffc 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -2,8 +2,6 @@ #include "zenserver.h" -#include "sentryintegration.h" - #include #include #include @@ -16,6 +14,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 0acb05d46435fd9f3b5497b268a96462bb69fb2f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 6 May 2025 18:19:15 +0200 Subject: optimize cache bucket state writing (#382) * optimize cache bucket snapshot and sidecar writing --- src/zenstore/cache/cachedisklayer.cpp | 101 ++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 42 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index da5038984..fbe559e58 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -318,10 +318,10 @@ private: #pragma pack(4) struct ManifestData { - uint32_t RawSize; // 4 - AccessTime Timestamp; // 4 - IoHash RawHash; // 20 - IoHash Key; // 20 + uint32_t RawSize; // 4 + uint32_t SecondsSinceEpoch; // 4 + IoHash RawHash; // 20 + IoHash Key; // 20 }; #pragma pack(pop) @@ -603,7 +603,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B ZenCacheDiskLayer::CacheBucket::BucketPayload& PayloadEntry = Payloads[PlIndex]; - AccessTimes[PlIndex] = Entry->Timestamp; + AccessTimes[PlIndex].SetSecondsSinceEpoch(Entry->SecondsSinceEpoch); if (Entry->RawSize && Entry->RawHash != IoHash::Zero) { @@ -630,6 +630,16 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, { ZEN_TRACE_CPU("Z$::WriteSidecarFile"); + ZEN_DEBUG("writing store sidecar for '{}'", SidecarPath); + const uint64_t EntryCount = Index.size(); + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("wrote store sidecar for '{}' containing {} entries in {}", + SidecarPath, + EntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + BucketMetaHeader Header; Header.EntryCount = m_ManifestEntryCount; Header.LogPosition = SnapshotLogPosition; @@ -647,43 +657,44 @@ BucketManifestSerializer::WriteSidecarFile(RwLock::SharedLockScope&, SidecarFile.Write(&Header, sizeof Header, 0); - // TODO: make this batching for better performance { uint64_t WriteOffset = sizeof Header; - // BasicFileWriter SidecarWriter(SidecarFile, 128 * 1024); + const size_t MaxManifestDataBufferCount = (512u * 1024u) / sizeof(ManifestData); - std::vector ManifestDataBuffer; - const size_t MaxManifestDataBufferCount = Min(Index.size(), 8192u); // 512 Kb - ManifestDataBuffer.reserve(MaxManifestDataBufferCount); + std::vector ManifestDataBuffer(Min(m_ManifestEntryCount, MaxManifestDataBufferCount)); + auto WriteIt = ManifestDataBuffer.begin(); for (auto& Kv : Index) { - const IoHash& Key = Kv.first; - const PayloadIndex PlIndex = Kv.second; + ManifestData& Data = *WriteIt++; - IoHash RawHash = IoHash::Zero; - uint32_t RawSize = 0; + const PayloadIndex PlIndex = Kv.second; + Data.Key = Kv.first; + Data.SecondsSinceEpoch = AccessTimes[PlIndex].GetSecondsSinceEpoch(); if (const MetaDataIndex MetaIndex = Payloads[PlIndex].MetaData) { - RawHash = MetaDatas[MetaIndex].RawHash; - RawSize = MetaDatas[MetaIndex].RawSize; + Data.RawHash = MetaDatas[MetaIndex].RawHash; + Data.RawSize = MetaDatas[MetaIndex].RawSize; + } + else + { + Data.RawHash = IoHash::Zero; + Data.RawSize = 0; } - ManifestDataBuffer.emplace_back( - ManifestData{.RawSize = RawSize, .Timestamp = AccessTimes[PlIndex], .RawHash = RawHash, .Key = Key}); - if (ManifestDataBuffer.size() == MaxManifestDataBufferCount) + if (WriteIt == ManifestDataBuffer.end()) { - const uint64_t WriteSize = sizeof(ManifestData) * ManifestDataBuffer.size(); + uint64_t WriteSize = std::distance(ManifestDataBuffer.begin(), WriteIt) * sizeof(ManifestData); SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset); WriteOffset += WriteSize; - ManifestDataBuffer.clear(); - ManifestDataBuffer.reserve(MaxManifestDataBufferCount); + WriteIt = ManifestDataBuffer.begin(); } } - if (ManifestDataBuffer.size() > 0) + if (WriteIt != ManifestDataBuffer.begin()) { - SidecarFile.Write(ManifestDataBuffer.data(), sizeof(ManifestData) * ManifestDataBuffer.size(), WriteOffset); + uint64_t WriteSize = std::distance(ManifestDataBuffer.begin(), WriteIt) * sizeof(ManifestData); + SidecarFile.Write(ManifestDataBuffer.data(), WriteSize, WriteOffset); } } @@ -867,32 +878,38 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, throw std::system_error(Ec, fmt::format("failed to create new snapshot file in '{}'", m_BucketDir)); } - { - // This is in a separate scope just to ensure IndexWriter goes out - // of scope before the file is flushed/closed, in order to ensure - // all data is written to the file - BasicFileWriter IndexWriter(ObjectIndexFile, 128 * 1024); - - cache::impl::CacheBucketIndexHeader Header = {.EntryCount = EntryCount, - .LogPosition = LogCount, - .PayloadAlignment = gsl::narrow(m_Configuration.PayloadAlignment)}; + cache::impl::CacheBucketIndexHeader Header = {.EntryCount = EntryCount, + .LogPosition = LogCount, + .PayloadAlignment = gsl::narrow(m_Configuration.PayloadAlignment)}; - Header.Checksum = cache::impl::CacheBucketIndexHeader::ComputeChecksum(Header); - IndexWriter.Write(&Header, sizeof(cache::impl::CacheBucketIndexHeader), 0); + Header.Checksum = cache::impl::CacheBucketIndexHeader::ComputeChecksum(Header); + ObjectIndexFile.Write(&Header, sizeof(cache::impl::CacheBucketIndexHeader), 0); + if (EntryCount > 0) + { uint64_t IndexWriteOffset = sizeof(cache::impl::CacheBucketIndexHeader); + size_t MaxWriteEntryCount = (512u * 1024u) / sizeof(DiskIndexEntry); + std::vector DiskEntryBuffer(Min(m_Index.size(), MaxWriteEntryCount)); + + auto WriteIt = DiskEntryBuffer.begin(); for (auto& Entry : m_Index) { - DiskIndexEntry IndexEntry; - IndexEntry.Key = Entry.first; - IndexEntry.Location = m_Payloads[Entry.second].Location; - IndexWriter.Write(&IndexEntry, sizeof(DiskIndexEntry), IndexWriteOffset); - - IndexWriteOffset += sizeof(DiskIndexEntry); + *WriteIt++ = {.Key = Entry.first, .Location = m_Payloads[Entry.second].Location}; + if (WriteIt == DiskEntryBuffer.end()) + { + uint64_t WriteSize = std::distance(DiskEntryBuffer.begin(), WriteIt) * sizeof(DiskIndexEntry); + ObjectIndexFile.Write(DiskEntryBuffer.data(), WriteSize, IndexWriteOffset); + IndexWriteOffset += WriteSize; + WriteIt = DiskEntryBuffer.begin(); + } } - IndexWriter.Flush(); + if (WriteIt != DiskEntryBuffer.begin()) + { + uint64_t WriteSize = std::distance(DiskEntryBuffer.begin(), WriteIt) * sizeof(DiskIndexEntry); + ObjectIndexFile.Write(DiskEntryBuffer.data(), WriteSize, IndexWriteOffset); + } } ObjectIndexFile.Flush(); -- cgit v1.2.3 From 7369ad887cf6055192c9234d8eec1550dfabd883 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 7 May 2025 07:52:32 +0100 Subject: added logic to handle empty directories correctly (#383) * added logic to handle empty directories correctly --- src/zen/cmds/wipe_cmd.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 9bf0c6f9e..269f95417 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -283,7 +283,18 @@ namespace { { DiscoveredItemCount++; } - if (!Content.FileNames.empty()) + if (Content.FileNames.empty()) + { + const std::filesystem::path ParentPath = Path / RelativeRoot; + bool KeepDirectory = RelativeRoot.empty(); + + bool Added = AddFoundDirectoryFunc(ParentPath, KeepDirectory); + if (Added) + { + ZEN_CONSOLE_VERBOSE("{} directory {}", KeepDirectory ? "Keeping" : "Removing", ParentPath); + } + } + else { DiscoveredItemCount += Content.FileNames.size(); -- cgit v1.2.3 From 68938614c95635045a394ff0a52786b82f01ffc4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 May 2025 10:23:42 +0200 Subject: optimize block store CompactBlocks (#384) - Improvement: Optimize block compact reducing memcpy operations - Improvement: Handle padding of block store blocks when compacting to avoid excessive flusing of write buffer - Improvement: Handle padding when writing oplog index snapshot to avoid unnecessary flushing of write buffer --- src/zencore/basicfile.cpp | 29 +++++++++++++++++++++++++ src/zencore/include/zencore/basicfile.h | 8 ++++--- src/zenserver/projectstore/projectstore.cpp | 24 +++++++-------------- src/zenstore/blockstore.cpp | 33 +++++++++++++++++------------ 4 files changed, 61 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index ea526399c..12ee26155 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -791,6 +791,35 @@ BasicFileWriter::~BasicFileWriter() Memory::Free(m_Buffer); } +void +BasicFileWriter::AddPadding(uint64_t Padding) +{ + while (Padding) + { + const uint64_t BufferOffset = m_BufferEnd - m_BufferStart; + const uint64_t RemainingBufferCapacity = m_BufferSize - BufferOffset; + const uint64_t BlockPadBytes = Min(RemainingBufferCapacity, Padding); + + memset(m_Buffer + BufferOffset, 0, BlockPadBytes); + m_BufferEnd += BlockPadBytes; + Padding -= BlockPadBytes; + + if ((BufferOffset + BlockPadBytes) == m_BufferSize) + { + Flush(); + } + } +} + +uint64_t +BasicFileWriter::AlignTo(uint64_t Alignment) +{ + uint64_t AlignedPos = RoundUp(m_BufferEnd, Alignment); + uint64_t Padding = AlignedPos - m_BufferEnd; + AddPadding(Padding); + return AlignedPos; +} + void BasicFileWriter::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 57798b6f4..465499d2b 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -174,9 +174,11 @@ public: BasicFileWriter(BasicFile& Base, uint64_t BufferSize); ~BasicFileWriter(); - void Write(const void* Data, uint64_t Size, uint64_t FileOffset); - void Write(const CompositeBuffer& Data, uint64_t FileOffset); - void Flush(); + void Write(const void* Data, uint64_t Size, uint64_t FileOffset); + void Write(const CompositeBuffer& Data, uint64_t FileOffset); + void AddPadding(uint64_t Padding); + uint64_t AlignTo(uint64_t Alignment); + void Flush(); private: BasicFile& m_Base; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index e91e6ac51..071aab137 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1748,36 +1748,28 @@ ProjectStore::Oplog::WriteIndexSnapshot() uint64_t Offset = 0; IndexFile.Write(&Header, sizeof(OplogIndexHeader), Offset); - Offset += sizeof(OplogIndexHeader); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(LSNEntries.data(), LSNEntries.size() * sizeof(uint32_t), Offset); - Offset += LSNEntries.size() * sizeof(uint32_t); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(Keys.data(), Keys.size() * sizeof(Oid), Offset); - Offset += Keys.size() * sizeof(Oid); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(AddressMapEntries.data(), AddressMapEntries.size() * sizeof(OplogEntryAddress), Offset); - Offset += AddressMapEntries.size() * sizeof(OplogEntryAddress); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(LatestOpMapEntries.data(), LatestOpMapEntries.size() * sizeof(uint32_t), Offset); - Offset += LatestOpMapEntries.size() * sizeof(uint32_t); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(ChunkMapEntries.data(), ChunkMapEntries.size() * sizeof(IoHash), Offset); - Offset += ChunkMapEntries.size() * sizeof(IoHash); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(MetaMapEntries.data(), MetaMapEntries.size() * sizeof(IoHash), Offset); - Offset += MetaMapEntries.size() * sizeof(IoHash); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); IndexFile.Write(FilePathLengths.data(), FilePathLengths.size() * sizeof(uint32_t), Offset); - Offset += FilePathLengths.size() * sizeof(uint32_t); - Offset = RoundUp(Offset, OplogIndexHeader::DataAlignment); + Offset = IndexFile.AlignTo(OplogIndexHeader::DataAlignment); for (const auto& FilePath : FilePaths) { diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index c58080e6a..e0f371061 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -1062,11 +1062,10 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, std::sort(SortedChunkIndexes.begin(), SortedChunkIndexes.end(), [&ChunkLocations](size_t Lhs, size_t Rhs) { return ChunkLocations[Lhs].Offset < ChunkLocations[Rhs].Offset; }); - BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(65536u, OldBlockSize)); + BasicFileBuffer SourceFileBuffer(OldBlockFile->GetBasicFile(), Min(256u * 1024u, OldBlockSize)); - uint64_t WrittenBytesToBlock = 0; - uint64_t MovedFromBlock = 0; - std::vector Chunk; + uint64_t MovedFromBlock = 0; + std::vector ChunkBuffer; for (const size_t& ChunkIndex : SortedChunkIndexes) { const BlockStoreLocation ChunkLocation = ChunkLocations[ChunkIndex]; @@ -1084,10 +1083,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, continue; } - Chunk.resize(ChunkLocation.Size); - SourceFileBuffer.Read(Chunk.data(), Chunk.size(), ChunkLocation.Offset); + MemoryView ChunkView = SourceFileBuffer.MakeView(ChunkLocation.Size, ChunkLocation.Offset); + if (ChunkView.GetSize() != ChunkLocation.Size) + { + ChunkBuffer.resize(ChunkLocation.Size); + SourceFileBuffer.Read(ChunkBuffer.data(), ChunkLocation.Size, ChunkLocation.Offset); + ChunkView = MemoryView(ChunkBuffer.data(), ChunkLocation.Size); + } - if ((WriteOffset + Chunk.size()) > m_MaxBlockSize) + if ((WriteOffset + ChunkView.GetSize()) > m_MaxBlockSize) { TargetFileBuffer.reset(); if (NewBlockFile) @@ -1174,20 +1178,21 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, NewBlockFile->Create(m_MaxBlockSize); NewBlockIndex = NextBlockIndex; WriteOffset = 0; - AddedSize += WrittenBytesToBlock; - WrittenBytesToBlock = 0; - TargetFileBuffer = std::make_unique(NewBlockFile->GetBasicFile(), Min(65536u, m_MaxBlockSize)); + AddedSize += WriteOffset; + WriteOffset = 0; + TargetFileBuffer = std::make_unique(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize)); } - TargetFileBuffer->Write(Chunk.data(), ChunkLocation.Size, WriteOffset); + WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment); + + TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset); MovedChunks.push_back( {ChunkIndex, {.BlockIndex = NewBlockIndex, .Offset = gsl::narrow(WriteOffset), .Size = ChunkLocation.Size}}); - WrittenBytesToBlock = WriteOffset + ChunkLocation.Size; + WriteOffset += ChunkLocation.Size; MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset; - WriteOffset = RoundUp(WriteOffset + ChunkLocation.Size, PayloadAlignment); } - AddedSize += WrittenBytesToBlock; + AddedSize += WriteOffset; ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}", LogPrefix, KeepChunkIndexes.size(), -- cgit v1.2.3 From be3324675705d45f111dfa47c1f4cf8508b1b383 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 May 2025 15:13:18 +0200 Subject: make RemoveExpiredData and PreCache serial to reduce CPU overhead / lock contention (#385) * make RemoveExpiredData and PreCache serial to reduce CPU overhead / lock contention --- src/zenstore/gc.cpp | 146 ++++++++++++++++++++++------------------------------ 1 file changed, 61 insertions(+), 85 deletions(-) (limited to 'src') diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 5bd34fc37..f6b3dca6f 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -709,7 +709,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) RwLock StoreCompactorsLock; std::unordered_map, size_t> ReferenceValidators; RwLock ReferenceValidatorsLock; - WorkerThreadPool& PreCachePhaseThreadPool = + WorkerThreadPool& ParallelWorkThreadPool = Settings.SingleThread ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Background); if (!m_GcReferencers.empty()) @@ -721,7 +721,6 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_INFO("GCV2: Removing expired data from {} referencers", m_GcReferencers.size()); ZEN_TRACE_CPU("GcV2::RemoveExpiredData"); - Latch WorkLeft(1); { // First remove any cache keys that may own references SCOPED_TIMER(Result.RemoveExpiredDataMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) { @@ -733,56 +732,45 @@ GcManager::CollectGarbage(const GcSettings& Settings) { if (CheckGCCancel()) { - WorkLeft.CountDown(); - WorkLeft.Wait(); return Sum(Result, true); } GcReferencer* Owner = m_GcReferencers[Index]; std::pair* Stats = &Result.ReferencerStats[Index]; - WorkLeft.AddCount(1); - PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, &WorkLeft, Owner, Stats, &StoreCompactorsLock, &StoreCompactors]() { - ZEN_MEMSCOPE(GetGcTag()); - - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - try + try + { + Stats->first = Owner->GetGcName(Ctx); + SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + std::unique_ptr StoreCompactor( + Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats)); + if (StoreCompactor) { - Stats->first = Owner->GetGcName(Ctx); - SCOPED_TIMER(Stats->second.RemoveExpiredDataStats.ElapsedMS = - std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - std::unique_ptr StoreCompactor( - Owner->RemoveExpiredData(Ctx, Stats->second.RemoveExpiredDataStats)); - if (StoreCompactor) - { - RwLock::ExclusiveLockScope __(StoreCompactorsLock); - StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats); - } + RwLock::ExclusiveLockScope __(StoreCompactorsLock); + StoreCompactors.insert_or_assign(std::move(StoreCompactor), &Stats->second.CompactStoreStats); } - catch (const std::system_error& Ex) + } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) { - if (IsOOD(Ex) || IsOOM(Ex)) - { - ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); - } - else - { - ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); - } - SetCancelGC(true); + ZEN_WARN("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); } - catch (const std::bad_alloc& Ex) - { - ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); - SetCancelGC(true); - } - catch (const std::exception& Ex) + else { ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); - SetCancelGC(true); } - }); + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("GCV2: Failed removing expired data for {}. Reason: '{}'", Owner->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } } - WorkLeft.CountDown(); - WorkLeft.Wait(); } } @@ -827,7 +815,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) GcReferenceStore* ReferenceStore = m_GcReferenceStores[Index]; std::pair* Stats = &Result.ReferenceStoreStats[Index]; WorkLeft.AddCount(1); - PreCachePhaseThreadPool.ScheduleWork( + ParallelWorkThreadPool.ScheduleWork( [this, &Ctx, ReferenceStore, Stats, Index, &WorkLeft, &ReferencePrunersLock, &ReferencePruners]() { ZEN_MEMSCOPE(GetGcTag()); @@ -925,7 +913,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) GcReferencer* Referencer = m_GcReferencers[Index]; std::pair* Stats = &Result.ReferencerStats[Index]; WorkLeft.AddCount(1); - PreCachePhaseThreadPool.ScheduleWork( + ParallelWorkThreadPool.ScheduleWork( [this, &Ctx, &WorkLeft, Referencer, Index, Stats, &ReferenceCheckersLock, &ReferenceCheckers]() { ZEN_MEMSCOPE(GetGcTag()); @@ -1031,15 +1019,15 @@ GcManager::CollectGarbage(const GcSettings& Settings) GcReferencer* Referencer = m_GcReferencers[Index]; std::pair* ReferemcerStats = &Result.ReferencerStats[Index]; WorkLeft.AddCount(1); - PreCachePhaseThreadPool.ScheduleWork([this, - &Ctx, - &WorkLeft, - Referencer, - Index, - Result = &Result, - ReferemcerStats, - &ReferenceValidatorsLock, - &ReferenceValidators]() { + ParallelWorkThreadPool.ScheduleWork([this, + &Ctx, + &WorkLeft, + Referencer, + Index, + Result = &Result, + ReferemcerStats, + &ReferenceValidatorsLock, + &ReferenceValidators]() { ZEN_MEMSCOPE(GetGcTag()); auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); @@ -1117,8 +1105,6 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_INFO("GCV2: Precaching state for {} reference checkers", ReferenceCheckers.size()); ZEN_TRACE_CPU("GcV2::PreCache"); - Latch WorkLeft(1); - { SCOPED_TIMER(Result.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs()); if (Ctx.Settings.Verbose) { @@ -1130,50 +1116,40 @@ GcManager::CollectGarbage(const GcSettings& Settings) { if (CheckGCCancel()) { - WorkLeft.CountDown(); - WorkLeft.Wait(); return Sum(Result, true); } GcReferenceChecker* Checker = It.first.get(); size_t Index = It.second; std::pair* Stats = &Result.ReferencerStats[Index]; - WorkLeft.AddCount(1); - PreCachePhaseThreadPool.ScheduleWork([this, &Ctx, Checker, Index, Stats, &WorkLeft]() { - ZEN_MEMSCOPE(GetGcTag()); - - auto _ = MakeGuard([&WorkLeft]() { WorkLeft.CountDown(); }); - try - { - SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); - Checker->PreCache(Ctx); - } - catch (const std::system_error& Ex) - { - if (IsOOD(Ex) || IsOOM(Ex)) - { - ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); - } - else - { - ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); - } - SetCancelGC(true); - } - catch (const std::bad_alloc& Ex) + try + { + SCOPED_TIMER(Stats->second.PreCacheStateMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); + Checker->PreCache(Ctx); + } + catch (const std::system_error& Ex) + { + if (IsOOD(Ex) || IsOOM(Ex)) { - ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); - SetCancelGC(true); + ZEN_WARN("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); } - catch (const std::exception& Ex) + else { ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); - SetCancelGC(true); } - }); + SetCancelGC(true); + } + catch (const std::bad_alloc& Ex) + { + ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("GCV2: Failed precaching for {}. Reason: '{}'", Checker->GetGcName(Ctx), Ex.what()); + SetCancelGC(true); + } } - WorkLeft.CountDown(); - WorkLeft.Wait(); } } -- cgit v1.2.3 From c00afaefd9dc359a0a20c64a50c2d0553e44a73b Mon Sep 17 00:00:00 2001 From: zousar Date: Wed, 7 May 2025 12:17:57 -0600 Subject: Change plugin config parsing to warn instead of throw --- src/zenserver/config.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index e81e8eb54..7b8e38e80 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -527,15 +527,15 @@ ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& Serv json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); if (!JsonError.empty()) { - throw std::runtime_error(fmt::format("failed parsing json file '{}'. Reason: '{}'", Path, JsonError)); + ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); + return; } for (const json11::Json& PluginInfo : PluginsInfo.array_items()) { if (!PluginInfo.is_object()) { - throw std::runtime_error(fmt::format("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", - Path, - PluginInfo.dump())); + ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); + continue; } HttpServerPluginConfig Config = {}; @@ -546,10 +546,10 @@ ParsePluginsConfigFile(const std::filesystem::path& Path, ZenServerOptions& Serv { if (!Items.second.is_string()) { - throw std::runtime_error( - fmt::format("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", - Path, - Items.second.dump())); + ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump()); + continue; } const std::string& Name = Items.first; -- cgit v1.2.3 From a535a19aae3e4c9573a16099ec19866f7d4b7bc1 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 9 May 2025 13:57:25 +0200 Subject: flush cas log file (#387) * make sure we remove the cas log file when writing full index at startup --- src/zenstore/compactcas.cpp | 75 ++++++++++++++++++--------------------------- src/zenstore/compactcas.h | 2 +- 2 files changed, 31 insertions(+), 46 deletions(-) (limited to 'src') diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 90e77e48a..a8914ed20 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -461,7 +461,7 @@ CasContainerStrategy::Flush() ZEN_TRACE_CPU("CasContainer::Flush"); m_BlockStore.Flush(/*ForceNewBlock*/ false); m_CasLog.Flush(); - MakeIndexSnapshot(); + MakeIndexSnapshot(/*FlushLockPosition*/ false); } void @@ -924,12 +924,12 @@ CasContainerStrategy::StorageSize() const } void -CasContainerStrategy::MakeIndexSnapshot() +CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) { ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::MakeIndexSnapshot"); - uint64_t LogCount = m_CasLog.GetLogCount(); + const uint64_t LogCount = FlushLockPosition ? 0 : m_CasLog.GetLogCount(); if (m_LogFlushPosition == LogCount) { return; @@ -947,27 +947,10 @@ CasContainerStrategy::MakeIndexSnapshot() namespace fs = std::filesystem; - fs::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); - fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName); - - // Move index away, we keep it if something goes wrong - if (IsFile(TempIndexPath)) - { - std::error_code Ec; - if (!RemoveFile(TempIndexPath, Ec) || Ec) - { - ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); - return; - } - } + fs::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); try { - if (IsFile(IndexPath)) - { - RenameFile(IndexPath, TempIndexPath); - } - // Write the current state of the location map to a new index state std::vector Entries; uint64_t IndexLogPosition = 0; @@ -993,7 +976,8 @@ CasContainerStrategy::MakeIndexSnapshot() { throw std::system_error(Ec, fmt::format("Failed to create temp file for index snapshot at '{}'", IndexPath)); } - CasDiskIndexHeader Header = {.EntryCount = Entries.size(), + EntryCount = Entries.size(); + CasDiskIndexHeader Header = {.EntryCount = EntryCount, .LogPosition = IndexLogPosition, .PayloadAlignment = gsl::narrow(m_PayloadAlignment)}; @@ -1005,36 +989,37 @@ CasContainerStrategy::MakeIndexSnapshot() ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec); if (Ec) { - throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath)); + ZEN_WARN("snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", + ObjectIndexFile.GetPath(), + IndexPath, + Ec.message()); } - EntryCount = Entries.size(); - m_LogFlushPosition = IndexLogPosition; - } - catch (const std::exception& Err) - { - ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what()); - - // Restore any previous snapshot - - if (IsFile(TempIndexPath)) + if (FlushLockPosition) { - std::error_code Ec; - RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - RenameFile(TempIndexPath, IndexPath, Ec); - if (Ec) + std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); + + if (IsFile(LogPath)) { - ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); + if (!RemoveFile(LogPath, Ec) || Ec) + { + ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", + LogPath, + IndexPath, + Ec.message()); + std::error_code RemoveIndexEc; + RemoveFile(IndexPath, RemoveIndexEc); + } } } - } - if (IsFile(TempIndexPath)) - { - std::error_code Ec; - if (!RemoveFile(TempIndexPath, Ec) || Ec) + if (!Ec) { - ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); + m_LogFlushPosition = LogCount; } } + catch (const std::exception& Err) + { + ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what()); + } } uint64_t @@ -1229,7 +1214,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore || (LogEntryCount > 0)) { - MakeIndexSnapshot(); + MakeIndexSnapshot(/*FlushLockPosition*/ true); } // TODO: should validate integrity of container files here diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 2eb4c233a..b7ea7591e 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -77,7 +77,7 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore private: CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); - void MakeIndexSnapshot(); + void MakeIndexSnapshot(bool FlushLockPosition); uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount); void OpenContainer(bool IsNewStore); -- cgit v1.2.3 From acc0b59e9cb349e3f9345467a3941ee25786d79e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 12 May 2025 10:07:40 +0200 Subject: handle exception in oplog mirror (#389) * gracefully handle errors in threaded part of oplog-mirror --- src/zen/cmds/projectstore_cmd.cpp | 114 +++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 46 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 540c4c9ac..066dac781 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2005,6 +2005,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::filesystem::path TmpPath = RootPath / ".tmp"; CreateDirectories(TmpPath); + auto _ = MakeGuard([&TmpPath]() { DeleteDirectories(TmpPath); }); std::atomic_int64_t FileCount = 0; int OplogEntryCount = 0; @@ -2017,6 +2018,8 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg std::unordered_set FileNames; std::atomic WrittenByteCount = 0; + std::atomic AbortFlag(false); + Stopwatch WriteStopWatch; auto EmitFilesForDataArray = [&](CbArrayView DataArray) { @@ -2045,38 +2048,50 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg EmitCount++; WorkRemaining.AddCount(1); WorkerPool.ScheduleWork( - [this, &RootPath, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining, &WrittenByteCount]() { + [this, &RootPath, &AbortFlag, FileName, &FileCount, ChunkId, &Http, TmpPath, &WorkRemaining, &WrittenByteCount]() { auto _ = MakeGuard([&WorkRemaining]() { WorkRemaining.CountDown(); }); - if (HttpClient::Response ChunkResponse = - Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath)) + if (!AbortFlag) { - auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize)) - { - return Compressed.Decompress().AsIoBuffer(); - }; - return std::move(Buffer); - }; - - IoBuffer ChunkData = - m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload; - std::filesystem::path TargetPath = RootPath / FileName; - if (!MoveToFile(TargetPath, ChunkData)) + try + { + if (HttpClient::Response ChunkResponse = + Http.Download(fmt::format("/prj/{}/oplog/{}/{}"sv, m_ProjectName, m_OplogName, ChunkId), TmpPath)) + { + auto TryDecompress = [](const IoBuffer& Buffer) -> IoBuffer { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(Buffer), RawHash, RawSize)) + { + return Compressed.Decompress().AsIoBuffer(); + }; + return std::move(Buffer); + }; + + IoBuffer ChunkData = + m_Decompress ? TryDecompress(ChunkResponse.ResponsePayload) : ChunkResponse.ResponsePayload; + + if (!MoveToFile(TargetPath, ChunkData)) + { + WriteFile(TargetPath, ChunkData); + } + WrittenByteCount.fetch_add(ChunkData.GetSize()); + ++FileCount; + } + else + { + throw std::runtime_error(fmt::format("Unable to fetch '{}' (chunk {}). Reason: '{}'", + FileName, + ChunkId, + ChunkResponse.ErrorMessage(""sv))); + } + } + catch (const std::exception& Ex) { - WriteFile(TargetPath, ChunkData); + AbortFlag.store(true); + ZEN_CONSOLE("Failed writing file to '{}'. Reason: '{}'", TargetPath, Ex.what()); } - WrittenByteCount.fetch_add(ChunkData.GetSize()); - ++FileCount; - } - else - { - ZEN_CONSOLE("Unable to fetch '{}' (chunk {}). Reason: '{}'", - FileName, - ChunkId, - ChunkResponse.ErrorMessage(""sv)); } }); } @@ -2096,28 +2111,31 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg uint64_t Remaining = Entries.Num(); for (auto EntryIter : Entries) { - CbObjectView Entry = EntryIter.AsObjectView(); - ParseProgressBar.UpdateState( - {.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining}, - false); - Remaining--; - if (!m_KeyFilter.empty()) + if (!AbortFlag) { - if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos) + CbObjectView Entry = EntryIter.AsObjectView(); + ParseProgressBar.UpdateState( + {.Task = "Parsing oplog", .Details = "", .TotalCount = Entries.Num(), .RemainingCount = Remaining}, + false); + Remaining--; + if (!m_KeyFilter.empty()) + { + if (Entry["key"].AsString().find(m_KeyFilter) == std::string_view::npos) + { + continue; + } + } + if (!EmitProgressBar) { - continue; + EmitProgressBar = std::make_unique(ProgressBar::Mode::Pretty, ""sv); + WriteStopWatch.Reset(); } - } - if (!EmitProgressBar) - { - EmitProgressBar = std::make_unique(ProgressBar::Mode::Pretty, ""sv); - WriteStopWatch.Reset(); - } - EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView()); - EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView()); + EmitFilesForDataArray(Entry["packagedata"sv].AsArrayView()); + EmitFilesForDataArray(Entry["bulkdata"sv].AsArrayView()); - ++OplogEntryCount; + ++OplogEntryCount; + } } ParseProgressBar.Finish(); } @@ -2149,6 +2167,12 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { WorkRemaining.Wait(); } + + if (AbortFlag) + { + // Error has already been reported by async code + return 1; + } } else { @@ -2162,8 +2186,6 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - DeleteDirectories(TmpPath); - ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); return 0; -- cgit v1.2.3 From e452381cf3fcff977e32541b1c3909294a8f6a4d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 12 May 2025 10:08:50 +0200 Subject: tweak iterate block parameters (#390) * tweak block iteration chunk sizes --- src/zenstore/cache/cachedisklayer.cpp | 77 ++++++++++++++++++++--------------- src/zenstore/cas.cpp | 7 +++- 2 files changed, 50 insertions(+), 34 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index fbe559e58..8b120fda5 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -1449,7 +1449,7 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept // Often we will find the metadata due to the thread setting the mem cached part doing it before us so it is worth // checking if it is present once more before spending time fetching and setting the RawHash and RawSize in metadata - auto FillOne = [&](const DiskLocation& Location, size_t KeyIndex, IoBuffer&& Value) { + auto FillOne = [&](const DiskLocation& Location, size_t KeyIndex, IoBuffer&& Value, bool UsesTemporaryMemory) { if (!Value) { return; @@ -1472,6 +1472,12 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept } } + if (AddToMemCache || UsesTemporaryMemory) + { + // We need to own it if we want to add it to the memcache or the buffer is just a range of the block iteration buffer + OutValue.Value.MakeOwned(); + } + if (SetMetaInfo) { // See ZenCacheDiskLayer::CacheBucket::Get - it sets the memcache part first and then if it needs to it set the @@ -1543,35 +1549,42 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept if (!InlineDiskLocations.empty()) { ZEN_TRACE_CPU("Z$::Bucket::EndGetBatch::ReadInline"); - m_BlockStore.IterateChunks( - std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, - [&](uint32_t, std::span ChunkIndexes) -> bool { - // Only read into memory the IoBuffers we could potentially add to memcache - const uint64_t LargeChunkSizeLimit = Max(m_Configuration.MemCacheSizeThreshold, 1u * 1024u); - m_BlockStore.IterateBlock( - std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, - ChunkIndexes, - [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, - const void* Data, - uint64_t Size) -> bool { - if (Data != nullptr) - { - FillOne(InlineDiskLocations[ChunkIndex], - InlineKeyIndexes[ChunkIndex], - IoBufferBuilder::MakeCloneFromMemory(Data, Size)); - } - return true; - }, - [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, - BlockStoreFile& File, - uint64_t Offset, - uint64_t Size) -> bool { - FillOne(InlineDiskLocations[ChunkIndex], InlineKeyIndexes[ChunkIndex], File.GetChunk(Offset, Size)); - return true; - }, - LargeChunkSizeLimit); - return true; - }); + m_BlockStore.IterateChunks(std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, + [&](uint32_t, std::span ChunkIndexes) -> bool { + // Up to 8KB or m_Configuration.MemCacheSizeThreshold depending on configuration + const uint64_t LargeChunkSizeLimit = + m_Configuration.MemCacheSizeThreshold == 0 + ? Min(m_Configuration.LargeObjectThreshold, 8u * 1024u) + : Max(m_Configuration.MemCacheSizeThreshold, 8u * 1024u); + + m_BlockStore.IterateBlock( + std::span{begin(InlineBlockLocations), end(InlineBlockLocations)}, + ChunkIndexes, + [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, + const void* Data, + uint64_t Size) -> bool { + if (Data != nullptr) + { + FillOne(InlineDiskLocations[ChunkIndex], + InlineKeyIndexes[ChunkIndex], + IoBufferBuilder::MakeFromMemory(MemoryView(Data, Size)), + /*UsesTemporaryMemory*/ true); + } + return true; + }, + [this, &FillOne, &InlineDiskLocations, &InlineKeyIndexes](size_t ChunkIndex, + BlockStoreFile& File, + uint64_t Offset, + uint64_t Size) -> bool { + FillOne(InlineDiskLocations[ChunkIndex], + InlineKeyIndexes[ChunkIndex], + File.GetChunk(Offset, Size), + /*UsesTemporaryMemory*/ false); + return true; + }, + LargeChunkSizeLimit); + return true; + }); } if (!StandaloneDiskLocations.empty()) @@ -1581,7 +1594,7 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept { size_t KeyIndex = StandaloneKeyIndexes[Index]; const DiskLocation& Location = StandaloneDiskLocations[Index]; - FillOne(Location, KeyIndex, GetStandaloneCacheValue(Location, Batch->Keys[KeyIndex])); + FillOne(Location, KeyIndex, GetStandaloneCacheValue(Location, Batch->Keys[KeyIndex]), /*UsesTemporaryMemory*/ false); } } @@ -3289,7 +3302,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, CaptureAttachments(ChunkIndex, File.GetChunk(Offset, Size).GetView()); return !IsCancelledFlag.load(); }, - 0); + 32u * 1024); if (Continue) { diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index ed42f254e..460f0e10d 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -412,6 +412,7 @@ CasImpl::IterateChunks(std::span DecompressedIds, uint64_t LargeSizeLimit) { ZEN_TRACE_CPU("CAS::IterateChunks"); + if (!m_SmallStrategy.IterateChunks( DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { @@ -420,10 +421,11 @@ CasImpl::IterateChunks(std::span DecompressedIds, return AsyncCallback(Index, Payload); }, OptionalWorkerPool, - LargeSizeLimit)) + LargeSizeLimit == 0 ? m_Config.HugeValueThreshold : Min(LargeSizeLimit, m_Config.HugeValueThreshold))) { return false; } + if (!m_TinyStrategy.IterateChunks( DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { @@ -432,10 +434,11 @@ CasImpl::IterateChunks(std::span DecompressedIds, return AsyncCallback(Index, Payload); }, OptionalWorkerPool, - LargeSizeLimit)) + LargeSizeLimit == 0 ? m_Config.TinyValueThreshold : Min(LargeSizeLimit, m_Config.TinyValueThreshold))) { return false; } + if (!m_LargeStrategy.IterateChunks( DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { -- cgit v1.2.3 From 00dc4e3d0976e0f0cbd8a09a7693bad31b8db511 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 12 May 2025 10:22:17 +0200 Subject: enable per bucket config (#388) Feature: Add per bucket cache configuration (Lua options file only) Improvement: --cache-memlayer-sizethreshold is now deprecated and has a new name: --cache-bucket-memlayer-sizethreshold to line up with per cache bucket configuration --- src/zenserver/config.cpp | 153 ++++++++++++++++++++- src/zenserver/config.h | 23 +++- src/zenserver/zenserver.cpp | 24 +++- src/zenstore/cache/cachedisklayer.cpp | 21 ++- .../include/zenstore/cache/cachedisklayer.h | 3 + 5 files changed, 208 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 7b8e38e80..e097147fc 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -313,6 +313,97 @@ public: ZenObjectStoreConfig& Value; }; +class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue +{ +public: + ZenStructuredCacheBucketsConfigOption(std::vector>& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, zen::StringBuilderBase& StringBuilder) override + { + if (Value.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const std::pair& Bucket : Value) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Bucket.first); + const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; + + Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); + Writer.BeginContainer("memlayer"); + { + Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); + } + Writer.EndContainer(); + + Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional Buckets = Object.as()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional Bucket = Kv.second.as()) + { + ZenStructuredCacheBucketConfig BucketConfig; + std::string Name = Kv.first.as(); + if (Name.empty()) + { + throw zen::OptionParseException(fmt::format("cache bucket option must have a name.")); + } + + const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); + if (MaxBlockSize == 0) + { + throw zen::OptionParseException( + fmt::format("maxblocksize option for cache bucket '{}' is invalid. It must be non-zero.", Name)); + } + BucketConfig.MaxBlockSize = MaxBlockSize; + + if (sol::optional Memlayer = Bucket.value().get_or("memlayer", sol::table())) + { + const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + if (MemCacheSizeThreshold == 0) + { + throw zen::OptionParseException( + fmt::format("memlayer.sizethreshold option for cache bucket '{}' is invalid. It must be non-zero.", Name)); + } + BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + } + + const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); + if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) + { + throw zen::OptionParseException(fmt::format( + "payloadalignment option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", + Name)); + } + BucketConfig.PayloadAlignment = PayloadAlignment; + + const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); + if (LargeObjectThreshold == 0) + { + throw zen::OptionParseException( + fmt::format("largeobjectthreshold option for cache bucket '{}' is invalid. It must be non-zero.", Name)); + } + BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); + } + } + } + } + std::vector>& Value; +}; + std::shared_ptr MakeOption(zen::UpstreamCachePolicy& Value) { @@ -331,6 +422,12 @@ MakeOption(zen::ZenObjectStoreConfig& Value) return std::make_shared(Value); }; +std::shared_ptr +MakeOption(std::vector>& Value) +{ + return std::make_shared(Value); +}; + void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, @@ -396,7 +493,7 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, - ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, "cache-memlayer-sizethreshold"sv); LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, @@ -406,6 +503,19 @@ ParseConfigFile(const std::filesystem::path& Path, "cache-memlayer-triminterval"sv); LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); + LuaOptions.AddOption("cache.bucket.maxblocksize"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + "cache-bucket-maxblocksize"sv); + LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-bucket-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.bucket.payloadalignment"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + "cache-bucket-payloadalignment"sv); + LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + "cache-bucket-largeobjectthreshold"sv); + ////// cache.upstream LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, @@ -493,6 +603,8 @@ ParseConfigFile(const std::filesystem::path& Path, ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, "workspaces-allow-changes"sv); + LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); + LuaOptions.Parse(Path, CmdLineResult); // These have special command line processing so we make sure we export them if they were configured on command line @@ -504,6 +616,10 @@ ParseConfigFile(const std::filesystem::path& Path, { LuaOptions.Touch("server.objectstore.buckets"sv); } + if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) + { + LuaOptions.Touch("cache.buckets"sv); + } if (!OutputConfigFile.empty()) { @@ -916,8 +1032,9 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) "cache", "", "cache-memlayer-sizethreshold", - "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", - cxxopts::value(ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold)->default_value("1024"), + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching. " + "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), ""); options.add_option("cache", @@ -941,6 +1058,36 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), ""); + options.add_option("cache", + "", + "cache-bucket-maxblocksize", + "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), + ""); + + options.add_option("cache", + "", + "cache-bucket-payloadalignment", + "Payload alignement for cache bucket blocks. Default set to 16.", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-largeobjectthreshold", + "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory caching.", + cxxopts::value(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + options.add_option("gc", "", "gc-cache-attachment-store", diff --git a/src/zenserver/config.h b/src/zenserver/config.h index bd277cd88..1d7d22ce9 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -119,15 +119,24 @@ struct ZenStatsConfig int StatsdPort = 8125; }; +struct ZenStructuredCacheBucketConfig +{ + uint64_t MaxBlockSize = 1ull << 30; + uint32_t PayloadAlignment = 1u << 4; + uint64_t MemCacheSizeThreshold = 1 * 1024; + uint64_t LargeObjectThreshold = 128 * 1024; +}; + struct ZenStructuredCacheConfig { - bool Enabled = true; - bool WriteLogEnabled = false; - bool AccessLogEnabled = false; - uint64_t MemCacheSizeThreshold = 1 * 1024; - uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; - uint64_t MemTrimIntervalSeconds = 60; - uint64_t MemMaxAgeSeconds = gsl::narrow(std::chrono::seconds(std::chrono::days(1)).count()); + bool Enabled = true; + bool WriteLogEnabled = false; + bool AccessLogEnabled = false; + std::vector> PerBucketConfigs; + ZenStructuredCacheBucketConfig BucketConfig; + uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; + uint64_t MemTrimIntervalSeconds = 60; + uint64_t MemMaxAgeSeconds = gsl::narrow(std::chrono::seconds(std::chrono::days(1)).count()); }; struct ZenProjectStoreConfig diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 366944ffc..1ad94ed63 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -547,10 +547,26 @@ ZenServer::InitializeStructuredCache(const ZenServerOptions& ServerOptions) Config.AllowAutomaticCreationOfNamespaces = true; Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; - Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = ServerOptions.StructuredCacheConfig.MemCacheSizeThreshold, - Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; - Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; - Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) + { + const std::string& BucketName = It.first; + const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; + ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, + .PayloadAlignment = ZenBucketConfig.PayloadAlignment, + .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold}; + Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); + } + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; + Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; if (ServerOptions.IsDedicated) { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 8b120fda5..e7b2e6bc6 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -3617,7 +3617,15 @@ ZenCacheDiskLayer::GetOrCreateBucket(std::string_view InBucket) // We create the bucket without holding a lock since contructor calls GcManager::AddGcReferencer which takes an exclusive lock. // This can cause a deadlock, if GC is running we would block while holding ZenCacheDiskLayer::m_Lock - std::unique_ptr Bucket(std::make_unique(m_Gc, m_TotalMemCachedSize, InBucket, m_Configuration.BucketConfig)); + BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; + if (auto It = m_Configuration.BucketConfigMap.find_as(InBucket, + std::hash(), + eastl::equal_to_2()); + It != m_Configuration.BucketConfigMap.end()) + { + BucketConfig = &It->second; + } + std::unique_ptr Bucket(std::make_unique(m_Gc, m_TotalMemCachedSize, InBucket, *BucketConfig)); RwLock::ExclusiveLockScope Lock(m_Lock); if (auto It = m_Buckets.find_as(InBucket, std::hash(), eastl::equal_to_2()); @@ -3944,8 +3952,17 @@ ZenCacheDiskLayer::DiscoverBuckets() const std::string BucketName = PathToUtf8(BucketPath.stem()); try { + BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; + if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName), + std::hash(), + eastl::equal_to_2()); + It != m_Configuration.BucketConfigMap.end()) + { + BucketConfig = &It->second; + } + std::unique_ptr NewBucket = - std::make_unique(m_Gc, m_TotalMemCachedSize, BucketName, m_Configuration.BucketConfig); + std::make_unique(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig); CacheBucket* Bucket = nullptr; { diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 5a51718d3..f36e780b9 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -119,6 +119,9 @@ public: struct Configuration { + typedef eastl::unordered_map, std::equal_to> + BucketConfigMap_t; + BucketConfigMap_t BucketConfigMap; BucketConfiguration BucketConfig; uint64_t MemCacheTargetFootprintBytes = 512 * 1024 * 1024; uint64_t MemCacheTrimIntervalSeconds = 60; -- cgit v1.2.3 From a417d19e6d2af229e7fd33c559f6fefee3a81042 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 12 May 2025 11:03:46 +0200 Subject: keep snapshot on log delete fail (#391) - Improvement: Cleaned up snapshot writing for CompactCAS/FileCas/Cache/Project stores - Improvement: Safer recovery when failing to delete log for CompactCAS/FileCas/Cache/Project stores - Improvement: Added log file reset when writing snapshot at startup for FileCas --- src/zenserver/projectstore/projectstore.cpp | 61 ++--------------- src/zenstore/cache/cachedisklayer.cpp | 47 ++++++------- src/zenstore/compactcas.cpp | 46 ++++++------- src/zenstore/compactcas.h | 2 +- src/zenstore/filecas.cpp | 78 +++++++++------------- src/zenstore/filecas.h | 2 +- .../include/zenstore/cache/cachedisklayer.h | 10 +-- 7 files changed, 88 insertions(+), 158 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 071aab137..6a55efdb7 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1643,31 +1643,9 @@ ProjectStore::Oplog::WriteIndexSnapshot() namespace fs = std::filesystem; - fs::path IndexPath = m_BasePath / "ops.zidx"; - fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp"; - - // Move index away, we keep it if something goes wrong - if (IsFile(TempIndexPath)) - { - std::error_code Ec; - if (!RemoveFile(TempIndexPath, Ec) || Ec) - { - ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'", - GetOuterProject()->Identifier, - m_OplogId, - TempIndexPath, - Ec.message()); - return; - } - } - + const fs::path IndexPath = m_BasePath / "ops.zidx"; try { - if (IsFile(IndexPath)) - { - RenameFile(IndexPath, TempIndexPath); - } - // Write the current state of the location map to a new index state std::vector LSNEntries; std::vector Keys; @@ -1781,7 +1759,11 @@ ProjectStore::Oplog::WriteIndexSnapshot() ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec); if (Ec) { - throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath)); + throw std::system_error(Ec, + fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", + ObjectIndexFile.GetPath(), + IndexPath, + Ec.message())); } EntryCount = LSNEntries.size(); m_LogFlushPosition = IndexLogPosition; @@ -1789,35 +1771,6 @@ ProjectStore::Oplog::WriteIndexSnapshot() catch (const std::exception& Err) { ZEN_WARN("oplog '{}/{}': snapshot FAILED, reason: '{}'", m_OuterProject->Identifier, m_OplogId, Err.what()); - - // Restore any previous snapshot - - if (IsFile(TempIndexPath)) - { - std::error_code Ec; - RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - RenameFile(TempIndexPath, IndexPath, Ec); - if (Ec) - { - ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'", - m_OuterProject->Identifier, - m_OplogId, - TempIndexPath, - Ec.message()); - } - } - } - if (IsFile(TempIndexPath)) - { - std::error_code Ec; - if (!RemoveFile(TempIndexPath, Ec) || Ec) - { - ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'", - m_OuterProject->Identifier, - m_OplogId, - TempIndexPath, - Ec.message()); - } } } @@ -1827,7 +1780,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot"); - std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; + const std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; if (IsFile(IndexPath)) { uint64_t EntryCount = 0; diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index e7b2e6bc6..91bd9cba8 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -824,12 +824,11 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } void -ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, const std::function& ClaimDiskReserveFunc) +ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool ResetLog, const std::function& ClaimDiskReserveFunc) { ZEN_TRACE_CPU("Z$::Bucket::WriteIndexSnapshot"); - const uint64_t LogCount = FlushLockPosition ? 0 : m_SlogFile.GetLogCount(); - if (m_LogFlushPosition == LogCount) + if (m_LogFlushPosition == m_SlogFile.GetLogCount()) { return; } @@ -846,7 +845,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, namespace fs = std::filesystem; - fs::path IndexPath = cache::impl::GetIndexPath(m_BucketDir, m_BucketName); + const fs::path IndexPath = cache::impl::GetIndexPath(m_BucketDir, m_BucketName); try { @@ -878,8 +877,10 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, throw std::system_error(Ec, fmt::format("failed to create new snapshot file in '{}'", m_BucketDir)); } + const uint64_t IndexLogPosition = ResetLog ? 0 : m_SlogFile.GetLogCount(); + cache::impl::CacheBucketIndexHeader Header = {.EntryCount = EntryCount, - .LogPosition = LogCount, + .LogPosition = IndexLogPosition, .PayloadAlignment = gsl::narrow(m_Configuration.PayloadAlignment)}; Header.Checksum = cache::impl::CacheBucketIndexHeader::ComputeChecksum(Header); @@ -916,34 +917,28 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec); if (Ec) { - std::filesystem::path TempFilePath = ObjectIndexFile.GetPath(); - ZEN_WARN("snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", TempFilePath, IndexPath, Ec.message()); + throw std::system_error(Ec, + fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", + ObjectIndexFile.GetPath(), + IndexPath, + Ec.message())); } - else + + if (ResetLog) { - // We must only update the log flush position once the snapshot write succeeds - if (FlushLockPosition) - { - std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); + const std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); - if (IsFile(LogPath)) + if (IsFile(LogPath)) + { + if (!RemoveFile(LogPath, Ec) || Ec) { - if (!RemoveFile(LogPath, Ec) || Ec) - { - ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", - LogPath, - IndexPath, - Ec.message()); - std::error_code RemoveIndexEc; - RemoveFile(IndexPath, RemoveIndexEc); - } + // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in + // the end it will be the same result + ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); } } - if (!Ec) - { - m_LogFlushPosition = LogCount; - } } + m_LogFlushPosition = IndexLogPosition; } catch (const std::exception& Err) { diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index a8914ed20..8cf241e34 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -461,7 +461,7 @@ CasContainerStrategy::Flush() ZEN_TRACE_CPU("CasContainer::Flush"); m_BlockStore.Flush(/*ForceNewBlock*/ false); m_CasLog.Flush(); - MakeIndexSnapshot(/*FlushLockPosition*/ false); + MakeIndexSnapshot(/*ResetLog*/ false); } void @@ -924,13 +924,12 @@ CasContainerStrategy::StorageSize() const } void -CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) +CasContainerStrategy::MakeIndexSnapshot(bool ResetLog) { ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::MakeIndexSnapshot"); - const uint64_t LogCount = FlushLockPosition ? 0 : m_CasLog.GetLogCount(); - if (m_LogFlushPosition == LogCount) + if (m_LogFlushPosition == m_CasLog.GetLogCount()) { return; } @@ -947,7 +946,7 @@ CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) namespace fs = std::filesystem; - fs::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); + const fs::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); try { @@ -957,7 +956,10 @@ CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) { RwLock::SharedLockScope ___(m_LocationMapLock); - IndexLogPosition = m_CasLog.GetLogCount(); + if (!ResetLog) + { + IndexLogPosition = m_CasLog.GetLogCount(); + } Entries.resize(m_LocationMap.size()); uint64_t EntryIndex = 0; @@ -967,6 +969,7 @@ CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) IndexEntry.Key = Entry.first; IndexEntry.Location = m_Locations[Entry.second]; } + EntryCount = m_LocationMap.size(); } TemporaryFile ObjectIndexFile; @@ -976,7 +979,6 @@ CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) { throw std::system_error(Ec, fmt::format("Failed to create temp file for index snapshot at '{}'", IndexPath)); } - EntryCount = Entries.size(); CasDiskIndexHeader Header = {.EntryCount = EntryCount, .LogPosition = IndexLogPosition, .PayloadAlignment = gsl::narrow(m_PayloadAlignment)}; @@ -989,32 +991,28 @@ CasContainerStrategy::MakeIndexSnapshot(bool FlushLockPosition) ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec); if (Ec) { - ZEN_WARN("snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", - ObjectIndexFile.GetPath(), - IndexPath, - Ec.message()); + throw std::system_error(Ec, + fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", + ObjectIndexFile.GetPath(), + IndexPath, + Ec.message())); } - if (FlushLockPosition) + + if (ResetLog) { - std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); + const std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); if (IsFile(LogPath)) { if (!RemoveFile(LogPath, Ec) || Ec) { - ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", - LogPath, - IndexPath, - Ec.message()); - std::error_code RemoveIndexEc; - RemoveFile(IndexPath, RemoveIndexEc); + // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in + // the end it will be the same result + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); } } } - if (!Ec) - { - m_LogFlushPosition = LogCount; - } + m_LogFlushPosition = IndexLogPosition; } catch (const std::exception& Err) { @@ -1214,7 +1212,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore || (LogEntryCount > 0)) { - MakeIndexSnapshot(/*FlushLockPosition*/ true); + MakeIndexSnapshot(/*ResetLog*/ true); } // TODO: should validate integrity of container files here diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index b7ea7591e..15e4cbf81 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -77,7 +77,7 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore private: CasStore::InsertResult InsertChunk(const void* ChunkData, size_t ChunkSize, const IoHash& ChunkHash); - void MakeIndexSnapshot(bool FlushLockPosition); + void MakeIndexSnapshot(bool ResetLog); uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntryCount); void OpenContainer(bool IsNewStore); diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 14bdc41f0..4911bffb9 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -251,7 +251,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN if (IsNewStore || LogEntryCount > 0) { - MakeIndexSnapshot(); + MakeIndexSnapshot(/*ResetLog*/ true); } } @@ -727,7 +727,7 @@ FileCasStrategy::Flush() ZEN_TRACE_CPU("FileCas::Flush"); m_CasLog.Flush(); - MakeIndexSnapshot(); + MakeIndexSnapshot(/*ResetLog*/ false); } void @@ -912,15 +912,14 @@ FileCasStrategy::ValidateEntry(const FileCasIndexEntry& Entry, std::string& OutR } void -FileCasStrategy::MakeIndexSnapshot() +FileCasStrategy::MakeIndexSnapshot(bool ResetLog) { ZEN_MEMSCOPE(GetFileCasTag()); ZEN_TRACE_CPU("FileCas::MakeIndexSnapshot"); using namespace filecas::impl; - uint64_t LogCount = m_CasLog.GetLogCount(); - if (m_LogFlushPosition == LogCount) + if (m_LogFlushPosition == m_CasLog.GetLogCount()) { return; } @@ -937,34 +936,20 @@ FileCasStrategy::MakeIndexSnapshot() namespace fs = std::filesystem; - fs::path IndexPath = GetIndexPath(m_RootDirectory); - fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory); - - // Move index away, we keep it if something goes wrong - if (IsFile(STmpIndexPath)) - { - std::error_code Ec; - if (!RemoveFile(STmpIndexPath, Ec) || Ec) - { - ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message()); - return; - } - } + const fs::path IndexPath = GetIndexPath(m_RootDirectory); try { - if (IsFile(IndexPath)) - { - RenameFile(IndexPath, STmpIndexPath); - } - // Write the current state of the location map to a new index state std::vector Entries; uint64_t IndexLogPosition = 0; { RwLock::SharedLockScope __(m_Lock); - IndexLogPosition = m_CasLog.GetLogCount(); + if (!ResetLog) + { + IndexLogPosition = m_CasLog.GetLogCount(); + } Entries.resize(m_Index.size()); uint64_t EntryIndex = 0; @@ -974,6 +959,7 @@ FileCasStrategy::MakeIndexSnapshot() IndexEntry.Key = Entry.first; IndexEntry.Size = Entry.second.Size; } + EntryCount = m_Index.size(); } TemporaryFile ObjectIndexFile; @@ -983,47 +969,45 @@ FileCasStrategy::MakeIndexSnapshot() { throw std::system_error(Ec, fmt::format("Failed to create temp file for index snapshot at '{}'", IndexPath)); } - filecas::impl::FileCasIndexHeader Header = {.EntryCount = Entries.size(), .LogPosition = IndexLogPosition}; + filecas::impl::FileCasIndexHeader Header = {.EntryCount = EntryCount, .LogPosition = IndexLogPosition}; Header.Checksum = filecas::impl::FileCasIndexHeader::ComputeChecksum(Header); ObjectIndexFile.Write(&Header, sizeof(filecas::impl::FileCasIndexHeader), 0); - ObjectIndexFile.Write(Entries.data(), Entries.size() * sizeof(FileCasIndexEntry), sizeof(filecas::impl::FileCasIndexHeader)); + ObjectIndexFile.Write(Entries.data(), EntryCount * sizeof(FileCasIndexEntry), sizeof(filecas::impl::FileCasIndexHeader)); ObjectIndexFile.Flush(); ObjectIndexFile.MoveTemporaryIntoPlace(IndexPath, Ec); if (Ec) { - throw std::system_error(Ec, fmt::format("Failed to move temp file '{}' to '{}'", ObjectIndexFile.GetPath(), IndexPath)); + throw std::system_error(Ec, + fmt::format("Snapshot failed to rename new snapshot '{}' to '{}', reason: '{}'", + ObjectIndexFile.GetPath(), + IndexPath, + Ec.message())); } - EntryCount = Entries.size(); - m_LogFlushPosition = IndexLogPosition; - } - catch (const std::exception& Err) - { - ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what()); - - // Restore any previous snapshot - if (IsFile(STmpIndexPath)) + if (ResetLog) { - std::error_code Ec; - RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - RenameFile(STmpIndexPath, IndexPath, Ec); - if (Ec) + const std::filesystem::path LogPath = GetLogPath(m_RootDirectory); + + if (IsFile(LogPath)) { - ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message()); + if (!RemoveFile(LogPath, Ec) || Ec) + { + // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in + // the end it will be the same result + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + } } } + m_LogFlushPosition = IndexLogPosition; } - if (IsFile(STmpIndexPath)) + catch (const std::exception& Err) { - std::error_code Ec; - if (!RemoveFile(STmpIndexPath, Ec) || Ec) - { - ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message()); - } + ZEN_WARN("snapshot FAILED, reason: '{}'", Err.what()); } } + uint64_t FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion) { diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h index 21d8c3b9e..e93356927 100644 --- a/src/zenstore/filecas.h +++ b/src/zenstore/filecas.h @@ -50,7 +50,7 @@ struct FileCasStrategy final : public GcStorage, public GcReferenceStore virtual GcReferencePruner* CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats& Stats) override; private: - void MakeIndexSnapshot(); + void MakeIndexSnapshot(bool ResetLog); uint64_t ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& OutVersion); uint64_t ReadLog(const std::filesystem::path& LogPath, uint64_t LogPosition); LoggerRef Log() { return m_Log; } diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index f36e780b9..54ebb324d 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -409,20 +409,20 @@ public: void SaveSnapshot(const std::function& ClaimDiskReserveFunc = []() { return 0; }); void WriteIndexSnapshot( RwLock::ExclusiveLockScope&, - bool FlushLockPosition, + bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }) { - WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc); + WriteIndexSnapshotLocked(ResetLog, ClaimDiskReserveFunc); } void WriteIndexSnapshot( RwLock::SharedLockScope&, - bool FlushLockPosition, + bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }) { - WriteIndexSnapshotLocked(FlushLockPosition, ClaimDiskReserveFunc); + WriteIndexSnapshotLocked(ResetLog, ClaimDiskReserveFunc); } void WriteIndexSnapshotLocked( - bool FlushLockPosition, + bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }); void CompactState(RwLock::ExclusiveLockScope& IndexLock, -- cgit v1.2.3 From 4ca731fe4445f2c84582a80a7c635e3d736b3cf3 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 13 May 2025 18:51:12 +0200 Subject: skip empty or single-space command line arguments (#393) --- src/zen/zen.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 7c21035c0..53fdf3d36 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -593,7 +593,11 @@ main(int argc, char** argv) Args.reserve(argc); for (int I = 0; I < argc; I++) { - Args.push_back(std::string(argv[I])); + std::string Arg(argv[I]); + if ((!Arg.empty()) && (Arg != " ")) + { + Args.emplace_back(std::move(Arg)); + } } #endif std::vector RawArgs = zen::StripCommandlineQuotes(Args); -- cgit v1.2.3 From a4676ccaf5a98b0d4204f70f43e88640db6ce29d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 13 May 2025 19:09:21 +0200 Subject: extend log on failed httpsys response (#394) * extend log on failed httpsys response * fix formatting for "Desired port is in use, retrying" * add warning log if port is remapped --- src/zenhttp/servers/httpasio.cpp | 21 ++++++++++++++------- src/zenhttp/servers/httpsys.cpp | 9 ++++++++- 2 files changed, 22 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index fe59e3a6f..c1b7294c9 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -678,7 +678,7 @@ struct HttpAcceptor if (BindErrorCode == asio::error::address_in_use) { // Do a retry after a short sleep on same port just to be sure - ZEN_INFO("Desired port %d is in use, retrying", BasePort); + ZEN_INFO("Desired port {} is in use, retrying", BasePort); Sleep(100); m_Acceptor.bind(asio::ip::tcp::endpoint(BindAddress, EffectivePort), BindErrorCode); } @@ -697,13 +697,20 @@ struct HttpAcceptor { ZEN_ERROR("Unable open asio service, error '{}'", BindErrorCode.message()); } - else if (BindAddress.is_loopback()) + else { - m_AlternateProtocolAcceptor.bind(asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), EffectivePort), BindErrorCode); - m_UseAlternateProtocolAcceptor = true; - ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts", - "localhost", - EffectivePort); + if (EffectivePort != BasePort) + { + ZEN_WARN("Desired port {} is in use, remapped to port {}", BasePort, EffectivePort); + } + if (BindAddress.is_loopback()) + { + m_AlternateProtocolAcceptor.bind(asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), EffectivePort), BindErrorCode); + m_UseAlternateProtocolAcceptor = true; + ZEN_INFO("Registered local-only handler 'http://{}:{}/' - this is not accessible from remote hosts", + "localhost", + EffectivePort); + } } #if ZEN_PLATFORM_WINDOWS diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index a315ea2df..62dab02c4 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -535,7 +535,14 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult != NO_ERROR) { - ZEN_WARN("response aborted due to error: {}", GetSystemErrorAsString(IoResult)); + ZEN_WARN("response '{}' ({}) aborted after transfering '{}', {} out of {} bytes, reason: {} ({})", + ReasonStringForHttpResultCode(m_ResponseCode), + m_ResponseCode, + ToString(m_ContentType), + NumberOfBytesTransferred, + m_TotalDataSize, + GetSystemErrorAsString(IoResult), + IoResult); // if one transmit failed there's really no need to go on return nullptr; -- cgit v1.2.3 From cab94c0edd7b7107e996f37e41c3a91fc905fa08 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 14 May 2025 22:42:39 +0200 Subject: `zen oplog-import` and `zen oplog-export` now supports `--oidctoken-exe-path` option (#395) --- src/zen/cmds/projectstore_cmd.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src') diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 066dac781..a26687147 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -930,6 +930,12 @@ ExportOplogCommand::ExportOplogCommand() "Path to json file that holds the cloud/builds Storage access token", cxxopts::value(m_JupiterAccessTokenPath), ""); + m_Options.add_option("", + "", + "oidctoken-exe-path", + "Path to OidcToken executable", + cxxopts::value(m_OidcTokenAuthExecutablePath)->default_value(""), + ""); m_Options.add_option("", "", "assume-http2", @@ -1404,6 +1410,12 @@ ImportOplogCommand::ImportOplogCommand() "Path to json file that holds the cloud/builds Storage access token", cxxopts::value(m_JupiterAccessTokenPath), ""); + m_Options.add_option("", + "", + "oidctoken-exe-path", + "Path to OidcToken executable", + cxxopts::value(m_OidcTokenAuthExecutablePath)->default_value(""), + ""); m_Options.add_option("", "", "assume-http2", -- cgit v1.2.3 From 5df6ebbf497401b6124a0b44470165bc14880585 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 15 May 2025 11:22:29 +0100 Subject: make sure tests initialize trace so we don't end up allocating tons of memory for no reason (#397) --- src/zencore-test/zencore-test.cpp | 2 ++ src/zenhttp-test/zenhttp-test.cpp | 2 ++ src/zennet-test/zennet-test.cpp | 2 ++ src/zenserver-test/zenserver-test.cpp | 1 + src/zenstore-test/zenstore-test.cpp | 4 ++-- src/zenstore/zenstore.cpp | 2 ++ src/zenutil-test/zenutil-test.cpp | 2 ++ 7 files changed, 13 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/zencore-test/zencore-test.cpp b/src/zencore-test/zencore-test.cpp index 40cb51156..928c4e5c2 100644 --- a/src/zencore-test/zencore-test.cpp +++ b/src/zencore-test/zencore-test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -20,6 +21,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) #if ZEN_WITH_TESTS zen::zencore_forcelinktests(); + zen::TraceInit("zencore-test"); zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); diff --git a/src/zenhttp-test/zenhttp-test.cpp b/src/zenhttp-test/zenhttp-test.cpp index 49db1ba54..a1327320e 100644 --- a/src/zenhttp-test/zenhttp-test.cpp +++ b/src/zenhttp-test/zenhttp-test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #if ZEN_WITH_TESTS @@ -16,6 +17,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) #if ZEN_WITH_TESTS zen::zenhttp_forcelinktests(); + zen::TraceInit("zenhttp-test"); zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); diff --git a/src/zennet-test/zennet-test.cpp b/src/zennet-test/zennet-test.cpp index 482d3c617..bac0dec8f 100644 --- a/src/zennet-test/zennet-test.cpp +++ b/src/zennet-test/zennet-test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -17,6 +18,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) #if ZEN_WITH_TESTS zen::zennet_forcelinktests(); + zen::TraceInit("zennet-test"); zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 78a735ea0..ded40a486 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -100,6 +100,7 @@ main(int argc, char** argv) using namespace std::literals; using namespace zen; + zen::TraceInit("zenserver-test"); zen::logging::InitializeLogging(); zen::logging::SetLogLevel(zen::logging::level::Debug); diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index c56971520..04f5a8e10 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include @@ -18,8 +18,8 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) #if ZEN_WITH_TESTS zen::zenstore_forcelinktests(); + zen::TraceInit("zenstore-test"); zen::logging::InitializeLogging(); - zen::buildstore_forcelink(); zen::MaximizeOpenFileCount(); return ZEN_RUN_TESTS(argc, argv); diff --git a/src/zenstore/zenstore.cpp b/src/zenstore/zenstore.cpp index c697647d2..654fb3510 100644 --- a/src/zenstore/zenstore.cpp +++ b/src/zenstore/zenstore.cpp @@ -5,6 +5,7 @@ #if ZEN_WITH_TESTS # include +# include # include # include # include @@ -19,6 +20,7 @@ namespace zen { void zenstore_forcelinktests() { + buildstore_forcelink(); CAS_forcelink(); filecas_forcelink(); blockstore_forcelink(); diff --git a/src/zenutil-test/zenutil-test.cpp b/src/zenutil-test/zenutil-test.cpp index fadaf0995..ca8208314 100644 --- a/src/zenutil-test/zenutil-test.cpp +++ b/src/zenutil-test/zenutil-test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -17,6 +18,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) #if ZEN_WITH_TESTS zen::zenutil_forcelinktests(); + zen::TraceInit("zencore-test"); zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); -- cgit v1.2.3 From f3d794f2a8f8ae96760bcab4880d34c589250b6a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 16 May 2025 12:03:21 +0200 Subject: sparse win file write (#398) * Added `--use-sparse-files` option to `zen builds` command improving write performance of large files. Enabled by default. --- src/zen/cmds/builds_cmd.cpp | 21 ++++++++++++++++++++- src/zen/cmds/builds_cmd.h | 1 + src/zencore/filesystem.cpp | 28 ++++++++++++++++++++++++++++ src/zencore/include/zencore/filesystem.h | 2 ++ 4 files changed, 51 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index c4a1344d4..117d0b291 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -95,6 +95,7 @@ namespace { const bool SingleThreaded = false; bool BoostWorkerThreads = false; + bool UseSparseFiles = false; WorkerThreadPool& GetIOWorkerPool() { @@ -4261,8 +4262,19 @@ namespace { ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); - OutputFile = std::move(NewOutputFile); + OutputFile = std::move(NewOutputFile); + if (UseSparseFiles) + { + void* Handle = OutputFile->Handle(); + if (!PrepareFileForScatteredWrite(Handle, TargetFinalSize)) + { + ZEN_DEBUG("Unable to to prepare file '{}' with size {} for random write", + GetTargetPath(TargetIndex), + TargetFinalSize); + } + } OpenFileWriter = std::make_unique(*OutputFile, Min(TargetFinalSize, 256u * 1024u)); + OpenFileWriter->Write(Buffer, FileOffset); m_DiskStats.WriteCount++; m_DiskStats.WriteByteCount += Buffer.GetSize(); @@ -8858,6 +8870,12 @@ BuildsCommand::BuildsCommand() auto AddSystemOptions = [this](cxxopts::Options& Ops) { Ops.add_option("", "", "system-dir", "Specify system root", cxxopts::value(m_SystemRootDir), ""); + Ops.add_option("", + "", + "use-sparse-files", + "Enable use of sparse files when writing large files. Defaults to true.", + cxxopts::value(m_UseSparseFiles), + ""); }; auto AddAuthOptions = [this](cxxopts::Options& Ops) { @@ -9872,6 +9890,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; BoostWorkerThreads = m_BoostWorkerThreads; + UseSparseFiles = m_UseSparseFiles; try { diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index de2d6fffc..41ed65105 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -31,6 +31,7 @@ private: bool m_LogProgress = false; bool m_Verbose = false; bool m_BoostWorkerThreads = false; + bool m_UseSparseFiles = true; std::filesystem::path m_ZenFolderPath; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 018330d9b..c1df6d53e 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2158,6 +2158,34 @@ MaximizeOpenFileCount() #endif } +bool +PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize) +{ + bool Result = true; +#if ZEN_PLATFORM_WINDOWS + DWORD _ = 0; + BOOL Ok = DeviceIoControl(FileHandle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &_, nullptr); + if (!Ok) + { + std::error_code DummyEc; + ZEN_DEBUG("Unable to set sparse mode for file '{}'", PathFromHandle(FileHandle, DummyEc)); + Result = false; + } + + FILE_ALLOCATION_INFO AllocationInfo = {}; + AllocationInfo.AllocationSize.QuadPart = FinalSize; + if (!SetFileInformationByHandle(FileHandle, FileAllocationInfo, &AllocationInfo, DWORD(sizeof(AllocationInfo)))) + { + std::error_code DummyEc; + ZEN_DEBUG("Unable to set file allocation size to {} for file '{}'", FinalSize, PathFromHandle(FileHandle, DummyEc)); + Result = false; + } +#else // ZEN_PLATFORM_WINDOWS + ZEN_UNUSED(FileHandle, FinalSize); +#endif // ZEN_PLATFORM_WINDOWS + return Result; +} + void GetDirectoryContent(const std::filesystem::path& RootDir, DirectoryContentFlags Flags, DirectoryContent& OutContent) { diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 1bc3943df..e62170eba 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -137,6 +137,8 @@ ZENCORE_API std::filesystem::path GetRunningExecutablePath(); */ ZENCORE_API void MaximizeOpenFileCount(); +ZENCORE_API bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize); + struct FileContents { std::vector Data; -- cgit v1.2.3 From 95fc783ae9914019fee03f12603e4c49f9ab6106 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 16 May 2025 12:08:05 +0100 Subject: validate custom fields (#399) * implemented validation of compact binary custom fields --- src/zencore/compactbinaryvalidation.cpp | 45 ++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index 6f53bba69..833649b88 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -134,6 +134,37 @@ ValidateCbFloat64(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) } } +/** + * Validate and read a fixed-size value from the view. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static MemoryView +ValidateCbFixedValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, uint64_t Size) +{ + ZEN_UNUSED(Mode); + + const MemoryView Value = View.Left(Size); + View += Size; + if (Value.GetSize() < Size) + { + AddError(Error, CbValidateError::OutOfBounds); + } + return Value; +}; + +/** + * Validate and read a value from the view where the view begins with the value size. + * + * Modifies the view to start at the end of the value, and adds error flags if applicable. + */ +static MemoryView +ValidateCbDynamicValue(MemoryView& View, CbValidateMode Mode, CbValidateError& Error) +{ + const uint64_t ValueSize = ValidateCbUInt(View, Mode, Error); + return ValidateCbFixedValue(View, Mode, Error, ValueSize); +} + /** * Validate and read a string from the view. * @@ -378,8 +409,20 @@ ValidateCbField(MemoryView& View, CbValidateMode Mode, CbValidateError& Error, c ValidateFixedPayload(12); break; case CbFieldType::CustomById: + { + MemoryView Value = ValidateCbDynamicValue(View, Mode, Error); + ValidateCbUInt(Value, Mode, Error); + } + break; case CbFieldType::CustomByName: - ZEN_NOT_IMPLEMENTED(); // TODO: FIX! + { + MemoryView Value = ValidateCbDynamicValue(View, Mode, Error); + const std::string_view TypeName = ValidateCbString(Value, Mode, Error); + if (TypeName.empty() && !EnumHasAnyFlags(Error, CbValidateError::OutOfBounds)) + { + AddError(Error, CbValidateError::InvalidType); + } + } break; } -- cgit v1.2.3 From 4e2efa1051e3eb86ab48d92b3f6ad5896cda5d81 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 16 May 2025 19:51:36 +0200 Subject: parallel work handle dispatch exception (#400) - Bugfix: Wait for async threads if dispatching of work using ParallellWork throws exception --- src/zen/cmds/builds_cmd.cpp | 2361 +++++++++++------------ src/zen/cmds/wipe_cmd.cpp | 4 +- src/zencore/filesystem.cpp | 5 + src/zenutil/buildstoragecache.cpp | 4 +- src/zenutil/chunkedcontent.cpp | 52 +- src/zenutil/include/zenutil/buildstoragecache.h | 2 +- src/zenutil/include/zenutil/chunkedcontent.h | 4 +- src/zenutil/include/zenutil/parallellwork.h | 119 -- src/zenutil/include/zenutil/parallelwork.h | 71 + src/zenutil/parallelwork.cpp | 192 ++ src/zenutil/zenutil.cpp | 2 + 11 files changed, 1444 insertions(+), 1372 deletions(-) delete mode 100644 src/zenutil/include/zenutil/parallellwork.h create mode 100644 src/zenutil/include/zenutil/parallelwork.h create mode 100644 src/zenutil/parallelwork.cpp (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 117d0b291..815bb7597 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include @@ -356,7 +356,7 @@ namespace { std::atomic DiscoveredItemCount = 0; std::atomic DeletedItemCount = 0; std::atomic DeletedByteCount = 0; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); struct AsyncVisitor : public GetDirectoryContentVisitor { @@ -1749,7 +1749,7 @@ namespace { const Oid& BuildId, const IoHash& ChunkHash, const std::uint64_t PreferredMultipartChunkSize, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& NetworkPool, DownloadStatistics& DownloadStats, std::function&& OnDownloadComplete) @@ -1800,16 +1800,13 @@ namespace { } for (auto& WorkItem : WorkItems) { - Work.ScheduleWork( - NetworkPool, - [WorkItem = std::move(WorkItem)](std::atomic&) { - ZEN_TRACE_CPU("DownloadLargeBlob_Work"); - if (!AbortFlag) - { - WorkItem(); - } - }, - Work.DefaultErrorFunction()); + Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic&) { + ZEN_TRACE_CPU("DownloadLargeBlob_Work"); + if (!AbortFlag) + { + WorkItem(); + } + }); } } @@ -1902,7 +1899,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& VerifyPool = GetIOWorkerPool(); - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1927,95 +1924,41 @@ namespace { for (const IoHash& ChunkAttachment : ChunkAttachments) { - Work.ScheduleWork( - NetworkPool, - [&, ChunkAttachment](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); - - FilteredDownloadedBytesPerSecond.Start(); - DownloadLargeBlob(Storage, - TempFolder, - BuildId, - ChunkAttachment, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) { - Payload.SetContentType(ZenContentType::kCompressedBinary); - if (!AbortFlag) - { - Work.ScheduleWork( - VerifyPool, - [&, Payload = std::move(Payload), ChunkHash](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_Validate"); - - if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == - AttachmentsToVerifyCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - FilteredVerifiedBytesPerSecond.Start(); - - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); - ValidateStats.VerifiedAttachmentCount++; - ValidateStats.VerifiedByteCount += DecompressedSize; - if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) - { - FilteredVerifiedBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); - } - }); - } - }, - Work.DefaultErrorFunction()); - } - - for (const IoHash& BlockAttachment : BlockAttachments) - { - Work.ScheduleWork( - NetworkPool, - [&, BlockAttachment](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); + Work.ScheduleWork(NetworkPool, [&, ChunkAttachment](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); - if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (!Payload) - { - throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); - } - if (!AbortFlag) - { - Work.ScheduleWork( - VerifyPool, - [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { + FilteredDownloadedBytesPerSecond.Start(); + DownloadLargeBlob( + Storage, + TempFolder, + BuildId, + ChunkAttachment, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) { + Payload.SetContentType(ZenContentType::kCompressedBinary); + if (!AbortFlag) + { + Work.ScheduleWork(VerifyPool, [&, Payload = std::move(Payload), ChunkHash](std::atomic&) mutable { if (!AbortFlag) { - ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); + ZEN_TRACE_CPU("ValidateBuildPart_Validate"); + + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == + AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); + ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); ValidateStats.VerifiedAttachmentCount++; ValidateStats.VerifiedByteCount += DecompressedSize; if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) @@ -2023,12 +1966,55 @@ namespace { FilteredVerifiedBytesPerSecond.Stop(); } } - }, - Work.DefaultErrorFunction()); - } + }); + } + }); + } + }); + } + + for (const IoHash& BlockAttachment : BlockAttachments) + { + Work.ScheduleWork(NetworkPool, [&, BlockAttachment](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); + + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); } - }, - Work.DefaultErrorFunction()); + if (!Payload) + { + throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); + } + if (!AbortFlag) + { + Work.ScheduleWork(VerifyPool, [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); + + FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); + ValidateStats.VerifiedAttachmentCount++; + ValidateStats.VerifiedByteCount += DecompressedSize; + if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredVerifiedBytesPerSecond.Stop(); + } + } + }); + } + } + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -2286,7 +2272,7 @@ namespace { FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); std::atomic QueuedPendingBlocksForUpload = 0; @@ -2297,139 +2283,134 @@ namespace { break; } const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; - Work.ScheduleWork( - GenerateBlobsPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); - - FilteredGeneratedBytesPerSecond.Start(); - // TODO: Convert ScheduleWork body to function - - Stopwatch GenerateTimer; - CompressedBuffer CompressedBlock = - GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); - ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(CompressedBlock.GetCompressedSize()), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); + Work.ScheduleWork(GenerateBlobsPool, [&, BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); - OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); - { - CbObjectWriter Writer; - Writer.AddString("createdBy", "zen"); - OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); - } - GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; - GenerateBlocksStats.GeneratedBlockCount++; + FilteredGeneratedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function - Lock.WithExclusiveLock([&]() { - OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockIndex); - }); + Stopwatch GenerateTimer; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); + ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlock.GetCompressedSize()), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } + OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); + { + CbObjectWriter Writer; + Writer.AddString("createdBy", "zen"); + OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); + } + GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; + GenerateBlocksStats.GeneratedBlockCount++; - if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) - { - FilteredGeneratedBytesPerSecond.Stop(); - } + Lock.WithExclusiveLock([&]() { + OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, BlockIndex); + }); - if (QueuedPendingBlocksForUpload.load() > 16) - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - else + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + { + FilteredGeneratedBytesPerSecond.Stop(); + } + + if (QueuedPendingBlocksForUpload.load() > 16) + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + if (!AbortFlag) { - if (!AbortFlag) - { - QueuedPendingBlocksForUpload++; + QueuedPendingBlocksForUpload++; - Work.ScheduleWork( - UploadBlocksPool, - [&, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { - auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); - if (!AbortFlag) + Work.ScheduleWork( + UploadBlocksPool, + [&, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { + auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); + if (!AbortFlag) + { + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { - if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); + ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); - FilteredUploadedBytesPerSecond.Stop(); - std::span Segments = Payload.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - else - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); + FilteredUploadedBytesPerSecond.Stop(); + std::span Segments = Payload.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); - FilteredUploadedBytesPerSecond.Start(); - // TODO: Convert ScheduleWork body to function + FilteredUploadedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function - const CbObject BlockMetaData = - BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], - OutBlocks.BlockMetaDatas[BlockIndex]); + const CbObject BlockMetaData = + BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], + OutBlocks.BlockMetaDatas[BlockIndex]); - const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; - const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); + const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - if (Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - Payload.GetCompressed()); - } + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); + } - Storage.BuildStorage->PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload).GetCompressed()); - UploadStats.BlocksBytes += CompressedBlockSize; + Storage.BuildStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); + UploadStats.BlocksBytes += CompressedBlockSize; - ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - BlockHash, - NiceBytes(CompressedBlockSize), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", + BlockHash, + NiceBytes(CompressedBlockSize), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - if (Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); - } + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - BlockHash, - NiceBytes(BlockMetaData.GetSize())); + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + BlockHash, + NiceBytes(BlockMetaData.GetSize())); - OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - UploadStats.BlockCount++; - if (UploadStats.BlockCount == NewBlockCount) - { - FilteredUploadedBytesPerSecond.Stop(); - } + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + UploadStats.BlockCount++; + if (UploadStats.BlockCount == NewBlockCount) + { + FilteredUploadedBytesPerSecond.Stop(); } } - }, - Work.DefaultErrorFunction()); - } + } + }); } } - }, - Work.DefaultErrorFunction()); + } + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -2491,7 +2472,7 @@ namespace { FilteredRate FilteredCompressedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); std::atomic UploadedBlockSize = 0; std::atomic UploadedBlockCount = 0; @@ -2606,8 +2587,7 @@ namespace { FilteredUploadedBytesPerSecond.Stop(); } } - }, - Work.DefaultErrorFunction()); + }); }; auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) { @@ -2658,16 +2638,13 @@ namespace { }); for (auto& WorkPart : MultipartWork) { - Work.ScheduleWork( - UploadChunkPool, - [Work = std::move(WorkPart)](std::atomic&) { - ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work"); - if (!AbortFlag) - { - Work(); - } - }, - Work.DefaultErrorFunction()); + Work.ScheduleWork(UploadChunkPool, [Work = std::move(WorkPart)](std::atomic&) { + ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work"); + if (!AbortFlag) + { + Work(); + } + }); } ZEN_CONSOLE_VERBOSE("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize)); } @@ -2687,8 +2664,7 @@ namespace { } } } - }, - Work.DefaultErrorFunction()); + }); }; std::vector GenerateBlockIndexes; @@ -2704,59 +2680,56 @@ namespace { const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; if (!AbortFlag) { - Work.ScheduleWork( - ReadChunkPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); + Work.ScheduleWork(ReadChunkPool, [&, BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); - FilteredGenerateBlockBytesPerSecond.Start(); + FilteredGenerateBlockBytesPerSecond.Start(); - Stopwatch GenerateTimer; - CompositeBuffer Payload; - if (NewBlocks.BlockHeaders[BlockIndex]) - { - Payload = RebuildBlock(Path, - Content, - Lookup, - std::move(NewBlocks.BlockHeaders[BlockIndex]), - NewBlockChunks[BlockIndex], - DiskStats) - .GetCompressed(); - } - else + Stopwatch GenerateTimer; + CompositeBuffer Payload; + if (NewBlocks.BlockHeaders[BlockIndex]) + { + Payload = RebuildBlock(Path, + Content, + Lookup, + std::move(NewBlocks.BlockHeaders[BlockIndex]), + NewBlockChunks[BlockIndex], + DiskStats) + .GetCompressed(); + } + else + { + ChunkBlockDescription BlockDescription; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); + if (!CompressedBlock) { - ChunkBlockDescription BlockDescription; - CompressedBuffer CompressedBlock = - GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); - if (!CompressedBlock) - { - throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); - } - ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); - Payload = std::move(CompressedBlock).GetCompressed(); + throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); } + ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); + Payload = std::move(CompressedBlock).GetCompressed(); + } - GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; - GeneratedBlockCount++; - if (GeneratedBlockCount == GenerateBlockIndexes.size()) - { - FilteredGenerateBlockBytesPerSecond.Stop(); - } - ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}", - NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(NewBlocks.BlockSizes[BlockIndex]), - NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - if (!AbortFlag) - { - AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); - } + GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; + GeneratedBlockCount++; + if (GeneratedBlockCount == GenerateBlockIndexes.size()) + { + FilteredGenerateBlockBytesPerSecond.Stop(); } - }, - Work.DefaultErrorFunction()); + ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}", + NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(NewBlocks.BlockSizes[BlockIndex]), + NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); + if (!AbortFlag) + { + AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); + } + } + }); } } @@ -2764,35 +2737,32 @@ namespace { for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; - Work.ScheduleWork( - ReadChunkPool, - [&, ChunkIndex](std::atomic&) { + Work.ScheduleWork(ReadChunkPool, [&, ChunkIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); + + FilteredCompressedBytesPerSecond.Start(); + Stopwatch CompressTimer; + CompositeBuffer Payload = + CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); + ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", + Content.ChunkedContent.ChunkHashes[ChunkIndex], + NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), + NiceBytes(Payload.GetSize()), + NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); + const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + UploadStats.ReadFromDiskBytes += ChunkRawSize; + if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) + { + FilteredCompressedBytesPerSecond.Stop(); + } if (!AbortFlag) { - ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); - - FilteredCompressedBytesPerSecond.Start(); - Stopwatch CompressTimer; - CompositeBuffer Payload = - CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); - ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", - Content.ChunkedContent.ChunkHashes[ChunkIndex], - NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), - NiceBytes(Payload.GetSize()), - NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); - const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - UploadStats.ReadFromDiskBytes += ChunkRawSize; - if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) - { - FilteredCompressedBytesPerSecond.Stop(); - } - if (!AbortFlag) - { - AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload)); - } + AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload)); } - }, - Work.DefaultErrorFunction()); + } + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -4045,7 +4015,7 @@ namespace { WorkerThreadPool& VerifyPool = GetIOWorkerPool(); - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); const uint32_t PathCount = gsl::narrow(Content.Paths.size()); @@ -4173,11 +4143,20 @@ namespace { VerifyFolderStats.FilesVerified++; } }, - [&, PathIndex](const std::exception& Ex, std::atomic&) { + [&, PathIndex](std::exception_ptr Ex, std::atomic&) { + std::string Description; + try + { + std::rethrow_exception(Ex); + } + catch (const std::exception& Ex) + { + Description = Ex.what(); + } ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}", (Path / Content.Paths[PathIndex]).make_preferred(), - Ex.what())); + Description)); }); VerifyFolderStats.FilesFailed++; }); @@ -4405,7 +4384,7 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& Lookup, std::span RemoteSequenceIndexes, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& VerifyPool) { if (RemoteSequenceIndexes.empty()) @@ -4416,21 +4395,18 @@ namespace { for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++) { const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; - Work.ScheduleWork( - VerifyPool, - [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic&) { + Work.ScheduleWork(VerifyPool, [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); if (!AbortFlag) { - ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); - VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); - if (!AbortFlag) - { - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - FinalizeChunkSequence(TargetFolder, SequenceRawHash); - } + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(TargetFolder, SequenceRawHash); } - }, - Work.DefaultErrorFunction()); + } + }); } const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; @@ -4480,7 +4456,7 @@ namespace { const ChunkedContentLookup& Lookup, std::span> SequenceIndexChunksLeftToWriteCounters, const BlockWriteOps& Ops, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& VerifyPool, DiskStatistics& DiskStats) { @@ -4672,7 +4648,7 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkBlockDescription& BlockDescription, std::span> SequenceIndexChunksLeftToWriteCounters, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& VerifyPool, CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, @@ -4745,7 +4721,7 @@ namespace { const ChunkedFolderContent& RemoteContent, const ChunkBlockDescription& BlockDescription, std::span> SequenceIndexChunksLeftToWriteCounters, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& VerifyPool, CompositeBuffer&& PartialBlockBuffer, uint32_t FirstIncludedBlockChunkIndex, @@ -4978,7 +4954,7 @@ namespace { const ChunkedContentLookup& RemoteLookup, uint32_t RemoteChunkIndex, std::vector&& ChunkTargetPtrs, - ParallellWork& Work, + ParallelWork& Work, WorkerThreadPool& WritePool, IoBuffer&& Payload, std::span> SequenceIndexChunksLeftToWriteCounters, @@ -5098,8 +5074,7 @@ namespace { } } } - }, - Work.DefaultErrorFunction()); + }); }; bool ReadStateFile(const std::filesystem::path& StateFilePath, @@ -5167,36 +5142,35 @@ namespace { ProgressBar ProgressBar(ProgressMode, "Check Files"); - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); std::atomic CompletedPathCount = 0; uint32_t PathIndex = 0; while (PathIndex < PathCount) { uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); - Work.ScheduleWork( - GetIOWorkerPool(), - [PathIndex, PathRangeCount, &PathsToCheck, &Path, &Result, &CompletedPathCount, &LocalFolderScanStats]( - std::atomic&) { - for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) - { - const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; - std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (TryGetFileProperties(LocalFilePath, - Result.RawSizes[PathRangeIndex], - Result.ModificationTicks[PathRangeIndex], - Result.Attributes[PathRangeIndex])) - { - Result.Paths[PathRangeIndex] = std::move(FilePath); - LocalFolderScanStats.FoundFileCount++; - LocalFolderScanStats.FoundFileByteCount += Result.RawSizes[PathRangeIndex]; - LocalFolderScanStats.AcceptedFileCount++; - LocalFolderScanStats.AcceptedFileByteCount += Result.RawSizes[PathRangeIndex]; - } - CompletedPathCount++; - } - }, - Work.DefaultErrorFunction()); + Work.ScheduleWork(GetIOWorkerPool(), + [PathIndex, PathRangeCount, &PathsToCheck, &Path, &Result, &CompletedPathCount, &LocalFolderScanStats]( + std::atomic&) { + for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; + PathRangeIndex++) + { + const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; + std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); + if (TryGetFileProperties(LocalFilePath, + Result.RawSizes[PathRangeIndex], + Result.ModificationTicks[PathRangeIndex], + Result.Attributes[PathRangeIndex])) + { + Result.Paths[PathRangeIndex] = std::move(FilePath); + LocalFolderScanStats.FoundFileCount++; + LocalFolderScanStats.FoundFileByteCount += Result.RawSizes[PathRangeIndex]; + LocalFolderScanStats.AcceptedFileCount++; + LocalFolderScanStats.AcceptedFileByteCount += Result.RawSizes[PathRangeIndex]; + } + CompletedPathCount++; + } + }); PathIndex += PathRangeCount; } Work.Wait(200, [&](bool, ptrdiff_t) { @@ -5831,8 +5805,8 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); - ParallellWork Work(AbortFlag); + ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); + ParallelWork Work(AbortFlag); struct LooseChunkHashWorkData { @@ -6199,41 +6173,38 @@ namespace { } if (!PrimeCacheOnly) { - Work.ScheduleWork( - WritePool, - [&, ScavengeOpIndex](std::atomic&) mutable { - if (!AbortFlag) - { - const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; - const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; - const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; + Work.ScheduleWork(WritePool, [&, ScavengeOpIndex](std::atomic&) mutable { + if (!AbortFlag) + { + const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; + const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; - const std::filesystem::path ScavengedFilePath = - (ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred(); - ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); + const std::filesystem::path ScavengedFilePath = + (ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred(); + ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); - const IoHash& RemoteSequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; - const std::filesystem::path TempFilePath = - GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + const IoHash& RemoteSequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; + const std::filesystem::path TempFilePath = + GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - CopyFile(ScavengedFilePath, TempFilePath, {.EnableClone = false}); + CopyFile(ScavengedFilePath, TempFilePath, {.EnableClone = false}); - DiskStats.WriteCount++; - DiskStats.WriteByteCount += ScavengeOp.RawSize; + DiskStats.WriteCount++; + DiskStats.WriteByteCount += ScavengeOp.RawSize; - const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - RenameFile(TempFilePath, CacheFilePath); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + RenameFile(TempFilePath, CacheFilePath); - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); } - }, - Work.DefaultErrorFunction()); + } + }); } } @@ -6256,287 +6227,279 @@ namespace { continue; } - Work.ScheduleWork( - WritePool, - [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { - if (!AbortFlag) + Work.ScheduleWork(WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); + std::filesystem::path ExistingCompressedChunkPath; + if (!PrimeCacheOnly) { - ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); - std::filesystem::path ExistingCompressedChunkPath; - if (!PrimeCacheOnly) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + if (IsFile(CompressedChunkPath)) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path CompressedChunkPath = - ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - if (IsFile(CompressedChunkPath)) + IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); + if (ExistingCompressedPart) { - IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); - if (ExistingCompressedPart) + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) - { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - ExistingCompressedChunkPath = std::move(CompressedChunkPath); - } - else + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - std::error_code DummyEc; - RemoveFile(CompressedChunkPath, DummyEc); + FilteredDownloadedBytesPerSecond.Stop(); } + ExistingCompressedChunkPath = std::move(CompressedChunkPath); + } + else + { + std::error_code DummyEc; + RemoveFile(CompressedChunkPath, DummyEc); } } } - if (!AbortFlag) + } + if (!AbortFlag) + { + if (!ExistingCompressedChunkPath.empty()) { - if (!ExistingCompressedChunkPath.empty()) - { - Work.ScheduleWork( - WritePool, - [&Path, - &ZenFolderPath, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); + Work.ScheduleWork( + WritePool, + [&Path, + &ZenFolderPath, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - FilteredWrittenBytesPerSecond.Start(); + FilteredWrittenBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error( - fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } - std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats); - WritePartsComplete++; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + DiskStats); + WritePartsComplete++; - if (!AbortFlag) + if (!AbortFlag) + { + if (WritePartsComplete == TotalPartWriteCount) { - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } + FilteredWrittenBytesPerSecond.Stop(); + } - RemoveFileWithRetry(CompressedChunkPath); + RemoveFileWithRetry(CompressedChunkPath); - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - if (NeedHashVerify) - { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - RemoteLookup, - CompletedSequences, - Work, - WritePool); - } - else - { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); - } + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); } } - }, - Work.DefaultErrorFunction()); - } - else - { - Work.ScheduleWork( - NetworkPool, - [&Path, - &ZenFolderPath, - &Storage, - BuildId, - &PrimeCacheOnly, - &RemoteContent, - &RemoteLookup, - &ExistsResult, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &NetworkPool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, - TotalRequestCount, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond, - LargeAttachmentSize, - PreferredMultipartChunkSize, - RemoteChunkIndex, - ChunkTargetPtrs, - &DownloadStats](std::atomic&) mutable { - if (!AbortFlag) + } + }); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&Path, + &ZenFolderPath, + &Storage, + BuildId, + &PrimeCacheOnly, + &RemoteContent, + &RemoteLookup, + &ExistsResult, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + LargeAttachmentSize, + PreferredMultipartChunkSize, + RemoteChunkIndex, + ChunkTargetPtrs, + &DownloadStats](std::atomic&) mutable { + if (!AbortFlag) + { + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) + { + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + } + if (BuildBlob) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BuildBlob; - const bool ExistsInCache = - Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); - if (ExistsInCache) + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + FilteredDownloadedBytesPerSecond.Stop(); } - if (BuildBlob) + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + else + { + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + *Storage.BuildStorage, + ZenTempDownloadFolderPath(ZenFolderPath), + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( + IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); } else { - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + if (BuildBlob && Storage.BuildCacheStorage) { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob( - *Storage.BuildStorage, - ZenTempDownloadFolderPath(ZenFolderPath), - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( - IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (Payload && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob( - BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); - } - if (!PrimeCacheOnly) - { - if (!AbortFlag) - { - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - } - }); + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + ChunkHash, + BuildBlob.GetContentType(), + CompositeBuffer(SharedBuffer(BuildBlob))); } - else + if (!BuildBlob) { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); - if (BuildBlob && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob( - BuildId, - ChunkHash, - BuildBlob.GetContentType(), - CompositeBuffer(SharedBuffer(BuildBlob))); - } - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - if (!PrimeCacheOnly) + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) { - if (!AbortFlag) + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); + FilteredDownloadedBytesPerSecond.Stop(); } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); } } } } - }, - Work.DefaultErrorFunction()); - } + } + }); } } - }, - Work.DefaultErrorFunction()); + } + }); } for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) @@ -6547,184 +6510,180 @@ namespace { break; } - Work.ScheduleWork( - WritePool, - [&, CopyDataIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); + Work.ScheduleWork(WritePool, [&, CopyDataIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); - FilteredWrittenBytesPerSecond.Start(); - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; + FilteredWrittenBytesPerSecond.Start(); + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - std::filesystem::path SourceFilePath; + std::filesystem::path SourceFilePath; - if (CopyData.ScavengeSourceIndex == (uint32_t)-1) - { - const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - } - else - { - const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; - const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; - const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; - const uint32_t ScavengedPathIndex = - ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); - } - ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); - ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); + if (CopyData.ScavengeSourceIndex == (uint32_t)-1) + { + const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + } + else + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; + const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; + const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); + } + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - uint64_t CacheLocalFileBytesRead = 0; + uint64_t CacheLocalFileBytesRead = 0; - size_t TargetStart = 0; - const std::span AllTargets( - CopyData.TargetChunkLocationPtrs); + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); - struct WriteOp - { - const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; - uint64_t CacheFileOffset = (uint64_t)-1; - uint32_t ChunkIndex = (uint32_t)-1; - }; + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + uint64_t CacheFileOffset = (uint64_t)-1; + uint32_t ChunkIndex = (uint32_t)-1; + }; - std::vector WriteOps; + std::vector WriteOps; - if (!AbortFlag) + if (!AbortFlag) + { + ZEN_TRACE_CPU("Sort"); + WriteOps.reserve(AllTargets.size()); + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) { - ZEN_TRACE_CPU("Sort"); - WriteOps.reserve(AllTargets.size()); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) - { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkIndex = ChunkTarget.RemoteChunkIndex}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkIndex = ChunkTarget.RemoteChunkIndex}); } + TargetStart += ChunkTarget.TargetChunkLocationCount; + } - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) + { + return true; + } + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) + { return false; - }); - } + } + if (Lhs.Target->Offset < Rhs.Target->Offset) + { + return true; + } + return false; + }); + } - if (!AbortFlag) - { - ZEN_TRACE_CPU("Write"); + if (!AbortFlag) + { + ZEN_TRACE_CPU("Write"); - tsl::robin_set ChunkIndexesWritten; + tsl::robin_set ChunkIndexesWritten; - BufferedOpenFile SourceFile(SourceFilePath, DiskStats); - WriteFileCache OpenFileCache(DiskStats); - for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) + BufferedOpenFile SourceFile(SourceFilePath, DiskStats); + WriteFileCache OpenFileCache(DiskStats); + for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) + { + if (AbortFlag) { - if (AbortFlag) + break; + } + const WriteOp& Op = WriteOps[WriteOpIndex]; + + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; + + uint64_t ReadLength = ChunkSize; + size_t WriteCount = 1; + uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; + uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; + while ((WriteOpIndex + WriteCount) < WriteOps.size()) + { + const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; + if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) { break; } - const WriteOp& Op = WriteOps[WriteOpIndex]; - - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); - const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; - - uint64_t ReadLength = ChunkSize; - size_t WriteCount = 1; - uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; - uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; - while ((WriteOpIndex + WriteCount) < WriteOps.size()) + if (NextOp.Target->Offset != OpTargetEnd) { - const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; - if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) - { - break; - } - if (NextOp.Target->Offset != OpTargetEnd) - { - break; - } - if (NextOp.CacheFileOffset != OpSourceEnd) - { - break; - } - const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; - if (ReadLength + NextChunkLength > 512u * 1024u) - { - break; - } - ReadLength += NextChunkLength; - OpSourceEnd += NextChunkLength; - OpTargetEnd += NextChunkLength; - WriteCount++; + break; + } + if (NextOp.CacheFileOffset != OpSourceEnd) + { + break; } + const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; + if (ReadLength + NextChunkLength > 512u * 1024u) + { + break; + } + ReadLength += NextChunkLength; + OpSourceEnd += NextChunkLength; + OpTargetEnd += NextChunkLength; + WriteCount++; + } - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - OpenFileCache.WriteToFile( - RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); + OpenFileCache.WriteToFile( + RemoteSequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName( + CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); - CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? + CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? - WriteOpIndex += WriteCount; - } + WriteOpIndex += WriteCount; } - if (!AbortFlag) + } + if (!AbortFlag) + { + // Write tracking, updating this must be done without any files open (WriteFileCache) + std::vector CompletedChunkSequences; + for (const WriteOp& Op : WriteOps) { - // Write tracking, updating this must be done without any files open (WriteFileCache) - std::vector CompletedChunkSequences; - for (const WriteOp& Op : WriteOps) + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) { - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) - { - CompletedChunkSequences.push_back(RemoteSequenceIndex); - } + CompletedChunkSequences.push_back(RemoteSequenceIndex); } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, - RemoteContent, - RemoteLookup, - CompletedChunkSequences, - Work, - WritePool); - ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); - } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); } + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, + RemoteContent, + RemoteLookup, + CompletedChunkSequences, + Work, + WritePool); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); } - }, - Work.DefaultErrorFunction()); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }); } for (uint32_t BlockIndex : CachedChunkBlockIndexes) @@ -6735,52 +6694,49 @@ namespace { break; } - Work.ScheduleWork( - WritePool, - [&, BlockIndex](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); + Work.ScheduleWork(WritePool, [&, BlockIndex](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - FilteredWrittenBytesPerSecond.Start(); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + FilteredWrittenBytesPerSecond.Start(); - std::filesystem::path BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + std::filesystem::path BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); + } + + if (!AbortFlag) + { + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) { - throw std::runtime_error( - fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } - - if (!AbortFlag) + WritePartsComplete++; + RemoveFileWithRetry(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) { - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - RemoveFileWithRetry(BlockChunkPath); - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } + FilteredWrittenBytesPerSecond.Stop(); } } - }, - Work.DefaultErrorFunction()); + } + }); } for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) @@ -6793,46 +6749,214 @@ namespace { const BlockRangeDescriptor BlockRange = BlockRangeWorks[BlockRangeIndex]; ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); const uint32_t BlockIndex = BlockRange.BlockIndex; - Work.ScheduleWork( - NetworkPool, - [&, BlockIndex, BlockRange](std::atomic&) { + Work.ScheduleWork(NetworkPool, [&, BlockIndex, BlockRange](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + } + if (!BlockBuffer) + { + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + } + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } if (!AbortFlag) { - ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + std::filesystem::path BlockChunkPath; - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + // Check if the dowloaded block is file based and we can move it directly without rewriting it { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + RenameFile(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; + + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } + } + } } - if (!BlockBuffer) + + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / + fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; } - if (!BlockBuffer) + + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( + std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } + } + + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } + + if (!BlockChunkPath.empty()) + { + RemoveFileWithRetry(BlockChunkPath); + } + + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }); + } + } + } + }); + } + + for (uint32_t BlockIndex : FullBlockWorks) + { + if (AbortFlag) + { + break; + } + + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + + Work.ScheduleWork(NetworkPool, [&, BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredDownloadedBytesPerSecond.Start(); + + IoBuffer BlockBuffer; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + } + if (!BlockBuffer) + { + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (BlockBuffer && Storage.BuildCacheStorage) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlockBuffer))); } - if (!AbortFlag) + } + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + if (!AbortFlag) + { + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!PrimeCacheOnly) + { std::filesystem::path BlockChunkPath; // Check if the dowloaded block is file based and we can move it directly without rewriting it @@ -6842,17 +6966,14 @@ namespace { (FileRef.FileChunkSize == BlockSize)) { ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -6871,10 +6992,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -6883,50 +7001,60 @@ namespace { { Work.ScheduleWork( WritePool, - [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( - std::atomic&) mutable { + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { if (!AbortFlag) { - ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; if (BlockChunkPath.empty()) { - ZEN_ASSERT(BlockPartialBuffer); + ZEN_ASSERT(BlockBuffer); } else { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); } } FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) @@ -6935,200 +7063,18 @@ namespace { } WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } } - }, - Work.DefaultErrorFunction()); + }); } } } - }, - Work.DefaultErrorFunction()); - } - - for (uint32_t BlockIndex : FullBlockWorks) - { - if (AbortFlag) - { - break; - } - - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) - { - DownloadStats.RequestsCompleteCount++; - continue; - } - - Work.ScheduleWork( - NetworkPool, - [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); - - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - FilteredDownloadedBytesPerSecond.Start(); - - IoBuffer BlockBuffer; - const bool ExistsInCache = - Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); - if (ExistsInCache) - { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); - } - if (!BlockBuffer) - { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); - if (BlockBuffer && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockBuffer.GetContentType(), - CompositeBuffer(SharedBuffer(BlockBuffer))); - } - } - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - if (!AbortFlag) - { - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - if (!PrimeCacheOnly) - { - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it - { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) - { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) - { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - RenameFile(TempBlobPath, BlockChunkPath, Ec); - if (Ec) - { - BlockChunkPath = std::filesystem::path{}; - - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); - } - } - } - } - - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } - - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) - { - throw std::runtime_error( - fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); - } - } - - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - - if (!BlockChunkPath.empty()) - { - RemoveFileWithRetry(BlockChunkPath); - } - - WritePartsComplete++; - - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); - } - } - } - } - }, - Work.DefaultErrorFunction()); + } + }); } { @@ -7321,8 +7267,8 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar CacheLocalProgressBar(ProgressMode, "Cache Local Data"); - ParallellWork Work(AbortFlag); + ProgressBar CacheLocalProgressBar(ProgressMode, "Cache Local Data"); + ParallelWork Work(AbortFlag); for (uint32_t LocalPathIndex : FilesToCache) { @@ -7330,23 +7276,20 @@ namespace { { break; } - Work.ScheduleWork( - WritePool, - [&, LocalPathIndex](std::atomic&) { - ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); - if (!AbortFlag) - { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); - const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - RenameFileWithRetry(LocalFilePath, CacheFilePath); - CachedCount++; - CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; - } - }, - Work.DefaultErrorFunction()); + Work.ScheduleWork(WritePool, [&, LocalPathIndex](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; + } + }); } { @@ -7409,8 +7352,8 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); - ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); - ParallellWork Work(AbortFlag); + ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); + ParallelWork Work(AbortFlag); OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size()); @@ -7425,18 +7368,15 @@ namespace { { break; } - Work.ScheduleWork( - WritePool, - [&, LocalPathIndex](std::atomic&) { - if (!AbortFlag) - { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); - DeletedCount++; - } - }, - Work.DefaultErrorFunction()); + Work.ScheduleWork(WritePool, [&, LocalPathIndex](std::atomic&) { + if (!AbortFlag) + { + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); + DeletedCount++; + } + }); } std::atomic TargetsComplete = 0; @@ -7479,166 +7419,158 @@ namespace { TargetCount++; } - Work.ScheduleWork( - WritePool, - [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("FinalizeTree_Work"); + Work.ScheduleWork(WritePool, [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("FinalizeTree_Work"); - size_t TargetOffset = BaseTargetOffset; - const IoHash& RawHash = Targets[TargetOffset].RawHash; + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; - if (RawHash == IoHash::Zero) + if (RawHash == IoHash::Zero) + { + ZEN_TRACE_CPU("ZeroSize"); + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - ZEN_TRACE_CPU("ZeroSize"); - while (TargetOffset < (BaseTargetOffset + TargetCount)) + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); - if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) + if (IsFileWithRetry(TargetFilePath)) { - if (IsFileWithRetry(TargetFilePath)) - { - SetFileReadOnlyWithRetry(TargetFilePath, false); - } - else - { - CreateDirectories(TargetFilePath.parent_path()); - } - BasicFile OutputFile; - OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + SetFileReadOnlyWithRetry(TargetFilePath, false); } - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; - - OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); - - TargetOffset++; - TargetsComplete++; + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); } + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } + } + else + { + ZEN_TRACE_CPU("Files"); + ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); } else { - ZEN_TRACE_CPU("Files"); - ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); - const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; - std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); - - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); - InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + if (IsFileWithRetry(FirstTargetFilePath)) { - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); } else { - if (IsFileWithRetry(FirstTargetFilePath)) - { - SetFileReadOnlyWithRetry(FirstTargetFilePath, false); - } - else - { - CreateDirectories(FirstTargetFilePath.parent_path()); - } + CreateDirectories(FirstTargetFilePath.parent_path()); + } - if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); - InplaceIt != SequenceHashToLocalPathIndex.end()) - { - ZEN_TRACE_CPU("Copy"); - const uint32_t LocalPathIndex = InplaceIt->second; - const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; - std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); - - ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); - CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - } - else - { - ZEN_TRACE_CPU("Rename"); - const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); + InplaceIt != SequenceHashToLocalPathIndex.end()) + { + ZEN_TRACE_CPU("Copy"); + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); + + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); + CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + ZEN_TRACE_CPU("Rename"); + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); - RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } + } - OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; - OutLocalFolderState.Attributes[FirstRemotePathIndex] = - RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstRemotePathIndex]); - OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = - GetModificationTickFromPath(FirstTargetFilePath); + OutLocalFolderState.Attributes[FirstRemotePathIndex] = + RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath); - TargetOffset++; - TargetsComplete++; + TargetOffset++; + TargetsComplete++; - while (TargetOffset < (BaseTargetOffset + TargetCount)) - { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + while (TargetOffset < (BaseTargetOffset + TargetCount)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); - InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); + } + else + { + ZEN_TRACE_CPU("Copy"); + if (IsFileWithRetry(TargetFilePath)) { - ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { - ZEN_TRACE_CPU("Copy"); - if (IsFileWithRetry(TargetFilePath)) - { - SetFileReadOnlyWithRetry(TargetFilePath, false); - } - else - { - CreateDirectories(TargetFilePath.parent_path()); - } - - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); - ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); - CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + CreateDirectories(TargetFilePath.parent_path()); } - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); + CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } - OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; - TargetOffset++; - TargetsComplete++; - } + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; } } - }, - Work.DefaultErrorFunction()); + } + }); TargetOffset += TargetCount; } @@ -10470,7 +10402,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return true; }; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -10512,8 +10444,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) SetFileReadOnly(FilePath, true); } } - }, - Work.DefaultErrorFunction()); + }); } } break; @@ -10677,14 +10608,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } } - catch (const ParallellWorkException& Ex) - { - for (const std::string& Error : Ex.m_Errors) - { - ZEN_ERROR("{}", Error); - } - return 3; - } catch (const std::exception& Ex) { ZEN_ERROR("{}", Ex.what()); diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 269f95417..9dfdca0a1 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include @@ -248,7 +248,7 @@ namespace { return Added; }; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); struct AsyncVisitor : public GetDirectoryContentVisitor { diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index c1df6d53e..f71397922 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2322,12 +2322,17 @@ GetDirectoryContent(const std::filesystem::path& RootDir, RelativeRoot = RelativeRoot / DirectoryName]() { ZEN_ASSERT(Visitor); auto _ = MakeGuard([&]() { PendingWorkCount->CountDown(); }); + try { MultithreadedVisitor SubVisitor(*WorkerPool, *PendingWorkCount, RelativeRoot, Flags, Visitor); FileSystemTraversal Traversal; Traversal.TraverseFileSystem(Path, SubVisitor); Visitor->AsyncVisitDirectory(SubVisitor.RelativeRoot, std::move(SubVisitor.Content)); } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed scheduling work to scan subfolder '{}'. Reason: '{}'", Path / RelativeRoot, Ex.what()); + } }); } catch (const std::exception Ex) diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp index f273ac699..88238effd 100644 --- a/src/zenutil/buildstoragecache.cpp +++ b/src/zenutil/buildstoragecache.cpp @@ -338,7 +338,7 @@ public: return {}; } - virtual void Flush(int32_t UpdateInteralMS, std::function&& UpdateCallback) override + virtual void Flush(int32_t UpdateIntervalMS, std::function&& UpdateCallback) override { if (IsFlushed) { @@ -358,7 +358,7 @@ public: intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining(); if (UpdateCallback(Remaining)) { - if (m_PendingBackgroundWorkCount.Wait(UpdateInteralMS)) + if (m_PendingBackgroundWorkCount.Wait(UpdateIntervalMS)) { UpdateCallback(0); return; diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 17b348f8d..ae129324e 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include ZEN_THIRD_PARTY_INCLUDES_START @@ -378,7 +378,7 @@ GetFolderContent(GetFolderContentStatistics& Stats, std::function&& AcceptDirectory, std::function&& AcceptFile, WorkerThreadPool& WorkerPool, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag) { @@ -467,7 +467,7 @@ GetFolderContent(GetFolderContentStatistics& Stats, WorkerPool, PendingWork); PendingWork.CountDown(); - while (!PendingWork.Wait(UpdateInteralMS)) + while (!PendingWork.Wait(UpdateIntervalMS)) { UpdateCallback(AbortFlag.load(), PendingWork.Remaining()); } @@ -731,7 +731,7 @@ ChunkFolderContent(ChunkingStatistics& Stats, const std::filesystem::path& RootPath, const FolderContent& Content, const ChunkingController& InChunkingController, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag) { @@ -772,7 +772,7 @@ ChunkFolderContent(ChunkingStatistics& Stats, RwLock Lock; - ParallellWork Work(AbortFlag); + ParallelWork Work(AbortFlag); for (uint32_t PathIndex : Order) { @@ -780,28 +780,26 @@ ChunkFolderContent(ChunkingStatistics& Stats, { break; } - Work.ScheduleWork( - WorkerPool, // GetSyncWorkerPool() - [&, PathIndex](std::atomic& AbortFlag) { - if (!AbortFlag) - { - IoHash RawHash = HashOneFile(Stats, - InChunkingController, - Result, - ChunkHashToChunkIndex, - RawHashToSequenceRawHashIndex, - Lock, - RootPath, - PathIndex, - AbortFlag); - Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; }); - Stats.FilesProcessed++; - } - }, - Work.DefaultErrorFunction()); - } - - Work.Wait(UpdateInteralMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + Work.ScheduleWork(WorkerPool, // GetSyncWorkerPool() + [&, PathIndex](std::atomic& AbortFlag) { + if (!AbortFlag) + { + IoHash RawHash = HashOneFile(Stats, + InChunkingController, + Result, + ChunkHashToChunkIndex, + RawHashToSequenceRawHashIndex, + Lock, + RootPath, + PathIndex, + AbortFlag); + Lock.WithExclusiveLock([&]() { Result.RawHashes[PathIndex] = RawHash; }); + Stats.FilesProcessed++; + } + }); + } + + Work.Wait(UpdateIntervalMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted); ZEN_UNUSED(PendingWork); UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining()); diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index cab35328d..e1fb73fd4 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -44,7 +44,7 @@ public: virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; virtual void Flush( - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback = [](intptr_t) { return true; }) = 0; }; diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 57b55cb8e..d33869be2 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -67,7 +67,7 @@ FolderContent GetFolderContent(GetFolderContentStatistics& Stats, std::function&& AcceptDirectory, std::function&& AcceptFile, WorkerThreadPool& WorkerPool, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag); @@ -116,7 +116,7 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, const std::filesystem::path& RootPath, const FolderContent& Content, const ChunkingController& InChunkingController, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag); diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h deleted file mode 100644 index 8ea77c65d..000000000 --- a/src/zenutil/include/zenutil/parallellwork.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include - -#include - -class ParallellWorkException : public std::runtime_error -{ -public: - explicit ParallellWorkException(std::vector&& Errors) : std::runtime_error(Errors.front()), m_Errors(std::move(Errors)) {} - - const std::vector m_Errors; -}; - -namespace zen { - -class ParallellWork -{ -public: - ParallellWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {} - - ~ParallellWork() - { - // Make sure to call Wait before destroying - ZEN_ASSERT(m_PendingWork.Remaining() == 0); - } - - std::function& AbortFlag)> DefaultErrorFunction() - { - return [&](const std::exception& Ex, std::atomic& AbortFlag) { - m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); }); - AbortFlag = true; - }; - } - - void ScheduleWork(WorkerThreadPool& WorkerPool, - std::function& AbortFlag)>&& Work, - std::function& AbortFlag)>&& OnError) - { - m_PendingWork.AddCount(1); - try - { - WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] { - try - { - Work(m_AbortFlag); - } - catch (const AssertException& AssertEx) - { - OnError( - std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())), - m_AbortFlag); - } - catch (const std::system_error& SystemError) - { - if (IsOOM(SystemError.code())) - { - OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag); - } - else if (IsOOD(SystemError.code())) - { - OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag); - } - else - { - OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag); - } - } - catch (const std::exception& Ex) - { - OnError(Ex, m_AbortFlag); - } - m_PendingWork.CountDown(); - }); - } - catch (const std::exception&) - { - m_PendingWork.CountDown(); - throw; - } - } - - void Abort() { m_AbortFlag = true; } - - bool IsAborted() const { return m_AbortFlag.load(); } - - void Wait(int32_t UpdateInteralMS, std::function&& UpdateCallback) - { - ZEN_ASSERT(m_PendingWork.Remaining() > 0); - m_PendingWork.CountDown(); - while (!m_PendingWork.Wait(UpdateInteralMS)) - { - UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); - } - if (m_Errors.size() == 1) - { - throw std::runtime_error(m_Errors.front()); - } - else if (m_Errors.size() > 1) - { - throw ParallellWorkException(std::move(m_Errors)); - } - } - Latch& PendingWork() { return m_PendingWork; } - -private: - std::atomic& m_AbortFlag; - Latch m_PendingWork; - - RwLock m_ErrorLock; - std::vector m_Errors; -}; - -} // namespace zen diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h new file mode 100644 index 000000000..08e730b28 --- /dev/null +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen { + +class ParallelWork +{ +public: + ParallelWork(std::atomic& AbortFlag); + + ~ParallelWork(); + + typedef std::function& AbortFlag)> WorkCallback; + typedef std::function& AbortFlag)> ExceptionCallback; + typedef std::function UpdateCallback; + + void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {}) + { + m_PendingWork.AddCount(1); + try + { + WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { + try + { + Work(m_AbortFlag); + } + catch (...) + { + OnError(std::current_exception(), m_AbortFlag); + } + m_PendingWork.CountDown(); + }); + } + catch (const std::exception&) + { + m_PendingWork.CountDown(); + throw; + } + } + + void Abort() { m_AbortFlag = true; } + + bool IsAborted() const { return m_AbortFlag.load(); } + + void Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback); + + void Wait(); + + Latch& PendingWork() { return m_PendingWork; } + +private: + ExceptionCallback DefaultErrorFunction(); + void RethrowErrors(); + + std::atomic& m_AbortFlag; + bool m_DispatchComplete = false; + Latch m_PendingWork; + + RwLock m_ErrorLock; + std::vector m_Errors; +}; + +void parallellwork_forcelink(); + +} // namespace zen diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp new file mode 100644 index 000000000..516d70e28 --- /dev/null +++ b/src/zenutil/parallelwork.cpp @@ -0,0 +1,192 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include + +#include + +#if ZEN_WITH_TESTS +# include +#endif // ZEN_WITH_TESTS + +namespace zen { + +ParallelWork::ParallelWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) +{ +} + +ParallelWork::~ParallelWork() +{ + try + { + if (!m_DispatchComplete) + { + ZEN_ASSERT(m_PendingWork.Remaining() > 0); + ZEN_WARN( + "ParallelWork disposed without explicit wait for completion, likely caused by an exception, waiting for dispatched threads " + "to complete"); + m_PendingWork.CountDown(); + } + m_AbortFlag.store(true); + m_PendingWork.Wait(); + ZEN_ASSERT(m_PendingWork.Remaining() == 0); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Exception in ~ParallelWork: {}", Ex.what()); + } +} + +ParallelWork::ExceptionCallback +ParallelWork::DefaultErrorFunction() +{ + return [&](std::exception_ptr Ex, std::atomic& AbortFlag) { + m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex); }); + AbortFlag = true; + }; +} + +void +ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback) +{ + ZEN_ASSERT(!m_DispatchComplete); + m_DispatchComplete = true; + + ZEN_ASSERT(m_PendingWork.Remaining() > 0); + m_PendingWork.CountDown(); + + while (!m_PendingWork.Wait(UpdateIntervalMS)) + { + UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); + } + + RethrowErrors(); +} + +void +ParallelWork::Wait() +{ + ZEN_ASSERT(!m_DispatchComplete); + m_DispatchComplete = true; + + ZEN_ASSERT(m_PendingWork.Remaining() > 0); + m_PendingWork.CountDown(); + m_PendingWork.Wait(); + + RethrowErrors(); +} + +void +ParallelWork::RethrowErrors() +{ + if (!m_Errors.empty()) + { + if (m_Errors.size() > 1) + { + ZEN_INFO("Multiple exceptions throwm during ParallelWork execution, dropping the following exceptions:"); + auto It = m_Errors.begin() + 1; + while (It != m_Errors.end()) + { + try + { + std::rethrow_exception(*It); + } + catch (const std::exception& Ex) + { + ZEN_INFO(" {}", Ex.what()); + } + It++; + } + } + std::exception_ptr Ex = m_Errors.front(); + m_Errors.clear(); + std::rethrow_exception(Ex); + } +} + +#if ZEN_WITH_TESTS + +TEST_CASE("parallellwork.nowork") +{ + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); + Work.Wait(); +} + +TEST_CASE("parallellwork.basic") +{ + WorkerThreadPool WorkerPool(2); + + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); + for (uint32_t I = 0; I < 5; I++) + { + Work.ScheduleWork(WorkerPool, [](std::atomic& AbortFlag) { CHECK(!AbortFlag); }); + } + Work.Wait(); +} + +TEST_CASE("parallellwork.throws_in_work") +{ + WorkerThreadPool WorkerPool(2); + + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); + for (uint32_t I = 0; I < 10; I++) + { + Work.ScheduleWork(WorkerPool, [I](std::atomic& AbortFlag) { + ZEN_UNUSED(AbortFlag); + if (I > 3) + { + throw std::runtime_error("We throw in async thread"); + } + else + { + Sleep(10); + } + }); + } + CHECK_THROWS_WITH(Work.Wait(), "We throw in async thread"); +} + +TEST_CASE("parallellwork.throws_in_dispatch") +{ + WorkerThreadPool WorkerPool(2); + std::atomic ExecutedCount; + try + { + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); + for (uint32_t I = 0; I < 5; I++) + { + Work.ScheduleWork(WorkerPool, [I, &ExecutedCount](std::atomic& AbortFlag) { + if (AbortFlag.load()) + { + return; + } + ExecutedCount++; + }); + if (I == 3) + { + throw std::runtime_error("We throw in dispatcher thread"); + } + } + CHECK(false); + } + catch (const std::runtime_error& Ex) + { + CHECK_EQ("We throw in dispatcher thread", std::string(Ex.what())); + CHECK_LE(ExecutedCount.load(), 4); + } +} + +void +parallellwork_forcelink() +{ +} +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index aff9156f4..fe23b00c1 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -8,6 +8,7 @@ # include # include # include +# include namespace zen { @@ -19,6 +20,7 @@ zenutil_forcelinktests() cacherequests_forcelink(); chunkedfile_forcelink(); commandlineoptions_forcelink(); + parallellwork_forcelink(); } } // namespace zen -- cgit v1.2.3 From 1756c3feb7f65d765ed1b86277eb50ef1816ec79 Mon Sep 17 00:00:00 2001 From: zousar Date: Sat, 17 May 2025 00:36:01 -0600 Subject: Fix oplog creation during oplog-import --- src/zen/cmds/projectstore_cmd.cpp | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index a26687147..58af0577e 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -1484,12 +1484,6 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - m_OplogName = ResolveOplog(Http, m_ProjectName, m_OplogName); - if (m_OplogName.empty()) - { - return 1; - } - size_t TargetCount = 0; TargetCount += m_CloudUrl.empty() ? 0 : 1; TargetCount += m_BuildsUrl.empty() ? 0 : 1; -- cgit v1.2.3 From 49701314f570da3622f11eb37cc889c7d39d9a93 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 19 May 2025 22:25:58 +0200 Subject: handle exception with batch work (#401) * use ParallelWork in rpc playback * use ParallelWork in projectstore * use ParallelWork in buildstore * use ParallelWork in cachedisklayer * use ParallelWork in compactcas * use ParallelWork in filecas * don't set abort flag in ParallelWork destructor * add PrepareFileForScatteredWrite for temp files in httpclient * Use PrepareFileForScatteredWrite when stream-decompressing files * be more relaxed when deleting temp files * allow explicit zen-cache when using direct host url without resolving * fix lambda capture when writing loose chunks * no delay when attempting to remove temp files --- src/zen/cmds/builds_cmd.cpp | 542 +++++++++++++++------------- src/zenhttp/httpclient.cpp | 6 +- src/zenserver/cache/httpstructuredcache.cpp | 17 +- src/zenserver/projectstore/projectstore.cpp | 40 +- src/zenstore/buildstore/buildstore.cpp | 58 ++- src/zenstore/cache/cachedisklayer.cpp | 25 +- src/zenstore/compactcas.cpp | 128 ++++--- src/zenstore/filecas.cpp | 70 ++-- src/zenutil/parallelwork.cpp | 1 - 9 files changed, 463 insertions(+), 424 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 815bb7597..78d362fdb 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -244,6 +244,24 @@ namespace { } } + void TryRemoveFile(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + if (Ec) + { + if (IsFile(Path, Ec)) + { + Ec.clear(); + RemoveFile(Path, Ec); + if (Ec) + { + ZEN_DEBUG("Failed removing file '{}', reason: {}", Path, Ec.message()); + } + } + } + } + void RemoveFileWithRetry(const std::filesystem::path& Path) { std::error_code Ec; @@ -4864,6 +4882,7 @@ namespace { { throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash)); } + PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize); IoHashStream Hash; bool CouldDecompress = Compressed.DecompressToStream( @@ -5055,7 +5074,10 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - RemoveFileWithRetry(CompressedChunkPath); + if (!CompressedChunkPath.empty()) + { + TryRemoveFile(CompressedChunkPath); + } std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -5304,7 +5326,7 @@ namespace { } } } - RemoveFileWithRetry(CacheDirContent.Files[Index]); + TryRemoveFile(CacheDirContent.Files[Index]); } CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); } @@ -5353,7 +5375,7 @@ namespace { } } } - RemoveFileWithRetry(BlockDirContent.Files[Index]); + TryRemoveFile(BlockDirContent.Files[Index]); } CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs(); @@ -6227,279 +6249,284 @@ namespace { continue; } - Work.ScheduleWork(WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); - std::filesystem::path ExistingCompressedChunkPath; - if (!PrimeCacheOnly) + Work.ScheduleWork( + WritePool, + [&, RemoteChunkIndex, ChunkTargetPtrs, BuildId, TotalRequestCount, TotalPartWriteCount](std::atomic&) mutable { + if (!AbortFlag) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - if (IsFile(CompressedChunkPath)) + ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); + std::filesystem::path ExistingCompressedChunkPath; + if (!PrimeCacheOnly) { - IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); - if (ExistingCompressedPart) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = + ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + if (IsFile(CompressedChunkPath)) { - IoHash RawHash; - uint64_t RawSize; - if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) + IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); + if (ExistingCompressedPart) { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart, RawHash, RawSize)) { - FilteredDownloadedBytesPerSecond.Stop(); + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + ExistingCompressedChunkPath = std::move(CompressedChunkPath); + } + else + { + std::error_code DummyEc; + RemoveFile(CompressedChunkPath, DummyEc); } - ExistingCompressedChunkPath = std::move(CompressedChunkPath); - } - else - { - std::error_code DummyEc; - RemoveFile(CompressedChunkPath, DummyEc); } } } - } - if (!AbortFlag) + if (!AbortFlag) - { - if (!ExistingCompressedChunkPath.empty()) { - Work.ScheduleWork( - WritePool, - [&Path, - &ZenFolderPath, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - - FilteredWrittenBytesPerSecond.Start(); - - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + WritePool, + [&Path, + &ZenFolderPath, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { + if (!AbortFlag) { - throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats); - WritePartsComplete++; + FilteredWrittenBytesPerSecond.Start(); - if (!AbortFlag) - { - if (WritePartsComplete == TotalPartWriteCount) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) { - FilteredWrittenBytesPerSecond.Stop(); + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); } - RemoveFileWithRetry(CompressedChunkPath); + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + DiskStats); + WritePartsComplete++; - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - if (NeedHashVerify) - { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - RemoteLookup, - CompletedSequences, - Work, - WritePool); - } - else + if (!AbortFlag) { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + + TryRemoveFile(CompressedChunkPath); + + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } } } - } - }); - } - else - { - Work.ScheduleWork( - NetworkPool, - [&Path, - &ZenFolderPath, - &Storage, - BuildId, - &PrimeCacheOnly, - &RemoteContent, - &RemoteLookup, - &ExistsResult, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &NetworkPool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - TotalPartWriteCount, - TotalRequestCount, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond, - LargeAttachmentSize, - PreferredMultipartChunkSize, - RemoteChunkIndex, - ChunkTargetPtrs, - &DownloadStats](std::atomic&) mutable { - if (!AbortFlag) - { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BuildBlob; - const bool ExistsInCache = - Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); - if (ExistsInCache) - { - BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); - } - if (BuildBlob) + }); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&Path, + &ZenFolderPath, + &Storage, + BuildId, + &PrimeCacheOnly, + &RemoteContent, + &RemoteLookup, + &ExistsResult, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + LargeAttachmentSize, + PreferredMultipartChunkSize, + RemoteChunkIndex, + ChunkTargetPtrs, + &DownloadStats](std::atomic&) mutable { + if (!AbortFlag) { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) { - FilteredDownloadedBytesPerSecond.Stop(); + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - else - { - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + if (BuildBlob) { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob( - *Storage.BuildStorage, - ZenTempDownloadFolderPath(ZenFolderPath), - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( - IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (Payload && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob( - BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); - } - if (!PrimeCacheOnly) - { - if (!AbortFlag) - { - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - } - }); + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); } else { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); - if (BuildBlob && Storage.BuildCacheStorage) + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, - ChunkHash, - BuildBlob.GetContentType(), - CompositeBuffer(SharedBuffer(BuildBlob))); - } - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + *Storage.BuildStorage, + ZenTempDownloadFolderPath(ZenFolderPath), + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( + IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); } - if (!PrimeCacheOnly) + else { - if (!AbortFlag) + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + if (BuildBlob && Storage.BuildCacheStorage) { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + BuildBlob.GetContentType(), + CompositeBuffer(SharedBuffer(BuildBlob))); + } + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) { - FilteredDownloadedBytesPerSecond.Stop(); + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); } - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); } } } } - } - }); + }); + } } } - } - }); + }); } for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) @@ -6728,8 +6755,11 @@ namespace { RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } + + TryRemoveFile(BlockChunkPath); + WritePartsComplete++; - RemoveFileWithRetry(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -6884,7 +6914,7 @@ namespace { if (!BlockChunkPath.empty()) { - RemoveFileWithRetry(BlockChunkPath); + TryRemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -7059,7 +7089,7 @@ namespace { if (!BlockChunkPath.empty()) { - RemoveFileWithRetry(BlockChunkPath); + TryRemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -9456,6 +9486,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::string CloudHost; + auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair { + HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); + if (TestResponse.IsSuccess()) + { + return {true, ""}; + } + return {false, TestResponse.ErrorMessage("")}; + }; + if (!m_Host.empty()) { if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) @@ -9535,22 +9581,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", m_OverrideHost, TestResult.second)); } - auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> std::pair { - HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{1000}, - .Timeout = std::chrono::milliseconds{2000}, - .AssumeHttp2 = AssumeHttp2, - .AllowResume = true, - .RetryCount = 0}; - HttpClient TestHttpClient(BaseUrl, TestClientSettings); - HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); - if (TestResponse.IsSuccess()) - { - return {true, ""}; - } - return {false, TestResponse.ErrorMessage("")}; - }; - if (m_ZenCacheHost.empty()) { CbArrayView CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView(); @@ -9680,7 +9710,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } - Result.CacheName = BuildCacheName; + Result.CacheName = BuildCacheName.empty() ? m_ZenCacheHost : BuildCacheName; } ZEN_CONSOLE("Remote: {}", StorageDescription); if (!Result.CacheName.empty()) diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index ca1b820c9..a6e4d9290 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -86,7 +86,7 @@ namespace detail { } } - std::error_code Open(const std::filesystem::path& TempFolderPath) + std::error_code Open(const std::filesystem::path& TempFolderPath, uint64_t FinalSize) { ZEN_TRACE_CPU("TempPayloadFile::Open"); ZEN_ASSERT(m_FileHandle == nullptr); @@ -128,6 +128,8 @@ namespace detail { #endif // ZEN_PLATFORM_WINDOWS m_FileHandle = FileHandle; + PrepareFileForScatteredWrite(m_FileHandle, FinalSize); + return {}; } @@ -1211,7 +1213,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold if (ContentLength.value() > 1024 * 1024) { PayloadFile = std::make_unique(); - std::error_code Ec = PayloadFile->Open(TempFolderPath); + std::error_code Ec = PayloadFile->Open(TempFolderPath, ContentLength.value()); if (Ec) { ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index b9a9ca380..f7e63433b 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "upstream/upstreamcache.h" @@ -1585,12 +1586,13 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co WorkerThreadPool WorkerPool(ThreadCount); uint64_t RequestCount = Replayer.GetRequestCount(); Stopwatch Timer; - auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); - Latch JobLatch(RequestCount); + auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); ZEN_INFO("Replaying {} requests", RequestCount); for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) { - WorkerPool.ScheduleWork([this, &Context, &JobLatch, &Replayer, RequestIndex]() { + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic&) { IoBuffer Body; zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); @@ -1634,16 +1636,15 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co } } } - JobLatch.CountDown(); }); } - while (!JobLatch.Wait(10000)) - { + Work.Wait(10000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted); ZEN_INFO("Replayed {} of {} requests, elapsed {}", - RequestCount - JobLatch.Remaining(), + RequestCount - PendingWork, RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); - } + }); } void diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 6a55efdb7..7d22da717 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -1588,17 +1589,14 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo } }; - Latch WorkLatch(1); - + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) { if (OptionalWorkerPool) { - WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([&, Index = OpIndex]() { + Work.ScheduleWork(*OptionalWorkerPool, [&, Index = OpIndex](std::atomic&) { ZEN_MEMSCOPE(GetProjectstoreTag()); - - auto _ = MakeGuard([&WorkLatch] { WorkLatch.CountDown(); }); ValidateOne(Index); }); } @@ -1607,9 +1605,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo ValidateOne(OpIndex); } } - - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); { // Check if we were deleted while we were checking the references without a lock... @@ -2106,8 +2102,9 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, } if (OptionalWorkerPool) { - std::atomic_bool Result = true; - Latch WorkLatch(1); + std::atomic_bool Result = true; + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { @@ -2115,10 +2112,10 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, { break; } - WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork( - [this, &WorkLatch, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]( + std::atomic&) { if (Result.load() == false) { return; @@ -2162,8 +2159,7 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, LargeSizeLimit); } - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); return Result.load(); } @@ -3886,13 +3882,12 @@ ProjectStore::Flush() } WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); - Latch WorkLatch(1); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); for (const Ref& Project : Projects) { - WorkLatch.AddCount(1); - WorkerPool.ScheduleWork([this, &WorkLatch, Project]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + Work.ScheduleWork(WorkerPool, [this, Project](std::atomic&) { try { Project->Flush(); @@ -3904,8 +3899,7 @@ ProjectStore::Flush() }); } - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); } void diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index b4891a742..6eb01dfc4 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include # include # include +# include #endif // ZEN_WITH_TESTS namespace zen { @@ -480,11 +482,12 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O if (!MetaLocations.empty()) { - Latch WorkLatch(1); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); m_MetadataBlockStore.IterateChunks( MetaLocations, - [this, OptionalWorkerPool, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock, &WorkLatch]( + [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( uint32_t BlockIndex, std::span ChunkIndexes) -> bool { ZEN_UNUSED(BlockIndex); @@ -496,40 +499,31 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O { ZEN_ASSERT(OptionalWorkerPool != nullptr); std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - WorkLatch.AddCount(1); - try - { - OptionalWorkerPool->ScheduleWork([this, - &Result, - &MetaLocations, - &MetaLocationResultIndexes, - DoOneBlock, - &WorkLatch, - ChunkIndexes = std::move(TmpChunkIndexes)]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &Result, &MetaLocations, &MetaLocationResultIndexes, DoOneBlock, ChunkIndexes = std::move(TmpChunkIndexes)]( + std::atomic& AbortFlag) { + if (AbortFlag) + { + return; + } try { - DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); + if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) + { + AbortFlag.store(true); + } } catch (const std::exception& Ex) { ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); } }); - } - catch (const std::exception& Ex) - { - WorkLatch.CountDown(); - ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", - ChunkIndexes.size(), - Ex.what()); - } - return true; + return !Work.IsAborted(); } }); - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); } for (size_t Index = 0; Index < Result.size(); Index++) { @@ -1661,6 +1655,8 @@ TEST_CASE("BuildStore.Metadata") ScopedTemporaryDirectory _; + WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); + BuildStoreConfig Config; Config.RootDirectory = _.Path() / "build_store"; @@ -1678,7 +1674,7 @@ TEST_CASE("BuildStore.Metadata") } Store.PutMetadatas(BlobHashes, MetaPayloads); - std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool); CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) { @@ -1689,7 +1685,7 @@ TEST_CASE("BuildStore.Metadata") { GcManager Gc; BuildStore Store(Config, Gc); - std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, &WorkerPool); CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) { @@ -1715,7 +1711,7 @@ TEST_CASE("BuildStore.Metadata") Store.PutBlob(CompressedBlobsHashes.back(), Payload); } - std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); for (const auto& MetadataIt : MetadataPayloads) { CHECK(!MetadataIt); @@ -1741,7 +1737,7 @@ TEST_CASE("BuildStore.Metadata") } Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); - std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); for (size_t I = 0; I < MetadataPayloads.size(); I++) { @@ -1754,7 +1750,7 @@ TEST_CASE("BuildStore.Metadata") GcManager Gc; BuildStore Store(Config, Gc); - std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); for (size_t I = 0; I < MetadataPayloads.size(); I++) { @@ -1783,7 +1779,7 @@ TEST_CASE("BuildStore.Metadata") GcManager Gc; BuildStore Store(Config, Gc); - std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, &WorkerPool); CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); for (size_t I = 0; I < MetadataPayloads.size(); I++) { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 91bd9cba8..d80da6ea6 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -3936,14 +3937,13 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock SyncLock; WorkerThreadPool& Pool = GetLargeWorkerPool(EWorkloadType::Burst); - Latch WorkLatch(1); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); for (auto& BucketPath : FoundBucketDirectories) { - WorkLatch.AddCount(1); - Pool.ScheduleWork([this, &WorkLatch, &SyncLock, BucketPath]() { + Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic&) { ZEN_MEMSCOPE(GetCacheDiskTag()); - auto _ = MakeGuard([&]() { WorkLatch.CountDown(); }); const std::string BucketName = PathToUtf8(BucketPath.stem()); try { @@ -3984,8 +3984,7 @@ ZenCacheDiskLayer::DiscoverBuckets() } }); } - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); } bool @@ -4062,16 +4061,15 @@ ZenCacheDiskLayer::Flush() } { WorkerThreadPool& Pool = GetMediumWorkerPool(EWorkloadType::Burst); - Latch WorkLatch(1); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); try { for (auto& Bucket : Buckets) { - WorkLatch.AddCount(1); - Pool.ScheduleWork([&WorkLatch, Bucket]() { + Work.ScheduleWork(Pool, [Bucket](std::atomic&) { ZEN_MEMSCOPE(GetCacheDiskTag()); - auto _ = MakeGuard([&]() { WorkLatch.CountDown(); }); try { Bucket->Flush(); @@ -4087,11 +4085,8 @@ ZenCacheDiskLayer::Flush() { ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what()); } - WorkLatch.CountDown(); - while (!WorkLatch.Wait(1000)) - { - ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", WorkLatch.Remaining(), m_RootDir); - } + Work.Wait(1000, + [&](std::ptrdiff_t Remaining, bool) { ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", Remaining, m_RootDir); }); } } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 8cf241e34..15bea272b 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -393,64 +394,75 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas LargeSizeLimit); }; - Latch WorkLatch(1); - std::atomic_bool AsyncContinue = true; - bool Continue = m_BlockStore.IterateChunks( - FoundChunkLocations, - [this, - &AsyncContinue, - &WorkLatch, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - &FoundChunkIndexes, - &FoundChunkLocations, - OptionalWorkerPool](uint32_t BlockIndex, std::span ChunkIndexes) { - if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) - { - std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([this, - &AsyncContinue, - &WorkLatch, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - BlockIndex, - &FoundChunkIndexes, - &FoundChunkLocations, - ChunkIndexes = std::move(TmpChunkIndexes)]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - if (!AsyncContinue) - { - return; - } - try - { - bool Continue = DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - if (!Continue) - { - AsyncContinue.store(false); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", - m_RootDirectory, - BlockIndex, - Ex.what()); - } - }); - return AsyncContinue.load(); - } - else - { - return DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - } - }); - WorkLatch.CountDown(); - WorkLatch.Wait(); - return AsyncContinue.load() && Continue; + std::atomic AsyncContinue = true; + { + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); + const bool Continue = m_BlockStore.IterateChunks( + FoundChunkLocations, + [this, + &Work, + &AsyncContinue, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + &FoundChunkIndexes, + &FoundChunkLocations, + OptionalWorkerPool](uint32_t BlockIndex, std::span ChunkIndexes) { + if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) + { + std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, + &AsyncContinue, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + BlockIndex, + &FoundChunkIndexes, + &FoundChunkLocations, + ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic& AbortFlag) { + if (AbortFlag) + { + AsyncContinue.store(false); + } + if (!AsyncContinue) + { + return; + } + try + { + bool Continue = + DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + if (!Continue) + { + AsyncContinue.store(false); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", + m_RootDirectory, + BlockIndex, + Ex.what()); + AsyncContinue.store(false); + } + }); + return AsyncContinue.load(); + } + else + { + return DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + } + }); + if (!Continue) + { + AsyncContinue.store(false); + } + Work.Wait(); + } + return AsyncContinue.load(); } void diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 4911bffb9..6354edf70 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #if ZEN_WITH_TESTS # include @@ -632,10 +633,11 @@ FileCasStrategy::IterateChunks(std::span ChunkHashes, } } } - std::atomic_bool Continue = true; + std::atomic AsyncContinue = true; if (!FoundChunkIndexes.empty()) { - auto ProcessOne = [this, &ChunkHashes, &Continue, &AsyncCallback](size_t ChunkIndex, uint64_t ExpectedSize) { + auto ProcessOne = [this, &ChunkHashes, &AsyncCallback](size_t ChunkIndex, uint64_t ExpectedSize) { + ZEN_ASSERT(ChunkIndex < ChunkHashes.size()); const IoHash& ChunkHash = ChunkHashes[ChunkIndex]; IoBuffer Payload = SafeOpenChunk(ChunkHash, ExpectedSize); if (!AsyncCallback(ChunkIndex, std::move(Payload))) @@ -645,49 +647,57 @@ FileCasStrategy::IterateChunks(std::span ChunkHashes, return true; }; - Latch WorkLatch(1); + std::atomic AbortFlag; + ParallelWork Work(AbortFlag); for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) { - size_t ChunkIndex = FoundChunkIndexes[Index]; - uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; - if (!Continue) + if (!AsyncContinue) { break; } + size_t ChunkIndex = FoundChunkIndexes[Index]; + uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; if (OptionalWorkerPool) { - WorkLatch.AddCount(1); - OptionalWorkerPool->ScheduleWork([this, &WorkLatch, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &Continue]() { - auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); - if (!Continue) - { - return; - } - try - { - if (!ProcessOne(ChunkIndex, ExpectedSize)) + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic& AbortFlag) { + if (AbortFlag) { - Continue = false; + AsyncContinue.store(false); } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", - m_RootDirectory, - ChunkHashes[ChunkIndex], - Ex.what()); - } - }); + if (!AsyncContinue) + { + return; + } + try + { + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", + m_RootDirectory, + ChunkHashes[ChunkIndex], + Ex.what()); + AsyncContinue.store(false); + } + }); } else { - Continue = Continue && ProcessOne(ChunkIndex, ExpectedSize); + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } } } - WorkLatch.CountDown(); - WorkLatch.Wait(); + Work.Wait(); } - return Continue; + return AsyncContinue.load(); } void diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index 516d70e28..91591375a 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -30,7 +30,6 @@ ParallelWork::~ParallelWork() "to complete"); m_PendingWork.CountDown(); } - m_AbortFlag.store(true); m_PendingWork.Wait(); ZEN_ASSERT(m_PendingWork.Remaining() == 0); } -- cgit v1.2.3 From 91515f65aec01f99931cc2b774fdb533f308d206 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 20 May 2025 16:28:17 +0200 Subject: replace copy file (#403) Custom CopyFile in zen builds command increasing throughput by 50% on Windows and give better progress update --- src/zen/cmds/builds_cmd.cpp | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 78d362fdb..1a570d6da 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -304,6 +304,26 @@ namespace { } } + void CopyFile(const std::filesystem::path& SourceFilePath, + const std::filesystem::path& TargetFilePath, + uint64_t RawSize, + std::atomic& WriteCount, + std::atomic& WriteByteCount) + { + BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate); + PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize); + uint64_t Offset = 0; + if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) { + TargetFile.Write(Data, Size, Offset); + Offset += Size; + WriteCount++; + WriteByteCount += Size; + })) + { + throw std::runtime_error(fmt::format("Failed to copy scavanged file '{}' to '{}'", SourceFilePath, TargetFilePath)); + } + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -6198,6 +6218,10 @@ namespace { Work.ScheduleWork(WritePool, [&, ScavengeOpIndex](std::atomic&) mutable { if (!AbortFlag) { + ZEN_TRACE_CPU("UpdateFolder_WriteScavanged"); + + FilteredWrittenBytesPerSecond.Start(); + const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; @@ -6211,10 +6235,8 @@ namespace { const std::filesystem::path TempFilePath = GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - CopyFile(ScavengedFilePath, TempFilePath, {.EnableClone = false}); - - DiskStats.WriteCount++; - DiskStats.WriteByteCount += ScavengeOp.RawSize; + const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex]; + CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount); const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); @@ -7527,7 +7549,10 @@ namespace { ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); - CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); + const uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + std::atomic WriteCount; + std::atomic WriteByteCount; + CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } else @@ -7581,7 +7606,10 @@ namespace { ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); - CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); + const uint64_t RawSize = RemoteContent.RawSizes[RemotePathIndex]; + std::atomic WriteCount; + std::atomic WriteByteCount; + CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } -- cgit v1.2.3 From 47cba21817608be9232e9d0a55bf01a313d4be2c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 20 May 2025 18:43:41 +0200 Subject: use explicit capture for lambdas (#404) --- src/zen/cmds/builds_cmd.cpp | 2107 +++++++++++++++++++------------- src/zenutil/jupiter/jupitersession.cpp | 47 +- 2 files changed, 1256 insertions(+), 898 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1a570d6da..d998ed3e8 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1962,41 +1962,131 @@ namespace { for (const IoHash& ChunkAttachment : ChunkAttachments) { - Work.ScheduleWork(NetworkPool, [&, ChunkAttachment](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); + Work.ScheduleWork(NetworkPool, + [&Storage, + &NetworkPool, + &VerifyPool, + &Work, + &DownloadStats, + &ValidateStats, + AttachmentsToVerifyCount, + &TempFolder, + BuildId = Oid(BuildId), + PreferredMultipartChunkSize, + &FilteredDownloadedBytesPerSecond, + &FilteredVerifiedBytesPerSecond, + ChunkAttachment](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetChunk"); + + FilteredDownloadedBytesPerSecond.Start(); + DownloadLargeBlob( + Storage, + TempFolder, + BuildId, + ChunkAttachment, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&Work, + &VerifyPool, + &DownloadStats, + &ValidateStats, + AttachmentsToVerifyCount, + &FilteredDownloadedBytesPerSecond, + &FilteredVerifiedBytesPerSecond, + ChunkHash = ChunkAttachment](IoBuffer&& Payload) { + Payload.SetContentType(ZenContentType::kCompressedBinary); + if (!AbortFlag) + { + Work.ScheduleWork( + VerifyPool, + [&DownloadStats, + &ValidateStats, + AttachmentsToVerifyCount, + &FilteredDownloadedBytesPerSecond, + &FilteredVerifiedBytesPerSecond, + Payload = std::move(Payload), + ChunkHash](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_Validate"); + + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == + AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + FilteredVerifiedBytesPerSecond.Start(); + + uint64_t CompressedSize; + uint64_t DecompressedSize; + ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); + ValidateStats.VerifiedAttachmentCount++; + ValidateStats.VerifiedByteCount += DecompressedSize; + if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) + { + FilteredVerifiedBytesPerSecond.Stop(); + } + } + }); + } + }); + } + }); + } - FilteredDownloadedBytesPerSecond.Start(); - DownloadLargeBlob( - Storage, - TempFolder, - BuildId, - ChunkAttachment, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, ChunkHash = ChunkAttachment](IoBuffer&& Payload) { - Payload.SetContentType(ZenContentType::kCompressedBinary); - if (!AbortFlag) - { - Work.ScheduleWork(VerifyPool, [&, Payload = std::move(Payload), ChunkHash](std::atomic&) mutable { + for (const IoHash& BlockAttachment : BlockAttachments) + { + Work.ScheduleWork( + NetworkPool, + [&Storage, + &BuildId, + &Work, + &VerifyPool, + &DownloadStats, + &ValidateStats, + AttachmentsToVerifyCount, + &FilteredDownloadedBytesPerSecond, + &FilteredVerifiedBytesPerSecond, + BlockAttachment](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); + + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!Payload) + { + throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); + } + if (!AbortFlag) + { + Work.ScheduleWork( + VerifyPool, + [&FilteredVerifiedBytesPerSecond, + AttachmentsToVerifyCount, + &ValidateStats, + Payload = std::move(Payload), + BlockAttachment](std::atomic&) mutable { if (!AbortFlag) { - ZEN_TRACE_CPU("ValidateBuildPart_Validate"); - - if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == - AttachmentsToVerifyCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } + ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); FilteredVerifiedBytesPerSecond.Start(); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(std::move(Payload), ChunkHash, CompressedSize, DecompressedSize); + ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); ValidateStats.VerifiedAttachmentCount++; ValidateStats.VerifiedByteCount += DecompressedSize; if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) @@ -2005,54 +2095,9 @@ namespace { } } }); - } - }); - } - }); - } - - for (const IoHash& BlockAttachment : BlockAttachments) - { - Work.ScheduleWork(NetworkPool, [&, BlockAttachment](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_GetBlock"); - - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlockAttachment); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); - if (DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (!Payload) - { - throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment)); - } - if (!AbortFlag) - { - Work.ScheduleWork(VerifyPool, [&, Payload = std::move(Payload), BlockAttachment](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock"); - - FilteredVerifiedBytesPerSecond.Start(); - - uint64_t CompressedSize; - uint64_t DecompressedSize; - ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize); - ValidateStats.VerifiedAttachmentCount++; - ValidateStats.VerifiedByteCount += DecompressedSize; - if (ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount) - { - FilteredVerifiedBytesPerSecond.Stop(); - } - } - }); + } } - } - }); + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -2321,134 +2366,163 @@ namespace { break; } const std::vector& ChunksInBlock = NewBlockChunks[BlockIndex]; - Work.ScheduleWork(GenerateBlobsPool, [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); - - FilteredGeneratedBytesPerSecond.Start(); - // TODO: Convert ScheduleWork body to function - - Stopwatch GenerateTimer; - CompressedBuffer CompressedBlock = - GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); - ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(CompressedBlock.GetCompressedSize()), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - - OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); + Work.ScheduleWork( + GenerateBlobsPool, + [&Storage, + BuildId, + &Path, + &Content, + &Lookup, + &Work, + &UploadBlocksPool, + NewBlockCount, + ChunksInBlock, + &Lock, + &OutBlocks, + &DiskStats, + &GenerateBlocksStats, + &UploadStats, + &FilteredGeneratedBytesPerSecond, + &QueuedPendingBlocksForUpload, + &FilteredUploadedBytesPerSecond, + BlockIndex](std::atomic&) { + if (!AbortFlag) { - CbObjectWriter Writer; - Writer.AddString("createdBy", "zen"); - OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); - } - GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; - GenerateBlocksStats.GeneratedBlockCount++; + ZEN_TRACE_CPU("GenerateBuildBlocks_Generate"); + + FilteredGeneratedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function + + Stopwatch GenerateTimer; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex], DiskStats); + ZEN_CONSOLE_VERBOSE("Generated block {} ({}) containing {} chunks in {}", + OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(CompressedBlock.GetCompressedSize()), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - Lock.WithExclusiveLock([&]() { - OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, BlockIndex); - }); + OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize(); + { + CbObjectWriter Writer; + Writer.AddString("createdBy", "zen"); + OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save(); + } + GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex]; + GenerateBlocksStats.GeneratedBlockCount++; - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } + Lock.WithExclusiveLock([&]() { + OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockIndex); + }); - if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) - { - FilteredGeneratedBytesPerSecond.Stop(); - } + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } - if (QueuedPendingBlocksForUpload.load() > 16) - { - std::span Segments = CompressedBlock.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - else - { - if (!AbortFlag) + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) { - QueuedPendingBlocksForUpload++; + FilteredGeneratedBytesPerSecond.Stop(); + } - Work.ScheduleWork( - UploadBlocksPool, - [&, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic&) mutable { - auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); - if (!AbortFlag) - { - if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) - { - ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); + if (QueuedPendingBlocksForUpload.load() > 16) + { + std::span Segments = CompressedBlock.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + if (!AbortFlag) + { + QueuedPendingBlocksForUpload++; - FilteredUploadedBytesPerSecond.Stop(); - std::span Segments = Payload.GetCompressed().GetSegments(); - ZEN_ASSERT(Segments.size() >= 2); - OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); - } - else + Work.ScheduleWork( + UploadBlocksPool, + [&Storage, + BuildId, + NewBlockCount, + &FilteredUploadedBytesPerSecond, + &QueuedPendingBlocksForUpload, + &GenerateBlocksStats, + &UploadStats, + &OutBlocks, + BlockIndex, + Payload = std::move(CompressedBlock)](std::atomic&) mutable { + auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; }); + if (!AbortFlag) { - ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); + if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount) + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Save"); + + FilteredUploadedBytesPerSecond.Stop(); + std::span Segments = Payload.GetCompressed().GetSegments(); + ZEN_ASSERT(Segments.size() >= 2); + OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]); + } + else + { + ZEN_TRACE_CPU("GenerateBuildBlocks_Upload"); - FilteredUploadedBytesPerSecond.Start(); - // TODO: Convert ScheduleWork body to function + FilteredUploadedBytesPerSecond.Start(); + // TODO: Convert ScheduleWork body to function - const CbObject BlockMetaData = - BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], - OutBlocks.BlockMetaDatas[BlockIndex]); + const CbObject BlockMetaData = + BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex], + OutBlocks.BlockMetaDatas[BlockIndex]); - const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; - const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); + const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; + const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - if (Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - Payload.GetCompressed()); - } + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); + } - Storage.BuildStorage->PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload).GetCompressed()); - UploadStats.BlocksBytes += CompressedBlockSize; + Storage.BuildStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); + UploadStats.BlocksBytes += CompressedBlockSize; - ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - BlockHash, - NiceBytes(CompressedBlockSize), - OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", + BlockHash, + NiceBytes(CompressedBlockSize), + OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - if (Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); - } + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - BlockHash, - NiceBytes(BlockMetaData.GetSize())); + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + BlockHash, + NiceBytes(BlockMetaData.GetSize())); - OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - UploadStats.BlockCount++; - if (UploadStats.BlockCount == NewBlockCount) - { - FilteredUploadedBytesPerSecond.Stop(); + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + UploadStats.BlockCount++; + if (UploadStats.BlockCount == NewBlockCount) + { + FilteredUploadedBytesPerSecond.Stop(); + } } } - } - }); + }); + } } } - } - }); + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -2558,10 +2632,22 @@ namespace { const size_t UploadBlockCount = BlockIndexes.size(); const uint32_t UploadChunkCount = gsl::narrow(LooseChunkOrderIndexes.size()); - auto AsyncUploadBlock = [&](const size_t BlockIndex, - const IoHash BlockHash, - CompositeBuffer&& Payload, - std::atomic& QueuedPendingInMemoryBlocksForUpload) { + auto AsyncUploadBlock = [&Storage, + &BuildId, + &Work, + &ZenFolderPath, + &NewBlocks, + UploadBlockCount, + &UploadedBlockCount, + UploadChunkCount, + &UploadedChunkCount, + &UploadedBlockSize, + &UploadStats, + &FilteredUploadedBytesPerSecond, + &UploadChunkPool](const size_t BlockIndex, + const IoHash BlockHash, + CompositeBuffer&& Payload, + std::atomic& QueuedPendingInMemoryBlocksForUpload) { bool IsInMemoryBlock = true; if (QueuedPendingInMemoryBlocksForUpload.load() > 16) { @@ -2576,7 +2662,21 @@ namespace { Work.ScheduleWork( UploadChunkPool, - [&, IsInMemoryBlock, BlockIndex, BlockHash, Payload = std::move(Payload)](std::atomic&) mutable { + [&Storage, + &BuildId, + &QueuedPendingInMemoryBlocksForUpload, + &NewBlocks, + UploadBlockCount, + &UploadedBlockCount, + UploadChunkCount, + &UploadedChunkCount, + &UploadedBlockSize, + &UploadStats, + &FilteredUploadedBytesPerSecond, + IsInMemoryBlock, + BlockIndex, + BlockHash, + Payload = std::move(Payload)](std::atomic&) mutable { auto _ = MakeGuard([IsInMemoryBlock, &QueuedPendingInMemoryBlocksForUpload] { if (IsInMemoryBlock) { @@ -2628,10 +2728,37 @@ namespace { }); }; - auto AsyncUploadLooseChunk = [&](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) { + auto AsyncUploadLooseChunk = [&Storage, + BuildId, + LargeAttachmentSize, + &Work, + &UploadChunkPool, + &FilteredUploadedBytesPerSecond, + &UploadedBlockCount, + &UploadedChunkCount, + UploadBlockCount, + UploadChunkCount, + &UploadedCompressedChunkSize, + &UploadedRawChunkSize, + &UploadStats](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) { Work.ScheduleWork( UploadChunkPool, - [&, RawHash, RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { + [&Storage, + BuildId, + &Work, + LargeAttachmentSize, + &FilteredUploadedBytesPerSecond, + &UploadChunkPool, + &UploadedBlockCount, + &UploadedChunkCount, + UploadBlockCount, + UploadChunkCount, + &UploadedCompressedChunkSize, + &UploadedRawChunkSize, + &UploadStats, + RawHash, + RawSize, + Payload = CompositeBuffer(std::move(Payload))](std::atomic&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("AsyncUploadLooseChunk"); @@ -2660,7 +2787,16 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&, RawSize](uint64_t SentBytes, bool IsComplete) { + [RawSize, + &UploadStats, + &UploadedCompressedChunkSize, + &UploadChunkPool, + &UploadedBlockCount, + UploadBlockCount, + &UploadedChunkCount, + UploadChunkCount, + &FilteredUploadedBytesPerSecond, + &UploadedRawChunkSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; UploadedCompressedChunkSize += SentBytes; if (IsComplete) @@ -2718,56 +2854,71 @@ namespace { const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; if (!AbortFlag) { - Work.ScheduleWork(ReadChunkPool, [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); + Work.ScheduleWork( + ReadChunkPool, + [BlockHash = IoHash(BlockHash), + BlockIndex, + &FilteredGenerateBlockBytesPerSecond, + Path, + &Content, + &Lookup, + &NewBlocks, + &NewBlockChunks, + &GenerateBlockIndexes, + &GeneratedBlockCount, + &GeneratedBlockByteCount, + &DiskStats, + &AsyncUploadBlock, + &QueuedPendingInMemoryBlocksForUpload](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock"); - FilteredGenerateBlockBytesPerSecond.Start(); + FilteredGenerateBlockBytesPerSecond.Start(); - Stopwatch GenerateTimer; - CompositeBuffer Payload; - if (NewBlocks.BlockHeaders[BlockIndex]) - { - Payload = RebuildBlock(Path, - Content, - Lookup, - std::move(NewBlocks.BlockHeaders[BlockIndex]), - NewBlockChunks[BlockIndex], - DiskStats) - .GetCompressed(); - } - else - { - ChunkBlockDescription BlockDescription; - CompressedBuffer CompressedBlock = - GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); - if (!CompressedBlock) + Stopwatch GenerateTimer; + CompositeBuffer Payload; + if (NewBlocks.BlockHeaders[BlockIndex]) { - throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + Payload = RebuildBlock(Path, + Content, + Lookup, + std::move(NewBlocks.BlockHeaders[BlockIndex]), + NewBlockChunks[BlockIndex], + DiskStats) + .GetCompressed(); + } + else + { + ChunkBlockDescription BlockDescription; + CompressedBuffer CompressedBlock = + GenerateBlock(Path, Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription, DiskStats); + if (!CompressedBlock) + { + throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash)); + } + ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); + Payload = std::move(CompressedBlock).GetCompressed(); } - ZEN_ASSERT(BlockDescription.BlockHash == BlockHash); - Payload = std::move(CompressedBlock).GetCompressed(); - } - GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; - GeneratedBlockCount++; - if (GeneratedBlockCount == GenerateBlockIndexes.size()) - { - FilteredGenerateBlockBytesPerSecond.Stop(); - } - ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}", - NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(NewBlocks.BlockSizes[BlockIndex]), - NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), - NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); - if (!AbortFlag) - { - AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); + GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex]; + GeneratedBlockCount++; + if (GeneratedBlockCount == GenerateBlockIndexes.size()) + { + FilteredGenerateBlockBytesPerSecond.Stop(); + } + ZEN_CONSOLE_VERBOSE("{} block {} ({}) containing {} chunks in {}", + NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated", + NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + NiceBytes(NewBlocks.BlockSizes[BlockIndex]), + NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(), + NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs())); + if (!AbortFlag) + { + AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload); + } } - } - }); + }); } } @@ -2775,32 +2926,43 @@ namespace { for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; - Work.ScheduleWork(ReadChunkPool, [&, ChunkIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); - - FilteredCompressedBytesPerSecond.Start(); - Stopwatch CompressTimer; - CompositeBuffer Payload = - CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); - ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", - Content.ChunkedContent.ChunkHashes[ChunkIndex], - NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), - NiceBytes(Payload.GetSize()), - NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); - const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; - UploadStats.ReadFromDiskBytes += ChunkRawSize; - if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) - { - FilteredCompressedBytesPerSecond.Stop(); - } + Work.ScheduleWork( + ReadChunkPool, + [&Path, + &Content, + &Lookup, + &ZenFolderPath, + &LooseChunksStats, + &LooseChunkOrderIndexes, + &FilteredCompressedBytesPerSecond, + &UploadStats, + &AsyncUploadLooseChunk, + ChunkIndex](std::atomic&) { if (!AbortFlag) { - AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload)); + ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); + + FilteredCompressedBytesPerSecond.Start(); + Stopwatch CompressTimer; + CompositeBuffer Payload = + CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); + ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", + Content.ChunkedContent.ChunkHashes[ChunkIndex], + NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), + NiceBytes(Payload.GetSize()), + NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs())); + const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; + UploadStats.ReadFromDiskBytes += ChunkRawSize; + if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) + { + FilteredCompressedBytesPerSecond.Stop(); + } + if (!AbortFlag) + { + AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload)); + } } - } - }); + }); } Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { @@ -3649,7 +3811,19 @@ namespace { PutBuildPartResult.second.size()); IoHash PartHash = PutBuildPartResult.first; - auto UploadAttachments = [&](std::span RawHashes) { + auto UploadAttachments = [&Storage, + &BuildId, + &Path, + &ZenFolderPath, + &LocalContent, + &LocalLookup, + &NewBlockChunks, + &NewBlocks, + &LooseChunkIndexes, + &LargeAttachmentSize, + &DiskStats, + &UploadStats, + &LooseChunksStats](std::span RawHashes) { if (!AbortFlag) { UploadStatistics TempUploadStats; @@ -4089,7 +4263,8 @@ namespace { Work.ScheduleWork( VerifyPool, - [&, PathIndex](std::atomic&) { + [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex]( + std::atomic&) { if (!AbortFlag) { ZEN_TRACE_CPU("VerifyFile_work"); @@ -5048,11 +5223,18 @@ namespace { Work.ScheduleWork( WritePool, - [&, + [&ZenFolderPath, + &RemoteContent, + &RemoteLookup, SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, CompressedChunkPath, RemoteChunkIndex, TotalPartWriteCount, + &DiskStats, + &WritePartsComplete, + &FilteredWrittenBytesPerSecond, ChunkTargetPtrs = std::move(ChunkTargetPtrs), CompressedPart = std::move(Payload)](std::atomic&) mutable { ZEN_TRACE_CPU("UpdateFolder_WriteChunk"); @@ -6215,40 +6397,51 @@ namespace { } if (!PrimeCacheOnly) { - Work.ScheduleWork(WritePool, [&, ScavengeOpIndex](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteScavanged"); + Work.ScheduleWork( + WritePool, + [&RemoteContent, + &CacheFolderPath, + &ScavengedPaths, + &ScavengeCopyOperations, + &ScavengedContents, + &FilteredWrittenBytesPerSecond, + ScavengeOpIndex, + &WritePartsComplete, + TotalPartWriteCount, + &DiskStats](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteScavanged"); - FilteredWrittenBytesPerSecond.Start(); + FilteredWrittenBytesPerSecond.Start(); - const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; - const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; - const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; + const ScavengeCopyOperation& ScavengeOp = ScavengeCopyOperations[ScavengeOpIndex]; + const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex]; + const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex]; - const std::filesystem::path ScavengedFilePath = - (ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred(); - ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); + const std::filesystem::path ScavengedFilePath = + (ScavengedPaths[ScavengeOp.ScavengedContentIndex] / ScavengedPath).make_preferred(); + ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize); - const IoHash& RemoteSequenceRawHash = - RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; - const std::filesystem::path TempFilePath = - GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + const IoHash& RemoteSequenceRawHash = + RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex]; + const std::filesystem::path TempFilePath = + GetTempChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex]; - CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount); + const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedContentIndex]; + CopyFile(ScavengedFilePath, TempFilePath, RawSize, DiskStats.WriteCount, DiskStats.WriteByteCount); - const std::filesystem::path CacheFilePath = - GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - RenameFile(TempFilePath, CacheFilePath); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + RenameFile(TempFilePath, CacheFilePath); - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }); + }); } } @@ -6273,7 +6466,31 @@ namespace { Work.ScheduleWork( WritePool, - [&, RemoteChunkIndex, ChunkTargetPtrs, BuildId, TotalRequestCount, TotalPartWriteCount](std::atomic&) mutable { + [&Storage, + &Path, + &ZenFolderPath, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + PrimeCacheOnly, + &ExistsResult, + &DiskStats, + &DownloadStats, + &WriteChunkStats, + &WritePartsComplete, + RemoteChunkIndex, + ChunkTargetPtrs, + BuildId = Oid(BuildId), + LargeAttachmentSize, + PreferredMultipartChunkSize, + TotalRequestCount, + TotalPartWriteCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond](std::atomic&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); @@ -6392,7 +6609,7 @@ namespace { [&Path, &ZenFolderPath, &Storage, - BuildId, + BuildId = Oid(BuildId), &PrimeCacheOnly, &RemoteContent, &RemoteLookup, @@ -6453,50 +6670,67 @@ namespace { if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob( - *Storage.BuildStorage, - ZenTempDownloadFolderPath(ZenFolderPath), - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, TotalPartWriteCount, TotalRequestCount, RemoteChunkIndex, ChunkTargetPtrs]( - IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (Payload && Storage.BuildCacheStorage) - { - Storage.BuildCacheStorage->PutBuildBlob( - BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); - } - if (!PrimeCacheOnly) - { - if (!AbortFlag) - { - AsyncWriteDownloadedChunk(ZenFolderPath, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - } - }); + DownloadLargeBlob(*Storage.BuildStorage, + ZenTempDownloadFolderPath(ZenFolderPath), + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&Storage, + &ZenFolderPath, + &RemoteContent, + &RemoteLookup, + BuildId, + PrimeCacheOnly, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + ChunkHash, + TotalPartWriteCount, + TotalRequestCount, + &WritePartsComplete, + &FilteredWrittenBytesPerSecond, + &FilteredDownloadedBytesPerSecond, + &DownloadStats, + &DiskStats, + RemoteChunkIndex, + ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk( + ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); } else { @@ -6559,456 +6793,343 @@ namespace { break; } - Work.ScheduleWork(WritePool, [&, CopyDataIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); - - FilteredWrittenBytesPerSecond.Start(); - const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - - std::filesystem::path SourceFilePath; - - if (CopyData.ScavengeSourceIndex == (uint32_t)-1) - { - const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - } - else + Work.ScheduleWork( + WritePool, + [&Path, + &LocalContent, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &LocalLookup, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &FilteredWrittenBytesPerSecond, + &CacheCopyDatas, + &ScavengedContents, + &ScavengedLookups, + &ScavengedPaths, + &WritePartsComplete, + TotalPartWriteCount, + &DiskStats, + CopyDataIndex](std::atomic&) { + if (!AbortFlag) { - const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; - const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; - const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; - const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; - SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); - } - ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); - ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - - uint64_t CacheLocalFileBytesRead = 0; + ZEN_TRACE_CPU("UpdateFolder_CopyLocal"); - size_t TargetStart = 0; - const std::span AllTargets( - CopyData.TargetChunkLocationPtrs); - - struct WriteOp - { - const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; - uint64_t CacheFileOffset = (uint64_t)-1; - uint32_t ChunkIndex = (uint32_t)-1; - }; + FilteredWrittenBytesPerSecond.Start(); + const CacheCopyData& CopyData = CacheCopyDatas[CopyDataIndex]; - std::vector WriteOps; + std::filesystem::path SourceFilePath; - if (!AbortFlag) - { - ZEN_TRACE_CPU("Sort"); - WriteOps.reserve(AllTargets.size()); - for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) + if (CopyData.ScavengeSourceIndex == (uint32_t)-1) { - std::span TargetRange = - AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); - for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) - { - WriteOps.push_back(WriteOp{.Target = Target, - .CacheFileOffset = ChunkTarget.CacheFileOffset, - .ChunkIndex = ChunkTarget.RemoteChunkIndex}); - } - TargetStart += ChunkTarget.TargetChunkLocationCount; + const uint32_t LocalPathIndex = LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + } + else + { + const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex]; + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex]; + const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex]; + const uint32_t ScavengedPathIndex = + ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex]; + SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred(); } + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty()); - std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { - if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) - { - return true; - } - else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) - { - return false; - } - if (Lhs.Target->Offset < Rhs.Target->Offset) - { - return true; - } - return false; - }); - } + uint64_t CacheLocalFileBytesRead = 0; - if (!AbortFlag) - { - ZEN_TRACE_CPU("Write"); + size_t TargetStart = 0; + const std::span AllTargets( + CopyData.TargetChunkLocationPtrs); + + struct WriteOp + { + const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr; + uint64_t CacheFileOffset = (uint64_t)-1; + uint32_t ChunkIndex = (uint32_t)-1; + }; - tsl::robin_set ChunkIndexesWritten; + std::vector WriteOps; - BufferedOpenFile SourceFile(SourceFilePath, DiskStats); - WriteFileCache OpenFileCache(DiskStats); - for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) + if (!AbortFlag) { - if (AbortFlag) - { - break; - } - const WriteOp& Op = WriteOps[WriteOpIndex]; - - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= - RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); - const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; - const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; - - uint64_t ReadLength = ChunkSize; - size_t WriteCount = 1; - uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; - uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; - while ((WriteOpIndex + WriteCount) < WriteOps.size()) + ZEN_TRACE_CPU("Sort"); + WriteOps.reserve(AllTargets.size()); + for (const CacheCopyData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets) { - const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; - if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) + std::span TargetRange = + AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount); + for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange) { - break; + WriteOps.push_back(WriteOp{.Target = Target, + .CacheFileOffset = ChunkTarget.CacheFileOffset, + .ChunkIndex = ChunkTarget.RemoteChunkIndex}); } - if (NextOp.Target->Offset != OpTargetEnd) + TargetStart += ChunkTarget.TargetChunkLocationCount; + } + + std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) { + if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex) { - break; + return true; } - if (NextOp.CacheFileOffset != OpSourceEnd) + else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex) { - break; + return false; } - const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; - if (ReadLength + NextChunkLength > 512u * 1024u) + if (Lhs.Target->Offset < Rhs.Target->Offset) { - break; + return true; } - ReadLength += NextChunkLength; - OpSourceEnd += NextChunkLength; - OpTargetEnd += NextChunkLength; - WriteCount++; - } - - CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); + return false; + }); + } - OpenFileCache.WriteToFile( - RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); + if (!AbortFlag) + { + ZEN_TRACE_CPU("Write"); - CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? + tsl::robin_set ChunkIndexesWritten; - WriteOpIndex += WriteCount; - } - } - if (!AbortFlag) - { - // Write tracking, updating this must be done without any files open (WriteFileCache) - std::vector CompletedChunkSequences; - for (const WriteOp& Op : WriteOps) - { - const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; - if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + BufferedOpenFile SourceFile(SourceFilePath, DiskStats); + WriteFileCache OpenFileCache(DiskStats); + for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) { - CompletedChunkSequences.push_back(RemoteSequenceIndex); - } - } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, - RemoteContent, - RemoteLookup, - CompletedChunkSequences, - Work, - WritePool); - ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); - } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }); - } + if (AbortFlag) + { + break; + } + const WriteOp& Op = WriteOps[WriteOpIndex]; + + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() <= + RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].load() > 0); + const uint32_t RemotePathIndex = RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ChunkSize = RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex]; + + uint64_t ReadLength = ChunkSize; + size_t WriteCount = 1; + uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize; + uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize; + while ((WriteOpIndex + WriteCount) < WriteOps.size()) + { + const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount]; + if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex) + { + break; + } + if (NextOp.Target->Offset != OpTargetEnd) + { + break; + } + if (NextOp.CacheFileOffset != OpSourceEnd) + { + break; + } + const uint64_t NextChunkLength = RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex]; + if (ReadLength + NextChunkLength > 512u * 1024u) + { + break; + } + ReadLength += NextChunkLength; + OpSourceEnd += NextChunkLength; + OpTargetEnd += NextChunkLength; + WriteCount++; + } - for (uint32_t BlockIndex : CachedChunkBlockIndexes) - { - ZEN_ASSERT(!PrimeCacheOnly); - if (AbortFlag) - { - break; - } + CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); + ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - Work.ScheduleWork(WritePool, [&, BlockIndex](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); + OpenFileCache.WriteToFile( + RemoteSequenceIndex, + [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { + return GetTempChunkedSequenceFileName( + CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + }, + ChunkSource, + Op.Target->Offset, + RemoteContent.RawSizes[RemotePathIndex]); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - FilteredWrittenBytesPerSecond.Start(); + CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? - std::filesystem::path BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) - { - throw std::runtime_error( - fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); - } - - if (!AbortFlag) - { - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + WriteOpIndex += WriteCount; + } + } + if (!AbortFlag) { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + // Write tracking, updating this must be done without any files open (WriteFileCache) + std::vector CompletedChunkSequences; + for (const WriteOp& Op : WriteOps) + { + const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex; + if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters)) + { + CompletedChunkSequences.push_back(RemoteSequenceIndex); + } + } + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, + RemoteContent, + RemoteLookup, + CompletedChunkSequences, + Work, + WritePool); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath); } - - TryRemoveFile(BlockChunkPath); - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } } - } - }); + }); } - for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) + for (uint32_t BlockIndex : CachedChunkBlockIndexes) { ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; } - const BlockRangeDescriptor BlockRange = BlockRangeWorks[BlockRangeIndex]; - ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); - const uint32_t BlockIndex = BlockRange.BlockIndex; - Work.ScheduleWork(NetworkPool, [&, BlockIndex, BlockRange](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) - { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - } - if (!BlockBuffer) - { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - } - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } + Work.ScheduleWork( + WritePool, + [&ZenFolderPath, + &CacheFolderPath, + &RemoteContent, + &RemoteLookup, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &BlockDescriptions, + &FilteredWrittenBytesPerSecond, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + BlockIndex](std::atomic&) mutable { if (!AbortFlag) { - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - std::filesystem::path BlockChunkPath; + ZEN_TRACE_CPU("UpdateFolder_WriteCachedBlock"); - // Check if the dowloaded block is file based and we can move it directly without rewriting it - { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) - { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) - { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - RenameFile(TempBlobPath, BlockChunkPath, Ec); - if (Ec) - { - BlockChunkPath = std::filesystem::path{}; - - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); - } - } - } - } + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + FilteredWrittenBytesPerSecond.Start(); - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + std::filesystem::path BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; + throw std::runtime_error( + fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); } if (!AbortFlag) { - Work.ScheduleWork( - WritePool, - [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( - std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else - { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) - { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); - } - } + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } - FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } + TryRemoveFile(BlockChunkPath); - if (!BlockChunkPath.empty()) - { - TryRemoveFile(BlockChunkPath); - } + WritePartsComplete++; - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } } - } - }); + }); } - for (uint32_t BlockIndex : FullBlockWorks) + for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; } + const BlockRangeDescriptor BlockRange = BlockRangeWorks[BlockRangeIndex]; + ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); + const uint32_t BlockIndex = BlockRange.BlockIndex; + Work.ScheduleWork( + NetworkPool, + [&Storage, + &ZenFolderPath, + BuildId, + &RemoteLookup, + &BlockDescriptions, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &CacheFolderPath, + &RemoteContent, + &SequenceIndexChunksLeftToWriteCounters, + &ExistsResult, + &FilteredDownloadedBytesPerSecond, + TotalRequestCount, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + &DiskStats, + &DownloadStats, + &Work, + &WritePool, + BlockIndex, + BlockRange](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_GetPartialBlock"); - if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) - { - DownloadStats.RequestsCompleteCount++; - continue; - } - - Work.ScheduleWork(NetworkPool, [&, BlockIndex](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); - - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - - FilteredDownloadedBytesPerSecond.Start(); + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - IoBuffer BlockBuffer; - const bool ExistsInCache = - Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); - if (ExistsInCache) - { - BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); - } - if (!BlockBuffer) - { - BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); - if (BlockBuffer && Storage.BuildCacheStorage) + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) { - Storage.BuildCacheStorage->PutBuildBlob(BuildId, - BlockDescription.BlockHash, - BlockBuffer.GetContentType(), - CompositeBuffer(SharedBuffer(BlockBuffer))); + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); } - } - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - if (!AbortFlag) - { - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); } - - if (!PrimeCacheOnly) + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + if (!AbortFlag) { + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + std::filesystem::path BlockChunkPath; // Check if the dowloaded block is file based and we can move it directly without rewriting it @@ -7018,14 +7139,17 @@ namespace { (FileRef.FileChunkSize == BlockSize)) { ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = - ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockBuffer = {}; + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -7044,7 +7168,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -7053,60 +7180,64 @@ namespace { { Work.ScheduleWork( WritePool, - [&Work, - &WritePool, + [&CacheFolderPath, &RemoteContent, &RemoteLookup, - CacheFolderPath, + &BlockDescriptions, &RemoteChunkIndexNeedsCopyFromSourceFlags, &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, &WritePartsComplete, + &Work, TotalPartWriteCount, + &WritePool, + &DiskStats, &FilteredWrittenBytesPerSecond, + BlockIndex, + BlockRange, BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + BlockPartialBuffer = std::move(BlockBuffer)](std::atomic&) mutable { if (!AbortFlag) { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; if (BlockChunkPath.empty()) { - ZEN_ASSERT(BlockBuffer); + ZEN_ASSERT(BlockPartialBuffer); } else { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath)); } } FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) @@ -7115,7 +7246,6 @@ namespace { } WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -7125,8 +7255,208 @@ namespace { } } } - } - }); + }); + } + + for (uint32_t BlockIndex : FullBlockWorks) + { + if (AbortFlag) + { + break; + } + + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + + Work.ScheduleWork( + NetworkPool, + [&Storage, + &ZenFolderPath, + BuildId, + PrimeCacheOnly, + &BlockDescriptions, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + &ExistsResult, + &Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &FilteredDownloadedBytesPerSecond, + &WriteChunkStats, + &DiskStats, + &DownloadStats, + TotalRequestCount, + BlockIndex](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_GetFullBlock"); + + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + + FilteredDownloadedBytesPerSecond.Start(); + + IoBuffer BlockBuffer; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + } + if (!BlockBuffer) + { + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (BlockBuffer && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlockBuffer))); + } + } + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + } + if (!AbortFlag) + { + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + + if (!PrimeCacheOnly) + { + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) + { + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + RenameFile(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; + + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } + } + } + } + + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } + + if (!AbortFlag) + { + Work.ScheduleWork(WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + + const ChunkBlockDescription& BlockDescription = + BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } + } + + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + + if (!BlockChunkPath.empty()) + { + TryRemoveFile(BlockChunkPath); + } + + WritePartsComplete++; + + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }); + } + } + } + } + }); } { @@ -7328,20 +7658,22 @@ namespace { { break; } - Work.ScheduleWork(WritePool, [&, LocalPathIndex](std::atomic&) { - ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); - if (!AbortFlag) - { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); - const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - RenameFileWithRetry(LocalFilePath, CacheFilePath); - CachedCount++; - CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; - } - }); + Work.ScheduleWork( + WritePool, + [&Path, &LocalContent, &CacheFolderPath, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; + } + }); } { @@ -7420,7 +7752,7 @@ namespace { { break; } - Work.ScheduleWork(WritePool, [&, LocalPathIndex](std::atomic&) { + Work.ScheduleWork(WritePool, [&Path, &LocalContent, &DeletedCount, LocalPathIndex](std::atomic&) { if (!AbortFlag) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); @@ -7471,164 +7803,183 @@ namespace { TargetCount++; } - Work.ScheduleWork(WritePool, [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { - if (!AbortFlag) - { - ZEN_TRACE_CPU("FinalizeTree_Work"); + Work.ScheduleWork( + WritePool, + [&Path, + &LocalContent, + &SequenceHashToLocalPathIndex, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &Targets, + &RemotePathIndexToLocalPathIndex, + &RebuildFolderStateStats, + &OutLocalFolderState, + BaseTargetOffset = TargetOffset, + TargetCount, + &TargetsComplete](std::atomic&) { + if (!AbortFlag) + { + ZEN_TRACE_CPU("FinalizeTree_Work"); - size_t TargetOffset = BaseTargetOffset; - const IoHash& RawHash = Targets[TargetOffset].RawHash; + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; - if (RawHash == IoHash::Zero) - { - ZEN_TRACE_CPU("ZeroSize"); - while (TargetOffset < (BaseTargetOffset + TargetCount)) + if (RawHash == IoHash::Zero) { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); - if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) + ZEN_TRACE_CPU("ZeroSize"); + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - if (IsFileWithRetry(TargetFilePath)) - { - SetFileReadOnlyWithRetry(TargetFilePath, false); - } - else + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - CreateDirectories(TargetFilePath.parent_path()); + if (IsFileWithRetry(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); } - BasicFile OutputFile; - OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; } - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; - - OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); - - TargetOffset++; - TargetsComplete++; - } - } - else - { - ZEN_TRACE_CPU("Files"); - ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); - const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; - std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); - - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); - InPlaceIt != RemotePathIndexToLocalPathIndex.end()) - { - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); } else { - if (IsFileWithRetry(FirstTargetFilePath)) - { - SetFileReadOnlyWithRetry(FirstTargetFilePath, false); - } - else - { - CreateDirectories(FirstTargetFilePath.parent_path()); - } + ZEN_TRACE_CPU("Files"); + ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); - if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); - InplaceIt != SequenceHashToLocalPathIndex.end()) + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_TRACE_CPU("Copy"); - const uint32_t LocalPathIndex = InplaceIt->second; - const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; - std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); - - ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); - const uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; - std::atomic WriteCount; - std::atomic WriteByteCount; - CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); } else { - ZEN_TRACE_CPU("Rename"); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); + if (IsFileWithRetry(FirstTargetFilePath)) + { + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); + } + else + { + CreateDirectories(FirstTargetFilePath.parent_path()); + } - RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); + InplaceIt != SequenceHashToLocalPathIndex.end()) + { + ZEN_TRACE_CPU("Copy"); + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath)); + + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); + const uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + std::atomic WriteCount; + std::atomic WriteByteCount; + CopyFile(SourceFilePath, FirstTargetFilePath, RawSize, WriteCount, WriteByteCount); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + ZEN_TRACE_CPU("Rename"); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath)); - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } - } + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + } + } - OutLocalFolderState.Attributes[FirstRemotePathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstRemotePathIndex]); - OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; - TargetOffset++; - TargetsComplete++; + OutLocalFolderState.Attributes[FirstRemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = + GetModificationTickFromPath(FirstTargetFilePath); - while (TargetOffset < (BaseTargetOffset + TargetCount)) - { - const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; - ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); - const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; - std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + TargetOffset++; + TargetsComplete++; - if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); - InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); - } - else - { - ZEN_TRACE_CPU("Copy"); - if (IsFileWithRetry(TargetFilePath)) + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - SetFileReadOnlyWithRetry(TargetFilePath, false); + ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath)); } else { - CreateDirectories(TargetFilePath.parent_path()); - } + ZEN_TRACE_CPU("Copy"); + if (IsFileWithRetry(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } - ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); - ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); - const uint64_t RawSize = RemoteContent.RawSizes[RemotePathIndex]; - std::atomic WriteCount; - std::atomic WriteByteCount; - CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - } + ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); + const uint64_t RawSize = RemoteContent.RawSizes[RemotePathIndex]; + std::atomic WriteCount; + std::atomic WriteByteCount; + CopyFile(FirstTargetFilePath, TargetFilePath, RawSize, WriteCount, WriteByteCount); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } - OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; - OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; - OutLocalFolderState.Attributes[RemotePathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(TargetFilePath) - : SetNativeFileAttributes(TargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[RemotePathIndex]); - OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); - TargetOffset++; - TargetsComplete++; + TargetOffset++; + TargetsComplete++; + } } } - } - }); + }); TargetOffset += TargetCount; } diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index de138f994..01a703a1b 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -697,28 +697,35 @@ JupiterSession::GetMultipartBuildBlob(std::string_view Namespace, while (Offset < TotalSize) { uint64_t PartSize = Min(ChunkSize, TotalSize - Offset); - OutWorkItems.emplace_back( - [this, Namespace, BucketId, BuildId, Hash, TotalSize, Workload, Offset, PartSize]() -> JupiterResult { - std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", - Namespace, - BucketId, - BuildId, - Hash.ToHexString(), - m_AllowRedirect ? "true"sv : "false"sv); - HttpClient::Response Response = m_HttpClient.Get( - RequestUrl, - HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); - if (Response.IsSuccess()) + OutWorkItems.emplace_back([this, + Namespace = std::string(Namespace), + BucketId = std::string(BucketId), + BuildId = Oid(BuildId), + Hash = IoHash(Hash), + TotalSize, + Workload, + Offset, + PartSize]() -> JupiterResult { + std::string RequestUrl = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + m_AllowRedirect ? "true"sv : "false"sv); + HttpClient::Response Response = m_HttpClient.Get( + RequestUrl, + HttpClient::KeyValueMap({{"Range", fmt::format("bytes={}-{}", Offset, Offset + PartSize - 1)}})); + if (Response.IsSuccess()) + { + Workload->OnReceive(Offset, Response.ResponsePayload); + uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize()); + if (ByteRemaning == Response.ResponsePayload.GetSize()) { - Workload->OnReceive(Offset, Response.ResponsePayload); - uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Response.ResponsePayload.GetSize()); - if (ByteRemaning == Response.ResponsePayload.GetSize()) - { - Workload->OnComplete(); - } + Workload->OnComplete(); } - return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); - }); + } + return detail::ConvertResponse(Response, "JupiterSession::GetMultipartBuildBlob"sv); + }); Offset += PartSize; } } -- cgit v1.2.3 From 57622d697585720a54af1c763d43ed782f6d996b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 26 May 2025 14:38:38 +0200 Subject: unblock cache bucket drop (#406) * don't hold exclusive locks while deleting files from a dropped bucket/namespace * cleaner detection of missing namespace when issuing a drop --- src/zenstore/cache/cachedisklayer.cpp | 108 ++++++++++++++++----- src/zenstore/cache/structuredcachestore.cpp | 42 +++++--- .../include/zenstore/cache/cachedisklayer.h | 24 ++--- .../include/zenstore/cache/structuredcachestore.h | 4 +- 4 files changed, 124 insertions(+), 54 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index d80da6ea6..d26c6dfd2 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -196,14 +196,14 @@ namespace cache::impl { return true; } - bool MoveAndDeleteDirectory(const std::filesystem::path& Dir) + std::filesystem::path MoveDroppedDirectory(const std::filesystem::path& Dir) { int DropIndex = 0; do { if (!IsDir(Dir)) { - return false; + return {}; } std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); @@ -218,12 +218,11 @@ namespace cache::impl { RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { - DeleteDirectories(DroppedBucketPath); - return true; + return DroppedBucketPath; } - // TODO: Do we need to bail at some point? zen::Sleep(100); - } while (true); + } while (DropIndex < 10); + return {}; } } // namespace cache::impl @@ -1944,7 +1943,7 @@ ZenCacheDiskLayer::CacheBucket::GetUsageByAccess(GcClock::TimePoint Now, GcClock } } -bool +std::function ZenCacheDiskLayer::CacheBucket::Drop() { ZEN_TRACE_CPU("Z$::Bucket::Drop"); @@ -1960,7 +1959,7 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_BlockStore.Close(); m_SlogFile.Close(); - const bool Deleted = cache::impl::MoveAndDeleteDirectory(m_BucketDir); + std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_BucketDir); m_Index.clear(); m_Payloads.clear(); @@ -1973,7 +1972,21 @@ ZenCacheDiskLayer::CacheBucket::Drop() m_OuterCacheMemoryUsage.fetch_sub(m_MemCachedSize.load()); m_MemCachedSize.store(0); - return Deleted; + if (DroppedPath.empty()) + { + return {}; + } + else + { + return [DroppedPath = std::move(DroppedPath)]() { + std::error_code Ec; + (void)DeleteDirectories(DroppedPath, Ec); + if (Ec) + { + ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message()); + } + }; + } } void @@ -3904,7 +3917,11 @@ ZenCacheDiskLayer::DiscoverBuckets() if (IsKnownBadBucketName(BucketName)) { BadBucketDirectories.push_back(BucketPath); - + continue; + } + else if (BucketName.starts_with("[dropped]")) + { + BadBucketDirectories.push_back(BucketPath); continue; } @@ -3987,7 +4004,7 @@ ZenCacheDiskLayer::DiscoverBuckets() Work.Wait(); } -bool +std::function ZenCacheDiskLayer::DropBucket(std::string_view InBucket) { ZEN_TRACE_CPU("Z$::DropBucket"); @@ -4005,33 +4022,72 @@ ZenCacheDiskLayer::DropBucket(std::string_view InBucket) return Bucket.Drop(); } - // Make sure we remove the folder even if we don't know about the bucket std::filesystem::path BucketPath = m_RootDir; BucketPath /= std::string(InBucket); - return cache::impl::MoveAndDeleteDirectory(BucketPath); + std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(BucketPath); + if (DroppedPath.empty()) + { + return {}; + } + else + { + return [DroppedPath = std::move(DroppedPath)]() { + std::error_code Ec; + (void)DeleteDirectories(DroppedPath, Ec); + if (Ec) + { + ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message()); + } + }; + } } -bool +std::function ZenCacheDiskLayer::Drop() { ZEN_TRACE_CPU("Z$::Drop"); - RwLock::ExclusiveLockScope _(m_Lock); - - std::vector> Buckets; - Buckets.reserve(m_Buckets.size()); - while (!m_Buckets.empty()) + std::vector> PostDropOps; { - const auto& It = m_Buckets.begin(); - CacheBucket& Bucket = *It->second; - m_DroppedBuckets.push_back(std::move(It->second)); - m_Buckets.erase(It->first); - if (!Bucket.Drop()) + RwLock::ExclusiveLockScope _(m_Lock); + PostDropOps.reserve(m_Buckets.size()); + while (!m_Buckets.empty()) { - return false; + const auto& It = m_Buckets.begin(); + CacheBucket& Bucket = *It->second; + m_DroppedBuckets.push_back(std::move(It->second)); + m_Buckets.erase(It->first); + if (std::function PostDropOp = Bucket.Drop(); !PostDropOp) + { + return {}; + } + else + { + PostDropOps.emplace_back(std::move(PostDropOp)); + } } } - return cache::impl::MoveAndDeleteDirectory(m_RootDir); + + std::filesystem::path DroppedPath = cache::impl::MoveDroppedDirectory(m_RootDir); + if (DroppedPath.empty()) + { + return {}; + } + else + { + return [DroppedPath = std::move(DroppedPath), PostDropOps = std::move(PostDropOps)]() { + for (auto& PostDropOp : PostDropOps) + { + PostDropOp(); + } + std::error_code Ec; + (void)DeleteDirectories(DroppedPath, Ec); + if (Ec) + { + ZEN_WARN("Failed to clean up dropped bucket directory '{}', reason: '{}'", DroppedPath, Ec.message()); + } + }; + } } void diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 7d277329e..5ce254fac 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -277,11 +277,14 @@ ZenCacheNamespace::DropBucket(std::string_view Bucket) { ZEN_INFO("dropping bucket '{}'", Bucket); - const bool Dropped = m_DiskLayer.DropBucket(Bucket); - - ZEN_INFO("bucket '{}' was {}", Bucket, Dropped ? "dropped" : "not found"); - - return Dropped; + std::function PostDropOp = m_DiskLayer.DropBucket(Bucket); + if (!PostDropOp) + { + ZEN_INFO("bucket '{}' was not found in {}", Bucket, m_RootDir); + return false; + } + PostDropOp(); + return true; } void @@ -291,7 +294,7 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view m_DiskLayer.EnumerateBucketContents(Bucket, Fn); } -bool +std::function ZenCacheNamespace::Drop() { return m_DiskLayer.Drop(); @@ -786,16 +789,27 @@ ZenCacheStore::DropBucket(std::string_view Namespace, std::string_view Bucket) bool ZenCacheStore::DropNamespace(std::string_view InNamespace) { - RwLock::SharedLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) + std::function PostDropOp; { - ZenCacheNamespace& Namespace = *It->second; - m_DroppedNamespaces.push_back(std::move(It->second)); - m_Namespaces.erase(It); - return Namespace.Drop(); + RwLock::SharedLockScope _(m_NamespacesLock); + if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) + { + ZenCacheNamespace& Namespace = *It->second; + m_DroppedNamespaces.push_back(std::move(It->second)); + m_Namespaces.erase(It); + PostDropOp = Namespace.Drop(); + } + else + { + ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); + return false; + } } - ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::DropNamespace", InNamespace); - return false; + if (PostDropOp) + { + PostDropOp(); + } + return true; } void diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 54ebb324d..b67a043df 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -183,15 +183,15 @@ public: PutBatchHandle* BeginPutBatch(std::vector& OutResult); void EndPutBatch(PutBatchHandle* Batch) noexcept; - void Put(std::string_view Bucket, - const IoHash& HashKey, - const ZenCacheValue& Value, - std::span References, - PutBatchHandle* OptionalBatchHandle); - bool Drop(); - bool DropBucket(std::string_view Bucket); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); + void Put(std::string_view Bucket, + const IoHash& HashKey, + const ZenCacheValue& Value, + std::span References, + PutBatchHandle* OptionalBatchHandle); + std::function Drop(); + std::function DropBucket(std::string_view Bucket); + void Flush(); + void ScrubStorage(ScrubContext& Ctx); void DiscoverBuckets(); GcStorageSize StorageSize() const; @@ -240,9 +240,9 @@ public: void EndPutBatch(PutBatchHandle* Batch) noexcept; void Put(const IoHash& HashKey, const ZenCacheValue& Value, std::span References, PutBatchHandle* OptionalBatchHandle); uint64_t MemCacheTrim(GcClock::TimePoint ExpireTime); - bool Drop(); - void Flush(); - void ScrubStorage(ScrubContext& Ctx); + std::function Drop(); + void Flush(); + void ScrubStorage(ScrubContext& Ctx); RwLock::SharedLockScope GetGcReferencerLock(); struct ReferencesStats diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 5e056cf2d..48fc17960 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -101,8 +101,8 @@ public: void EnumerateBucketContents(std::string_view Bucket, std::function& Fn) const; - bool Drop(); - void Flush(); + std::function Drop(); + void Flush(); // GcStorage virtual void ScrubStorage(ScrubContext& ScrubCtx) override; -- cgit v1.2.3 From 459a8c7a46d1392f04007bf30315dfc86cba7abb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 26 May 2025 22:34:32 +0200 Subject: made fmt use of format_context more consistent (#409) fixes compilation issues on Linux in some cases --- src/zencore/include/zencore/compactbinaryfmt.h | 3 ++- src/zencore/include/zencore/fmtutils.h | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h index ae0c3eb42..b03683db4 100644 --- a/src/zencore/include/zencore/compactbinaryfmt.h +++ b/src/zencore/include/zencore/compactbinaryfmt.h @@ -14,7 +14,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::CbObject& a, format_context& ctx) const + template + auto format(const zen::CbObject& a, FormatContext& ctx) const { zen::ExtendableStringBuilder<1024> ObjStr; zen::CompactBinaryToJson(a, ObjStr); diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h index 10dfa5393..404e570fd 100644 --- a/src/zencore/include/zencore/fmtutils.h +++ b/src/zencore/include/zencore/fmtutils.h @@ -21,7 +21,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::StringBuilderBase& a, format_context& ctx) const + template + auto format(const zen::StringBuilderBase& a, FormatContext& ctx) const { return fmt::formatter::format(a.ToView(), ctx); } @@ -31,7 +32,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::NiceBase& a, format_context& ctx) const + template + auto format(const zen::NiceBase& a, FormatContext& ctx) const { return fmt::formatter::format(std::string_view(a), ctx); } @@ -98,7 +100,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::PathBuilderBase& a, format_context& ctx) const + template + auto format(const zen::PathBuilderBase& a, FormatContext& ctx) const { return fmt::formatter::format(a.ToView(), ctx); } -- cgit v1.2.3 From 9b4928b7196738950bc71dddd4eb5b59f535f188 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 27 May 2025 11:36:46 +0200 Subject: frequent disk space check (#407) * check low disk space condition more frequently and trigger GC when low water mark is reached * show waited time when waiting for zenserver instance to exit --- src/zenstore/gc.cpp | 109 ++++++++++++++++++++++++++++++--------- src/zenutil/zenserverprocess.cpp | 6 ++- 2 files changed, 89 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index f6b3dca6f..a15a2e084 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -2068,17 +2068,46 @@ GcScheduler::SchedulerThread() ZEN_MEMSCOPE(GetGcTag()); SetCurrentThreadName("GcScheduler"); - std::chrono::seconds WaitTime{0}; - - bool SilenceErrors = false; + std::chrono::seconds WaitTime{0}; + const std::chrono::seconds ShortWaitTime{5}; + bool SilenceErrors = false; for (;;) { - bool Timeout = false; + (void)CheckDiskSpace(); + + std::chrono::seconds WaitedTime{0}; + bool Timeout = false; { ZEN_ASSERT(WaitTime.count() >= 0); std::unique_lock Lock(m_GcMutex); - Timeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, WaitTime); + while (!Timeout) + { + std::chrono::seconds ShortWait = Min(WaitTime, ShortWaitTime); + bool ShortTimeout = std::cv_status::timeout == m_GcSignal.wait_for(Lock, ShortWait); + if (ShortTimeout) + { + if (WaitTime > ShortWaitTime) + { + DiskSpace Space = CheckDiskSpace(); + if (!AreDiskWritesAllowed()) + { + ZEN_INFO("Triggering GC due to low disk space ({}) on {}", NiceBytes(Space.Free), m_Config.RootDirectory); + Timeout = true; + } + WaitTime -= ShortWaitTime; + } + else + { + Timeout = true; + } + } + else + { + // We got a signal + break; + } + } } if (Status() == GcSchedulerStatus::kStopped) @@ -2110,6 +2139,7 @@ GcScheduler::SchedulerThread() std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; std::chrono::seconds MaxBuildStoreDuration = m_Config.MaxBuildStoreDuration; uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; + uint64_t MinimumFreeDiskSpaceToAllowWrites = m_Config.MinimumFreeDiskSpaceToAllowWrites; bool SkipCid = false; GcVersion UseGCVersion = m_Config.UseGCVersion; uint32_t CompactBlockUsageThresholdPercent = m_Config.CompactBlockUsageThresholdPercent; @@ -2124,8 +2154,9 @@ GcScheduler::SchedulerThread() uint8_t NextAttachmentPassIndex = ComputeAttachmentRange(m_AttachmentPassIndex, m_Config.AttachmentPassCount, AttachmentRangeMin, AttachmentRangeMax); - bool DiskSpaceGCTriggered = false; - bool TimeBasedGCTriggered = false; + bool LowDiskSpaceGCTriggered = false; + bool HighDiskSpaceUsageGCTriggered = false; + bool TimeBasedGCTriggered = false; GcClock::TimePoint Now = GcClock::Now(); @@ -2262,12 +2293,32 @@ GcScheduler::SchedulerThread() } } - uint64_t GcDiskSpaceGoal = 0; + uint64_t MaximumDiskUseGcSpaceGoal = 0; + uint64_t MinimumFreeDiskGcSpaceGoal = 0; + if (DiskSizeSoftLimit != 0 && TotalSize.DiskSize > DiskSizeSoftLimit) { - GcDiskSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit; + MaximumDiskUseGcSpaceGoal = TotalSize.DiskSize - DiskSizeSoftLimit; + HighDiskSpaceUsageGCTriggered = true; + } + + if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free < MinimumFreeDiskSpaceToAllowWrites) + { + MinimumFreeDiskGcSpaceGoal = MinimumFreeDiskSpaceToAllowWrites - Space.Free; + if (MinimumFreeDiskGcSpaceGoal > MaximumDiskUseGcSpaceGoal) + { + LowDiskSpaceGCTriggered = true; + EnableValidation = false; + } + } + + if (MaximumDiskUseGcSpaceGoal > 0 || MinimumFreeDiskGcSpaceGoal > 0) + { + const uint64_t GcDiskSpaceRemoveGoal = Max(MaximumDiskUseGcSpaceGoal, MinimumFreeDiskGcSpaceGoal); + std::unique_lock Lock(m_GcMutex); - GcClock::Tick AgeTick = m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceGoal, Now.time_since_epoch().count()); + GcClock::Tick AgeTick = + m_DiskUsageWindow.FindTimepointThatRemoves(GcDiskSpaceRemoveGoal, Now.time_since_epoch().count()); GcClock::TimePoint SizeBasedExpireTime = GcClock::TimePointFromTick(AgeTick); if (SizeBasedExpireTime > CacheExpireTime) { @@ -2309,29 +2360,33 @@ GcScheduler::SchedulerThread() RemainingTimeUntilLightweightGc = RemainingTimeUntilGc; } - if (GcDiskSpaceGoal > 0) + if (MaximumDiskUseGcSpaceGoal == 0 && MinimumFreeDiskGcSpaceGoal == 0) { - DiskSpaceGCTriggered = true; - } - else if (RemainingTimeUntilGc.count() == 0) - { - TimeBasedGCTriggered = true; - } - else if (RemainingTimeUntilLightweightGc.count() == 0) - { - TimeBasedGCTriggered = true; - SkipCid = true; + if (RemainingTimeUntilGc.count() == 0) + { + TimeBasedGCTriggered = true; + } + else if (RemainingTimeUntilLightweightGc.count() == 0) + { + TimeBasedGCTriggered = true; + SkipCid = true; + } } std::string NextTriggerStatus; - if (GcInterval.count() != 0 || LightweightGcInterval.count() != 0 || DiskSizeSoftLimit != 0) { ExtendableStringBuilder<256> Sb; - if (DiskSpaceGCTriggered) + if (LowDiskSpaceGCTriggered) + { + Sb.Append(fmt::format(" Free disk space is below {}, trying to reclaim {}.", + NiceBytes(MinimumFreeDiskSpaceToAllowWrites), + NiceBytes(MinimumFreeDiskGcSpaceGoal))); + } + else if (HighDiskSpaceUsageGCTriggered) { Sb.Append(fmt::format(" Disk space exceeds {}, trying to reclaim {}.", NiceBytes(DiskSizeSoftLimit), - NiceBytes(GcDiskSpaceGoal))); + NiceBytes(MaximumDiskUseGcSpaceGoal))); } else if (TimeBasedGCTriggered) { @@ -2361,6 +2416,10 @@ GcScheduler::SchedulerThread() { Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(DiskSizeSoftLimit - TotalSize.DiskSize))); } + else if (MinimumFreeDiskSpaceToAllowWrites != 0 && Space.Free > MinimumFreeDiskSpaceToAllowWrites) + { + Sb.Append(fmt::format(" Disk usage GC in {}.", NiceBytes(Space.Free - MinimumFreeDiskSpaceToAllowWrites))); + } } NextTriggerStatus = Sb; } @@ -2377,7 +2436,7 @@ GcScheduler::SchedulerThread() NiceBytes(MaxLoad / uint64_t(std::chrono::seconds(m_Config.MonitorInterval).count())), NextTriggerStatus); - if (!DiskSpaceGCTriggered && !TimeBasedGCTriggered) + if (!HighDiskSpaceUsageGCTriggered && !LowDiskSpaceGCTriggered && !TimeBasedGCTriggered) { WaitTime = m_Config.MonitorInterval; if (RemainingTimeUntilGc < WaitTime) diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index c0c2a754a..bfa0d3c49 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -619,6 +619,7 @@ ZenServerInstance::Shutdown() std::error_code Ec; if (SignalShutdown(Ec)) { + Stopwatch Timer; ZEN_DEBUG("Waiting for zenserver process {} ({}) to shut down", m_Name, m_Process.Pid()); while (!m_Process.Wait(1000)) { @@ -632,7 +633,10 @@ ZenServerInstance::Shutdown() ZEN_WARN("Wait abandoned by exited process"); return 0; } - ZEN_WARN("Waiting for zenserver process {} ({}) timed out", m_Name, m_Process.Pid()); + ZEN_WARN("Waited for zenserver process {} ({}) to exit for {}", + m_Name, + m_Process.Pid(), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } ZEN_DEBUG("zenserver process {} ({}) exited", m_Name, m_Process.Pid()); int ExitCode = m_Process.GetExitCode(); -- cgit v1.2.3 From 42aa2c9a8124ada33c88f5c608e4865b1ff84c64 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 30 May 2025 11:51:05 +0200 Subject: faster oplog validate (#408) Improvement: Faster oplog validate to reduce GC wall time and disk I/O pressure --- src/zenserver/projectstore/projectstore.cpp | 25 +++++++++------ src/zenstore/blockstore.cpp | 47 ++++++++++++++++++++++++++--- src/zenstore/compactcas.cpp | 7 ++++- src/zenstore/filecas.cpp | 20 ++++++++++-- src/zenstore/include/zenstore/blockstore.h | 5 +-- src/zenutil/parallelwork.cpp | 2 +- 6 files changed, 86 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 7d22da717..3ec4373a2 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1511,11 +1511,17 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo ValidationResult Result; + const size_t OpCount = OplogCount(); + std::vector KeyHashes; std::vector Keys; std::vector> Attachments; std::vector Mappings; + KeyHashes.reserve(OpCount); + Keys.reserve(OpCount); + Mappings.reserve(OpCount); + IterateOplogWithKey([&](uint32_t LSN, const Oid& Key, CbObjectView OpView) { Result.LSNLow = Min(Result.LSNLow, LSN); Result.LSNHigh = Max(Result.LSNHigh, LSN); @@ -1540,20 +1546,22 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo bool HasMissingEntries = false; for (const ChunkMapping& Chunk : Mapping.Chunks) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(Chunk.Hash); !Payload) + if (!m_CidStore.ContainsChunk(Chunk.Hash)) { ResultLock.WithExclusiveLock([&]() { Result.MissingChunks.push_back({KeyHash, Chunk}); }); HasMissingEntries = true; } } + for (const ChunkMapping& Meta : Mapping.Meta) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(Meta.Hash); !Payload) + if (!m_CidStore.ContainsChunk(Meta.Hash)) { ResultLock.WithExclusiveLock([&]() { Result.MissingMetas.push_back({KeyHash, Meta}); }); HasMissingEntries = true; } } + for (const FileMapping& File : Mapping.Files) { if (File.Hash == IoHash::Zero) @@ -1565,24 +1573,23 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo HasMissingEntries = true; } } - else + else if (!m_CidStore.ContainsChunk(File.Hash)) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(File.Hash); !Payload) - { - ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); - HasMissingEntries = true; - } + ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); + HasMissingEntries = true; } } + const std::vector& OpAttachments = Attachments[OpIndex]; for (const IoHash& Attachment : OpAttachments) { - if (IoBuffer Payload = m_CidStore.FindChunkByCid(Attachment); !Payload) + if (!m_CidStore.ContainsChunk(Attachment)) { ResultLock.WithExclusiveLock([&]() { Result.MissingAttachments.push_back({KeyHash, Attachment}); }); HasMissingEntries = true; } } + if (HasMissingEntries) { ResultLock.WithExclusiveLock([&]() { Result.OpKeys.push_back({KeyHash, Key}); }); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e0f371061..86dbcc971 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -153,14 +153,28 @@ void BlockStoreFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset) { ZEN_TRACE_CPU("BlockStoreFile::Write"); +#if ZEN_BUILD_DEBUG + if (uint64_t CachedFileSize = m_CachedFileSize.load(); CachedFileSize > 0) + { + ZEN_ASSERT(FileOffset + Size <= CachedFileSize); + } +#endif // ZEN_BUILD_DEBUG m_File.Write(Data, Size, FileOffset); } void -BlockStoreFile::Flush() +BlockStoreFile::Flush(uint64_t FinalSize) { ZEN_TRACE_CPU("BlockStoreFile::Flush"); m_File.Flush(); + if (FinalSize != (uint64_t)-1) + { + uint64_t ExpectedSize = 0; + if (!m_CachedFileSize.compare_exchange_weak(ExpectedSize, FinalSize)) + { + ZEN_ASSERT(m_CachedFileSize.load() == FinalSize); + } + } } BasicFile& @@ -540,7 +554,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons { if (m_WriteBlock) { - m_WriteBlock->Flush(); + m_WriteBlock->Flush(m_CurrentInsertOffset); m_WriteBlock = nullptr; } @@ -674,6 +688,27 @@ BlockStore::WriteChunks(std::span Datas, uint32_t Alignment, con } } +bool +BlockStore::HasChunk(const BlockStoreLocation& Location) const +{ + ZEN_TRACE_CPU("BlockStore::TryGetChunk"); + RwLock::SharedLockScope InsertLock(m_InsertLock); + if (auto BlockIt = m_ChunkBlocks.find(Location.BlockIndex); BlockIt != m_ChunkBlocks.end()) + { + if (const Ref& Block = BlockIt->second; Block) + { + InsertLock.ReleaseNow(); + + const uint64_t BlockSize = Block->FileSize(); + if (Location.Offset + Location.Size <= BlockSize) + { + return true; + } + } + } + return false; +} + IoBuffer BlockStore::TryGetChunk(const BlockStoreLocation& Location) const { @@ -706,7 +741,7 @@ BlockStore::Flush(bool ForceNewBlock) { if (m_WriteBlock) { - m_WriteBlock->Flush(); + m_WriteBlock->Flush(m_CurrentInsertOffset); } m_WriteBlock = nullptr; m_CurrentInsertOffset = 0; @@ -1097,7 +1132,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, if (NewBlockFile) { ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); - NewBlockFile->Flush(); + NewBlockFile->Flush(WriteOffset); uint64_t NewBlockSize = NewBlockFile->FileSize(); MovedSize += NewBlockSize; NewBlockFile = nullptr; @@ -1228,7 +1263,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, if (NewBlockFile) { ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); - NewBlockFile->Flush(); + NewBlockFile->Flush(WriteOffset); uint64_t NewBlockSize = NewBlockFile->FileSize(); MovedSize += NewBlockSize; NewBlockFile = nullptr; @@ -1359,6 +1394,8 @@ TEST_CASE("blockstore.blockfile") CHECK(std::string(Boop) == "boop"); File1.Flush(); CHECK(File1.FileSize() == 10); + File1.Flush(10); + CHECK(File1.FileSize() == 10); } { BlockStoreFile File1(RootDirectory / "1"); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 15bea272b..730bdf143 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -307,7 +307,12 @@ bool CasContainerStrategy::HaveChunk(const IoHash& ChunkHash) { RwLock::SharedLockScope _(m_LocationMapLock); - return m_LocationMap.contains(ChunkHash); + if (auto KeyIt = m_LocationMap.find(ChunkHash); KeyIt != m_LocationMap.end()) + { + const BlockStoreLocation& Location = m_Locations[KeyIt->second].Get(m_PayloadAlignment); + return m_BlockStore.HasChunk(Location); + } + return false; } void diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 6354edf70..56979f267 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -564,8 +564,24 @@ FileCasStrategy::HaveChunk(const IoHash& ChunkHash) { ZEN_ASSERT(m_IsInitialized); - RwLock::SharedLockScope _(m_Lock); - return m_Index.contains(ChunkHash); + { + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_Index.find(ChunkHash); It == m_Index.end()) + { + return false; + } + } + + ShardingHelper Name(m_RootDirectory, ChunkHash); + const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath(); + RwLock::SharedLockScope ShardLock(LockForHash(ChunkHash)); + + if (IsFile(ChunkPath)) + { + return true; + } + + return false; } void diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 0c72a13aa..b0713fea1 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -94,7 +94,7 @@ struct BlockStoreFile : public RefCounted IoBuffer GetChunk(uint64_t Offset, uint64_t Size); void Read(void* Data, uint64_t Size, uint64_t FileOffset); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); - void Flush(); + void Flush(uint64_t FinalSize = (uint64_t)-1); BasicFile& GetBasicFile(); void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); bool IsOpen() const; @@ -107,7 +107,7 @@ private: const std::filesystem::path m_Path; IoBuffer m_IoBuffer; BasicFile m_File; - uint64_t m_CachedFileSize = 0; + std::atomic m_CachedFileSize = 0; }; class BlockStoreCompactState; @@ -158,6 +158,7 @@ public: typedef std::function Locations)> WriteChunksCallback; void WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback); + bool HasChunk(const BlockStoreLocation& Location) const; IoBuffer TryGetChunk(const BlockStoreLocation& Location) const; void Flush(bool ForceNewBlock); diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index 91591375a..ecacc4b5a 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -85,7 +85,7 @@ ParallelWork::RethrowErrors() { if (m_Errors.size() > 1) { - ZEN_INFO("Multiple exceptions throwm during ParallelWork execution, dropping the following exceptions:"); + ZEN_INFO("Multiple exceptions thrown during ParallelWork execution, dropping the following exceptions:"); auto It = m_Errors.begin() + 1; while (It != m_Errors.end()) { -- cgit v1.2.3 From 6a958656168bdab4a02a34f9697e713fb34a8047 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 30 May 2025 18:28:18 +0200 Subject: add missing flush inblockstore compact (#411) - Bugfix: Flush the last block before closing the last new block written to during blockstore compact. UE-291196 - Feature: Drop unreachable CAS data during GC pass. UE-291196 --- src/zenstore/blockstore.cpp | 78 +++++++++++++++++++++++++----- src/zenstore/buildstore/buildstore.cpp | 24 ++++++++- src/zenstore/cache/cachedisklayer.cpp | 24 ++++++++- src/zenstore/compactcas.cpp | 16 +++++- src/zenstore/include/zenstore/blockstore.h | 5 +- 5 files changed, 130 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 86dbcc971..5081ae65d 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -1004,6 +1004,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, uint32_t NewBlockIndex = 0; MovedChunksArray MovedChunks; + ChunkIndexArray ScrubbedChunks; uint64_t AddedSize = 0; uint64_t RemovedSize = 0; @@ -1032,14 +1033,15 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, auto ReportChanges = [&]() -> bool { bool Continue = true; - if (!MovedChunks.empty() || RemovedSize > 0) + if (!MovedChunks.empty() || !ScrubbedChunks.empty() || RemovedSize > 0) { - Continue = ChangeCallback(MovedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0); + Continue = ChangeCallback(MovedChunks, ScrubbedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0); DeletedSize += RemovedSize; RemovedSize = 0; AddedSize = 0; MovedCount += MovedChunks.size(); MovedChunks.clear(); + ScrubbedChunks.clear(); } return Continue; }; @@ -1068,6 +1070,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, LogPrefix, m_BlocksBasePath, BlockIndex); + ScrubbedChunks.insert(ScrubbedChunks.end(), KeepChunkIndexes.begin(), KeepChunkIndexes.end()); return true; } if (!It->second) @@ -1076,6 +1079,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, LogPrefix, m_BlocksBasePath, BlockIndex); + ScrubbedChunks.insert(ScrubbedChunks.end(), KeepChunkIndexes.begin(), KeepChunkIndexes.end()); return true; } OldBlockFile = It->second; @@ -1115,6 +1119,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, ChunkLocation.Size, OldBlockFile->GetPath(), OldBlockSize); + ScrubbedChunks.push_back(ChunkIndex); continue; } @@ -1128,7 +1133,11 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, if ((WriteOffset + ChunkView.GetSize()) > m_MaxBlockSize) { - TargetFileBuffer.reset(); + if (TargetFileBuffer) + { + TargetFileBuffer->Flush(); + TargetFileBuffer.reset(); + } if (NewBlockFile) { ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); @@ -1260,6 +1269,12 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, return true; }); + if (TargetFileBuffer) + { + TargetFileBuffer->Flush(); + TargetFileBuffer.reset(); + } + if (NewBlockFile) { ZEN_ASSERT_SLOW(NewBlockFile->IsOpen()); @@ -1853,7 +1868,7 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray&, uint64_t) { + [&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray&, uint64_t) { CHECK(false); return true; }, @@ -1878,9 +1893,10 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) { + [&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) { RemovedSize += Removed; CHECK(Moved.empty()); + CHECK(Scrubbed.empty()); return true; }, []() { return 0; }); @@ -1903,9 +1919,10 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) { + [&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) { RemovedSize += Removed; CHECK(Moved.empty()); + CHECK(Scrubbed.empty()); return true; }, []() { return 0; }); @@ -1913,7 +1930,7 @@ TEST_CASE("blockstore.compact.blocks") CHECK_LE(Store.TotalSize(), 1088); CHECK_GT(Store.TotalSize(), 0); } - SUBCASE("keep everthing") + SUBCASE("keep everything") { Store.Flush(true); @@ -1926,7 +1943,7 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray&, uint64_t) { + [&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray&, uint64_t) { CHECK(false); return true; }, @@ -1957,8 +1974,9 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) { + [&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) { CHECK(Moved.empty()); + CHECK(Scrubbed.empty()); RemovedSize += Removed; return true; }, @@ -1992,7 +2010,8 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) { + [&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) { + CHECK(Scrubbed.empty()); for (const auto& Move : Moved) { const BlockStoreLocation& OldLocation = State.GetLocation(Move.first); @@ -2069,7 +2088,8 @@ TEST_CASE("blockstore.compact.blocks") Store.CompactBlocks( State, Alignment, - [&](const BlockStore::MovedChunksArray& Moved, uint64_t Removed) { + [&](const BlockStore::MovedChunksArray& Moved, const BlockStore::ChunkIndexArray& Scrubbed, uint64_t Removed) { + CHECK(Scrubbed.empty()); for (const auto& Move : Moved) { const BlockStoreLocation& OldLocation = State.GetLocation(Move.first); @@ -2104,6 +2124,42 @@ TEST_CASE("blockstore.compact.blocks") } CHECK_LT(Store.TotalSize(), PreSize); } + SUBCASE("scrub") + { + Store.Flush(true); + + BlockStoreCompactState State; + for (const BlockStoreLocation& Location : ChunkLocations) + { + State.IncludeBlock(Location.BlockIndex); + CHECK(State.AddKeepLocation(Location)); + } + State.IncludeBlock(0); + State.IncludeBlock(999); + std::vector ExpectedScrubbedIndexes; + ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 0); + State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 0, .Offset = 2000, .Size = 322}); + ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 1); + State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 0, .Offset = 10, .Size = 3220}); + ExpectedScrubbedIndexes.push_back(ChunkLocations.size() + 2); + State.AddKeepLocation(BlockStoreLocation{.BlockIndex = 999, .Offset = 2, .Size = 40}); + + std::vector ScrubbedIndexes; + + Store.CompactBlocks( + State, + Alignment, + [&](const BlockStore::MovedChunksArray&, const BlockStore::ChunkIndexArray& ScrubbedArray, uint64_t) { + ScrubbedIndexes.insert(ScrubbedIndexes.end(), ScrubbedArray.begin(), ScrubbedArray.end()); + return true; + }, + []() { + CHECK(false); + return 0; + }); + std::sort(ScrubbedIndexes.begin(), ScrubbedIndexes.end()); + CHECK_EQ(ExpectedScrubbedIndexes, ScrubbedIndexes); + } } #endif diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 6eb01dfc4..41c747e08 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -1237,7 +1237,9 @@ public: m_Store.m_MetadataBlockStore.CompactBlocks( BlockCompactState, m_Store.m_Config.MetadataBlockStoreAlignement, - [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + [&](const BlockStore::MovedChunksArray& MovedArray, + const BlockStore::ChunkIndexArray& ScrubbedArray, + uint64_t FreedDiskSpace) { std::vector MovedEntries; MovedEntries.reserve(MovedArray.size()); RwLock::ExclusiveLockScope _(m_Store.m_Lock); @@ -1264,7 +1266,27 @@ public: } } } + + for (size_t Scrubbed : ScrubbedArray) + { + const IoHash& Key = BlockCompactStateKeys[Scrubbed]; + if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + MovedEntries.push_back( + MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); + MovedEntries.back().Entry.Flags |= MetadataEntry::kTombStone; + m_Store.m_MetadataEntries[Meta] = {}; + m_Store.m_BlobEntries[Index].Metadata = {}; + } + } + } + m_Store.m_MetadatalogFile.Append(MovedEntries); + Stats.RemovedDisk += FreedDiskSpace; if (Ctx.IsCancelledFlag.load()) { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index d26c6dfd2..f76ad5c7d 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2952,10 +2952,12 @@ public: m_Bucket.m_BlockStore.CompactBlocks( BlockCompactState, m_Bucket.m_Configuration.PayloadAlignment, - [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + [&](const BlockStore::MovedChunksArray& MovedArray, + const BlockStore::ChunkIndexArray& ScrubbedArray, + uint64_t FreedDiskSpace) { std::vector MovedEntries; MovedEntries.reserve(MovedArray.size()); - RwLock::ExclusiveLockScope _(m_Bucket.m_IndexLock); + RwLock::ExclusiveLockScope IndexLock(m_Bucket.m_IndexLock); for (const std::pair& Moved : MovedArray) { size_t ChunkIndex = Moved.first; @@ -2977,6 +2979,24 @@ public: MovedEntries.push_back({.Key = Key, .Location = Payload.Location}); } } + + for (size_t ScrubbedIndex : ScrubbedArray) + { + const IoHash& Key = BlockCompactStateKeys[ScrubbedIndex]; + + if (auto It = m_Bucket.m_Index.find(Key); It != m_Bucket.m_Index.end()) + { + BucketPayload& Payload = m_Bucket.m_Payloads[It->second]; + DiskLocation Location = Payload.Location; + + m_Bucket.RemoveMemCachedData(IndexLock, Payload); + m_Bucket.RemoveMetaData(IndexLock, Payload); + + Location.Flags |= DiskLocation::kTombStone; + MovedEntries.push_back(DiskIndexEntry{.Key = Key, .Location = Location}); + } + } + m_Bucket.m_SlogFile.Append(MovedEntries); Stats.RemovedDisk += FreedDiskSpace; if (Ctx.IsCancelledFlag.load()) diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 730bdf143..75562176e 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -718,7 +718,9 @@ public: m_CasContainerStrategy.m_BlockStore.CompactBlocks( BlockCompactState, m_CasContainerStrategy.m_PayloadAlignment, - [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + [&](const BlockStore::MovedChunksArray& MovedArray, + const BlockStore::ChunkIndexArray& ScrubbedArray, + uint64_t FreedDiskSpace) { std::vector MovedEntries; RwLock::ExclusiveLockScope _(m_CasContainerStrategy.m_LocationMapLock); for (const std::pair& Moved : MovedArray) @@ -743,6 +745,18 @@ public: MovedEntries.push_back(CasDiskIndexEntry{.Key = Key, .Location = Location}); } } + for (size_t ScrubbedIndex : ScrubbedArray) + { + const IoHash& Key = BlockCompactStateKeys[ScrubbedIndex]; + if (auto It = m_CasContainerStrategy.m_LocationMap.find(Key); + It != m_CasContainerStrategy.m_LocationMap.end()) + { + BlockStoreDiskLocation& Location = m_CasContainerStrategy.m_Locations[It->second]; + MovedEntries.push_back( + CasDiskIndexEntry{.Key = Key, .Location = Location, .Flags = CasDiskIndexEntry::kTombstone}); + m_CasContainerStrategy.m_LocationMap.erase(It); + } + } m_CasContainerStrategy.m_CasLog.Append(MovedEntries); Stats.RemovedDisk += FreedDiskSpace; if (Ctx.IsCancelledFlag.load()) diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index b0713fea1..8cbcad11b 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -127,7 +127,8 @@ public: typedef std::vector> MovedChunksArray; typedef std::vector ChunkIndexArray; - typedef std::function CompactCallback; + typedef std::function + CompactCallback; typedef std::function ClaimDiskReserveCallback; typedef std::function IterateChunksSmallSizeCallback; typedef std::function IterateChunksLargeSizeCallback; @@ -173,7 +174,7 @@ public: void CompactBlocks( const BlockStoreCompactState& CompactState, uint32_t PayloadAlignment, - const CompactCallback& ChangeCallback = [](const MovedChunksArray&, uint64_t) { return true; }, + const CompactCallback& ChangeCallback = [](const MovedChunksArray&, const ChunkIndexArray&, uint64_t) { return true; }, const ClaimDiskReserveCallback& DiskReserveCallback = []() { return 0; }, std::string_view LogPrefix = {}); -- cgit v1.2.3 From 8494a3c61ae23c74510c5327f0c1e31bc1e42bb9 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 14:41:30 +0200 Subject: use system temp dir (#412) * use system temp dir when uploading builds --- src/zen/cmds/builds_cmd.cpp | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d998ed3e8..f6f15acb0 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -121,9 +121,11 @@ namespace { { return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks } - std::filesystem::path ZenTempChunkFolderPath(const std::filesystem::path& ZenFolderPath) + std::filesystem::path UploadTempDirectory(const std::filesystem::path& Path) { - return ZenTempFolderPath(ZenFolderPath) / "chunks"; // Temp storage for decompressed and validated chunks + const std::u8string LocalPathString = Path.generic_u8string(); + IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); + return std::filesystem::temp_directory_path() / fmt::format("zen_{}", PathHash); } std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) @@ -2561,7 +2563,7 @@ namespace { void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, + const std::filesystem::path& TempDir, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span RawHashes, @@ -2635,7 +2637,7 @@ namespace { auto AsyncUploadBlock = [&Storage, &BuildId, &Work, - &ZenFolderPath, + &TempDir, &NewBlocks, UploadBlockCount, &UploadedBlockCount, @@ -2652,7 +2654,7 @@ namespace { if (QueuedPendingInMemoryBlocksForUpload.load() > 16) { ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); - Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), ZenTempBlockFolderPath(ZenFolderPath), BlockHash)); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), TempDir, BlockHash)); IsInMemoryBlock = false; } else @@ -2931,7 +2933,7 @@ namespace { [&Path, &Content, &Lookup, - &ZenFolderPath, + &TempDir, &LooseChunksStats, &LooseChunkOrderIndexes, &FilteredCompressedBytesPerSecond, @@ -2944,8 +2946,7 @@ namespace { FilteredCompressedBytesPerSecond.Start(); Stopwatch CompressTimer; - CompositeBuffer Payload = - CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); + CompositeBuffer Payload = CompressChunk(Path, Content, Lookup, ChunkIndex, TempDir, LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {}) in {}", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), @@ -3174,7 +3175,7 @@ namespace { const Oid& BuildPartId, const std::string_view BuildPartName, const std::filesystem::path& Path, - const std::filesystem::path& ZenFolderPath, + const std::filesystem::path& TempDir, const std::filesystem::path& ManifestPath, const uint64_t FindBlockMaxCount, const uint8_t BlockReuseMinPercentLimit, @@ -3203,18 +3204,15 @@ namespace { Stopwatch ProcessTimer; - const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); - CreateDirectories(ZenTempFolder); - CleanDirectory(ZenTempFolder, {}); + CreateDirectories(TempDir); + CleanDirectory(TempDir, {}); auto _ = MakeGuard([&]() { - if (CleanDirectory(ZenTempFolder, {})) + if (CleanDirectory(TempDir, {})) { std::error_code DummyEc; - RemoveDir(ZenTempFolder, DummyEc); + RemoveDir(TempDir, DummyEc); } }); - CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); - CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::PrepareBuild, TaskSteps::StepCount); @@ -3814,7 +3812,7 @@ namespace { auto UploadAttachments = [&Storage, &BuildId, &Path, - &ZenFolderPath, + &TempDir, &LocalContent, &LocalLookup, &NewBlockChunks, @@ -3853,7 +3851,7 @@ namespace { UploadPartBlobs(Storage, BuildId, Path, - ZenFolderPath, + TempDir, LocalContent, LocalLookup, RawHashes, @@ -10413,12 +10411,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CbObject MetaData = ParseBuildMetadata(); + const std::filesystem::path TempDir = UploadTempDirectory(m_Path); + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, m_Path, - m_ZenFolderPath, + TempDir, m_ManifestPath, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10720,12 +10720,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); } + const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); + // std::filesystem::path UploadTempDir = m_ZenFolderPath / "upload_tmp"; + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, m_Path, - m_ZenFolderPath, + UploadTempDir, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, @@ -10919,7 +10922,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, - m_ZenFolderPath, + UploadTempDir, {}, m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, -- cgit v1.2.3 From db6a98ba01b7e3b0dd79c725dbaadd5f465c6799 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 14:55:35 +0200 Subject: streaming none compressor (#414) * add proper streaming to none compressor type --- src/zencore/compress.cpp | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index ad6b6103c..62b64bc9d 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -217,6 +218,20 @@ public: { UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); + + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + { + ZEN_ASSERT(FileRef.FileHandle != nullptr); + uint64_t CallbackOffset = 0; + ScanFile(FileRef.FileHandle, 0, RawData.GetSize(), 512u * 1024u, [&](const void* Data, size_t Size) { + CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); + Callback(CallbackOffset, Size, HeaderData.GetSize() + CallbackOffset, Tmp); + CallbackOffset += Size; + }); + return true; + } + Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData); return true; } @@ -299,11 +314,29 @@ public: Header.TotalCompressedSize == Header.TotalRawSize + sizeof(BufferHeader) && RawOffset < Header.TotalRawSize && (RawOffset + RawSize) <= Header.TotalRawSize) { - if (!Callback(sizeof(BufferHeader) + RawOffset, RawSize, 0, CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize))) + bool Result = true; + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) { - return false; + ZEN_ASSERT(FileRef.FileHandle != nullptr); + uint64_t CallbackOffset = 0; + ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) { + if (Result) + { + CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); + Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp); + } + CallbackOffset += Size; + }); + return Result; + } + else + { + return Callback(sizeof(BufferHeader) + RawOffset, + RawSize, + 0, + CompressedData.Mid(sizeof(BufferHeader) + RawOffset, RawSize)); } - return true; } return false; } -- cgit v1.2.3 From ac31b9db1055cad4e2226448c4d2f6d3af8b13f4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 19:14:28 +0200 Subject: fix cachbucket mem hit count (#415) * Don't count a miss twice for memory stats if the entry can't be found * changelog --- src/zenstore/cache/cachedisklayer.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index f76ad5c7d..e973cee77 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -1436,6 +1436,13 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept } } } + else + { + if (m_Configuration.MemCacheSizeThreshold > 0) + { + m_MemoryMissCount++; + } + } } } @@ -1669,10 +1676,6 @@ ZenCacheDiskLayer::CacheBucket::EndGetBatch(GetBatchHandle* Batch) noexcept else { m_DiskMissCount++; - if (m_Configuration.MemCacheSizeThreshold > 0) - { - m_MemoryMissCount++; - } } } } -- cgit v1.2.3 From 33d443f5361d007f4971bf0d98585b81ee691437 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 19:14:52 +0200 Subject: http client streaming upload (#413) - Improvement: Add streaming upload from HttpClient to reduce I/O caused by excessive MMap usage --- src/zencore/basicfile.cpp | 149 ++------------- src/zencore/filesystem.cpp | 126 ++++++++++++- src/zencore/include/zencore/filesystem.h | 11 ++ src/zencore/iobuffer.cpp | 42 +---- src/zenhttp/httpclient.cpp | 310 ++++++++++++++++++++++++++++--- 5 files changed, 438 insertions(+), 200 deletions(-) (limited to 'src') diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 12ee26155..993f2b616 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -181,58 +181,18 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount) void BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - - while (BytesToRead) + const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; + std::error_code Ec; + ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec); + if (Ec) { - const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); - int32_t Error = 0; - size_t BytesRead = 0; - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); - if (Success) - { - BytesRead = size_t(dwNumberOfBytesRead); - } - else - { - Error = zen::GetLastError(); - } -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset); - if (ReadResult != -1) - { - BytesRead = size_t(ReadResult); - } - else - { - Error = zen::GetLastError(); - } -#endif - - if (Error || (BytesRead != NumberOfBytesToRead)) - { - std::error_code DummyEc; - throw std::system_error(std::error_code(Error, std::system_category()), - fmt::format("ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", - FileOffset, - NumberOfBytesToRead, - PathFromHandle(m_FileHandle, DummyEc), - FileSizeFromHandle(m_FileHandle))); - } - - BytesToRead -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast(Data) + NumberOfBytesToRead; + std::error_code DummyEc; + throw std::system_error(Ec, + fmt::format("BasicFile::Read: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", + FileOffset, + BytesToRead, + PathFromHandle(m_FileHandle, DummyEc), + FileSizeFromHandle(m_FileHandle))); } } @@ -323,43 +283,9 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { - ZEN_ASSERT(m_FileHandle != nullptr); - - Ec.clear(); - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; - while (Size) - { - const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); - -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(FileOffset >> 32); - - DWORD dwNumberOfBytesWritten = 0; - - BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); - bool Success = (BytesWritten > 0); -#endif - - if (!Success) - { - Ec = MakeErrorCodeFromLastError(); - - return; - } - - Size -= NumberOfBytesToWrite; - FileOffset += NumberOfBytesToWrite; - Data = reinterpret_cast(Data) + NumberOfBytesToWrite; - } + WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec); } void @@ -405,59 +331,20 @@ BasicFile::Flush() uint64_t BasicFile::FileSize() const { -#if ZEN_PLATFORM_WINDOWS - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart == INVALID_FILE_SIZE) - { - int Error = zen::GetLastError(); - if (Error) - { - std::error_code Dummy; - ThrowSystemError(Error, fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); - } - } - return uint64_t(liFileSize.QuadPart); -#else - int Fd = int(uintptr_t(m_FileHandle)); - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - if (fstat(Fd, &Stat) == -1) + std::error_code Ec; + uint64_t FileSize = FileSizeFromHandle(m_FileHandle, Ec); + if (Ec) { std::error_code Dummy; - ThrowSystemError(GetLastError(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); + ThrowSystemError(Ec.value(), fmt::format("Failed to get file size from file '{}'", PathFromHandle(m_FileHandle, Dummy))); } - return uint64_t(Stat.st_size); -#endif + return FileSize; } uint64_t BasicFile::FileSize(std::error_code& Ec) const { -#if ZEN_PLATFORM_WINDOWS - ULARGE_INTEGER liFileSize; - liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); - if (liFileSize.LowPart == INVALID_FILE_SIZE) - { - int Error = zen::GetLastError(); - if (Error) - { - Ec = MakeErrorCode(Error); - return 0; - } - } - return uint64_t(liFileSize.QuadPart); -#else - int Fd = int(uintptr_t(m_FileHandle)); - static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); - struct stat Stat; - if (fstat(Fd, &Stat) == -1) - { - Ec = MakeErrorCodeFromLastError(); - return 0; - } - return uint64_t(Stat.st_size); -#endif + return FileSizeFromHandle(m_FileHandle, Ec); } void diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index f71397922..0a9b2a73a 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1051,6 +1051,100 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop } } +void +WriteFile(void* NativeHandle, const void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec) +{ + ZEN_ASSERT(NativeHandle != nullptr); + + Ec.clear(); + + while (Size) + { + const uint64_t NumberOfBytesToWrite = Min(Size, ChunkSize); + +#if ZEN_PLATFORM_WINDOWS + OVERLAPPED Ovl{}; + + Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); + Ovl.OffsetHigh = DWORD(FileOffset >> 32); + + DWORD dwNumberOfBytesWritten = 0; + + BOOL Success = ::WriteFile(NativeHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(NativeHandle)); + int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); + bool Success = (BytesWritten > 0); +#endif + + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + return; + } + + Size -= NumberOfBytesToWrite; + FileOffset += NumberOfBytesToWrite; + Data = reinterpret_cast(Data) + NumberOfBytesToWrite; + } +} + +void +ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec) +{ + while (Size) + { + const uint64_t NumberOfBytesToRead = Min(Size, ChunkSize); + size_t BytesRead = 0; + +#if ZEN_PLATFORM_WINDOWS + OVERLAPPED Ovl{}; + + Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); + Ovl.OffsetHigh = DWORD(FileOffset >> 32); + + DWORD dwNumberOfBytesRead = 0; + BOOL Success = ::ReadFile(NativeHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); + if (Success) + { + BytesRead = size_t(dwNumberOfBytesRead); + } + else if ((BytesRead != NumberOfBytesToRead)) + { + Ec = MakeErrorCode(ERROR_HANDLE_EOF); + return; + } + else + { + Ec = MakeErrorCodeFromLastError(); + return; + } +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(NativeHandle)); + ssize_t ReadResult = pread(Fd, Data, NumberOfBytesToRead, FileOffset); + if (ReadResult != -1) + { + BytesRead = size_t(ReadResult); + } + else if ((BytesRead != NumberOfBytesToRead)) + { + Ec = MakeErrorCode(EIO); + return; + } + else + { + Ec = MakeErrorCodeFromLastError(); + return; + } +#endif + Size -= NumberOfBytesToRead; + FileOffset += NumberOfBytesToRead; + Data = reinterpret_cast(Data) + NumberOfBytesToRead; + } +} + void WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t BufferCount) { @@ -1921,23 +2015,41 @@ FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec) } uint64_t -FileSizeFromHandle(void* NativeHandle) +FileSizeFromHandle(void* NativeHandle, std::error_code& Ec) { - uint64_t FileSize = ~0ull; - #if ZEN_PLATFORM_WINDOWS BY_HANDLE_FILE_INFORMATION Bhfh = {}; if (GetFileInformationByHandle(NativeHandle, &Bhfh)) { - FileSize = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow; + return uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow; + } + else + { + Ec = MakeErrorCodeFromLastError(); + return 0; } #else - int Fd = int(intptr_t(NativeHandle)); + int Fd = int(intptr_t(NativeHandle)); + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); struct stat Stat; - fstat(Fd, &Stat); - FileSize = size_t(Stat.st_size); + if (fstat(Fd, &Stat) == -1) + { + Ec = MakeErrorCodeFromLastError(); + return 0; + } + return uint64_t(Stat.st_size); #endif +} +uint64_t +FileSizeFromHandle(void* NativeHandle) +{ + std::error_code Ec; + uint64_t FileSize = FileSizeFromHandle(NativeHandle, Ec); + if (Ec) + { + return ~0ull; + } return FileSize; } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index e62170eba..dfd0eedc9 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -102,6 +102,10 @@ ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::er */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); +/** Query file size from native file handle + */ +ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); + /** Get a native time tick of last modification time */ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); @@ -167,6 +171,13 @@ ZENCORE_API void ScanFile(void* NativeHandle, uint64_t Size, uint64_t ChunkSize, std::function&& ProcessFunc); +ZENCORE_API void WriteFile(void* NativeHandle, + const void* Data, + uint64_t Size, + uint64_t FileOffset, + uint64_t ChunkSize, + std::error_code& Ec); +ZENCORE_API void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); struct CopyFileOptions { diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index 3b5c89c3e..8e9a37a27 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -297,49 +297,21 @@ IoBufferExtendedCore::Materialize() const AllocateBuffer(m_DataBytes, sizeof(void*)); NewFlags |= kIsOwnedByThis; - int32_t Error = 0; - size_t BytesRead = 0; -#if ZEN_PLATFORM_WINDOWS - OVERLAPPED Ovl{}; - - Ovl.Offset = DWORD(m_FileOffset & 0xffff'ffffu); - Ovl.OffsetHigh = DWORD(m_FileOffset >> 32); - - DWORD dwNumberOfBytesRead = 0; - BOOL Success = ::ReadFile(m_FileHandle, (void*)m_DataPtr, DWORD(m_DataBytes), &dwNumberOfBytesRead, &Ovl) == TRUE; - if (Success) - { - BytesRead = size_t(dwNumberOfBytesRead); - } - else - { - Error = zen::GetLastError(); - } -#else - static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); - int Fd = int(uintptr_t(m_FileHandle)); - ssize_t ReadResult = pread(Fd, (void*)m_DataPtr, m_DataBytes, m_FileOffset); - if (ReadResult != -1) - { - BytesRead = size_t(ReadResult); - } - else - { - Error = zen::GetLastError(); - } -#endif // ZEN_PLATFORM_WINDOWS - if (Error || (BytesRead != m_DataBytes)) + std::error_code Ec; + ReadFile(m_FileHandle, (void*)m_DataPtr, m_DataBytes, m_FileOffset, DisableMMapSizeLimit, Ec); + if (Ec) { std::error_code DummyEc; - ZEN_WARN("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {}", + ZEN_WARN("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x}), {} ({})", m_FileOffset, m_DataBytes, zen::PathFromHandle(m_FileHandle, DummyEc), zen::FileSizeFromHandle(m_FileHandle), - GetSystemErrorAsString(Error)); + Ec.message(), + Ec.value()); throw std::system_error( - std::error_code(Error, std::system_category()), + Ec, fmt::format("IoBufferExtendedCore::Materialize: ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", m_FileOffset, m_DataBytes, diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index a6e4d9290..f2b26b922 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -12,13 +12,19 @@ #include #include #include +#include #include #include #include #include -#include #include +#if ZEN_WITH_TESTS +# include +# include +# include +#endif // ZEN_WITH_TESTS + ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END @@ -42,6 +48,9 @@ namespace detail { class TempPayloadFile { public: + TempPayloadFile(const TempPayloadFile&) = delete; + TempPayloadFile& operator=(const TempPayloadFile&) = delete; + TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {} ~TempPayloadFile() { @@ -271,6 +280,167 @@ namespace detail { std::uint64_t m_CacheBufferOffset = 0; }; + class BufferedReadFileStream + { + public: + BufferedReadFileStream(const BufferedReadFileStream&) = delete; + BufferedReadFileStream& operator=(const BufferedReadFileStream&) = delete; + + BufferedReadFileStream(void* FileHandle, uint64_t FileOffset, uint64_t FileSize, uint64_t BufferSize) + : m_FileHandle(FileHandle) + , m_FileSize(FileSize) + , m_FileEnd(FileOffset + FileSize) + , m_BufferSize(Min(BufferSize, FileSize)) + , m_FileOffset(FileOffset) + { + } + + ~BufferedReadFileStream() { Memory::Free(m_Buffer); } + void Read(void* Data, uint64_t Size) + { + ZEN_ASSERT(Data != nullptr); + if (Size > m_BufferSize) + { + Read(Data, Size, m_FileOffset); + m_FileOffset += Size; + return; + } + uint8_t* WritePtr = ((uint8_t*)Data); + uint64_t Begin = m_FileOffset; + uint64_t End = m_FileOffset + Size; + ZEN_ASSERT(m_FileOffset >= m_BufferStart); + if (m_FileOffset < m_BufferEnd) + { + ZEN_ASSERT(m_Buffer != nullptr); + uint64_t Count = Min(m_BufferEnd, End) - m_FileOffset; + memcpy(WritePtr + Begin - m_FileOffset, m_Buffer + Begin - m_BufferStart, Count); + Begin += Count; + if (Begin == End) + { + m_FileOffset = End; + return; + } + } + if (End == m_FileEnd) + { + Read(WritePtr + Begin - m_FileOffset, End - Begin, Begin); + } + else + { + if (!m_Buffer) + { + m_BufferSize = Min(m_FileEnd - m_FileOffset, m_BufferSize); + m_Buffer = (uint8_t*)Memory::Alloc(gsl::narrow(m_BufferSize)); + } + m_BufferStart = Begin; + m_BufferEnd = Min(Begin + m_BufferSize, m_FileSize); + Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart); + uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart; + memcpy(WritePtr + Begin - m_FileOffset, m_Buffer, Count); + ZEN_ASSERT(Begin + Count == End); + } + m_FileOffset = End; + } + + private: + void Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) + { + const uint64_t MaxChunkSize = 1u * 1024 * 1024; + std::error_code Ec; + ReadFile(m_FileHandle, Data, BytesToRead, FileOffset, MaxChunkSize, Ec); + + if (Ec) + { + std::error_code DummyEc; + throw std::system_error( + Ec, + fmt::format( + "HttpClient::BufferedReadFileStream ReadFile/pread failed (offset {:#x}, size {:#x}) file: '{}' (size {:#x})", + FileOffset, + BytesToRead, + PathFromHandle(m_FileHandle, DummyEc).generic_string(), + m_FileSize)); + } + } + + void* m_FileHandle = nullptr; + const uint64_t m_FileSize = 0; + const uint64_t m_FileEnd = 0; + uint64_t m_BufferSize = 0; + uint8_t* m_Buffer = nullptr; + uint64_t m_BufferStart = 0; + uint64_t m_BufferEnd = 0; + uint64_t m_FileOffset = 0; + }; + + class CompositeBufferReadStream + { + public: + CompositeBufferReadStream(const CompositeBuffer& Data, uint64_t BufferSize) + : m_Data(Data) + , m_BufferSize(BufferSize) + , m_SegmentIndex(0) + , m_BytesLeftInSegment(0) + { + } + uint64_t Read(void* Data, uint64_t Size) + { + uint64_t Result = 0; + uint8_t* WritePtr = (uint8_t*)Data; + while ((Size > 0) && (m_SegmentIndex < m_Data.GetSegments().size())) + { + if (m_BytesLeftInSegment == 0) + { + const SharedBuffer& Segment = m_Data.GetSegments()[m_SegmentIndex]; + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if (Segment.AsIoBuffer().GetFileReference(FileRef)) + { + m_SegmentDiskBuffer = std::make_unique(FileRef.FileHandle, + FileRef.FileChunkOffset, + FileRef.FileChunkSize, + m_BufferSize); + } + else + { + m_SegmentMemoryBuffer = Segment.GetView(); + } + m_BytesLeftInSegment = Segment.GetSize(); + } + uint64_t BytesToRead = Min(m_BytesLeftInSegment, Size); + if (m_SegmentDiskBuffer) + { + m_SegmentDiskBuffer->Read(WritePtr, BytesToRead); + } + else + { + ZEN_ASSERT_SLOW(m_SegmentMemoryBuffer.GetSize() >= BytesToRead); + memcpy(WritePtr, m_SegmentMemoryBuffer.GetData(), BytesToRead); + m_SegmentMemoryBuffer.MidInline(BytesToRead); + } + WritePtr += BytesToRead; + Size -= BytesToRead; + Result += BytesToRead; + + m_BytesLeftInSegment -= BytesToRead; + if (m_BytesLeftInSegment == 0) + { + m_SegmentDiskBuffer.reset(); + m_SegmentMemoryBuffer.Reset(); + m_SegmentIndex++; + } + } + return Result; + } + + private: + const CompositeBuffer& m_Data; + const uint64_t m_BufferSize; + size_t m_SegmentIndex; + std::unique_ptr m_SegmentDiskBuffer; + MemoryView m_SegmentMemoryBuffer; + uint64_t m_BytesLeftInSegment; + }; + } // namespace detail ////////////////////////////////////////////////////////////////////////// @@ -1005,9 +1175,22 @@ HttpClient::Post(std::string_view Url, const IoBuffer& Payload, ZenContentType C [&]() { Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Sess->UpdateHeader({HeaderContentType(ContentType)}); + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if (Payload.GetFileReference(FileRef)) + { + uint64_t Offset = 0; + detail::BufferedReadFileStream Buffer(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, 512u * 1024u); + auto ReadCallback = [&Payload, &Offset, &Buffer](char* buffer, size_t& size, intptr_t) { + size = Min(size, Payload.GetSize() - Offset); + Buffer.Read(buffer, size); + Offset += size; + return true; + }; + return Sess.Post(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); + } Sess->SetBody(AsCprBody(Payload)); - Sess->UpdateHeader({HeaderContentType(ContentType)}); return Sess.Post(); }, m_ConnectionSettings.RetryCount)); @@ -1049,19 +1232,15 @@ HttpClient::Post(std::string_view Url, const CompositeBuffer& Payload, ZenConten DoWithRetry( m_SessionId, [&]() { - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; - return true; - }; Impl::Session Sess = m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(ContentType)}); + detail::CompositeBufferReadStream Reader(Payload, 512u * 1024u); + auto ReadCallback = [&Reader](char* buffer, size_t& size, intptr_t) { + size = Reader.Read(buffer, size); + return true; + }; return Sess.Post(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); }, m_ConnectionSettings.RetryCount)); @@ -1081,16 +1260,16 @@ HttpClient::Upload(std::string_view Url, const IoBuffer& Payload, const KeyValue m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(Payload.GetContentType())}); - uint64_t Offset = 0; - if (Payload.IsWholeFile()) + IoBufferFileReference FileRef = {nullptr, 0, 0}; + if (Payload.GetFileReference(FileRef)) { - auto ReadCallback = [&Payload, &Offset](char* buffer, size_t& size, intptr_t) { - size = Min(size, Payload.GetSize() - Offset); - IoBuffer PayloadRange = IoBuffer(Payload, Offset, size); - MutableMemoryView Data(buffer, size); - Data.CopyFrom(PayloadRange.GetView()); - Offset += size; - return true; + uint64_t Offset = 0; + detail::BufferedReadFileStream Buffer(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, 512u * 1024u); + auto ReadCallback = [&Payload, &Offset, &Buffer](char* buffer, size_t& size, intptr_t) { + size = Min(size, Payload.GetSize() - Offset); + Buffer.Read(buffer, size); + Offset += size; + return true; }; return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); } @@ -1114,13 +1293,9 @@ HttpClient::Upload(std::string_view Url, const CompositeBuffer& Payload, ZenCont m_Impl->AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); Sess->UpdateHeader({HeaderContentType(ContentType)}); - uint64_t SizeLeft = Payload.GetSize(); - CompositeBuffer::Iterator BufferIt = Payload.GetIterator(0); - auto ReadCallback = [&Payload, &BufferIt, &SizeLeft](char* buffer, size_t& size, intptr_t) { - size = Min(size, SizeLeft); - MutableMemoryView Data(buffer, size); - Payload.CopyTo(Data, BufferIt); - SizeLeft -= size; + detail::CompositeBufferReadStream Reader(Payload, 512u * 1024u); + auto ReadCallback = [&Reader](char* buffer, size_t& size, intptr_t) { + size = Reader.Read(buffer, size); return true; }; return Sess.Put(cpr::ReadCallback(gsl::narrow(Payload.GetSize()), ReadCallback)); @@ -1479,6 +1654,40 @@ HttpClient::Response::ThrowError(std::string_view ErrorPrefix) #if ZEN_WITH_TESTS +namespace testutil { + IoHash HashComposite(const CompositeBuffer& Payload) + { + IoHashStream Hasher; + const uint64_t PayloadSize = Payload.GetSize(); + std::vector Buffer(64u * 1024u); + detail::CompositeBufferReadStream Stream(Payload, 137u * 1024u); + for (uint64_t Offset = 0; Offset < PayloadSize;) + { + uint64_t Count = Min(64u * 1024u, PayloadSize - Offset); + Stream.Read(Buffer.data(), Count); + Hasher.Append(Buffer.data(), Count); + Offset += Count; + } + return Hasher.GetHash(); + }; + + IoHash HashFileStream(void* FileHandle, uint64_t FileOffset, uint64_t FileSize) + { + IoHashStream Hasher; + std::vector Buffer(64u * 1024u); + detail::BufferedReadFileStream Stream(FileHandle, FileOffset, FileSize, 137u * 1024u); + for (uint64_t Offset = 0; Offset < FileSize;) + { + uint64_t Count = Min(64u * 1024u, FileSize - Offset); + Stream.Read(Buffer.data(), Count); + Hasher.Append(Buffer.data(), Count); + Offset += Count; + } + return Hasher.GetHash(); + } + +} // namespace testutil + TEST_CASE("responseformat") { using namespace std::literals; @@ -1525,6 +1734,53 @@ TEST_CASE("responseformat") } } +TEST_CASE("BufferedReadFileStream") +{ + ScopedTemporaryDirectory TmpDir; + + IoBuffer DiskBuffer = WriteToTempFile(CompositeBuffer(CreateRandomBlob(496 * 5 * 1024)), TmpDir.Path() / "diskbuffer1"); + + IoBufferFileReference FileRef = {nullptr, 0, 0}; + CHECK(DiskBuffer.GetFileReference(FileRef)); + CHECK_EQ(IoHash::HashBuffer(DiskBuffer), testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize)); + + IoBuffer Partial(DiskBuffer, 37 * 1024, 512 * 1024); + CHECK(Partial.GetFileReference(FileRef)); + CHECK_EQ(IoHash::HashBuffer(Partial), testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize)); + + IoBuffer SmallDiskBuffer = WriteToTempFile(CompositeBuffer(CreateRandomBlob(63 * 1024)), TmpDir.Path() / "diskbuffer2"); + CHECK(SmallDiskBuffer.GetFileReference(FileRef)); + CHECK_EQ(IoHash::HashBuffer(SmallDiskBuffer), + testutil::HashFileStream(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize)); +} + +TEST_CASE("CompositeBufferReadStream") +{ + ScopedTemporaryDirectory TmpDir; + + IoBuffer MemoryBuffer1 = CreateRandomBlob(64); + CHECK_EQ(IoHash::HashBuffer(MemoryBuffer1), testutil::HashComposite(CompositeBuffer(SharedBuffer(MemoryBuffer1)))); + + IoBuffer MemoryBuffer2 = CreateRandomBlob(561 * 1024); + CHECK_EQ(IoHash::HashBuffer(MemoryBuffer2), testutil::HashComposite(CompositeBuffer(SharedBuffer(MemoryBuffer2)))); + + IoBuffer DiskBuffer1 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(267 * 3 * 1024)), TmpDir.Path() / "diskbuffer1"); + CHECK_EQ(IoHash::HashBuffer(DiskBuffer1), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer1)))); + + IoBuffer DiskBuffer2 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(3 * 1024)), TmpDir.Path() / "diskbuffer2"); + CHECK_EQ(IoHash::HashBuffer(DiskBuffer2), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer2)))); + + IoBuffer DiskBuffer3 = WriteToTempFile(CompositeBuffer(CreateRandomBlob(496 * 5 * 1024)), TmpDir.Path() / "diskbuffer3"); + CHECK_EQ(IoHash::HashBuffer(DiskBuffer3), testutil::HashComposite(CompositeBuffer(SharedBuffer(DiskBuffer3)))); + + CompositeBuffer Data(SharedBuffer(std::move(MemoryBuffer1)), + SharedBuffer(std::move(DiskBuffer1)), + SharedBuffer(std::move(DiskBuffer2)), + SharedBuffer(std::move(MemoryBuffer2)), + SharedBuffer(std::move(DiskBuffer3))); + CHECK_EQ(IoHash::HashBuffer(Data), testutil::HashComposite(Data)); +} + TEST_CASE("httpclient") { using namespace std::literals; -- cgit v1.2.3 From e70338d57be3694b95f4c359a3e852e273ad5e2d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 3 Jun 2025 15:09:49 +0200 Subject: minor: fix unused variable warning on some compilers --- src/zen/cmds/admin_cmd.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index a7cfa6a4e..b3f8a990e 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -58,8 +58,8 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (int StatusCode = (int)Response.StatusCode) { ZEN_ERROR("scrub start failed: {}: {} ({})", - (int)Response.StatusCode, - ReasonStringForHttpResultCode((int)Response.StatusCode), + StatusCode, + ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else @@ -646,8 +646,8 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (int StatusCode = (int)Response.StatusCode) { ZEN_ERROR("flush failed: {}: {} ({})", - (int)Response.StatusCode, - ReasonStringForHttpResultCode((int)Response.StatusCode), + StatusCode, + ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else -- cgit v1.2.3 From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- src/zen/cmds/builds_cmd.cpp | 549 +++++++++++---------- src/zencore/basicfile.cpp | 10 +- src/zencore/blake3.cpp | 22 + src/zencore/compress.cpp | 50 +- src/zencore/filesystem.cpp | 23 +- src/zencore/include/zencore/blake3.h | 1 + src/zencore/include/zencore/iohash.h | 6 + src/zenutil/bufferedwritefilecache.cpp | 177 +++++++ src/zenutil/chunkedcontent.cpp | 4 + src/zenutil/chunkingcontroller.cpp | 289 ++++++----- src/zenutil/filebuildstorage.cpp | 19 +- .../include/zenutil/bufferedwritefilecache.h | 106 ++++ src/zenutil/include/zenutil/chunkedcontent.h | 1 + src/zenutil/include/zenutil/chunkingcontroller.h | 45 +- 14 files changed, 873 insertions(+), 429 deletions(-) create mode 100644 src/zenutil/bufferedwritefilecache.cpp create mode 100644 src/zenutil/include/zenutil/bufferedwritefilecache.h (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f6f15acb0..e13c90b4b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +106,40 @@ namespace { } WorkerThreadPool& GetNetworkPool() { return SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); } + static const std::vector NonCompressableExtensions({HashStringDjb2(".mp4"sv), + HashStringDjb2(".zip"sv), + HashStringDjb2(".7z"sv), + HashStringDjb2(".bzip"sv), + HashStringDjb2(".rar"sv), + HashStringDjb2(".gzip"sv), + HashStringDjb2(".apk"sv), + HashStringDjb2(".nsp"sv), + HashStringDjb2(".xvc"sv), + HashStringDjb2(".pkg"sv), + HashStringDjb2(".dmg"sv), + HashStringDjb2(".ipa"sv)}); + + static const tsl::robin_set NonCompressableExtensionSet(NonCompressableExtensions.begin(), NonCompressableExtensions.end()); + + static bool IsExtensionHashCompressable(const uint32_t PathHash) { return !NonCompressableExtensionSet.contains(PathHash); } + + static bool IsChunkCompressable(const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) + { + ZEN_UNUSED(Content); + const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex]; + if (ChunkLocationCount == 0) + { + return false; + } + const size_t ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex]; + const uint32_t SequenceIndex = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex; + const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const uint32_t ExtensionHash = Lookup.PathExtensionHash[PathIndex]; + + const bool IsCompressable = IsExtensionHashCompressable(ExtensionHash); + return IsCompressable; + } + const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; const std::string ZenFolderName = ".zen"; @@ -313,7 +348,10 @@ namespace { std::atomic& WriteByteCount) { BasicFile TargetFile(TargetFilePath, BasicFile::Mode::kTruncate); - PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize); + if (UseSparseFiles) + { + PrepareFileForScatteredWrite(TargetFile.Handle(), RawSize); + } uint64_t Offset = 0; if (!ScanFile(SourceFilePath, 512u * 1024u, [&](const void* Data, size_t Size) { TargetFile.Write(Data, Size, Offset); @@ -600,24 +638,6 @@ namespace { return AuthToken; } - bool IsBufferDiskBased(const IoBuffer& Buffer) - { - IoBufferFileReference FileRef; - if (Buffer.GetFileReference(FileRef)) - { - return true; - } - return false; - } - - bool IsBufferDiskBased(const CompositeBuffer& Buffer) - { - // If this is a file based buffer or a compressed buffer with a memory-based header, we don't need to rewrite to disk to save memory - std::span Segments = Buffer.GetSegments(); - ZEN_ASSERT(Buffer.GetSegments().size() > 0); - return IsBufferDiskBased(Segments.back().AsIoBuffer()); - } - IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& TempFolderPath, const IoHash& Hash, @@ -1729,16 +1749,13 @@ namespace { { ZEN_ASSERT(false); } - uint64_t RawSize = Chunk.GetSize(); - if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) - { - // Standalone chunk, not part of a sequence - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid)}; - } - else - { - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; - } + uint64_t RawSize = Chunk.GetSize(); + const bool ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) && + (RawSize >= MinimumSizeForCompressInBlock) && + IsChunkCompressable(Content, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = + ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel)}; })); } @@ -1768,18 +1785,15 @@ namespace { Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); - const uint64_t RawSize = Chunk.GetSize(); - if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) - { - CompositeBuffer CompressedChunk = CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); - } - else - { - CompositeBuffer CompressedChunk = - CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); - } + const uint64_t RawSize = Chunk.GetSize(); + const bool ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) && + (RawSize >= MinimumSizeForCompressInBlock) && IsChunkCompressable(Content, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = + ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); } return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); }; @@ -2253,7 +2267,11 @@ namespace { { throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash)); } - ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); + + const bool ShouldCompressChunk = IsChunkCompressable(Content, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; + + if (ShouldCompressChunk) { std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString(); @@ -2266,6 +2284,9 @@ namespace { fmt::format("Failed creating temporary file for compressing blob {}. Reason: {}", ChunkHash, Ec.message())); } + uint64_t StreamRawBytes = 0; + uint64_t StreamCompressedBytes = 0; + bool CouldCompress = CompressedBuffer::CompressToStream( CompositeBuffer(SharedBuffer(RawSource)), [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { @@ -2273,7 +2294,11 @@ namespace { LooseChunksStats.CompressedChunkRawBytes += SourceSize; CompressedFile.Write(RangeBuffer, Offset); LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); - }); + StreamRawBytes += SourceSize; + StreamCompressedBytes += RangeBuffer.GetSize(); + }, + OodleCompressor::Mermaid, + CompressionLevel); if (CouldCompress) { uint64_t CompressedSize = CompressedFile.FileSize(); @@ -2296,22 +2321,36 @@ namespace { return Compressed.GetCompressed(); } + else + { + LooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes; + LooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes; + } CompressedFile.Close(); RemoveFile(TempFilePath, Ec); ZEN_UNUSED(Ec); } - // Try regular compress - decompress may fail if compressed data is larger than non-compressed - CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(RawSource))); + CompressedBuffer CompressedBlob = + CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel); if (!CompressedBlob) { throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash)); } - if (!IsBufferDiskBased(CompressedBlob.GetCompressed())) + ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash); + ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize); + + LooseChunksStats.CompressedChunkRawBytes += ChunkSize; + LooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize(); + + // If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk + if (ShouldCompressChunk) { IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFolderPath, ChunkHash); CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload)); } + + LooseChunksStats.CompressedChunkCount++; return std::move(CompressedBlob).GetCompressed(); } @@ -3431,7 +3470,8 @@ namespace { LocalFolderScanStats.ElapsedWallTimeUS = ManifestParseTimer.GetElapsedTimeUs(); } - std::unique_ptr ChunkController = CreateBasicChunkingController(); + std::unique_ptr ChunkController = + CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); { CbObjectWriter ChunkParametersWriter; ChunkParametersWriter.AddString("name"sv, ChunkController->GetName()); @@ -4399,106 +4439,6 @@ namespace { } } - class WriteFileCache - { - public: - WriteFileCache(DiskStatistics& DiskStats) : m_DiskStats(DiskStats) {} - ~WriteFileCache() { Flush(); } - - template - void WriteToFile(uint32_t TargetIndex, - std::function&& GetTargetPath, - const TBufferType& Buffer, - uint64_t FileOffset, - uint64_t TargetFinalSize) - { - ZEN_TRACE_CPU("WriteFileCache_WriteToFile"); - if (!SeenTargetIndexes.empty() && SeenTargetIndexes.back() == TargetIndex) - { - ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); - ZEN_ASSERT(OpenFileWriter); - OpenFileWriter->Write(Buffer, FileOffset); - m_DiskStats.WriteCount++; - m_DiskStats.WriteByteCount += Buffer.GetSize(); - } - else - { - std::unique_ptr NewOutputFile; - { - ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Open"); - Flush(); - const std::filesystem::path& TargetPath = GetTargetPath(TargetIndex); - CreateDirectories(TargetPath.parent_path()); - uint32_t Tries = 5; - NewOutputFile = - std::make_unique(TargetPath, BasicFile::Mode::kWrite, [&Tries, TargetPath](std::error_code& Ec) { - if (Tries < 3) - { - ZEN_CONSOLE("Failed opening file '{}': {}{}", TargetPath, Ec.message(), Tries > 1 ? " Retrying"sv : ""sv); - } - if (Tries > 1) - { - Sleep(100); - } - return --Tries > 0; - }); - m_DiskStats.OpenWriteCount++; - m_DiskStats.CurrentOpenFileCount++; - } - - const bool CacheWriter = TargetFinalSize > Buffer.GetSize(); - if (CacheWriter) - { - ZEN_TRACE_CPU("WriteFileCache_WriteToFile_CacheWrite"); - ZEN_ASSERT_SLOW(std::find(SeenTargetIndexes.begin(), SeenTargetIndexes.end(), TargetIndex) == SeenTargetIndexes.end()); - - OutputFile = std::move(NewOutputFile); - if (UseSparseFiles) - { - void* Handle = OutputFile->Handle(); - if (!PrepareFileForScatteredWrite(Handle, TargetFinalSize)) - { - ZEN_DEBUG("Unable to to prepare file '{}' with size {} for random write", - GetTargetPath(TargetIndex), - TargetFinalSize); - } - } - OpenFileWriter = std::make_unique(*OutputFile, Min(TargetFinalSize, 256u * 1024u)); - - OpenFileWriter->Write(Buffer, FileOffset); - m_DiskStats.WriteCount++; - m_DiskStats.WriteByteCount += Buffer.GetSize(); - SeenTargetIndexes.push_back(TargetIndex); - } - else - { - ZEN_TRACE_CPU("WriteFileCache_WriteToFile_Write"); - NewOutputFile->Write(Buffer, FileOffset); - m_DiskStats.WriteCount++; - m_DiskStats.WriteByteCount += Buffer.GetSize(); - NewOutputFile = {}; - m_DiskStats.CurrentOpenFileCount--; - } - } - } - - void Flush() - { - ZEN_TRACE_CPU("WriteFileCache_Flush"); - if (OutputFile) - { - m_DiskStats.CurrentOpenFileCount--; - } - - OpenFileWriter = {}; - OutputFile = {}; - } - DiskStatistics& m_DiskStats; - std::vector SeenTargetIndexes; - std::unique_ptr OutputFile; - std::unique_ptr OpenFileWriter; - }; - std::vector GetRemainingChunkTargets( std::span> SequenceIndexChunksLeftToWriteCounters, const ChunkedContentLookup& Lookup, @@ -4581,7 +4521,7 @@ namespace { VerifySize, ExpectedSize)); } - ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer)); if (VerifyChunkHash != SequenceRawHash) { @@ -4651,6 +4591,77 @@ namespace { return CompletedSequenceIndexes; } + void WriteSequenceChunk(const std::filesystem::path& TargetFolderPath, + const ChunkedFolderContent& RemoteContent, + BufferedWriteFileCache::Local& LocalWriter, + const CompositeBuffer& Chunk, + const uint32_t SequenceIndex, + const uint64_t FileOffset, + const uint32_t PathIndex, + DiskStatistics& DiskStats) + { + ZEN_TRACE_CPU("WriteSequenceChunk"); + + const uint64_t SequenceSize = RemoteContent.RawSizes[PathIndex]; + + auto OpenFile = [&](BasicFile& File) { + const std::filesystem::path FileName = + GetTempChunkedSequenceFileName(TargetFolderPath, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + File.Open(FileName, BasicFile::Mode::kWrite); + if (UseSparseFiles) + { + PrepareFileForScatteredWrite(File.Handle(), SequenceSize); + } + }; + + const uint64_t ChunkSize = Chunk.GetSize(); + ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize); + if (ChunkSize == SequenceSize) + { + BasicFile SingleChunkFile; + OpenFile(SingleChunkFile); + + DiskStats.CurrentOpenFileCount++; + auto _ = MakeGuard([&DiskStats]() { DiskStats.CurrentOpenFileCount--; }); + SingleChunkFile.Write(Chunk, FileOffset); + + DiskStats.WriteCount++; + DiskStats.WriteByteCount += ChunkSize; + } + else + { + const uint64_t MaxWriterBufferSize = 256u * 1025u; + + BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex); + if (Writer) + { + if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize)) + { + Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); + } + Writer->Write(Chunk, FileOffset); + + DiskStats.WriteCount++; + DiskStats.WriteByteCount += ChunkSize; + } + else + { + Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique()); + + Writer->File = std::make_unique(); + OpenFile(*Writer->File); + if (ChunkSize < MaxWriterBufferSize) + { + Writer->Writer = std::make_unique(*Writer->File, Min(SequenceSize, MaxWriterBufferSize)); + } + Writer->Write(Chunk, FileOffset); + + DiskStats.WriteCount++; + DiskStats.WriteByteCount += ChunkSize; + } + } + } + struct BlockWriteOps { std::vector ChunkBuffers; @@ -4667,13 +4678,15 @@ namespace { const ChunkedContentLookup& Lookup, std::span> SequenceIndexChunksLeftToWriteCounters, const BlockWriteOps& Ops, + BufferedWriteFileCache& WriteCache, ParallelWork& Work, WorkerThreadPool& VerifyPool, DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockChunkOps"); + { - WriteFileCache OpenFileCache(DiskStats); + BufferedWriteFileCache::Local LocalWriter(WriteCache); for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) { if (AbortFlag) @@ -4685,25 +4698,15 @@ namespace { ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <= RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]); ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0); - const uint64_t ChunkSize = Chunk.GetSize(); const uint64_t FileOffset = WriteOp.Target->Offset; const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; - ZEN_ASSERT(FileOffset + ChunkSize <= RemoteContent.RawSizes[PathIndex]); - OpenFileCache.WriteToFile( - SequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName(CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - Chunk, - FileOffset, - RemoteContent.RawSizes[PathIndex]); + WriteSequenceChunk(CacheFolderPath, RemoteContent, LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex, DiskStats); } } if (!AbortFlag) { - // Write tracking, updating this must be done without any files open (WriteFileCache) + // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local) std::vector CompletedChunkSequences; for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps) { @@ -4713,6 +4716,7 @@ namespace { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } + WriteCache.Close(CompletedChunkSequences); VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, Lookup, CompletedChunkSequences, Work, VerifyPool); } } @@ -4864,6 +4868,7 @@ namespace { CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache, DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockToDisk"); @@ -4896,6 +4901,7 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + WriteCache, Work, VerifyPool, DiskStats); @@ -4920,6 +4926,7 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + WriteCache, Work, VerifyPool, DiskStats); @@ -4939,6 +4946,7 @@ namespace { uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, + BufferedWriteFileCache& WriteCache, DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); @@ -4963,6 +4971,7 @@ namespace { Lookup, SequenceIndexChunksLeftToWriteCounters, Ops, + WriteCache, Work, VerifyPool, DiskStats); @@ -4974,75 +4983,15 @@ namespace { } } - SharedBuffer Decompress(CompositeBuffer&& CompressedChunk, const IoHash& ChunkHash, const uint64_t ChunkRawSize) - { - ZEN_TRACE_CPU("Decompress"); - - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedChunk, RawHash, RawSize); - if (!Compressed) - { - throw std::runtime_error(fmt::format("Invalid build blob format for chunk {}", ChunkHash)); - } - if (RawHash != ChunkHash) - { - throw std::runtime_error(fmt::format("Mismatching build blob {}, but compressed header rawhash is {}", ChunkHash, RawHash)); - } - if (RawSize != ChunkRawSize) - { - throw std::runtime_error( - fmt::format("Mismatching build blob {}, expected raw size {} but recevied raw size {}", ChunkHash, ChunkRawSize, RawSize)); - } - if (!Compressed) - { - throw std::runtime_error(fmt::format("Invalid build blob {}, not a compressed buffer", ChunkHash)); - } - - SharedBuffer Decompressed = Compressed.Decompress(); - - if (!Decompressed) - { - throw std::runtime_error(fmt::format("Decompression of build blob {} failed", ChunkHash)); - } - return Decompressed; - } - - void WriteChunkToDisk(const std::filesystem::path& CacheFolderPath, - const ChunkedFolderContent& Content, - const ChunkedContentLookup& Lookup, - std::span ChunkTargets, - CompositeBuffer&& ChunkData, - WriteFileCache& OpenFileCache) - { - ZEN_TRACE_CPU("WriteChunkToDisk"); - - for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargets) - { - const auto& Target = *TargetPtr; - const uint64_t FileOffset = Target.Offset; - const uint32_t SequenceIndex = Target.SequenceIndex; - const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; - - OpenFileCache.WriteToFile( - SequenceIndex, - [&CacheFolderPath, &Content](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName(CacheFolderPath, Content.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkData, - FileOffset, - Content.RawSizes[PathIndex]); - } - } - - bool CanDecompressDirectToSequence(const ChunkedFolderContent& RemoteContent, - const std::vector Locations) + bool IsSingleFileChunk(const ChunkedFolderContent& RemoteContent, + const std::vector Locations) { if (Locations.size() == 1) { const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex; - if (Locations[0]->Offset == 0 && RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1) + if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1) { + ZEN_ASSERT_SLOW(Locations[0]->Offset == 0); return true; } } @@ -5087,13 +5036,14 @@ namespace { DiskStats.ReadByteCount += SourceSize; if (!AbortFlag) { - DecompressedTemp.Write(RangeBuffer, Offset); for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) { Hash.Append(Segment.GetView()); + DecompressedTemp.Write(Segment, Offset); + Offset += Segment.GetSize(); + DiskStats.WriteByteCount += Segment.GetSize(); + DiskStats.WriteCount++; } - DiskStats.WriteByteCount += RangeBuffer.GetSize(); - DiskStats.WriteCount++; return true; } return false; @@ -5128,37 +5078,80 @@ namespace { const ChunkedContentLookup& RemoteLookup, const IoHash& ChunkHash, const std::vector& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, IoBuffer&& CompressedPart, DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteCompressedChunk"); auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); - const uint32_t ChunkIndex = ChunkHashToChunkIndexIt->second; - const uint64_t ChunkRawSize = RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]; - if (CanDecompressDirectToSequence(RemoteContent, ChunkTargetPtrs)) + if (IsSingleFileChunk(RemoteContent, ChunkTargetPtrs)) { const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats); + return false; } else { - SharedBuffer Chunk = Decompress(CompositeBuffer(std::move(CompressedPart)), ChunkHash, ChunkRawSize); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize); + if (!Compressed) + { + throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash)); + } + if (RawHash != ChunkHash) + { + throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash)); + } - if (!AbortFlag) + BufferedWriteFileCache::Local LocalWriter(WriteCache); + + IoHashStream Hash; + bool CouldDecompress = Compressed.DecompressToStream( + 0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset); + ZEN_TRACE_CPU("StreamDecompress_Write"); + DiskStats.ReadByteCount += SourceSize; + if (!AbortFlag) + { + for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs) + { + const auto& Target = *TargetPtr; + const uint64_t FileOffset = Target.Offset + Offset; + const uint32_t SequenceIndex = Target.SequenceIndex; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + + WriteSequenceChunk(TargetFolder, + RemoteContent, + LocalWriter, + RangeBuffer, + SequenceIndex, + FileOffset, + PathIndex, + DiskStats); + } + + return true; + } + return false; + }); + + if (AbortFlag) { - WriteFileCache OpenFileCache(DiskStats); - WriteChunkToDisk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkTargetPtrs, - CompositeBuffer(std::move(Chunk)), - OpenFileCache); - return true; + return false; + } + + if (!CouldDecompress) + { + throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash)); } + + return true; } - return false; } void AsyncWriteDownloadedChunk(const std::filesystem::path& ZenFolderPath, @@ -5166,6 +5159,7 @@ namespace { const ChunkedContentLookup& RemoteLookup, uint32_t RemoteChunkIndex, std::vector&& ChunkTargetPtrs, + BufferedWriteFileCache& WriteCache, ParallelWork& Work, WorkerThreadPool& WritePool, IoBuffer&& Payload, @@ -5230,6 +5224,7 @@ namespace { CompressedChunkPath, RemoteChunkIndex, TotalPartWriteCount, + &WriteCache, &DiskStats, &WritePartsComplete, &FilteredWrittenBytesPerSecond, @@ -5264,6 +5259,7 @@ namespace { RemoteLookup, ChunkHash, ChunkTargetPtrs, + WriteCache, std::move(CompressedPart), DiskStats); if (!AbortFlag) @@ -5281,6 +5277,7 @@ namespace { std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + WriteCache.Close(CompletedSequences); if (NeedHashVerify) { VerifyAndCompleteChunkSequencesAsync(TargetFolder, @@ -6387,6 +6384,8 @@ namespace { } } + BufferedWriteFileCache WriteCache; + for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < ScavengeCopyOperations.size(); ScavengeOpIndex++) { if (AbortFlag) @@ -6487,6 +6486,7 @@ namespace { PreferredMultipartChunkSize, TotalRequestCount, TotalPartWriteCount, + &WriteCache, &FilteredDownloadedBytesPerSecond, &FilteredWrittenBytesPerSecond](std::atomic&) mutable { if (!AbortFlag) @@ -6535,6 +6535,7 @@ namespace { &RemoteLookup, &CacheFolderPath, &SequenceIndexChunksLeftToWriteCounters, + &WriteCache, &Work, &WritePool, &DiskStats, @@ -6568,6 +6569,7 @@ namespace { RemoteLookup, ChunkHash, ChunkTargetPtrs, + WriteCache, std::move(CompressedPart), DiskStats); WritePartsComplete++; @@ -6583,6 +6585,7 @@ namespace { std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + WriteCache.Close(CompletedSequences); if (NeedHashVerify) { VerifyAndCompleteChunkSequencesAsync(TargetFolder, @@ -6608,11 +6611,12 @@ namespace { &ZenFolderPath, &Storage, BuildId = Oid(BuildId), - &PrimeCacheOnly, + PrimeCacheOnly, &RemoteContent, &RemoteLookup, &ExistsResult, &SequenceIndexChunksLeftToWriteCounters, + &WriteCache, &Work, &WritePool, &NetworkPool, @@ -6654,6 +6658,7 @@ namespace { RemoteLookup, RemoteChunkIndex, std::move(ChunkTargetPtrs), + WriteCache, Work, WritePool, std::move(BuildBlob), @@ -6683,6 +6688,7 @@ namespace { BuildId, PrimeCacheOnly, &SequenceIndexChunksLeftToWriteCounters, + &WriteCache, &Work, &WritePool, ChunkHash, @@ -6718,6 +6724,7 @@ namespace { RemoteLookup, RemoteChunkIndex, std::move(ChunkTargetPtrs), + WriteCache, Work, WritePool, std::move(Payload), @@ -6763,6 +6770,7 @@ namespace { RemoteLookup, RemoteChunkIndex, std::move(ChunkTargetPtrs), + WriteCache, Work, WritePool, std::move(BuildBlob), @@ -6800,6 +6808,7 @@ namespace { &CacheFolderPath, &LocalLookup, &SequenceIndexChunksLeftToWriteCounters, + &WriteCache, &Work, &WritePool, &FilteredWrittenBytesPerSecond, @@ -6892,8 +6901,9 @@ namespace { tsl::robin_set ChunkIndexesWritten; - BufferedOpenFile SourceFile(SourceFilePath, DiskStats); - WriteFileCache OpenFileCache(DiskStats); + BufferedOpenFile SourceFile(SourceFilePath, DiskStats); + BufferedWriteFileCache::Local LocalWriter(WriteCache); + for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();) { if (AbortFlag) @@ -6940,18 +6950,17 @@ namespace { } CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength); - ZEN_ASSERT(Op.Target->Offset + ChunkSource.GetSize() <= RemoteContent.RawSizes[RemotePathIndex]); - - OpenFileCache.WriteToFile( - RemoteSequenceIndex, - [&CacheFolderPath, &RemoteContent](uint32_t SequenceIndex) { - return GetTempChunkedSequenceFileName( - CacheFolderPath, - RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - }, - ChunkSource, - Op.Target->Offset, - RemoteContent.RawSizes[RemotePathIndex]); + + const uint64_t FileOffset = Op.Target->Offset; + + WriteSequenceChunk(CacheFolderPath, + RemoteContent, + LocalWriter, + ChunkSource, + RemoteSequenceIndex, + FileOffset, + RemotePathIndex, + DiskStats); CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? @@ -6960,7 +6969,7 @@ namespace { } if (!AbortFlag) { - // Write tracking, updating this must be done without any files open (WriteFileCache) + // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local) std::vector CompletedChunkSequences; for (const WriteOp& Op : WriteOps) { @@ -6970,6 +6979,7 @@ namespace { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } + WriteCache.Close(CompletedChunkSequences); VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, RemoteLookup, @@ -7003,6 +7013,7 @@ namespace { &RemoteLookup, &RemoteChunkIndexNeedsCopyFromSourceFlags, &SequenceIndexChunksLeftToWriteCounters, + &WriteCache, &Work, &WritePool, &BlockDescriptions, @@ -7038,6 +7049,7 @@ namespace { CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, + WriteCache, DiskStats)) { std::error_code DummyEc; @@ -7080,6 +7092,7 @@ namespace { &RemoteContent, &SequenceIndexChunksLeftToWriteCounters, &ExistsResult, + &WriteCache, &FilteredDownloadedBytesPerSecond, TotalRequestCount, &WritePartsComplete, @@ -7185,6 +7198,7 @@ namespace { &RemoteChunkIndexNeedsCopyFromSourceFlags, &SequenceIndexChunksLeftToWriteCounters, &WritePartsComplete, + &WriteCache, &Work, TotalPartWriteCount, &WritePool, @@ -7230,6 +7244,7 @@ namespace { BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, + WriteCache, DiskStats)) { std::error_code DummyEc; @@ -7284,6 +7299,7 @@ namespace { &WritePool, &RemoteContent, &RemoteLookup, + &WriteCache, &CacheFolderPath, &RemoteChunkIndexNeedsCopyFromSourceFlags, &SequenceIndexChunksLeftToWriteCounters, @@ -7388,6 +7404,7 @@ namespace { &SequenceIndexChunksLeftToWriteCounters, BlockIndex, &BlockDescriptions, + &WriteCache, &WriteChunkStats, &DiskStats, &WritePartsComplete, @@ -7429,6 +7446,7 @@ namespace { CompositeBuffer(std::move(BlockBuffer)), RemoteLookup, RemoteChunkIndexNeedsCopyFromSourceFlags, + WriteCache, DiskStats)) { std::error_code DummyEc; @@ -7753,6 +7771,7 @@ namespace { Work.ScheduleWork(WritePool, [&Path, &LocalContent, &DeletedCount, LocalPathIndex](std::atomic&) { if (!AbortFlag) { + ZEN_TRACE_CPU("FinalizeTree_Remove"); const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); SetFileReadOnlyWithRetry(LocalFilePath, false); RemoveFileWithRetry(LocalFilePath); @@ -8815,7 +8834,7 @@ namespace { if (!ChunkController) { ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); - ChunkController = CreateBasicChunkingController(); + ChunkController = CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); } LocalContent = GetLocalContent(LocalFolderScanStats, @@ -8899,7 +8918,10 @@ namespace { { BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path); + + uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0)); + + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -9037,8 +9059,9 @@ namespace { ChunkedFolderContent CompareFolderContent; { - std::unique_ptr ChunkController = CreateBasicChunkingController(); - std::vector ExcludeExtensions = DefaultExcludeExtensions; + std::unique_ptr ChunkController = + CreateChunkingControllerWithFixedChunking(ChunkingControllerWithFixedChunkingSettings{}); + std::vector ExcludeExtensions = DefaultExcludeExtensions; if (OnlyChunked) { ExcludeExtensions.insert(ExcludeExtensions.end(), @@ -10717,7 +10740,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ExtendableStringBuilder<256> SB; CompactBinaryToJson(MetaData, SB); - ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); + ZEN_CONSOLE("Upload Build {}, Part {} ({}) from '{}'\n{}", m_BuildId, BuildPartId, m_BuildPartName, m_Path, SB.ToView()); } const std::filesystem::path UploadTempDir = UploadTempDirectory(m_Path); diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index 993f2b616..6989da67e 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -283,7 +283,7 @@ BasicFile::Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec) void BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec) { - const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; + const uint64_t MaxChunkSize = 2u * 1024 * 1024; WriteFile(m_FileHandle, Data, Size, FileOffset, MaxChunkSize, Ec); } @@ -794,7 +794,7 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) { uint64_t Offset = 0; static const uint64_t BufferingSize = 256u * 1024u; - // BasicFileWriter BufferedOutput(BlockFile, BufferingSize / 2); + BasicFileWriter BufferedOutput(Temp, Min(BufferingSize, BufferSize)); for (const SharedBuffer& Segment : Buffer.GetSegments()) { size_t SegmentSize = Segment.GetSize(); @@ -806,14 +806,14 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, - [&Temp, &Offset](const void* Data, size_t Size) { - Temp.Write(Data, Size, Offset); + [&BufferedOutput, &Offset](const void* Data, size_t Size) { + BufferedOutput.Write(Data, Size, Offset); Offset += Size; }); } else { - Temp.Write(Segment.GetData(), SegmentSize, Offset); + BufferedOutput.Write(Segment.GetData(), SegmentSize, Offset); Offset += SegmentSize; } } diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 4a77aa49a..054f0d3a0 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -151,6 +151,28 @@ BLAKE3Stream::Append(const void* data, size_t byteCount) return *this; } +BLAKE3Stream& +BLAKE3Stream::Append(const IoBuffer& Buffer) +{ + blake3_hasher* b3h = reinterpret_cast(m_HashState); + + size_t BufferSize = Buffer.GetSize(); + static const uint64_t BufferingSize = 256u * 1024u; + IoBufferFileReference FileRef; + if (BufferSize >= (BufferingSize + BufferingSize / 2) && Buffer.GetFileReference(FileRef)) + { + ScanFile(FileRef.FileHandle, FileRef.FileChunkOffset, FileRef.FileChunkSize, BufferingSize, [&b3h](const void* Data, size_t Size) { + blake3_hasher_update(b3h, Data, Size); + }); + } + else + { + blake3_hasher_update(b3h, Buffer.GetData(), BufferSize); + } + + return *this; +} + BLAKE3 BLAKE3Stream::GetHash() { diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 62b64bc9d..d9f381811 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -216,23 +216,45 @@ public: std::function&& Callback, uint64_t /* BlockSize */) const final { - UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), BLAKE3::HashBuffer(RawData)); - Callback(0, 0, 0, CompositeBuffer(IoBuffer(IoBuffer::Wrap, HeaderData.GetData(), HeaderData.GetSize()))); + const uint64_t HeaderSize = CompressedBuffer::GetHeaderSizeForNoneEncoder(); - IoBufferFileReference FileRef = {nullptr, 0, 0}; - if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef)) + uint64_t RawOffset = 0; + BLAKE3Stream HashStream; + + for (const SharedBuffer& Segment : RawData.GetSegments()) { - ZEN_ASSERT(FileRef.FileHandle != nullptr); - uint64_t CallbackOffset = 0; - ScanFile(FileRef.FileHandle, 0, RawData.GetSize(), 512u * 1024u, [&](const void* Data, size_t Size) { - CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); - Callback(CallbackOffset, Size, HeaderData.GetSize() + CallbackOffset, Tmp); - CallbackOffset += Size; - }); - return true; + IoBufferFileReference FileRef = {nullptr, 0, 0}; + IoBuffer SegmentBuffer = Segment.AsIoBuffer(); + if (SegmentBuffer.GetFileReference(FileRef)) + { + ZEN_ASSERT(FileRef.FileHandle != nullptr); + + ScanFile(FileRef.FileHandle, + FileRef.FileChunkOffset, + FileRef.FileChunkSize, + 512u * 1024u, + [&](const void* Data, size_t Size) { + HashStream.Append(Data, Size); + CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size)); + Callback(RawOffset, Size, HeaderSize + RawOffset, Tmp); + RawOffset += Size; + }); + } + else + { + const uint64_t Size = SegmentBuffer.GetSize(); + HashStream.Append(SegmentBuffer); + Callback(RawOffset, Size, HeaderSize + RawOffset, CompositeBuffer(Segment)); + RawOffset += Size; + } } - Callback(0, RawData.GetSize(), HeaderData.GetSize(), RawData); + ZEN_ASSERT(RawOffset == RawData.GetSize()); + + UniqueBuffer HeaderData = CompressedBuffer::CreateHeaderForNoneEncoder(RawData.GetSize(), HashStream.GetHash()); + ZEN_ASSERT(HeaderData.GetSize() == HeaderSize); + Callback(0, 0, 0, CompositeBuffer(HeaderData.MoveToShared())); + return true; } }; @@ -323,7 +345,7 @@ public: ScanFile(FileRef.FileHandle, sizeof(BufferHeader) + RawOffset, RawSize, 512u * 1024u, [&](const void* Data, size_t Size) { if (Result) { - CompositeBuffer Tmp(SharedBuffer(IoBuffer(IoBuffer::Wrap, Data, Size))); + CompositeBuffer Tmp(SharedBuffer::MakeView(Data, Size)); Result = Callback(sizeof(BufferHeader) + RawOffset + CallbackOffset, Size, CallbackOffset, Tmp); } CallbackOffset += Size; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 0a9b2a73a..c4264bc29 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2275,23 +2275,32 @@ PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize) { bool Result = true; #if ZEN_PLATFORM_WINDOWS - DWORD _ = 0; - BOOL Ok = DeviceIoControl(FileHandle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &_, nullptr); - if (!Ok) + + BY_HANDLE_FILE_INFORMATION Information; + if (GetFileInformationByHandle(FileHandle, &Information)) { - std::error_code DummyEc; - ZEN_DEBUG("Unable to set sparse mode for file '{}'", PathFromHandle(FileHandle, DummyEc)); - Result = false; + if ((Information.dwFileAttributes & FILE_ATTRIBUTE_SPARSE_FILE) == 0) + { + DWORD _ = 0; + BOOL Ok = DeviceIoControl(FileHandle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &_, nullptr); + if (!Ok) + { + std::error_code DummyEc; + ZEN_DEBUG("Unable to set sparse mode for file '{}'", PathFromHandle(FileHandle, DummyEc)); + Result = false; + } + } } FILE_ALLOCATION_INFO AllocationInfo = {}; - AllocationInfo.AllocationSize.QuadPart = FinalSize; + AllocationInfo.AllocationSize.QuadPart = LONGLONG(FinalSize); if (!SetFileInformationByHandle(FileHandle, FileAllocationInfo, &AllocationInfo, DWORD(sizeof(AllocationInfo)))) { std::error_code DummyEc; ZEN_DEBUG("Unable to set file allocation size to {} for file '{}'", FinalSize, PathFromHandle(FileHandle, DummyEc)); Result = false; } + #else // ZEN_PLATFORM_WINDOWS ZEN_UNUSED(FileHandle, FinalSize); #endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/include/zencore/blake3.h b/src/zencore/include/zencore/blake3.h index 28bb348c0..f01e45266 100644 --- a/src/zencore/include/zencore/blake3.h +++ b/src/zencore/include/zencore/blake3.h @@ -53,6 +53,7 @@ struct BLAKE3Stream void Reset(); // Begin streaming hash compute (not needed on freshly constructed instance) BLAKE3Stream& Append(const void* data, size_t byteCount); // Append another chunk BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); } // Append another chunk + BLAKE3Stream& Append(const IoBuffer& Buffer); // Append another chunk BLAKE3 GetHash(); // Obtain final hash. If you wish to reuse the instance call reset() private: diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h index 7443e17b7..a619b0053 100644 --- a/src/zencore/include/zencore/iohash.h +++ b/src/zencore/include/zencore/iohash.h @@ -102,6 +102,12 @@ struct IoHashStream return *this; } + IoHashStream& Append(const IoBuffer& Buffer) + { + m_Blake3Stream.Append(Buffer); + return *this; + } + /// Append another chunk IoHashStream& Append(MemoryView Data) { diff --git a/src/zenutil/bufferedwritefilecache.cpp b/src/zenutil/bufferedwritefilecache.cpp new file mode 100644 index 000000000..a52850314 --- /dev/null +++ b/src/zenutil/bufferedwritefilecache.cpp @@ -0,0 +1,177 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +BufferedWriteFileCache::BufferedWriteFileCache() : m_CacheHitCount(0), m_CacheMissCount(0), m_OpenHandleCount(0), m_DroppedHandleCount(0) +{ +} + +BufferedWriteFileCache::~BufferedWriteFileCache() +{ + ZEN_TRACE_CPU("~BufferedWriteFileCache()"); + + try + { + for (TOpenHandles& OpenHandles : m_OpenFiles) + { + while (BasicFile* File = OpenHandles.Pop()) + { + std::unique_ptr FileToClose(File); + m_OpenHandleCount--; + } + } + m_OpenFiles.clear(); + m_ChunkWriters.clear(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BufferedWriteFileCache() threw exeption: {}", Ex.what()); + } +} + +std::unique_ptr +BufferedWriteFileCache::Get(uint32_t FileIndex) +{ + ZEN_TRACE_CPU("BufferedWriteFileCache::Get"); + + RwLock::ExclusiveLockScope _(m_WriterLock); + if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end()) + { + const uint32_t HandleIndex = It->second; + TOpenHandles& OpenHandles = m_OpenFiles[HandleIndex]; + if (BasicFile* File = OpenHandles.Pop(); File != nullptr) + { + m_OpenHandleCount--; + m_CacheHitCount++; + return std::unique_ptr(File); + } + } + m_CacheMissCount++; + return nullptr; +} + +void +BufferedWriteFileCache::Put(uint32_t FileIndex, std::unique_ptr&& Writer) +{ + ZEN_TRACE_CPU("BufferedWriteFileCache::Put"); + + if (m_OpenHandleCount.load() >= MaxBufferedCount) + { + m_DroppedHandleCount++; + return; + } + RwLock::ExclusiveLockScope _(m_WriterLock); + if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end()) + { + const uint32_t HandleIndex = It->second; + TOpenHandles& OpenHandles = m_OpenFiles[HandleIndex]; + if (OpenHandles.Push(Writer.get())) + { + Writer.release(); + m_OpenHandleCount++; + } + else + { + m_DroppedHandleCount++; + } + } + else + { + const uint32_t HandleIndex = gsl::narrow(m_OpenFiles.size()); + m_OpenFiles.push_back(TOpenHandles{}); + m_OpenFiles.back().Push(Writer.release()); + m_ChunkWriters.insert_or_assign(FileIndex, HandleIndex); + m_OpenHandleCount++; + } +} + +void +BufferedWriteFileCache::Close(std::span FileIndexes) +{ + ZEN_TRACE_CPU("BufferedWriteFileCache::Close"); + + std::vector> FilesToClose; + FilesToClose.reserve(FileIndexes.size()); + { + RwLock::ExclusiveLockScope _(m_WriterLock); + for (uint32_t FileIndex : FileIndexes) + { + if (auto It = m_ChunkWriters.find(FileIndex); It != m_ChunkWriters.end()) + { + const uint32_t HandleIndex = It->second; + TOpenHandles& OpenHandles = m_OpenFiles[HandleIndex]; + while (BasicFile* File = OpenHandles.Pop()) + { + FilesToClose.emplace_back(std::unique_ptr(File)); + m_OpenHandleCount--; + } + m_ChunkWriters.erase(It); + } + } + } + FilesToClose.clear(); +} + +BufferedWriteFileCache::Local::Local(BufferedWriteFileCache& Cache) : m_Cache(Cache) +{ +} + +BufferedWriteFileCache::Local::Writer* +BufferedWriteFileCache::Local::GetWriter(uint32_t FileIndex) +{ + if (auto It = m_FileIndexToWriterIndex.find(FileIndex); It != m_FileIndexToWriterIndex.end()) + { + return m_ChunkWriters[It->second].get(); + } + std::unique_ptr File = m_Cache.Get(FileIndex); + if (File) + { + const uint32_t WriterIndex = gsl::narrow(m_ChunkWriters.size()); + m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex); + m_ChunkWriters.emplace_back(std::make_unique(Writer{.File = std::move(File)})); + return m_ChunkWriters.back().get(); + } + return nullptr; +} + +BufferedWriteFileCache::Local::Writer* +BufferedWriteFileCache::Local::PutWriter(uint32_t FileIndex, std::unique_ptr Writer) +{ + ZEN_ASSERT(!m_FileIndexToWriterIndex.contains(FileIndex)); + const uint32_t WriterIndex = gsl::narrow(m_ChunkWriters.size()); + m_FileIndexToWriterIndex.insert_or_assign(FileIndex, WriterIndex); + m_ChunkWriters.emplace_back(std::move(Writer)); + return m_ChunkWriters.back().get(); +} + +BufferedWriteFileCache::Local::~Local() +{ + ZEN_TRACE_CPU("BufferedWriteFileCache::~Local()"); + try + { + for (auto& It : m_FileIndexToWriterIndex) + { + const uint32_t FileIndex = It.first; + const uint32_t WriterIndex = It.second; + m_ChunkWriters[WriterIndex]->Writer.reset(); + std::unique_ptr File; + File.swap(m_ChunkWriters[WriterIndex]->File); + m_Cache.Put(FileIndex, std::move(File)); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("BufferedWriteFileCache::~Local() threw exeption: {}", Ex.what()); + } +} + +} // namespace zen diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index ae129324e..4bec4901a 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -891,8 +891,12 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) } Result.SequenceIndexFirstPathIndex.resize(Content.ChunkedContent.SequenceRawHashes.size(), (uint32_t)-1); + Result.PathExtensionHash.resize(Content.Paths.size()); for (uint32_t PathIndex = 0; PathIndex < Content.Paths.size(); PathIndex++) { + std::string LowercaseExtension = Content.Paths[PathIndex].extension().string(); + std::transform(LowercaseExtension.begin(), LowercaseExtension.end(), LowercaseExtension.begin(), ::tolower); + Result.PathExtensionHash[PathIndex] = HashStringDjb2(LowercaseExtension); if (Content.RawSizes[PathIndex] > 0) { const IoHash& RawHash = Content.RawHashes[PathIndex]; diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index a5ebce193..6fb4182c0 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -4,6 +4,7 @@ #include #include +#include #include ZEN_THIRD_PARTY_INCLUDES_START @@ -35,32 +36,54 @@ namespace { return ChunkedParams{.UseThreshold = UseThreshold, .MinSize = MinSize, .MaxSize = MaxSize, .AvgSize = AvgSize}; } -} // namespace + void WriteChunkParams(CbObjectWriter& Writer, const ChunkedParams& Params) + { + Writer.BeginObject("ChunkingParams"sv); + { + Writer.AddBool("UseThreshold"sv, Params.UseThreshold); -class BasicChunkingController : public ChunkingController -{ -public: - BasicChunkingController(std::span ExcludeExtensions, - bool ExcludeElfFiles, - bool ExcludeMachOFiles, - uint64_t ChunkFileSizeLimit, - const ChunkedParams& ChunkingParams) - : m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end()) - , m_ExcludeElfFiles(ExcludeElfFiles) - , m_ExcludeMachOFiles(ExcludeMachOFiles) - , m_ChunkFileSizeLimit(ChunkFileSizeLimit) - , m_ChunkingParams(ChunkingParams) + Writer.AddInteger("MinSize"sv, (uint64_t)Params.MinSize); + Writer.AddInteger("MaxSize"sv, (uint64_t)Params.MaxSize); + Writer.AddInteger("AvgSize"sv, (uint64_t)Params.AvgSize); + } + Writer.EndObject(); // ChunkingParams + } + + bool IsElfFile(BasicFile& Buffer) { + if (Buffer.FileSize() > 4) + { + uint32_t ElfCheck = 0; + Buffer.Read(&ElfCheck, 4, 0); + if (ElfCheck == 0x464c457f) + { + return true; + } + } + return false; } - BasicChunkingController(CbObjectView Parameters) - : m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView())) - , m_ExcludeElfFiles(Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles)) - , m_ExcludeMachOFiles(Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles)) - , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) - , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) + bool IsMachOFile(BasicFile& Buffer) { + if (Buffer.FileSize() > 4) + { + uint32_t MachOCheck = 0; + Buffer.Read(&MachOCheck, 4, 0); + if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe)) + { + return true; + } + } + return false; } +} // namespace + +class BasicChunkingController : public ChunkingController +{ +public: + BasicChunkingController(const BasicChunkingControllerSettings& Settings) : m_Settings(Settings) {} + + BasicChunkingController(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {} virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, @@ -70,35 +93,25 @@ public: { ZEN_TRACE_CPU("BasicChunkingController::ProcessFile"); const bool ExcludeFromChunking = - std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) != - m_ChunkExcludeExtensions.end(); + std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) != + m_Settings.ExcludeExtensions.end(); - if (ExcludeFromChunking || (RawSize < m_ChunkFileSizeLimit)) + if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit)) { return false; } BasicFile Buffer(InputPath, BasicFile::Mode::kRead); - if (m_ExcludeElfFiles && Buffer.FileSize() > 4) + if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer)) { - uint32_t ElfCheck = 0; - Buffer.Read(&ElfCheck, 4, 0); - if (ElfCheck == 0x464c457f) - { - return false; - } + return false; } - if (m_ExcludeMachOFiles && Buffer.FileSize() > 4) + if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer)) { - uint32_t MachOCheck = 0; - Buffer.Read(&MachOCheck, 4, 0); - if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe)) - { - return false; - } + return false; } - OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag); + OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag); return true; } @@ -109,59 +122,43 @@ public: CbObjectWriter Writer; Writer.BeginArray("ChunkExcludeExtensions"sv); { - for (const std::string& Extension : m_ChunkExcludeExtensions) + for (const std::string& Extension : m_Settings.ExcludeExtensions) { Writer.AddString(Extension); } } Writer.EndArray(); // ChunkExcludeExtensions - Writer.AddBool("ExcludeElfFiles"sv, m_ExcludeElfFiles); - Writer.AddBool("ExcludeMachOFiles"sv, m_ExcludeMachOFiles); + Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles); + Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles); + Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit); - Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); - Writer.BeginObject("ChunkingParams"sv); - { - Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold); + WriteChunkParams(Writer, m_Settings.ChunkingParams); - Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize); - Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize); - Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize); - } - Writer.EndObject(); // ChunkingParams return Writer.Save(); } static constexpr std::string_view Name = "BasicChunkingController"sv; -protected: - const std::vector m_ChunkExcludeExtensions; - const bool m_ExcludeElfFiles = false; - const bool m_ExcludeMachOFiles = false; - const uint64_t m_ChunkFileSizeLimit; - const ChunkedParams m_ChunkingParams; +private: + static BasicChunkingControllerSettings ReadSettings(CbObjectView Parameters) + { + return BasicChunkingControllerSettings{ + .ExcludeExtensions = ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()), + .ExcludeElfFiles = Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles), + .ExcludeMachOFiles = Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles), + .ChunkFileSizeLimit = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit), + .ChunkingParams = ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())}; + } + + const BasicChunkingControllerSettings m_Settings; }; class ChunkingControllerWithFixedChunking : public ChunkingController { public: - ChunkingControllerWithFixedChunking(std::span FixedChunkingExtensions, - uint64_t ChunkFileSizeLimit, - const ChunkedParams& ChunkingParams, - uint32_t FixedChunkingChunkSize) - : m_FixedChunkingExtensions(FixedChunkingExtensions.begin(), FixedChunkingExtensions.end()) - , m_ChunkFileSizeLimit(ChunkFileSizeLimit) - , m_ChunkingParams(ChunkingParams) - , m_FixedChunkingChunkSize(FixedChunkingChunkSize) - { - } + ChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Settings) : m_Settings(Settings) {} - ChunkingControllerWithFixedChunking(CbObjectView Parameters) - : m_FixedChunkingExtensions(ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView())) - , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) - , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) - , m_FixedChunkingChunkSize(Parameters["FixedChunkingChunkSize"sv].AsUInt32(16u * 1024u * 1024u)) - { - } + ChunkingControllerWithFixedChunking(CbObjectView Parameters) : m_Settings(ReadSettings(Parameters)) {} virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, @@ -170,33 +167,71 @@ public: std::atomic& AbortFlag) const override { ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile"); - if (RawSize < m_ChunkFileSizeLimit) + const bool ExcludeFromChunking = + std::find(m_Settings.ExcludeExtensions.begin(), m_Settings.ExcludeExtensions.end(), InputPath.extension()) != + m_Settings.ExcludeExtensions.end(); + + if (ExcludeFromChunking || (RawSize < m_Settings.ChunkFileSizeLimit)) { return false; } - const bool FixedChunking = std::find(m_FixedChunkingExtensions.begin(), m_FixedChunkingExtensions.end(), InputPath.extension()) != - m_FixedChunkingExtensions.end(); - if (FixedChunking) + const bool FixedChunkingExtension = + std::find(m_Settings.FixedChunkingExtensions.begin(), m_Settings.FixedChunkingExtensions.end(), InputPath.extension()) != + m_Settings.FixedChunkingExtensions.end(); + + if (FixedChunkingExtension) { + if (RawSize < m_Settings.MinSizeForFixedChunking) + { + return false; + } ZEN_TRACE_CPU("FixedChunking"); - IoHashStream FullHash; - IoBuffer Source = IoBufferBuilder::MakeFromFile(InputPath); + IoHashStream FullHasher; + BasicFile Source(InputPath, BasicFile::Mode::kRead); uint64_t Offset = 0; tsl::robin_map ChunkHashToChunkIndex; - ChunkHashToChunkIndex.reserve(1 + (RawSize / m_FixedChunkingChunkSize)); + const uint64_t ExpectedChunkCount = 1 + (RawSize / m_Settings.FixedChunkingChunkSize); + ChunkHashToChunkIndex.reserve(ExpectedChunkCount); + OutChunked.Info.ChunkHashes.reserve(ExpectedChunkCount); + OutChunked.Info.ChunkSequence.reserve(ExpectedChunkCount); + OutChunked.ChunkSources.reserve(ExpectedChunkCount); + + static const uint64_t BufferingSize = 256u * 1024u; + + IoHashStream ChunkHasher; + while (Offset < RawSize) { if (AbortFlag) { return false; } - uint64_t ChunkSize = std::min(RawSize - Offset, m_FixedChunkingChunkSize); - IoBuffer Chunk(Source, Offset, ChunkSize); - MemoryView ChunkData = Chunk.GetView(); - FullHash.Append(ChunkData); - IoHash ChunkHash = IoHash::HashBuffer(ChunkData); + ChunkHasher.Reset(); + + uint64_t ChunkSize = std::min(RawSize - Offset, m_Settings.FixedChunkingChunkSize); + if (ChunkSize >= (BufferingSize + BufferingSize / 2)) + { + ScanFile(Source.Handle(), + Offset, + ChunkSize, + BufferingSize, + [&FullHasher, &ChunkHasher, &BytesProcessed](const void* Data, size_t Size) { + FullHasher.Append(Data, Size); + ChunkHasher.Append(Data, Size); + BytesProcessed.fetch_add(Size); + }); + } + else + { + IoBuffer ChunkData = Source.ReadRange(Offset, ChunkSize); + FullHasher.Append(ChunkData); + ChunkHasher.Append(ChunkData); + BytesProcessed.fetch_add(ChunkSize); + } + + const IoHash ChunkHash = ChunkHasher.GetHash(); if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end()) { OutChunked.Info.ChunkSequence.push_back(It->second); @@ -209,16 +244,24 @@ public: OutChunked.ChunkSources.push_back({.Offset = Offset, .Size = gsl::narrow(ChunkSize)}); } Offset += ChunkSize; - BytesProcessed.fetch_add(ChunkSize); } OutChunked.Info.RawSize = RawSize; - OutChunked.Info.RawHash = FullHash.GetHash(); + OutChunked.Info.RawHash = FullHasher.GetHash(); return true; } else { BasicFile Buffer(InputPath, BasicFile::Mode::kRead); - OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed); + if (m_Settings.ExcludeElfFiles && IsElfFile(Buffer)) + { + return false; + } + if (m_Settings.ExcludeMachOFiles && IsMachOFile(Buffer)) + { + return false; + } + + OutChunked = ChunkData(Buffer, 0, RawSize, m_Settings.ChunkingParams, &BytesProcessed, &AbortFlag); return true; } } @@ -230,47 +273,57 @@ public: CbObjectWriter Writer; Writer.BeginArray("FixedChunkingExtensions"); { - for (const std::string& Extension : m_FixedChunkingExtensions) + for (const std::string& Extension : m_Settings.FixedChunkingExtensions) { Writer.AddString(Extension); } } Writer.EndArray(); // ChunkExcludeExtensions - Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); - Writer.BeginObject("ChunkingParams"sv); - { - Writer.AddBool("UseThreshold"sv, m_ChunkingParams.UseThreshold); - Writer.AddInteger("MinSize"sv, (uint64_t)m_ChunkingParams.MinSize); - Writer.AddInteger("MaxSize"sv, (uint64_t)m_ChunkingParams.MaxSize); - Writer.AddInteger("AvgSize"sv, (uint64_t)m_ChunkingParams.AvgSize); + Writer.BeginArray("ChunkExcludeExtensions"sv); + { + for (const std::string& Extension : m_Settings.ExcludeExtensions) + { + Writer.AddString(Extension); + } } - Writer.EndObject(); // ChunkingParams - Writer.AddInteger("FixedChunkingChunkSize"sv, m_FixedChunkingChunkSize); + Writer.EndArray(); // ChunkExcludeExtensions + + Writer.AddBool("ExcludeElfFiles"sv, m_Settings.ExcludeElfFiles); + Writer.AddBool("ExcludeMachOFiles"sv, m_Settings.ExcludeMachOFiles); + + Writer.AddInteger("ChunkFileSizeLimit"sv, m_Settings.ChunkFileSizeLimit); + + WriteChunkParams(Writer, m_Settings.ChunkingParams); + + Writer.AddInteger("FixedChunkingChunkSize"sv, m_Settings.FixedChunkingChunkSize); + Writer.AddInteger("MinSizeForFixedChunking"sv, m_Settings.MinSizeForFixedChunking); return Writer.Save(); } static constexpr std::string_view Name = "ChunkingControllerWithFixedChunking"sv; -protected: - const std::vector m_FixedChunkingExtensions; - const uint64_t m_ChunkFileSizeLimit; - const ChunkedParams m_ChunkingParams; - const uint32_t m_FixedChunkingChunkSize; +private: + static ChunkingControllerWithFixedChunkingSettings ReadSettings(CbObjectView Parameters) + { + return ChunkingControllerWithFixedChunkingSettings{ + .FixedChunkingExtensions = ReadStringArray(Parameters["FixedChunkingExtensions"sv].AsArrayView()), + .ExcludeExtensions = ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView()), + .ExcludeElfFiles = Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles), + .ExcludeMachOFiles = Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles), + .ChunkFileSizeLimit = Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit), + .ChunkingParams = ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView()), + .FixedChunkingChunkSize = Parameters["FixedChunkingChunkSize"sv].AsUInt64(DefaultFixedChunkingChunkSize), + .MinSizeForFixedChunking = Parameters["MinSizeForFixedChunking"sv].AsUInt64(DefaultFixedChunkingChunkSize)}; + } + + const ChunkingControllerWithFixedChunkingSettings m_Settings; }; std::unique_ptr -CreateBasicChunkingController(std::span ExcludeExtensions, - bool ExcludeElfFiles, - bool ExcludeMachOFiles, - uint64_t ChunkFileSizeLimit, - const ChunkedParams& ChunkingParams) +CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings) { - return std::make_unique(ExcludeExtensions, - ExcludeElfFiles, - ExcludeMachOFiles, - ChunkFileSizeLimit, - ChunkingParams); + return std::make_unique(Settings); } std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters) @@ -279,15 +332,9 @@ CreateBasicChunkingController(CbObjectView Parameters) } std::unique_ptr -CreateChunkingControllerWithFixedChunking(std::span FixedChunkingExtensions, - uint64_t ChunkFileSizeLimit, - const ChunkedParams& ChunkingParams, - uint32_t FixedChunkingChunkSize) +CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting) { - return std::make_unique(FixedChunkingExtensions, - ChunkFileSizeLimit, - ChunkingParams, - FixedChunkingChunkSize); + return std::make_unique(Setting); } std::unique_ptr CreateChunkingControllerWithFixedChunking(CbObjectView Parameters) diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index badfb4840..c389d16c5 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -678,13 +678,24 @@ protected: { return false; } - CompositeBuffer Decompressed = ValidateBuffer.DecompressToComposite(); - if (!Decompressed) + + IoHashStream Hash; + bool CouldDecompress = ValidateBuffer.DecompressToStream( + 0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset, SourceSize, Offset); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + return true; + }); + if (!CouldDecompress) { return false; } - IoHash Hash = IoHash::HashBuffer(Decompressed); - if (Hash != RawHash) + if (Hash.GetHash() != VerifyHash) { return false; } diff --git a/src/zenutil/include/zenutil/bufferedwritefilecache.h b/src/zenutil/include/zenutil/bufferedwritefilecache.h new file mode 100644 index 000000000..68d6c375e --- /dev/null +++ b/src/zenutil/include/zenutil/bufferedwritefilecache.h @@ -0,0 +1,106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CompositeBuffer; + +class BufferedWriteFileCache +{ +public: + BufferedWriteFileCache(const BufferedWriteFileCache&) = delete; + BufferedWriteFileCache& operator=(const BufferedWriteFileCache&) = delete; + + BufferedWriteFileCache(); + + ~BufferedWriteFileCache(); + + std::unique_ptr Get(uint32_t FileIndex); + + void Put(uint32_t FileIndex, std::unique_ptr&& Writer); + + void Close(std::span FileIndexes); + + class Local + { + public: + struct Writer + { + std::unique_ptr File; + std::unique_ptr Writer; + + inline void Write(const CompositeBuffer& Chunk, uint64_t FileOffset) + { + if (Writer) + { + Writer->Write(Chunk, FileOffset); + } + else + { + File->Write(Chunk, FileOffset); + } + } + }; + + Local(const Local&) = delete; + Local& operator=(const Local&) = delete; + + explicit Local(BufferedWriteFileCache& Cache); + ~Local(); + + Writer* GetWriter(uint32_t FileIndex); + Writer* PutWriter(uint32_t FileIndex, std::unique_ptr Writer); + + private: + tsl::robin_map m_FileIndexToWriterIndex; + std::vector> m_ChunkWriters; + BufferedWriteFileCache& m_Cache; + }; + +private: + static constexpr size_t MaxHandlesPerPath = 7; + static constexpr size_t MaxBufferedCount = 1024; + struct TOpenHandles + { + BasicFile* Files[MaxHandlesPerPath]; + uint64_t Size = 0; + inline BasicFile* Pop() + { + if (Size > 0) + { + return Files[--Size]; + } + else + { + return nullptr; + } + } + inline bool Push(BasicFile* File) + { + if (Size < MaxHandlesPerPath) + { + Files[Size++] = File; + return true; + } + return false; + } + }; + static_assert(sizeof(TOpenHandles) == 64); + + RwLock m_WriterLock; + tsl::robin_map m_ChunkWriters; + std::vector m_OpenFiles; + std::atomic m_CacheHitCount; + std::atomic m_CacheMissCount; + std::atomic m_OpenHandleCount; + std::atomic m_DroppedHandleCount; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index d33869be2..03f52e5f6 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -135,6 +135,7 @@ struct ChunkedContentLookup ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash + std::vector PathExtensionHash; }; ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 970917fb0..315502265 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -11,9 +11,11 @@ namespace zen { -const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; -const bool DefaultChunkingExcludeElfFiles = true; -const bool DefaultChunkingExcludeMachOFiles = true; +const std::vector DefaultChunkingExcludeExtensions = + {".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"}; +const std::vector DefaultFixedChunkingExtensions = {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -21,7 +23,8 @@ const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128 const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize; -const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u; +const uint64_t DefaultFixedChunkingChunkSize = 32u * 1024u * 1024u; +const uint64_t DefaultMinSizeForFixedChunking = DefaultFixedChunkingChunkSize * 8u; struct ChunkedInfoWithSource; @@ -40,19 +43,31 @@ public: virtual CbObject GetParameters() const = 0; }; -std::unique_ptr CreateBasicChunkingController( - std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, - bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, - bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, - uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, - const ChunkedParams& ChunkingParams = DefaultChunkedParams); +struct BasicChunkingControllerSettings +{ + std::vector ExcludeExtensions = DefaultChunkingExcludeExtensions; + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles; + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles; + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit; + ChunkedParams ChunkingParams = DefaultChunkedParams; +}; + +std::unique_ptr CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings); std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); -std::unique_ptr CreateChunkingControllerWithFixedChunking( - std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, - uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, - const ChunkedParams& ChunkingParams = DefaultChunkedParams, - uint32_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize); +struct ChunkingControllerWithFixedChunkingSettings +{ + std::vector FixedChunkingExtensions = DefaultFixedChunkingExtensions; + std::vector ExcludeExtensions = DefaultChunkingExcludeExtensions; + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles; + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles; + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit; + ChunkedParams ChunkingParams = DefaultChunkedParams; + uint64_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize; + uint64_t MinSizeForFixedChunking = DefaultMinSizeForFixedChunking; +}; + +std::unique_ptr CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting); std::unique_ptr CreateChunkingControllerWithFixedChunking(CbObjectView Parameters); std::unique_ptr CreateChunkingController(std::string_view Name, CbObjectView Parameters); -- cgit v1.2.3 From 937510356143f83ecd15d0a9f58b611c7418ed61 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Jun 2025 08:59:44 +0200 Subject: faster scavenge (#417) - Improvement: Multithreaded scavenge pass for zen builds download - Improvement: Optimized check for modified files when verifying state of scavenged paths --- src/zen/cmds/admin_cmd.cpp | 10 +- src/zen/cmds/builds_cmd.cpp | 212 +++++++++++++++++++++------ src/zenutil/chunkedcontent.cpp | 91 ++++++++---- src/zenutil/include/zenutil/chunkedcontent.h | 39 ++--- 4 files changed, 251 insertions(+), 101 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index b3f8a990e..fe2bbbdc7 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -57,10 +57,7 @@ ScrubCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (int StatusCode = (int)Response.StatusCode) { - ZEN_ERROR("scrub start failed: {}: {} ({})", - StatusCode, - ReasonStringForHttpResultCode(StatusCode), - Response.ToText()); + ZEN_ERROR("scrub start failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else { @@ -645,10 +642,7 @@ FlushCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (int StatusCode = (int)Response.StatusCode) { - ZEN_ERROR("flush failed: {}: {} ({})", - StatusCode, - ReasonStringForHttpResultCode(StatusCode), - Response.ToText()); + ZEN_ERROR("flush failed: {}: {} ({})", StatusCode, ReasonStringForHttpResultCode(StatusCode), Response.ToText()); } else { diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e13c90b4b..fbcb6b900 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -360,7 +360,7 @@ namespace { WriteByteCount += Size; })) { - throw std::runtime_error(fmt::format("Failed to copy scavanged file '{}' to '{}'", SourceFilePath, TargetFilePath)); + throw std::runtime_error(fmt::format("Failed to copy scavenged file '{}' to '{}'", SourceFilePath, TargetFilePath)); } } @@ -5342,9 +5342,10 @@ namespace { return HasLocalState; } - FolderContent GetValidFolderContent(GetFolderContentStatistics& LocalFolderScanStats, - const std::filesystem::path& Path, - std::span PathsToCheck) + FolderContent GetValidFolderContent(GetFolderContentStatistics& LocalFolderScanStats, + const std::filesystem::path& Path, + std::span PathsToCheck, + std::function&& ProgressCallback) { ZEN_TRACE_CPU("GetValidFolderContent"); FolderContent Result; @@ -5359,8 +5360,6 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); - ProgressBar ProgressBar(ProgressMode, "Check Files"); - ParallelWork Work(AbortFlag); std::atomic CompletedPathCount = 0; uint32_t PathIndex = 0; @@ -5393,18 +5392,11 @@ namespace { PathIndex += PathRangeCount; } Work.Wait(200, [&](bool, ptrdiff_t) { - // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} checked, {} found", - CompletedPathCount.load(), - PathCount, - LocalFolderScanStats.FoundFileCount.load()); - ProgressBar.UpdateState({.Task = "Checking files ", - .Details = Details, - .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount.load()}, - false); + if (ProgressCallback) + { + ProgressCallback(PathCount, CompletedPathCount.load()); + } }); - ProgressBar.Finish(); } uint32_t WritePathIndex = 0; @@ -5675,40 +5667,153 @@ namespace { ScavengedContents.resize(ScavengePathCount); ScavengedLookups.resize(ScavengePathCount); ScavengedPaths.resize(ScavengePathCount); + + ProgressBar ScavengeProgressBar(ProgressMode, "Scavenging"); + ParallelWork Work(AbortFlag); + + std::atomic PathsFound(0); + std::atomic ChunksFound(0); + std::atomic PathsScavenged(0); + for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++) { - const ScavengeSource& Source = ScavengeSources[ScavengeIndex]; + Work.ScheduleWork( + GetIOWorkerPool(), + [&RemoteLookup, + &ScavengeSources, + &ScavengedContents, + &ScavengedPaths, + &ScavengedLookups, + &PathsFound, + &ChunksFound, + &PathsScavenged, + ScavengeIndex](std::atomic&) { + if (!AbortFlag) + { + const ScavengeSource& Source = ScavengeSources[ScavengeIndex]; - ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengeIndex]; - std::filesystem::path& ScavengePath = ScavengedPaths[ScavengeIndex]; - FolderContent LocalFolderState; - if (ReadStateFile(Source.StateFilePath, LocalFolderState, ScavengedLocalContent)) - { - GetFolderContentStatistics ScavengedFolderScanStats; + ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengeIndex]; + std::filesystem::path& ScavengePath = ScavengedPaths[ScavengeIndex]; - FolderContent ValidFolderContent = - GetValidFolderContent(ScavengedFolderScanStats, Source.Path, LocalFolderState.Paths); + FolderContent LocalFolderState; + if (ReadStateFile(Source.StateFilePath, LocalFolderState, ScavengedLocalContent)) + { + if (IsDir(Source.Path)) + { + ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengeIndex]; + ScavengedLookup = BuildChunkedContentLookup(ScavengedLocalContent); + + std::vector PathIndexesToScavange; + uint32_t ScavengedStatePathCount = gsl::narrow(ScavengedLocalContent.Paths.size()); + PathIndexesToScavange.reserve(ScavengedStatePathCount); + for (uint32_t ScavengedStatePathIndex = 0; ScavengedStatePathIndex < ScavengedStatePathCount; + ScavengedStatePathIndex++) + { + const IoHash& SequenceHash = ScavengedLocalContent.RawHashes[ScavengedStatePathIndex]; + if (auto ScavengeSequenceIt = ScavengedLookup.RawHashToSequenceIndex.find(SequenceHash); + ScavengeSequenceIt != ScavengedLookup.RawHashToSequenceIndex.end()) + { + const uint32_t ScavengeSequenceIndex = ScavengeSequenceIt->second; + if (RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash)) + { + PathIndexesToScavange.push_back(ScavengedStatePathIndex); + } + else + { + const uint32_t ScavengeChunkCount = + ScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex]; + for (uint32_t ScavengeChunkIndexOffset = 0; + ScavengeChunkIndexOffset < ScavengeChunkCount; + ScavengeChunkIndexOffset++) + { + const size_t ScavengeChunkOrderIndex = + ScavengedLookup.ChunkSequenceLocationOffset[ScavengeSequenceIndex] + + ScavengeChunkIndexOffset; + const uint32_t ScavengeChunkIndex = + ScavengedLocalContent.ChunkedContent.ChunkOrders[ScavengeChunkOrderIndex]; + const IoHash& ScavengeChunkHash = + ScavengedLocalContent.ChunkedContent.ChunkHashes[ScavengeChunkIndex]; + if (RemoteLookup.ChunkHashToChunkIndex.contains(ScavengeChunkHash)) + { + PathIndexesToScavange.push_back(ScavengedStatePathIndex); + break; + } + } + } + } + } - if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent)) - { - std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths); + if (!PathIndexesToScavange.empty()) + { + std::vector PathsToScavenge; + PathsToScavenge.reserve(PathIndexesToScavange.size()); + for (uint32_t ScavengedStatePathIndex : PathIndexesToScavange) + { + PathsToScavenge.push_back(ScavengedLocalContent.Paths[ScavengedStatePathIndex]); + } - // If the files are modified since the state was saved we ignore the files since we don't want to incur the - // cost of scanning/hashing scavenged files - DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end()); - if (!DeletedPaths.empty()) - { - ScavengedLocalContent = DeletePathsFromChunkedContent(ScavengedLocalContent, DeletedPaths); + GetFolderContentStatistics ScavengedFolderScanStats; + + FolderContent ValidFolderContent = + GetValidFolderContent(ScavengedFolderScanStats, Source.Path, PathsToScavenge, {}); + + if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent)) + { + std::vector DeletedPaths; + FolderContent UpdatedContent = + GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths); + + // If the files are modified since the state was saved we ignore the files since we don't + // want to incur the cost of scanning/hashing scavenged files + DeletedPaths.insert(DeletedPaths.end(), + UpdatedContent.Paths.begin(), + UpdatedContent.Paths.end()); + if (!DeletedPaths.empty()) + { + ScavengedLocalContent = + DeletePathsFromChunkedContent(ScavengedLocalContent, ScavengedLookup, DeletedPaths); + ScavengedLookup = BuildChunkedContentLookup(ScavengedLocalContent); + } + } + + if (!ScavengedLocalContent.Paths.empty()) + { + ScavengePath = Source.Path; + PathsFound += ScavengedLocalContent.Paths.size(); + ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size(); + } + } + + if (ScavengePath.empty()) + { + ScavengedLocalContent = {}; + ScavengedLookups[ScavengeIndex] = {}; + ScavengedPaths[ScavengeIndex].clear(); + } + } + } + PathsScavenged++; } - } + }); + } + { + ZEN_TRACE_CPU("ScavengeScan_Wait"); - if (!ScavengedLocalContent.Paths.empty()) - { - ScavengePath = Source.Path; - } - } + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavanging", + PathsScavenged.load(), + ScavengePathCount, + PathsFound.load(), + ChunksFound.load()); + ScavengeProgressBar.UpdateState({.Task = "Scavenging ", + .Details = Details, + .TotalCount = ScavengePathCount, + .RemainingCount = ScavengePathCount - PathsScavenged.load()}, + false); + }); } + ScavengeProgressBar.Finish(); for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (!SequenceIndexesLeftToFindToRemoteIndex.empty()); @@ -5718,8 +5823,7 @@ namespace { if (!ScavengePath.empty()) { const ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengedContentIndex]; - ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; - ScavengedLookup = BuildChunkedContentLookup(ScavengedLocalContent); + const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex]; for (uint32_t ScavengedSequenceIndex = 0; ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size(); @@ -5980,7 +6084,7 @@ namespace { } if (CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0) { - ZEN_CONSOLE("Scavenge of {} paths found {} ({}) chunk sequences, {} ({}) chunks in {}", + ZEN_CONSOLE("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}", ScavengedPathsCount, CacheMappingStats.ScavengedPathsMatchingSequencesCount, NiceBytes(CacheMappingStats.ScavengedPathsMatchingSequencesByteCount), @@ -6408,7 +6512,7 @@ namespace { &DiskStats](std::atomic&) mutable { if (!AbortFlag) { - ZEN_TRACE_CPU("UpdateFolder_WriteScavanged"); + ZEN_TRACE_CPU("UpdateFolder_WriteScavenged"); FilteredWrittenBytesPerSecond.Start(); @@ -8574,7 +8678,21 @@ namespace { } } - OutLocalFolderContent = GetValidFolderContent(LocalFolderScanStats, Path, PathsToCheck); + ProgressBar ProgressBar(ProgressMode, "Check Files"); + OutLocalFolderContent = GetValidFolderContent( + LocalFolderScanStats, + Path, + PathsToCheck, + [&ProgressBar, &LocalFolderScanStats](uint64_t PathCount, uint64_t CompletedPathCount) { + std::string Details = + fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount}, + false); + }); + ProgressBar.Finish(); } bool ScanContent = true; diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 4bec4901a..c7532e098 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -662,7 +662,9 @@ MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span DeletedPaths) +DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, + const ChunkedContentLookup& BaseContentLookup, + std::span DeletedPaths) { ZEN_TRACE_CPU("DeletePathsFromChunkedContent"); @@ -676,14 +678,18 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span { DeletedPathSet.insert(PathCompareString(DeletedPath)); } - const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); - tsl::robin_map ChunkHashToChunkIndex; - const size_t ExpectedCount = BaseContent.Paths.size() - DeletedPaths.size(); - Result.Paths.reserve(ExpectedCount); - Result.RawSizes.reserve(ExpectedCount); - Result.Attributes.reserve(ExpectedCount); - Result.RawHashes.reserve(ExpectedCount); + const size_t BaseChunkCount = BaseContent.ChunkedContent.ChunkHashes.size(); + std::vector NewChunkIndexes(BaseChunkCount, (uint32_t)-1); + + const size_t ExpectedPathCount = BaseContent.Paths.size() - DeletedPaths.size(); + Result.Paths.reserve(ExpectedPathCount); + Result.RawSizes.reserve(ExpectedPathCount); + Result.Attributes.reserve(ExpectedPathCount); + Result.RawHashes.reserve(ExpectedPathCount); + + Result.ChunkedContent.ChunkHashes.reserve(BaseChunkCount); + Result.ChunkedContent.ChunkRawSizes.reserve(BaseChunkCount); tsl::robin_map RawHashToSequenceRawHashIndex; for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) @@ -703,20 +709,33 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span { RawHashToSequenceRawHashIndex.insert( {RawHash, gsl::narrow(Result.ChunkedContent.SequenceRawHashes.size())}); - const uint32_t SequenceRawHashIndex = BaseLookup.RawHashToSequenceIndex.at(RawHash); - const uint32_t OrderIndexOffset = BaseLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex]; - const uint32_t ChunkCount = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; - ChunkingStatistics Stats; + const uint32_t SequenceRawHashIndex = BaseContentLookup.RawHashToSequenceIndex.at(RawHash); + const uint32_t OrderIndexOffset = BaseContentLookup.SequenceIndexChunkOrderOffset[SequenceRawHashIndex]; + const uint32_t ChunkCount = BaseContent.ChunkedContent.ChunkCounts[SequenceRawHashIndex]; + std::span OriginalChunkOrder = std::span(BaseContent.ChunkedContent.ChunkOrders).subspan(OrderIndexOffset, ChunkCount); - AddChunkSequence(Stats, - Result.ChunkedContent, - ChunkHashToChunkIndex, - RawHash, - OriginalChunkOrder, - BaseContent.ChunkedContent.ChunkHashes, - BaseContent.ChunkedContent.ChunkRawSizes); - Stats.UniqueSequencesFound++; + + Result.ChunkedContent.ChunkCounts.push_back(gsl::narrow(OriginalChunkOrder.size())); + + for (uint32_t OldChunkIndex : OriginalChunkOrder) + { + if (uint32_t FoundChunkIndex = NewChunkIndexes[OldChunkIndex]; FoundChunkIndex != (uint32_t)-1) + { + Result.ChunkedContent.ChunkOrders.push_back(FoundChunkIndex); + } + else + { + const uint32_t NewChunkIndex = gsl::narrow(Result.ChunkedContent.ChunkHashes.size()); + NewChunkIndexes[OldChunkIndex] = NewChunkIndex; + const IoHash& ChunkHash = BaseContent.ChunkedContent.ChunkHashes[OldChunkIndex]; + const uint64_t OldChunkSize = BaseContent.ChunkedContent.ChunkRawSizes[OldChunkIndex]; + Result.ChunkedContent.ChunkHashes.push_back(ChunkHash); + Result.ChunkedContent.ChunkRawSizes.push_back(OldChunkSize); + Result.ChunkedContent.ChunkOrders.push_back(NewChunkIndex); + } + } + Result.ChunkedContent.SequenceRawHashes.push_back(RawHash); } } } @@ -725,6 +744,19 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span return Result; } +ChunkedFolderContent +DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span DeletedPaths) +{ + ZEN_TRACE_CPU("DeletePathsFromChunkedContent"); + ZEN_ASSERT(DeletedPaths.size() <= BaseContent.Paths.size()); + if (DeletedPaths.size() == BaseContent.Paths.size()) + { + return {}; + } + const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); + return DeletePathsFromChunkedContent(BaseContent, BaseLookup, DeletedPaths); +} + ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, WorkerThreadPool& WorkerPool, @@ -815,8 +847,9 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) struct ChunkLocationReference { - uint32_t ChunkIndex = (uint32_t)-1; - ChunkedContentLookup::ChunkSequenceLocation Location; + uint32_t ChunkIndex = (uint32_t)-1; + uint32_t SequenceIndex = (uint32_t)-1; + uint64_t Offset = (uint64_t)-1; }; ChunkedContentLookup Result; @@ -845,8 +878,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) { uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex]; - Locations.push_back( - ChunkLocationReference{ChunkIndex, ChunkedContentLookup::ChunkSequenceLocation{SequenceIndex, LocationOffset}}); + Locations.push_back(ChunkLocationReference{.ChunkIndex = ChunkIndex, .SequenceIndex = SequenceIndex, .Offset = LocationOffset}); LocationOffset += Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; } @@ -861,15 +893,15 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) { return false; } - if (Lhs.Location.SequenceIndex < Rhs.Location.SequenceIndex) + if (Lhs.SequenceIndex < Rhs.SequenceIndex) { return true; } - if (Lhs.Location.SequenceIndex > Rhs.Location.SequenceIndex) + if (Lhs.SequenceIndex > Rhs.SequenceIndex) { return false; } - return Lhs.Location.Offset < Rhs.Location.Offset; + return Lhs.Offset < Rhs.Offset; }); Result.ChunkSequenceLocations.reserve(Locations.size()); @@ -882,7 +914,10 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content) uint32_t Count = 0; while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex)) { - Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location); + const ChunkLocationReference& LocationReference = Locations[RangeOffset + Count]; + Result.ChunkSequenceLocations.push_back( + ChunkedContentLookup::ChunkSequenceLocation{.SequenceIndex = LocationReference.SequenceIndex, + .Offset = LocationReference.Offset}); Count++; } Result.ChunkSequenceLocationOffset.push_back(RangeOffset); diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 03f52e5f6..225b1a3a5 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -94,10 +94,31 @@ struct ChunkedFolderContent ChunkedContentData ChunkedContent; }; +struct ChunkedContentLookup +{ + struct ChunkSequenceLocation + { + uint32_t SequenceIndex = (uint32_t)-1; + uint64_t Offset = (uint64_t)-1; + }; + tsl::robin_map ChunkHashToChunkIndex; + tsl::robin_map RawHashToSequenceIndex; + std::vector SequenceIndexChunkOrderOffset; + std::vector ChunkSequenceLocations; + std::vector + ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash + std::vector PathExtensionHash; +}; + void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output); ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input); ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays); +ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, + const ChunkedContentLookup& BaseContentLookup, + std::span DeletedPaths); ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span DeletedPaths); struct ChunkingStatistics @@ -120,24 +141,6 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, std::function&& UpdateCallback, std::atomic& AbortFlag); -struct ChunkedContentLookup -{ - struct ChunkSequenceLocation - { - uint32_t SequenceIndex = (uint32_t)-1; - uint64_t Offset = (uint64_t)-1; - }; - tsl::robin_map ChunkHashToChunkIndex; - tsl::robin_map RawHashToSequenceIndex; - std::vector SequenceIndexChunkOrderOffset; - std::vector ChunkSequenceLocations; - std::vector - ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex - std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex - std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash - std::vector PathExtensionHash; -}; - ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); inline std::pair -- cgit v1.2.3 From 99814bacd1788bfd670f6abd9dd017dbdf483d70 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Jun 2025 12:41:43 +0200 Subject: new builds search (#418) * don't require bucket for search operations to allow multi-bucket search --- src/zen/cmds/builds_cmd.cpp | 44 +++++++++++++++++++++------------- src/zenutil/jupiter/jupitersession.cpp | 7 +++--- 2 files changed, 32 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index fbcb6b900..372291fcc 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9433,7 +9433,6 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); - Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); Ops.add_option("cloud build", "", @@ -9829,17 +9828,21 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseSystemOptions(); - auto ParseStorageOptions = [&]() { + auto ParseStorageOptions = [&](bool RequireBucket) { if (!m_OverrideHost.empty() || !m_Host.empty()) { if (!m_StoragePath.empty()) { - throw zen::OptionParseException(fmt::format("url is not compatible with the storage-path option\n{}", m_Options.help())); + throw zen::OptionParseException( + fmt::format("host/url/override-host is not compatible with the storage-path option\n{}", m_Options.help())); } - if (SubOption != &m_ListNamespacesOptions && (m_Namespace.empty() || m_Bucket.empty())) + if (m_Namespace.empty()) { - throw zen::OptionParseException( - fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("namespace option is required for this storage option\n{}", m_Options.help())); + } + if (RequireBucket && m_Bucket.empty()) + { + throw zen::OptionParseException(fmt::format("bucket option is required for this storage option\n{}", m_Options.help())); } } else if (m_StoragePath.empty()) @@ -9986,8 +9989,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, BuildStorageCache::Statistics& StorageCacheStats, - const std::filesystem::path& TempPath) -> StorageInstance { - ParseStorageOptions(); + const std::filesystem::path& TempPath, + bool RequireBucket) -> StorageInstance { + ParseStorageOptions(RequireBucket); StorageInstance Result; @@ -10396,7 +10400,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath), /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -10480,7 +10485,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -10548,7 +10554,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); CbObject MetaData = ParseBuildMetadata(); @@ -10627,7 +10634,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); DownloadFolder(Storage, BuildId, @@ -10679,7 +10687,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); uint64_t CompressedSize; uint64_t DecompressedSize; @@ -10724,7 +10733,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); @@ -10753,7 +10763,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -10838,7 +10849,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath)); + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 01a703a1b..1fd59acdf 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -379,9 +379,10 @@ JupiterResult JupiterSession::ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload) { ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCbObject); - HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/search", Namespace, BucketId), - Payload, - {HttpClient::Accept(ZenContentType::kCbObject)}); + std::string OptionalBucketPath = BucketId.empty() ? "" : fmt::format("/{}", BucketId); + HttpClient::Response Response = m_HttpClient.Post(fmt::format("/api/v2/builds/{}{}/search", Namespace, OptionalBucketPath), + Payload, + {HttpClient::Accept(ZenContentType::kCbObject)}); return detail::ConvertResponse(Response, "JupiterSession::ListBuilds"sv); } -- cgit v1.2.3 From c44b9f7151a047cde5edd369f9adb09518a0bc6f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Jun 2025 14:03:55 +0200 Subject: builds download url (#419) * RemoveQuotes helper * `--url` option for `zen builds` command has been reworked to accept a "Cloud Artifact URL", removing the need to specify "host", "namespace" and "bucket" separately --- src/zen/cmds/builds_cmd.cpp | 195 ++++++++++++++++------- src/zen/cmds/builds_cmd.h | 2 +- src/zenutil/commandlineoptions.cpp | 20 ++- src/zenutil/include/zenutil/commandlineoptions.h | 1 + 4 files changed, 154 insertions(+), 64 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 372291fcc..1c1193a3f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -37,6 +37,7 @@ #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -9434,12 +9435,7 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); - Ops.add_option("cloud build", - "", - "url", - "Cloud Builds host url (legacy - use --override-host)", - cxxopts::value(m_OverrideHost), - ""); + Ops.add_option("cloud build", "", "url", "Cloud Artifact URL", cxxopts::value(m_Url), ""); Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), ""); Ops.add_option("cloud build", "", @@ -9829,6 +9825,55 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParseSystemOptions(); auto ParseStorageOptions = [&](bool RequireBucket) { + m_Host = RemoveQuotes(m_Host); + m_OverrideHost = RemoveQuotes(m_OverrideHost); + m_Url = RemoveQuotes(m_Url); + m_Namespace = RemoveQuotes(m_Url); + m_Bucket = RemoveQuotes(m_Bucket); + if (!m_Url.empty()) + { + if (!m_Host.empty()) + { + throw zen::OptionParseException(fmt::format("host is not compatible with the url option\n{}", m_Options.help())); + } + if (!m_Bucket.empty()) + { + throw zen::OptionParseException(fmt::format("bucket is not compatible with the url option\n{}", m_Options.help())); + } + if (!m_BuildId.empty()) + { + throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", m_Options.help())); + } + const std::string ArtifactURLRegExString = R"((.*?:\/\/.*?)\/api\/v2\/builds\/(.*?)\/(.*?)\/(.*))"; + const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript); + std::match_results MatchResults; + const std::string_view Url(RemoveQuotes(m_Url)); + if (regex_match(begin(Url), end(Url), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) + { + auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { + ZEN_ASSERT(Index < MatchResults.size()); + + const auto& Match = MatchResults[Index]; + + return std::string_view(&*Match.first, Match.second - Match.first); + }; + + const std::string_view Host = GetMatch(1); + const std::string_view Namespace = GetMatch(2); + const std::string_view Bucket = GetMatch(3); + const std::string_view BuildId = GetMatch(4); + + m_Host = Host; + m_Namespace = Namespace; + m_Bucket = Bucket; + m_BuildId = BuildId; + } + else + { + throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", m_Options.help())); + } + } + if (!m_OverrideHost.empty() || !m_Host.empty()) { if (!m_StoragePath.empty()) @@ -9890,6 +9935,19 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseAuthOptions = [&]() { + m_OpenIdProviderUrl = RemoveQuotes(m_OpenIdProviderUrl); + m_OpenIdClientId = RemoveQuotes(m_OpenIdClientId); + + m_AccessToken = RemoveQuotes(m_AccessToken); + m_EncryptionKey = RemoveQuotes(m_EncryptionKey); + m_EncryptionIV = RemoveQuotes(m_EncryptionIV); + + m_OAuthUrl = RemoveQuotes(m_OAuthUrl); + m_OAuthClientId = RemoveQuotes(m_OAuthClientId); + m_OAuthClientSecret = RemoveQuotes(m_OAuthClientSecret); + + m_OidcTokenAuthExecutablePath = RemoveQuotes(m_OidcTokenAuthExecutablePath); + if (!m_OpenIdProviderUrl.empty() && !m_OpenIdClientId.empty()) { CreateAuthMgr(); @@ -9993,6 +10051,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) bool RequireBucket) -> StorageInstance { ParseStorageOptions(RequireBucket); + m_ZenCacheHost = RemoveQuotes(m_ZenCacheHost); + StorageInstance Result; std::string BuildStorageName = ZEN_CLOUD_STORAGE; @@ -10259,6 +10319,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseBlobHash = [&]() -> IoHash { + m_BlobHash = RemoveQuotes(m_BlobHash); if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -10274,6 +10335,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseBuildId = [&]() -> Oid { + m_BuildId = RemoveQuotes(m_BuildId); if (m_BuildId.length() != Oid::StringLength) { throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); @@ -10289,6 +10351,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; auto ParseBuildPartId = [&]() -> Oid { + m_BuildPartId = RemoveQuotes(m_BuildPartId); if (m_BuildPartId.length() != Oid::StringLength) { throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); @@ -10307,7 +10370,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::vector BuildPartIds; for (const std::string& BuildPartId : m_BuildPartIds) { - BuildPartIds.push_back(Oid::TryFromHexString(BuildPartId)); + BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId))); if (BuildPartIds.back() == Oid::Zero) { throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); @@ -10316,6 +10379,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return BuildPartIds; }; + auto ParseBuildPartNames = [&]() -> std::vector { + std::vector BuildPartNames; + for (const std::string& BuildPartName : m_BuildPartNames) + { + BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName))); + if (BuildPartNames.back().empty()) + { + throw zen::OptionParseException( + fmt::format("build-part-names '{}' is invalid\n{}", BuildPartName, m_DownloadOptions.help())); + } + } + return BuildPartNames; + }; + auto ParseBuildMetadata = [&]() -> CbObject { if (m_CreateBuild) { @@ -10342,6 +10419,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } return MetaData; } + m_BuildMetadata = RemoveQuotes(m_BuildMetadata); if (!m_BuildMetadata.empty()) { CbObjectWriter MetaDataWriter(1024); @@ -10520,22 +10598,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); - if (m_BuildPartName.empty()) - { - m_BuildPartName = m_Path.filename().string(); - } - - const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); - if (m_BuildId.empty()) - { - m_BuildId = BuildId.ToString(); - } - const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); - if (m_BuildPartId.empty()) - { - m_BuildPartId = BuildPartId.ToString(); - } - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10557,10 +10619,28 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + if (m_BuildPartName.empty()) + { + m_BuildPartName = m_Path.filename().string(); + } + + const Oid BuildId = m_BuildId.empty() ? Oid::NewOid() : ParseBuildId(); + if (m_BuildId.empty()) + { + m_BuildId = BuildId.ToString(); + } + const Oid BuildPartId = m_BuildPartId.empty() ? Oid::NewOid() : ParseBuildPartId(); + if (m_BuildPartId.empty()) + { + m_BuildPartId = BuildPartId.ToString(); + } + CbObject MetaData = ParseBuildMetadata(); const std::filesystem::path TempDir = UploadTempDirectory(m_Path); + m_ManifestPath = RemoveQuotes(m_ManifestPath); + UploadFolder(Storage, BuildId, BuildPartId, @@ -10605,6 +10685,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); + if (m_ZenFolderPath.empty()) + { + m_ZenFolderPath = m_Path / ZenFolderName; + } + MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = + CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + const Oid BuildId = ParseBuildId(); if (m_PostDownloadVerify && m_PrimeCacheOnly) @@ -10623,24 +10715,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); } - std::vector BuildPartIds = ParseBuildPartIds(); - - if (m_ZenFolderPath.empty()) - { - m_ZenFolderPath = m_Path / ZenFolderName; - } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - - BuildStorage::Statistics StorageStats; - BuildStorageCache::Statistics StorageCacheStats; - - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + std::vector BuildPartIds = ParseBuildPartIds(); + std::vector BuildPartNames = ParseBuildPartNames(); DownloadFolder(Storage, BuildId, BuildPartIds, - m_BuildPartNames, + BuildPartNames, m_Path, m_ZenFolderPath, m_SystemRootDir, @@ -10665,10 +10746,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - IoHash BlobHash = ParseBlobHash(); - - const Oid BuildId = Oid::FromHexString(m_BuildId); - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10690,6 +10767,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + IoHash BlobHash = ParseBlobHash(); + + const Oid BuildId = Oid::FromHexString(m_BuildId); + uint64_t CompressedSize; uint64_t DecompressedSize; ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); @@ -10708,13 +10789,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - Oid BuildId = ParseBuildId(); - - if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) - { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); - } - BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10736,6 +10810,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + Oid BuildId = ParseBuildId(); + + if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) + { + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + } + const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); ValidateStatistics ValidateStats; @@ -10769,7 +10850,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) { - Oid BuildId = Oid::FromHexString(BuildIdString); + Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); if (BuildId == Oid::Zero) { throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); @@ -10807,14 +10888,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); - m_BuildId = Oid::NewOid().ToString(); - m_BuildPartName = m_Path.filename().string(); - m_BuildPartId = Oid::NewOid().ToString(); - m_CreateBuild = true; - - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - if (m_OverrideHost.empty() && m_StoragePath.empty()) { m_StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); @@ -10852,6 +10925,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + m_BuildId = Oid::NewOid().ToString(); + m_BuildPartName = m_Path.filename().string(); + m_BuildPartId = Oid::NewOid().ToString(); + m_CreateBuild = true; + + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; { diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 41ed65105..378810155 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -38,6 +38,7 @@ private: // cloud builds std::string m_OverrideHost; std::string m_Host; + std::string m_Url; bool m_AssumeHttp2 = false; bool m_AllowRedirect = false; std::string m_Namespace; @@ -88,7 +89,6 @@ private: std::string m_Verb; // list, upload, download cxxopts::Options m_ListNamespacesOptions{"list-namespaces", "List available build namespaces"}; - std::string m_ListNamespacesResultPath; bool m_ListNamespacesRecursive = false; cxxopts::Options m_ListOptions{"list", "List available builds"}; diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp index 0dffa42f0..afef7f6f2 100644 --- a/src/zenutil/commandlineoptions.cpp +++ b/src/zenutil/commandlineoptions.cpp @@ -157,12 +157,7 @@ MakeSafeAbsolutePath(const std::filesystem::path& Path) std::filesystem::path StringToPath(const std::string_view& Path) { - std::string_view UnquotedPath = Path; - - if (UnquotedPath.length() > 2 && UnquotedPath.front() == '\"' && UnquotedPath.back() == '\"') - { - UnquotedPath = UnquotedPath.substr(1, UnquotedPath.length() - 2); - } + std::string_view UnquotedPath = RemoveQuotes(Path); if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) { @@ -172,6 +167,19 @@ StringToPath(const std::string_view& Path) return std::filesystem::path(UnquotedPath).make_preferred(); } +std::string_view +RemoveQuotes(const std::string_view& Arg) +{ + if (Arg.length() > 2) + { + if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"') + { + return Arg.substr(1, Arg.length() - 2); + } + } + return Arg; +} + #if ZEN_WITH_TESTS void diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h index b7581f6cd..f927d41e5 100644 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -22,6 +22,7 @@ std::vector StripCommandlineQuotes(std::vector& InOutArg void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); std::filesystem::path StringToPath(const std::string_view& Path); +std::string_view RemoveQuotes(const std::string_view& Arg); void commandlineoptions_forcelink(); // internal -- cgit v1.2.3 From a584b6cbb5a3239488a38e7705c5443a77cf16ab Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Jun 2025 14:12:45 +0200 Subject: Initialize sentry after command line parsing is done (#420) Previously the option variables were used before options parsing had a chance to set them according to command line options --- src/zen/zen.cpp | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 53fdf3d36..2dc0fa98c 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -844,7 +844,6 @@ main(int argc, char** argv) .add_option("ue-trace", "", "tracefile", "Path to write a trace to", cxxopts::value(TraceFile)->default_value(""), ""); #endif // ZEN_WITH_TRACE - Options.parse_positional({"command"}); #if ZEN_USE_SENTRY bool NoSentry = false; bool SentryAllowPII = false; @@ -854,29 +853,7 @@ main(int argc, char** argv) cxxopts::value(SentryAllowPII)->default_value("false")); #endif -#if ZEN_USE_SENTRY - SentryIntegration Sentry; - - if (NoSentry == false) - { - std::string SentryDatabasePath = (GetRunningExecutablePath().parent_path() / ".sentry-native").string(); - - ExtendableStringBuilder<512> SB; - for (int i = 0; i < argc; ++i) - { - if (i) - { - SB.Append(' '); - } - - SB.Append(argv[i]); - } - - Sentry.Initialize(SentryDatabasePath, {}, SentryAllowPII, SB.ToString()); - - SentryIntegration::ClearCaches(); - } -#endif + Options.parse_positional({"command"}); const bool IsNullInvoke = (argc == 1); // If no arguments are passed we want to print usage information @@ -917,6 +894,30 @@ main(int argc, char** argv) exit(0); } +#if ZEN_USE_SENTRY + SentryIntegration Sentry; + + if (NoSentry == false) + { + std::string SentryDatabasePath = (GetRunningExecutablePath().parent_path() / ".sentry-native").string(); + + ExtendableStringBuilder<512> SB; + for (int i = 0; i < argc; ++i) + { + if (i) + { + SB.Append(' '); + } + + SB.Append(argv[i]); + } + + Sentry.Initialize(SentryDatabasePath, {}, SentryAllowPII, SB.ToString()); + + SentryIntegration::ClearCaches(); + } +#endif + zen::LoggingOptions LogOptions; LogOptions.IsDebug = GlobalOptions.IsDebug; LogOptions.IsVerbose = GlobalOptions.IsVerbose; -- cgit v1.2.3 From a705d9135fc6b10c22f64a0d1fb715e21b77076c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Jun 2025 13:31:52 +0200 Subject: revert system temp dir for builds upload (#422) * revert temp upload folder to be inside source directory to avoid filling up system disk * use selected subcommand when displaying help for failed command line options in zen builds --- src/zen/cmds/builds_cmd.cpp | 56 ++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1c1193a3f..b04575009 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9834,15 +9834,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_Host.empty()) { - throw zen::OptionParseException(fmt::format("host is not compatible with the url option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("host is not compatible with the url option\n{}", SubOption->help())); } if (!m_Bucket.empty()) { - throw zen::OptionParseException(fmt::format("bucket is not compatible with the url option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("bucket is not compatible with the url option\n{}", SubOption->help())); } if (!m_BuildId.empty()) { - throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", SubOption->help())); } const std::string ArtifactURLRegExString = R"((.*?:\/\/.*?)\/api\/v2\/builds\/(.*?)\/(.*?)\/(.*))"; const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript); @@ -9870,7 +9870,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", SubOption->help())); } } @@ -9879,20 +9879,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_StoragePath.empty()) { throw zen::OptionParseException( - fmt::format("host/url/override-host is not compatible with the storage-path option\n{}", m_Options.help())); + fmt::format("host/url/override-host is not compatible with the storage-path option\n{}", SubOption->help())); } if (m_Namespace.empty()) { - throw zen::OptionParseException(fmt::format("namespace option is required for this storage option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("namespace option is required for this storage option\n{}", SubOption->help())); } if (RequireBucket && m_Bucket.empty()) { - throw zen::OptionParseException(fmt::format("bucket option is required for this storage option\n{}", m_Options.help())); + throw zen::OptionParseException(fmt::format("bucket option is required for this storage option\n{}", SubOption->help())); } } else if (m_StoragePath.empty()) { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", SubOption->help())); } MakeSafeAbsolutePathÍnPlace(m_StoragePath); }; @@ -10262,7 +10262,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", SubOption->help())); } if (!m_ZenCacheHost.empty()) { @@ -10305,7 +10305,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto ParsePath = [&]() { if (m_Path.empty()) { - throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("local-path is required\n{}", SubOption->help())); } MakeSafeAbsolutePathÍnPlace(m_Path); }; @@ -10313,7 +10313,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto ParseDiffPath = [&]() { if (m_DiffPath.empty()) { - throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); + throw zen::OptionParseException(fmt::format("compare-path is required\n{}", SubOption->help())); } MakeSafeAbsolutePathÍnPlace(m_DiffPath); }; @@ -10322,13 +10322,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BlobHash = RemoveQuotes(m_BlobHash); if (m_BlobHash.empty()) { - throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", SubOption->help())); } IoHash BlobHash; if (!IoHash::TryParse(m_BlobHash, BlobHash)) { - throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", SubOption->help())); } return BlobHash; @@ -10338,11 +10338,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildId = RemoveQuotes(m_BuildId); if (m_BuildId.length() != Oid::StringLength) { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", SubOption->help())); } else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) { - throw zen::OptionParseException(fmt::format("Invalid build id\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Invalid build id\n{}", SubOption->help())); } else { @@ -10354,11 +10354,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildPartId = RemoveQuotes(m_BuildPartId); if (m_BuildPartId.length() != Oid::StringLength) { - throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", SubOption->help())); } else if (Oid BuildPartId = Oid::FromHexString(m_BuildPartId); BuildPartId == Oid::Zero) { - throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", SubOption->help())); } else { @@ -10373,7 +10373,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartIds.push_back(Oid::TryFromHexString(RemoveQuotes(BuildPartId))); if (BuildPartIds.back() == Oid::Zero) { - throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, m_DownloadOptions.help())); + throw zen::OptionParseException(fmt::format("build-part-id '{}' is invalid\n{}", BuildPartId, SubOption->help())); } } return BuildPartIds; @@ -10386,8 +10386,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartNames.push_back(std::string(RemoveQuotes(BuildPartName))); if (BuildPartNames.back().empty()) { - throw zen::OptionParseException( - fmt::format("build-part-names '{}' is invalid\n{}", BuildPartName, m_DownloadOptions.help())); + throw zen::OptionParseException(fmt::format("build-part-names '{}' is invalid\n{}", BuildPartName, SubOption->help())); } } return BuildPartNames; @@ -10398,11 +10397,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (m_BuildMetadataPath.empty() && m_BuildMetadata.empty()) { - throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Options for builds target are missing\n{}", SubOption->help())); } if (!m_BuildMetadataPath.empty() && !m_BuildMetadata.empty()) { - throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("Conflicting options for builds target\n{}", SubOption->help())); } if (!m_BuildMetadataPath.empty()) @@ -10440,12 +10439,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildMetadataPath.empty()) { throw zen::OptionParseException( - fmt::format("metadata-path option is only valid if creating a build\n{}", m_UploadOptions.help())); + fmt::format("metadata-path option is only valid if creating a build\n{}", SubOption->help())); } if (!m_BuildMetadata.empty()) { - throw zen::OptionParseException( - fmt::format("metadata option is only valid if creating a build\n{}", m_UploadOptions.help())); + throw zen::OptionParseException(fmt::format("metadata option is only valid if creating a build\n{}", SubOption->help())); } } return {}; @@ -10637,7 +10635,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) CbObject MetaData = ParseBuildMetadata(); - const std::filesystem::path TempDir = UploadTempDirectory(m_Path); + const std::filesystem::path TempDir = ZenTempFolderPath(m_ZenFolderPath); m_ManifestPath = RemoveQuotes(m_ManifestPath); @@ -10702,7 +10700,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_PostDownloadVerify && m_PrimeCacheOnly) { throw zen::OptionParseException( - fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", m_DownloadOptions.help())); + fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", SubOption->help())); } if (m_Clean && m_PrimeCacheOnly) @@ -10814,7 +10812,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildPartName.empty() && !m_BuildPartId.empty()) { - throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); + throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", SubOption->help())); } const Oid BuildPartId = m_BuildPartName.empty() ? Oid::Zero : ParseBuildPartId(); @@ -10853,7 +10851,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Oid BuildId = Oid::FromHexString(RemoveQuotes(BuildIdString)); if (BuildId == Oid::Zero) { - throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); + throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, SubOption->help())); } DownloadFolder(Storage, BuildId, -- cgit v1.2.3 From 40b9386054de3c23f77da74eefaa743240d164fd Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Jun 2025 14:40:02 +0200 Subject: pause, resume and abort running builds cmd (#421) - Feature: `zen builds pause`, `zen builds resume` and `zen builds abort` commands to control a running `zen builds` command - `--process-id` the process id to control, if omitted it tries to find a running process using the same executable as itself - Improvement: Process report now indicates if it is pausing or aborting --- src/zen/cmds/builds_cmd.cpp | 437 ++++++++++++++++++++++----- src/zen/cmds/builds_cmd.h | 27 +- src/zen/cmds/up_cmd.cpp | 2 +- src/zen/cmds/wipe_cmd.cpp | 12 +- src/zen/zen.cpp | 43 ++- src/zen/zen.h | 21 ++ src/zencore/filesystem.cpp | 230 ++++++++++++++ src/zencore/include/zencore/filesystem.h | 10 + src/zencore/include/zencore/process.h | 2 +- src/zencore/process.cpp | 64 ++-- src/zenserver/cache/httpstructuredcache.cpp | 7 +- src/zenserver/projectstore/projectstore.cpp | 9 +- src/zenstore/buildstore/buildstore.cpp | 3 +- src/zenstore/cache/cachedisklayer.cpp | 12 +- src/zenstore/compactcas.cpp | 3 +- src/zenstore/filecas.cpp | 3 +- src/zenutil/chunkedcontent.cpp | 24 +- src/zenutil/include/zenutil/chunkedcontent.h | 17 +- src/zenutil/include/zenutil/parallelwork.h | 13 +- src/zenutil/parallelwork.cpp | 19 +- src/zenutil/zenserverprocess.cpp | 2 +- 21 files changed, 799 insertions(+), 161 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b04575009..49b032ab1 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -59,23 +60,205 @@ ZEN_THIRD_PARTY_INCLUDES_END #define ZEN_CLOUD_STORAGE "Cloud Storage" namespace zen { + +using namespace std::literals; + namespace { + namespace zenutil { +#if ZEN_PLATFORM_WINDOWS + class SecurityAttributes + { + public: + inline SECURITY_ATTRIBUTES* Attributes() { return &m_Attributes; } + + protected: + SECURITY_ATTRIBUTES m_Attributes{}; + SECURITY_DESCRIPTOR m_Sd{}; + }; + + // Security attributes which allows any user access + + class AnyUserSecurityAttributes : public SecurityAttributes + { + public: + AnyUserSecurityAttributes() + { + m_Attributes.nLength = sizeof m_Attributes; + m_Attributes.bInheritHandle = false; // Disable inheritance + + const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); + + if (Success) + { + if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE)) + { + ThrowLastError("SetSecurityDescriptorDacl failed"); + } + + m_Attributes.lpSecurityDescriptor = &m_Sd; + } + } + }; +#endif // ZEN_PLATFORM_WINDOWS + + } // namespace zenutil + static std::atomic AbortFlag = false; - static void SignalCallbackHandler(int SigNum) + static std::atomic PauseFlag = false; + + static void SignalCallbackHandler(int SigNum) { if (SigNum == SIGINT) { + PauseFlag = false; AbortFlag = true; } #if ZEN_PLATFORM_WINDOWS if (SigNum == SIGBREAK) { + PauseFlag = false; AbortFlag = true; } #endif // ZEN_PLATFORM_WINDOWS } - using namespace std::literals; + struct ZenStateSharedData + { + static constexpr uint64_t kMagicV1 = 0x3176646d636e657a; // zencmdv1 + + uint64_t Magic = 0; // Implies the size and layout of this struct - changes to the data requires change to Magic constant + std::atomic Pid; + uint8_t SessionId[12]; + uint8_t Padding1[4]; + std::atomic Abort; + std::atomic Pause; + uint8_t Padding2[2]; + }; + + struct MemMap + { + void* Handle = nullptr; + void* Data = nullptr; + size_t Size = 0; + std::string Name; + }; + + class ZenState + { + public: + ZenState(const ZenState&) = delete; + ZenState& operator=(const ZenState&) = delete; + + ZenState(); + explicit ZenState(uint32_t Pid); + ~ZenState(); + + const ZenStateSharedData& StateData() const + { + ZEN_ASSERT(m_Data); + return *m_Data; + } + ZenStateSharedData& StateData() + { + ZEN_ASSERT(m_Data); + return *m_Data; + } + + private: + static constexpr std::string_view MapBaseName = "UnrealEngineZenCmd_"sv; + static constexpr size_t MapSize = sizeof(ZenStateSharedData); + + bool m_Created = false; + std::unique_ptr m_MemMap; + ZenStateSharedData* m_Data = nullptr; + + std::thread m_StateMonitor; + Event m_ExitStateMonitorEvent; + }; + + ZenState::ZenState(uint32_t Pid) + { + const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid); + + if (!IsProcessRunning(Pid)) + { + throw std::runtime_error(fmt::format("The process {} is not running", Pid)); + } + std::unique_ptr MemMap = OpenSharedMemory(ZenStateMapName, MapSize, false); + if (!MemMap) + { + throw std::runtime_error(fmt::format("The process {} is not a running zen process", Pid)); + } + ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData(); + if (uint64_t MemMagic = data->Magic; MemMagic != ZenStateSharedData::kMagicV1) + { + throw std::runtime_error(fmt::format("The mem map for process {} has an unsupported magic {:x}, expected {:x}", + Pid, + MemMagic, + ZenStateSharedData::kMagicV1)); + } + if (uint32_t MemPid = data->Pid.load(); MemPid != Pid) + { + throw std::runtime_error( + fmt::format("The mem map for process {} has an missmatching pid of {}, expected {}", Pid, MemPid, Pid)); + } + m_MemMap = std::move(MemMap); + m_Data = data; + } + + ZenState::ZenState() + { + int Pid = GetCurrentProcessId(); + + const std::string ZenStateMapName = fmt::format("{}{}", MapBaseName, Pid); + + std::unique_ptr MemMap = CreateSharedMemory(ZenStateMapName, MapSize, false); + if (!MemMap) + { + throw std::runtime_error(fmt::format("The mem map for process {} could not be created", Pid)); + } + + ZenStateSharedData* data = (ZenStateSharedData*)MemMap->GetData(); + memset(data, 0, sizeof(ZenStateSharedData)); + data->Magic = ZenStateSharedData::kMagicV1; + data->Pid.store(gsl::narrow(Pid)); + data->SessionId; + data->Abort.store(false); + data->Pause.store(false); + const Oid SessionId = GetSessionId(); + memcpy(data->SessionId, &SessionId, sizeof SessionId); + + m_MemMap = std::move(MemMap); + m_Data = data; + + m_StateMonitor = std::thread([this]() { + while (!m_ExitStateMonitorEvent.Wait(500)) + { + if (m_Data->Abort.load()) + { + AbortFlag.store(true); + } + PauseFlag.store(m_Data->Pause.load()); + } + }); + } + + ZenState::~ZenState() + { + try + { + if (m_StateMonitor.joinable()) + { + m_ExitStateMonitorEvent.Set(); + m_StateMonitor.join(); + } + m_MemMap.reset(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("ZenState::~ZenState threw exception: {}", Ex.what()); + } + } static const size_t DefaultMaxBlockSize = 64u * 1024u * 1024u; static const size_t DefaultMaxChunkEmbedSize = 3u * 512u * 1024u; @@ -435,7 +618,7 @@ namespace { std::atomic DiscoveredItemCount = 0; std::atomic DeletedItemCount = 0; std::atomic DeletedByteCount = 0; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); struct AsyncVisitor : public GetDirectoryContentVisitor { @@ -549,8 +732,8 @@ namespace { uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); uint64_t Deleted = DeletedItemCount.load(); @@ -559,7 +742,8 @@ namespace { Progress.UpdateState({.Task = "Cleaning folder ", .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted}, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); @@ -598,12 +782,17 @@ namespace { Progress.UpdateState({.Task = "Cleaning folder ", .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted}, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, false); } } Progress.Finish(); + if (AbortFlag) + { + return false; + } uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (ElapsedTimeMs >= 200) @@ -1954,7 +2143,7 @@ namespace { WorkerThreadPool& NetworkPool = GetNetworkPool(); WorkerThreadPool& VerifyPool = GetIOWorkerPool(); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -2117,8 +2306,8 @@ namespace { }); } - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); const uint64_t DownloadedAttachmentCount = DownloadStats.DownloadedChunkCount + DownloadStats.DownloadedBlockCount; const uint64_t DownloadedByteCount = DownloadStats.DownloadedChunkByteCount + DownloadStats.DownloadedBlockByteCount; @@ -2141,7 +2330,8 @@ namespace { .Details = Details, .TotalCount = gsl::narrow(AttachmentsToVerifyCount * 2), .RemainingCount = gsl::narrow(AttachmentsToVerifyCount * 2 - - (DownloadedAttachmentCount + ValidateStats.VerifiedAttachmentCount.load()))}, + (DownloadedAttachmentCount + ValidateStats.VerifiedAttachmentCount.load())), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); @@ -2397,7 +2587,7 @@ namespace { FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); std::atomic QueuedPendingBlocksForUpload = 0; @@ -2567,8 +2757,8 @@ namespace { }); } - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load()); @@ -2587,7 +2777,8 @@ namespace { {.Task = "Generating blocks", .Details = Details, .TotalCount = gsl::narrow(NewBlockCount), - .RemainingCount = gsl::narrow(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load())}, + .RemainingCount = gsl::narrow(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); @@ -2626,7 +2817,7 @@ namespace { FilteredRate FilteredCompressedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); std::atomic UploadedBlockSize = 0; std::atomic UploadedBlockCount = 0; @@ -3006,8 +3197,8 @@ namespace { }); } - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); @@ -3035,13 +3226,15 @@ namespace { ProgressBar.UpdateState({.Task = "Uploading blobs ", .Details = Details, .TotalCount = gsl::narrow(TotalRawSize), - .RemainingCount = gsl::narrow(TotalRawSize - UploadedRawSize)}, + .RemainingCount = gsl::narrow(TotalRawSize - UploadedRawSize), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); ZEN_ASSERT(AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0); ProgressBar.Finish(); + UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS(); LooseChunksStats.CompressChunksElapsedWallTimeUS = FilteredCompressedBytesPerSecond.GetElapsedTimeUS(); } @@ -3493,7 +3686,7 @@ namespace { Content, *ChunkController, GetUpdateDelayMS(ProgressMode), - [&](bool, std::ptrdiff_t) { + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), @@ -3506,16 +3699,18 @@ namespace { ProgressBar.UpdateState({.Task = "Scanning files ", .Details = Details, .TotalCount = TotalRawSize, - .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load()}, + .RemainingCount = TotalRawSize - ChunkingStats.BytesHashed.load(), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }, - AbortFlag); + AbortFlag, + PauseFlag); + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); if (AbortFlag) { return; } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); } ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec", @@ -4266,7 +4461,7 @@ namespace { WorkerThreadPool& VerifyPool = GetIOWorkerPool(); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); const uint32_t PathCount = gsl::narrow(Content.Paths.size()); @@ -4414,8 +4609,8 @@ namespace { }); } - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}", VerifyFolderStats.FilesVerified.load(), PathCount, @@ -4424,12 +4619,18 @@ namespace { ProgressBar.UpdateState({.Task = "Verifying files ", .Details = Details, .TotalCount = gsl::narrow(PathCount), - .RemainingCount = gsl::narrow(PathCount - VerifyFolderStats.FilesVerified.load())}, + .RemainingCount = gsl::narrow(PathCount - VerifyFolderStats.FilesVerified.load()), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs(); ProgressBar.Finish(); + if (AbortFlag) + { + return; + } + for (const std::string& Error : Errors) { ZEN_CONSOLE("{}", Error); @@ -5361,7 +5562,7 @@ namespace { Stopwatch Timer; auto _ = MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); std::atomic CompletedPathCount = 0; uint32_t PathIndex = 0; @@ -5392,7 +5593,7 @@ namespace { }); PathIndex += PathRangeCount; } - Work.Wait(200, [&](bool, ptrdiff_t) { + Work.Wait(200, [&](bool, bool, ptrdiff_t) { if (ProgressCallback) { ProgressCallback(PathCount, CompletedPathCount.load()); @@ -5670,7 +5871,7 @@ namespace { ScavengedPaths.resize(ScavengePathCount); ProgressBar ScavengeProgressBar(ProgressMode, "Scavenging"); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); std::atomic PathsFound(0); std::atomic ChunksFound(0); @@ -5800,8 +6001,8 @@ namespace { { ZEN_TRACE_CPU("ScavengeScan_Wait"); - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavanging", PathsScavenged.load(), ScavengePathCount, @@ -5810,11 +6011,17 @@ namespace { ScavengeProgressBar.UpdateState({.Task = "Scavenging ", .Details = Details, .TotalCount = ScavengePathCount, - .RemainingCount = ScavengePathCount - PathsScavenged.load()}, + .RemainingCount = ScavengePathCount - PathsScavenged.load(), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); } + ScavengeProgressBar.Finish(); + if (AbortFlag) + { + return; + } for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (!SequenceIndexesLeftToFindToRemoteIndex.empty()); @@ -6130,7 +6337,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar WriteProgressBar(ProgressMode, PrimeCacheOnly ? "Downloading" : "Writing"); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); struct LooseChunkHashWorkData { @@ -7583,8 +7790,8 @@ namespace { { ZEN_TRACE_CPU("WriteChunks_Wait"); - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); @@ -7610,7 +7817,8 @@ namespace { .Details = Details, .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) - : (BytesToWrite - DiskStats.WriteByteCount.load())}, + : (BytesToWrite - DiskStats.WriteByteCount.load()), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); } @@ -7618,13 +7826,12 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); FilteredDownloadedBytesPerSecond.Stop(); + WriteProgressBar.Finish(); if (AbortFlag) { return; } - WriteProgressBar.Finish(); - if (!PrimeCacheOnly) { uint32_t RawSequencesMissingWriteCount = 0; @@ -7771,7 +7978,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar CacheLocalProgressBar(ProgressMode, "Cache Local Data"); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t LocalPathIndex : FilesToCache) { @@ -7800,26 +8007,26 @@ namespace { { ZEN_TRACE_CPU("CacheLocal_Wait"); - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); const uint64_t WorkTotal = FilesToCache.size(); const uint64_t WorkComplete = CachedCount.load(); std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount)); CacheLocalProgressBar.UpdateState({.Task = "Caching local ", .Details = Details, .TotalCount = gsl::narrow(WorkTotal), - .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); } + CacheLocalProgressBar.Finish(); if (AbortFlag) { return; } - CacheLocalProgressBar.Finish(); - ZEN_DEBUG( "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " "Delete: {}", @@ -7858,7 +8065,7 @@ namespace { WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar RebuildProgressBar(ProgressMode, "Rebuild State"); - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); OutLocalFolderState.Paths.resize(RemoteContent.Paths.size()); OutLocalFolderState.RawSizes.resize(RemoteContent.Paths.size()); @@ -8109,26 +8316,21 @@ namespace { { ZEN_TRACE_CPU("FinalizeTree_Wait"); - Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted, PendingWork); + Work.Wait(GetUpdateDelayMS(ProgressMode), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", .Details = Details, .TotalCount = gsl::narrow(WorkTotal), - .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); } RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs(); - - if (AbortFlag) - { - return; - } - RebuildProgressBar.Finish(); } } @@ -8690,10 +8892,15 @@ namespace { ProgressBar.UpdateState({.Task = "Checking files ", .Details = Details, .TotalCount = PathCount, - .RemainingCount = PathCount - CompletedPathCount}, + .RemainingCount = PathCount - CompletedPathCount, + .Status = ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)}, false); }); ProgressBar.Finish(); + if (AbortFlag) + { + return {}; + } } bool ScanContent = true; @@ -8731,7 +8938,7 @@ namespace { UpdatedContent, ChunkController, GetUpdateDelayMS(ProgressMode), - [&](bool, std::ptrdiff_t) { + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), @@ -8744,16 +8951,19 @@ namespace { ProgressBar.UpdateState({.Task = "Scanning files ", .Details = Details, .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load(), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }, - AbortFlag); + AbortFlag, + PauseFlag); + + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); if (AbortFlag) { return {}; } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); } } @@ -8799,7 +9009,7 @@ namespace { OutLocalFolderContent, ChunkController, GetUpdateDelayMS(ProgressMode), - [&](bool, std::ptrdiff_t) { + [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), @@ -8812,18 +9022,19 @@ namespace { ProgressBar.UpdateState({.Task = "Scanning files ", .Details = Details, .TotalCount = ByteCountToScan, - .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load())}, + .RemainingCount = (ByteCountToScan - ChunkingStats.BytesHashed.load()), + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }, - AbortFlag); + AbortFlag, + PauseFlag); + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); if (AbortFlag) { return {}; } - - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); } return LocalContent; } @@ -9740,6 +9951,21 @@ BuildsCommand::BuildsCommand() m_FetchBlobOptions.parse_positional({"build-id", "blob-hash"}); m_FetchBlobOptions.positional_help("build-id blob-hash"); + auto AddZenProcessOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", "", "process-id", "Process id of running process", cxxopts::value(m_ZenProcessId), ""); + }; + AddZenProcessOptions(m_PauseOptions); + m_PauseOptions.parse_positional({"process-id"}); + m_PauseOptions.positional_help("process-id"); + + AddZenProcessOptions(m_ResumeOptions); + m_ResumeOptions.parse_positional({"process-id"}); + m_ResumeOptions.positional_help("process-id"); + + AddZenProcessOptions(m_AbortOptions); + m_AbortOptions.parse_positional({"process-id"}); + m_AbortOptions.positional_help("process-id"); + AddSystemOptions(m_ValidateBuildPartOptions); AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); @@ -10458,7 +10684,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); } BuildStorage::Statistics StorageStats; @@ -10513,7 +10742,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_ListResultPath.empty()) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", + GetRunningExecutablePath(), + ZEN_CFG_VERSION_BUILD_STRING_FULL, + GetCurrentProcessId()); } CbObject QueryObject; if (m_ListQueryPath.empty()) @@ -10592,7 +10824,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + + ZenState InstanceState; ParsePath(); @@ -10679,7 +10913,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + + ZenState InstanceState; ParsePath(); @@ -10743,7 +10979,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10785,7 +11022,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + ZEN_CONSOLE("Running {}: {} (pid {})", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL, GetCurrentProcessId()); + + ZenState InstanceState; BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -10877,6 +11116,50 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } + auto ParseZenProcessId = [&]() { + if (m_ZenProcessId == -1) + { + const std::filesystem::path RunningExecutablePath = GetRunningExecutablePath(); + ProcessHandle RunningProcess; + std::error_code Ec = FindProcess(RunningExecutablePath, RunningProcess, /*IncludeSelf*/ false); + if (Ec) + { + throw std::runtime_error( + fmt::format("Failed finding process running '{}', reason: '{}'", RunningExecutablePath, Ec.message())); + } + if (!RunningProcess.IsValid()) + { + throw zen::OptionParseException( + fmt::format("Unable to find a running instance of the zen executable '{}'", RunningExecutablePath)); + } + m_ZenProcessId = RunningProcess.Pid(); + } + }; + + if (SubOption == &m_PauseOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(true); + return 0; + } + + if (SubOption == &m_ResumeOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Pause.store(false); + return 0; + } + + if (SubOption == &m_AbortOptions) + { + ParseZenProcessId(); + ZenState RunningState(m_ZenProcessId); + RunningState.StateData().Abort.store(true); + return 0; + } + if (SubOption == &m_TestOptions) { m_SystemRootDir = (GetRunningExecutablePath().parent_path() / ".tmpzensystem").make_preferred(); @@ -11046,7 +11329,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return true; }; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); uint32_t Randomizer = 0; auto FileSizeIt = DownloadContent.FileSizes.begin(); @@ -11104,8 +11387,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } FileSizeIt++; } - Work.Wait(5000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted); + Work.Wait(5000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); ZEN_CONSOLE("Scrambling files, {} remaining", PendingWork); }); ZEN_ASSERT(!AbortFlag.load()); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 378810155..b3466c0d3 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -114,6 +114,12 @@ private: cxxopts::Options m_FetchBlobOptions{"fetch-blob", "Fetch a blob from remote store"}; std::string m_BlobHash; + cxxopts::Options m_PauseOptions{"pause", "Pause an ongoing zen builds process"}; + cxxopts::Options m_ResumeOptions{"resume", "Resume a paused zen builds process"}; + cxxopts::Options m_AbortOptions{"abort", "Abort an ongoing zen builds process"}; + + int m_ZenProcessId = -1; + cxxopts::Options m_ValidateBuildPartOptions{"validate-part", "Fetch a build part and validate all referenced attachments"}; cxxopts::Options m_TestOptions{"test", "Test upload and download with verify"}; @@ -121,15 +127,18 @@ private: cxxopts::Options m_MultiTestDownloadOptions{"multi-test-download", "Test multiple sequenced downloads with verify"}; std::vector m_BuildIds; - cxxopts::Options* m_SubCommands[9] = {&m_ListNamespacesOptions, - &m_ListOptions, - &m_UploadOptions, - &m_DownloadOptions, - &m_DiffOptions, - &m_FetchBlobOptions, - &m_ValidateBuildPartOptions, - &m_TestOptions, - &m_MultiTestDownloadOptions}; + cxxopts::Options* m_SubCommands[12] = {&m_ListNamespacesOptions, + &m_ListOptions, + &m_UploadOptions, + &m_DownloadOptions, + &m_PauseOptions, + &m_ResumeOptions, + &m_AbortOptions, + &m_DiffOptions, + &m_FetchBlobOptions, + &m_ValidateBuildPartOptions, + &m_TestOptions, + &m_MultiTestDownloadOptions}; }; } // namespace zen diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index f3bf2f66d..d0763701e 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -311,7 +311,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // Try to find the running executable by path name std::filesystem::path ServerExePath = m_ProgramBaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; - if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec) + if (std::error_code Ec = FindProcess(ServerExePath, RunningProcess); !Ec, /*IncludeSelf*/ false) { ZEN_WARN("attempting hard terminate of zen process with pid ({})", RunningProcess.Pid()); try diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index 9dfdca0a1..fcc18df2b 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -33,6 +33,7 @@ namespace zen { namespace { static std::atomic AbortFlag = false; + static std::atomic PauseFlag = false; static bool IsVerbose = false; static bool Quiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; @@ -72,11 +73,13 @@ namespace { { if (SigNum == SIGINT) { + PauseFlag = false; AbortFlag = true; } #if ZEN_PLATFORM_WINDOWS if (SigNum == SIGBREAK) { + PauseFlag = false; AbortFlag = true; } #endif // ZEN_PLATFORM_WINDOWS @@ -248,7 +251,7 @@ namespace { return Added; }; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); struct AsyncVisitor : public GetDirectoryContentVisitor { @@ -423,12 +426,12 @@ namespace { GetIOWorkerPool(), Work.PendingWork()); - Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, ptrdiff_t PendingWork) { + Work.Wait(ProgressMode == ProgressBar::Mode::Pretty ? 200 : 5000, [&](bool IsAborted, bool IsPaused, ptrdiff_t PendingWork) { + ZEN_UNUSED(PendingWork); if (Quiet) { return; } - ZEN_UNUSED(IsAborted, PendingWork); LastUpdateTimeMs = Timer.GetElapsedTimeMs(); uint64_t Deleted = DeletedItemCount.load(); @@ -437,7 +440,8 @@ namespace { Progress.UpdateState({.Task = "Removing files ", .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), .TotalCount = Discovered, - .RemainingCount = Discovered - Deleted}, + .RemainingCount = Discovered - Deleted, + .Status = ProgressBar::State::CalculateStatus(IsAborted, IsPaused)}, false); }); diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 2dc0fa98c..119215d40 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -376,7 +376,8 @@ ProgressBar::SetLogOperationProgress(Mode InMode, uint32_t StepIndex, uint32_t S ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) : m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) -, m_LastUpdateMS(m_SW.GetElapsedTimeMs() - 10000) +, m_LastUpdateMS((uint64_t)-1) +, m_PausedMS(0) , m_SubTask(InSubTask) { if (!m_SubTask.empty() && InMode == Mode::Log) @@ -413,20 +414,46 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) } uint64_t ElapsedTimeMS = m_SW.GetElapsedTimeMs(); - if (!DoLinebreak && (NewState.Task == m_State.Task) && ((m_LastUpdateMS + 200) > ElapsedTimeMS)) + if (m_LastUpdateMS != (uint64_t)-1) { - return; + if (!DoLinebreak && (NewState.Status == m_State.Status) && (NewState.Task == m_State.Task) && + ((m_LastUpdateMS + 200) > ElapsedTimeMS)) + { + return; + } + if (m_State.Status == State::EStatus::Paused) + { + uint64_t ElapsedSinceLast = ElapsedTimeMS - m_LastUpdateMS; + m_PausedMS += ElapsedSinceLast; + } } m_LastUpdateMS = ElapsedTimeMS; + std::string Task = NewState.Task; + switch (NewState.Status) + { + case State::EStatus::Aborted: + Task = "Aborting"; + break; + case State::EStatus::Paused: + Task = "Paused"; + break; + default: + break; + } + if (NewState.Task.length() > Task.length()) + { + Task += std::string(NewState.Task.length() - Task.length(), ' '); + } + const size_t PercentDone = NewState.TotalCount > 0u ? gsl::narrow((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; if (m_Mode == Mode::Plain) { const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; - const std::string Output = fmt::format("{} {}% ({}){}\n", NewState.Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); + const std::string Output = fmt::format("{} {}% ({}){}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); OutputToConsoleRaw(Output); } else if (m_Mode == Mode::Pretty) @@ -435,12 +462,12 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; - uint64_t ETAMS = (PercentDone > 5) ? (ElapsedTimeMS * NewState.RemainingCount) / Completed : 0; + uint64_t ETAElapsedMS = ElapsedTimeMS -= m_PausedMS; + uint64_t ETAMS = + (NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0; uint32_t ConsoleColumns = GetConsoleColumns(); - std::string_view TaskString = NewState.Task; - const std::string PercentString = fmt::format("{:#3}%", PercentDone); const std::string ProgressBarString = @@ -454,7 +481,7 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) ExtendableStringBuilder<256> OutputBuilder; - OutputBuilder << "\r" << TaskString << PercentString; + OutputBuilder << "\r" << Task << PercentString; if (OutputBuilder.Size() + 1 < ConsoleColumns) { size_t RemainingSpace = ConsoleColumns - (OutputBuilder.Size() + 1); diff --git a/src/zen/zen.h b/src/zen/zen.h index 9ca228e37..995bd13b6 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -79,6 +79,26 @@ public: std::string Details; uint64_t TotalCount = 0; uint64_t RemainingCount = 0; + enum class EStatus + { + Running, + Aborted, + Paused + }; + EStatus Status = EStatus::Running; + + static EStatus CalculateStatus(bool IsAborted, bool IsPaused) + { + if (IsAborted) + { + return EStatus::Aborted; + } + if (IsPaused) + { + return EStatus::Paused; + } + return EStatus::Running; + } }; enum class Mode @@ -104,6 +124,7 @@ private: const Mode m_Mode; Stopwatch m_SW; uint64_t m_LastUpdateMS; + uint64_t m_PausedMS; State m_State; const std::string m_SubTask; size_t m_LastOutputLength = 0; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index c4264bc29..46337ffc8 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -33,6 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include # include # include +# include # include # include # include @@ -43,6 +44,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include # include # include +# include # include # include # include @@ -2824,6 +2826,218 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) return Result; } +class SharedMemoryImpl : public SharedMemory +{ +public: + struct Data + { + void* Handle = nullptr; + void* DataPtr = nullptr; + size_t Size = 0; + std::string Name; + }; + + static Data Open(std::string_view Name, size_t Size, bool SystemGlobal) + { +#if ZEN_PLATFORM_WINDOWS + std::wstring InstanceMapName = Utf8ToWide(fmt::format("{}\\{}", SystemGlobal ? "Global" : "Local", Name)); + + HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, InstanceMapName.c_str()); + if (hMap == NULL) + { + return {}; + } + void* pBuf = MapViewOfFile(hMap, // handle to map object + FILE_MAP_ALL_ACCESS, // read/write permission + 0, // offset high + 0, // offset low + DWORD(Size)); // size + + if (pBuf == NULL) + { + CloseHandle(hMap); + } + return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + ZEN_UNUSED(SystemGlobal); + std::string InstanceMapName = fmt::format("/{}", Name); + + int Fd = shm_open(InstanceMapName.c_str(), O_RDWR, 0666); + if (Fd < 0) + { + return {}; + } + void* hMap = (void*)intptr_t(Fd); + + struct stat Stat; + fstat(Fd, &Stat); + + if (size_t(Stat.st_size) < Size) + { + close(Fd); + return {}; + } + + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + close(Fd); + return {}; + } + return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + } + + static Data Create(std::string_view Name, size_t Size, bool SystemGlobal) + { +#if ZEN_PLATFORM_WINDOWS + std::wstring InstanceMapName = Utf8ToWide(fmt::format("{}\\{}", SystemGlobal ? "Global" : "Local", Name)); + + SECURITY_ATTRIBUTES m_Attributes{}; + SECURITY_DESCRIPTOR m_Sd{}; + + m_Attributes.nLength = sizeof m_Attributes; + m_Attributes.bInheritHandle = false; // Disable inheritance + + const BOOL Success = InitializeSecurityDescriptor(&m_Sd, SECURITY_DESCRIPTOR_REVISION); + + if (Success) + { + if (!SetSecurityDescriptorDacl(&m_Sd, TRUE, (PACL)NULL, FALSE)) + { + ThrowLastError("SetSecurityDescriptorDacl failed"); + } + + m_Attributes.lpSecurityDescriptor = &m_Sd; + } + + HANDLE hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file + &m_Attributes, // allow anyone to access + PAGE_READWRITE, // read/write access + 0, // maximum object size (high-order DWORD) + DWORD(Size), // maximum object size (low-order DWORD) + InstanceMapName.c_str()); + if (hMap == NULL) + { + return {}; + } + void* pBuf = MapViewOfFile(hMap, // handle to map object + FILE_MAP_ALL_ACCESS, // read/write permission + 0, // offset high + 0, // offset low + DWORD(Size)); // size + + if (pBuf == NULL) + { + CloseHandle(hMap); + return {}; + } + return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + ZEN_UNUSED(SystemGlobal); + std::string InstanceMapName = fmt::format("/{}", Name); + + int Fd = shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); + if (Fd < 0) + { + return {}; + } + fchmod(Fd, 0666); + void* hMap = (void*)intptr_t(Fd); + + int Result = ftruncate(Fd, Size); + ZEN_UNUSED(Result); + + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + close(Fd); + return {}; + } + return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + } + + static void Close(Data&& MemMap, bool Delete) + { +#if ZEN_PLATFORM_WINDOWS + ZEN_UNUSED(Delete); + if (MemMap.DataPtr != nullptr) + { + UnmapViewOfFile(MemMap.DataPtr); + MemMap.DataPtr = nullptr; + } + if (MemMap.Handle != nullptr) + { + CloseHandle(MemMap.Handle); + MemMap.Handle = nullptr; + } +#endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + if (MemMap.DataPtr != nullptr) + { + munmap(MemMap.DataPtr, MemMap.Size); + MemMap.DataPtr = nullptr; + } + + if (MemMap.Handle != nullptr) + { + int Fd = int(intptr_t(MemMap.Handle)); + close(Fd); + MemMap.Handle = nullptr; + } + if (Delete) + { + std::string InstanceMapName = fmt::format("/{}", MemMap.Name); + shm_unlink(InstanceMapName.c_str()); + } +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + } + + SharedMemoryImpl(Data&& MemMap, bool IsOwned) : m_MemMap(std::move(MemMap)), m_IsOwned(IsOwned) {} + virtual ~SharedMemoryImpl() + { + try + { + Close(std::move(m_MemMap), /*Delete*/ m_IsOwned); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("SharedMemoryImpl::~SharedMemoryImpl threw exception: {}", Ex.what()); + } + } + + virtual void* GetData() override { return m_MemMap.DataPtr; } + +private: + Data m_MemMap; + const bool m_IsOwned = false; +}; + +std::unique_ptr +OpenSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal) +{ + SharedMemoryImpl::Data MemMap = SharedMemoryImpl::Open(Name, Size, SystemGlobal); + if (MemMap.DataPtr) + { + return std::make_unique(std::move(MemMap), /*IsOwned*/ false); + } + return {}; +} + +std::unique_ptr +CreateSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal) +{ + SharedMemoryImpl::Data MemMap = SharedMemoryImpl::Create(Name, Size, SystemGlobal); + if (MemMap.DataPtr) + { + return std::make_unique(std::move(MemMap), /*IsOwned*/ true); + } + return {}; +} + ////////////////////////////////////////////////////////////////////////// // // Testing related code follows... @@ -3108,6 +3322,22 @@ TEST_CASE("RotateDirectories") } } +TEST_CASE("SharedMemory") +{ + CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); + CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, true)); + + { + auto Mem0 = CreateSharedMemory("SharedMemoryTest0", 482, false); + CHECK(Mem0); + strcpy((char*)Mem0->GetData(), "this is the string we are looking for"); + auto Mem1 = OpenSharedMemory("SharedMemoryTest0", 482, false); + CHECK_EQ(std::string((char*)Mem0->GetData()), std::string((char*)Mem1->GetData())); + } + + CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); +} + #endif } // namespace zen diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index dfd0eedc9..36d4d1b68 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -385,6 +385,16 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); +class SharedMemory +{ +public: + virtual ~SharedMemory() {} + virtual void* GetData() = 0; +}; + +std::unique_ptr OpenSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal); +std::unique_ptr CreateSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index d1394cd9a..d3e1de703 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -98,7 +98,7 @@ ZENCORE_API int GetCurrentProcessId(); int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); -std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); +std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); void process_forcelink(); // internal diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 48efc3f85..fcbe657cb 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -929,7 +929,7 @@ GetProcessExecutablePath(int Pid, std::error_code& OutEc) } std::error_code -FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle) +FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf) { #if ZEN_PLATFORM_WINDOWS HANDLE ProcessSnapshotHandle = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); @@ -939,13 +939,15 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand } auto _ = MakeGuard([&]() { CloseHandle(ProcessSnapshotHandle); }); + const DWORD ThisProcessId = ::GetCurrentProcessId(); + PROCESSENTRY32 Entry; Entry.dwSize = sizeof(PROCESSENTRY32); if (Process32First(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry)) { do { - if (ExecutableImage.filename() == Entry.szExeFile) + if ((IncludeSelf || (Entry.th32ProcessID != ThisProcessId)) && (ExecutableImage.filename() == Entry.szExeFile)) { std::error_code Ec; std::filesystem::path EntryPath = GetProcessExecutablePath(Entry.th32ProcessID, Ec); @@ -970,6 +972,7 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand } } } while (::Process32Next(ProcessSnapshotHandle, (LPPROCESSENTRY32)&Entry)); + return {}; } return MakeErrorCodeFromLastError(); #endif // ZEN_PLATFORM_WINDOWS @@ -980,6 +983,8 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand struct kinfo_proc* Processes = nullptr; uint32_t ProcCount = 0; + const pid_t ThisProcessId = getpid(); + if (sysctl(Mib, 4, NULL, &BufferSize, NULL, 0) != -1 && BufferSize > 0) { struct kinfo_proc* Processes = (struct kinfo_proc*)malloc(BufferSize); @@ -990,36 +995,46 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand char Buffer[PROC_PIDPATHINFO_MAXSIZE]; for (uint32_t ProcIndex = 0; ProcIndex < ProcCount; ProcIndex++) { - pid_t Pid = Processes[ProcIndex].kp_proc.p_pid; - std::error_code Ec; - std::filesystem::path EntryPath = GetProcessExecutablePath(Pid, Ec); - if (!Ec) + pid_t Pid = Processes[ProcIndex].kp_proc.p_pid; + if (IncludeSelf || (Pid != ThisProcessId)) { - if (EntryPath == ExecutableImage) + std::error_code Ec; + std::filesystem::path EntryPath = GetProcessExecutablePath(Pid, Ec); + if (!Ec) { - if (Processes[ProcIndex].kp_proc.p_stat != SZOMB) + if (EntryPath == ExecutableImage) { - OutHandle.Initialize(Pid, Ec); - return Ec; + if (Processes[ProcIndex].kp_proc.p_stat != SZOMB) + { + OutHandle.Initialize(Pid, Ec); + return Ec; + } } } + Ec.clear(); } } + return {}; } } return MakeErrorCodeFromLastError(); #endif // ZEN_PLATFORM_MAC #if ZEN_PLATFORM_LINUX + const pid_t ThisProcessId = getpid(); + std::vector RunningPids; DirectoryContent ProcList; GetDirectoryContent("/proc", DirectoryContentFlags::IncludeDirs, ProcList); for (const std::filesystem::path& EntryPath : ProcList.Directories) { std::string EntryName = EntryPath.stem(); - std::optional Pid = ParseInt(EntryName); - if (Pid.has_value()) + std::optional PidMaybe = ParseInt(EntryName); + if (PidMaybe.has_value()) { - RunningPids.push_back(Pid.value()); + if (pid_t Pid = PidMaybe.value(); IncludeSelf || (Pid != ThisProcessId)) + { + RunningPids.push_back(Pid); + } } } @@ -1042,6 +1057,7 @@ FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHand } } } + Ec.clear(); } return {}; #endif // ZEN_PLATFORM_LINUX @@ -1065,10 +1081,24 @@ TEST_CASE("Process") TEST_CASE("FindProcess") { - ProcessHandle Process; - std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process); - CHECK(!Ec); - CHECK(Process.IsValid()); + { + ProcessHandle Process; + std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process, /*IncludeSelf*/ true); + CHECK(!Ec); + CHECK(Process.IsValid()); + } + { + ProcessHandle Process; + std::error_code Ec = FindProcess(GetRunningExecutablePath(), Process, /*IncludeSelf*/ false); + CHECK(!Ec); + CHECK(!Process.IsValid()); + } + { + ProcessHandle Process; + std::error_code Ec = FindProcess("this/does\\not/exist\\123914921929412312312312asdad\\12134.no", Process, /*IncludeSelf*/ false); + CHECK(!Ec); + CHECK(!Process.IsValid()); + } } TEST_CASE("BuildArgV") diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index f7e63433b..9f2e826d6 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -1588,7 +1588,8 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co Stopwatch Timer; auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); ZEN_INFO("Replaying {} requests", RequestCount); for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) { @@ -1638,8 +1639,8 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co } }); } - Work.Wait(10000, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted); + Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); ZEN_INFO("Replayed {} of {} requests, elapsed {}", RequestCount - PendingWork, RequestCount, diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 3ec4373a2..a2e73380f 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1597,7 +1597,8 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo }; std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) { if (OptionalWorkerPool) @@ -2111,7 +2112,8 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, { std::atomic_bool Result = true; std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { @@ -3890,7 +3892,8 @@ ProjectStore::Flush() WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (const Ref& Project : Projects) { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 41c747e08..afb7e4bee 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -483,7 +483,8 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O if (!MetaLocations.empty()) { std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); m_MetadataBlockStore.IterateChunks( MetaLocations, diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index e973cee77..3f1f0e34a 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -3978,7 +3978,8 @@ ZenCacheDiskLayer::DiscoverBuckets() WorkerThreadPool& Pool = GetLargeWorkerPool(EWorkloadType::Burst); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (auto& BucketPath : FoundBucketDirectories) { Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic&) { @@ -4141,7 +4142,8 @@ ZenCacheDiskLayer::Flush() { WorkerThreadPool& Pool = GetMediumWorkerPool(EWorkloadType::Burst); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); try { for (auto& Bucket : Buckets) @@ -4164,8 +4166,10 @@ ZenCacheDiskLayer::Flush() { ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what()); } - Work.Wait(1000, - [&](std::ptrdiff_t Remaining, bool) { ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", Remaining, m_RootDir); }); + Work.Wait(1000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t RemainingWork) { + ZEN_UNUSED(IsAborted, IsPaused); + ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", RemainingWork, m_RootDir); + }); } } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 75562176e..0c9302ec8 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -402,7 +402,8 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas std::atomic AsyncContinue = true; { std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); const bool Continue = m_BlockStore.IterateChunks( FoundChunkLocations, [this, diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 56979f267..539b5e95b 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -664,7 +664,8 @@ FileCasStrategy::IterateChunks(std::span ChunkHashes, }; std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) { if (!AsyncContinue) diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index c7532e098..cd1bf7dd7 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -758,14 +758,15 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span } ChunkedFolderContent -ChunkFolderContent(ChunkingStatistics& Stats, - WorkerThreadPool& WorkerPool, - const std::filesystem::path& RootPath, - const FolderContent& Content, - const ChunkingController& InChunkingController, - int32_t UpdateIntervalMS, - std::function&& UpdateCallback, - std::atomic& AbortFlag) +ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateIntervalMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag, + std::atomic& PauseFlag) { ZEN_TRACE_CPU("ChunkFolderContent"); @@ -804,7 +805,7 @@ ChunkFolderContent(ChunkingStatistics& Stats, RwLock Lock; - ParallelWork Work(AbortFlag); + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t PathIndex : Order) { @@ -831,10 +832,9 @@ ChunkFolderContent(ChunkingStatistics& Stats, }); } - Work.Wait(UpdateIntervalMS, [&](bool IsAborted, std::ptrdiff_t PendingWork) { - ZEN_UNUSED(IsAborted); + Work.Wait(UpdateIntervalMS, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { ZEN_UNUSED(PendingWork); - UpdateCallback(Work.IsAborted(), Work.PendingWork().Remaining()); + UpdateCallback(IsAborted, IsPaused, Work.PendingWork().Remaining()); }); } return Result; diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 225b1a3a5..306a5d990 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -132,14 +132,15 @@ struct ChunkingStatistics uint64_t ElapsedWallTimeUS = 0; }; -ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, - WorkerThreadPool& WorkerPool, - const std::filesystem::path& RootPath, - const FolderContent& Content, - const ChunkingController& InChunkingController, - int32_t UpdateIntervalMS, - std::function&& UpdateCallback, - std::atomic& AbortFlag); +ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateIntervalMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag, + std::atomic& PauseFlag); ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h index 08e730b28..d7e986551 100644 --- a/src/zenutil/include/zenutil/parallelwork.h +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -12,13 +12,13 @@ namespace zen { class ParallelWork { public: - ParallelWork(std::atomic& AbortFlag); + ParallelWork(std::atomic& AbortFlag, std::atomic& PauseFlag); ~ParallelWork(); - typedef std::function& AbortFlag)> WorkCallback; - typedef std::function& AbortFlag)> ExceptionCallback; - typedef std::function UpdateCallback; + typedef std::function& AbortFlag)> WorkCallback; + typedef std::function& AbortFlag)> ExceptionCallback; + typedef std::function UpdateCallback; void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {}) { @@ -28,6 +28,10 @@ public: WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { try { + while (m_PauseFlag && !m_AbortFlag) + { + Sleep(2000); + } Work(m_AbortFlag); } catch (...) @@ -59,6 +63,7 @@ private: void RethrowErrors(); std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; bool m_DispatchComplete = false; Latch m_PendingWork; diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index ecacc4b5a..67fc03c04 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -14,7 +14,10 @@ namespace zen { -ParallelWork::ParallelWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) +ParallelWork::ParallelWork(std::atomic& AbortFlag, std::atomic& PauseFlag) +: m_AbortFlag(AbortFlag) +, m_PauseFlag(PauseFlag) +, m_PendingWork(1) { } @@ -59,7 +62,7 @@ ParallelWork::Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback) while (!m_PendingWork.Wait(UpdateIntervalMS)) { - UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); + UpdateCallback(m_AbortFlag.load(), m_PauseFlag.load(), m_PendingWork.Remaining()); } RethrowErrors(); @@ -111,7 +114,8 @@ ParallelWork::RethrowErrors() TEST_CASE("parallellwork.nowork") { std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); Work.Wait(); } @@ -120,7 +124,8 @@ TEST_CASE("parallellwork.basic") WorkerThreadPool WorkerPool(2); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t I = 0; I < 5; I++) { Work.ScheduleWork(WorkerPool, [](std::atomic& AbortFlag) { CHECK(!AbortFlag); }); @@ -133,7 +138,8 @@ TEST_CASE("parallellwork.throws_in_work") WorkerThreadPool WorkerPool(2); std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t I = 0; I < 10; I++) { Work.ScheduleWork(WorkerPool, [I](std::atomic& AbortFlag) { @@ -158,7 +164,8 @@ TEST_CASE("parallellwork.throws_in_dispatch") try { std::atomic AbortFlag; - ParallelWork Work(AbortFlag); + std::atomic PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag); for (uint32_t I = 0; I < 5; I++) { Work.ScheduleWork(WorkerPool, [I, &ExecutedCount](std::atomic& AbortFlag) { diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index bfa0d3c49..a5b342cb0 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -1059,7 +1059,7 @@ ZenServerInstance::Terminate() const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL; ProcessHandle RunningProcess; - std::error_code Ec = FindProcess(Executable, RunningProcess); + std::error_code Ec = FindProcess(Executable, RunningProcess, /*IncludeSelf*/ false); if (Ec) { throw std::system_error(Ec, fmt::format("failed to look up running server executable '{}'", Executable)); -- cgit v1.2.3 From 92d5b9ffcf6e19b7dd843655bfe06728d8770372 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Jun 2025 14:54:29 +0200 Subject: silence oom ood errors for sentry (#423) - Improvement: Don't report OOD or OOM errors to Sentry when running `zen builds` commands --- src/zen/cmds/builds_cmd.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 49b032ab1..1f7dbd91b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -11535,6 +11535,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 0; } } + catch (const std::system_error& SysErr) + { + if (IsOOD(SysErr)) + { + ZEN_CONSOLE("Operation failed due to out of disk space: {}", SysErr.what()); + return 3; + } + else if (IsOOM(SysErr)) + { + ZEN_CONSOLE("Operation failed due to out of memory: {}", SysErr.what()); + return 3; + } + else + { + ZEN_ERROR("{}", SysErr.what()); + return 3; + } + } catch (const std::exception& Ex) { ZEN_ERROR("{}", Ex.what()); -- cgit v1.2.3 From 6f2d68d2c11011d541259d0037908dd76eadeb8a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Jun 2025 09:03:39 +0200 Subject: missing chunks bugfix (#424) * make sure to close log file when resetting log * drop entries that refers to missing blocks * Don't scrub keys that has been rewritten * currectly count added bytes / m_TotalSize * fix negative sleep time in BlockStoreFile::Open() * be defensive when fetching log position * append to log files *after* we updated all state successfully * explicitly close stuff in destructors with exception catching * clean up empty size block store files --- src/zenserver/projectstore/projectstore.cpp | 2 +- src/zenstore/blockstore.cpp | 53 ++-- src/zenstore/buildstore/buildstore.cpp | 98 +++++-- src/zenstore/cache/cachedisklayer.cpp | 84 +++++- src/zenstore/compactcas.cpp | 306 +++++++++++++++++++-- src/zenstore/filecas.cpp | 9 +- src/zenstore/include/zenstore/blockstore.h | 6 +- .../include/zenstore/cache/cachedisklayer.h | 7 +- 8 files changed, 480 insertions(+), 85 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index a2e73380f..6359b9db9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1010,8 +1010,8 @@ struct ProjectStore::OplogStorage : public RefCounted .OpCoreHash = OpData.OpCoreHash, .OpKeyHash = OpData.KeyHash}; - m_Oplog.Append(Entry); m_OpBlobs.Write(OpData.Buffer.GetData(), WriteSize, WriteOffset); + m_Oplog.Append(Entry); return Entry; } diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 5081ae65d..7b56c64bd 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -70,7 +70,7 @@ BlockStoreFile::Open() return false; } ZEN_WARN("Failed to open cas block '{}', reason: '{}', retries left: {}.", m_Path, Ec.message(), RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + Sleep(100 + (3 - RetriesLeft) * 100); // Total 600 ms RetriesLeft--; return true; }); @@ -286,6 +286,14 @@ BlockStore::BlockStore() BlockStore::~BlockStore() { + try + { + Close(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BlockStore() failed with: ", Ex.what()); + } } void @@ -307,6 +315,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max if (IsDir(m_BlocksBasePath)) { + std::vector EmptyBlockFiles; uint32_t NextBlockIndex = 0; std::vector FoldersToScan; FoldersToScan.push_back(m_BlocksBasePath); @@ -334,6 +343,12 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max { continue; } + if (Entry.file_size() == 0) + { + EmptyBlockFiles.push_back(Path); + continue; + } + Ref BlockFile{new BlockStoreFile(Path)}; BlockFile->Open(); m_TotalSize.fetch_add(BlockFile->TotalSize(), std::memory_order::relaxed); @@ -347,6 +362,17 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max } ++FolderOffset; } + + for (const std::filesystem::path& EmptyBlockFile : EmptyBlockFiles) + { + std::error_code Ec; + RemoveFile(EmptyBlockFile, Ec); + if (Ec) + { + ZEN_WARN("Unable to remove empty block file {}. Reason: {}", EmptyBlockFile, Ec.message()); + } + } + m_WriteBlockIndex.store(NextBlockIndex, std::memory_order_release); } else @@ -355,7 +381,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max } } -void +BlockStore::BlockIndexSet BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks) { ZEN_MEMSCOPE(GetBlocksTag()); @@ -363,8 +389,8 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks) RwLock::ExclusiveLockScope InsertLock(m_InsertLock); - tsl::robin_set MissingBlocks; - tsl::robin_set DeleteBlocks; + BlockIndexSet MissingBlocks; + BlockIndexSet DeleteBlocks; DeleteBlocks.reserve(m_ChunkBlocks.size()); for (auto It : m_ChunkBlocks) { @@ -383,13 +409,6 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks) MissingBlocks.insert(BlockIndex); } } - for (std::uint32_t BlockIndex : MissingBlocks) - { - std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex); - Ref NewBlockFile(new BlockStoreFile(BlockPath)); - NewBlockFile->Create(0); - m_ChunkBlocks[BlockIndex] = NewBlockFile; - } for (std::uint32_t BlockIndex : DeleteBlocks) { std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex); @@ -400,6 +419,7 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks) } m_ChunkBlocks.erase(BlockIndex); } + return MissingBlocks; } BlockStore::BlockEntryCountMap @@ -1037,6 +1057,7 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, { Continue = ChangeCallback(MovedChunks, ScrubbedChunks, RemovedSize > AddedSize ? RemovedSize - AddedSize : 0); DeletedSize += RemovedSize; + m_TotalSize.fetch_add(AddedSize); RemovedSize = 0; AddedSize = 0; MovedCount += MovedChunks.size(); @@ -1220,14 +1241,13 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, NiceBytes(Space.Free + ReclaimedSpace)); } NewBlockFile->Create(m_MaxBlockSize); - NewBlockIndex = NextBlockIndex; - WriteOffset = 0; - AddedSize += WriteOffset; + NewBlockIndex = NextBlockIndex; WriteOffset = 0; TargetFileBuffer = std::make_unique(NewBlockFile->GetBasicFile(), Min(256u * 1024u, m_MaxBlockSize)); } - WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment); + const uint64_t OldWriteOffset = WriteOffset; + WriteOffset = TargetFileBuffer->AlignTo(PayloadAlignment); TargetFileBuffer->Write(ChunkView.GetData(), ChunkLocation.Size, WriteOffset); MovedChunks.push_back( @@ -1235,8 +1255,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, WriteOffset += ChunkLocation.Size; MovedFromBlock += RoundUp(ChunkLocation.Offset + ChunkLocation.Size, PayloadAlignment) - ChunkLocation.Offset; + uint64_t WrittenBytes = WriteOffset - OldWriteOffset; + AddedSize += WrittenBytes; } - AddedSize += WriteOffset; ZEN_INFO("{}moved {} chunks ({}) from '{}' to new block, freeing {}", LogPrefix, KeepChunkIndexes.size(), diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index afb7e4bee..c25f762f5 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -177,22 +177,60 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) m_Config.SmallBlobBlockStoreAlignement, IsNew); m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20); + + BlockStore::BlockIndexSet KnownBlocks; + for (const BlobEntry& Blob : m_BlobEntries) { - BlockStore::BlockIndexSet KnownBlocks; - for (const BlobEntry& Blob : m_BlobEntries) + if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) { - if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) - { - const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; - KnownBlocks.insert(Metadata.Location.BlockIndex); - } + const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; + KnownBlocks.insert(Metadata.Location.BlockIndex); } - m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); } + BlockStore::BlockIndexSet MissingBlocks = m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite); m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite); + if (!MissingBlocks.empty()) + { + std::vector MissingMetadatas; + for (auto& It : m_BlobLookup) + { + const IoHash& BlobHash = It.first; + const BlobIndex ReadBlobIndex = It.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& MetaData = m_MetadataEntries[ReadBlobEntry.Metadata]; + if (MissingBlocks.contains(MetaData.Location.BlockIndex)) + { + MissingMetadatas.push_back( + MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = BlobHash}); + MissingMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; + m_MetadataEntries[ReadBlobEntry.Metadata] = {}; + m_BlobEntries[ReadBlobIndex].Metadata = {}; + } + } + } + ZEN_ASSERT(!MissingMetadatas.empty()); + + for (const MetadataDiskEntry& Entry : MissingMetadatas) + { + auto It = m_BlobLookup.find(Entry.BlobHash); + ZEN_ASSERT(It != m_BlobLookup.end()); + + const BlobIndex ReadBlobIndex = It->second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + if (!ReadBlobEntry.Payload) + { + m_BlobLookup.erase(It); + } + } + m_MetadatalogFile.Append(MissingMetadatas); + CompactState(); + } + m_Gc.AddGcReferencer(*this); m_Gc.AddGcReferenceLocker(*this); m_Gc.AddGcStorage(this); @@ -256,34 +294,36 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) ZEN_UNUSED(Result); Entry = PayloadEntry(0, PayloadSize); } - m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); - RwLock::ExclusiveLockScope _(m_Lock); - if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) { - const BlobIndex ExistingBlobIndex = It->second; - BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; - if (Blob.Payload) + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) { - m_PayloadEntries[Blob.Payload] = Entry; + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Payload) + { + m_PayloadEntries[Blob.Payload] = Entry; + } + else + { + Blob.Payload = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); } else { - Blob.Payload = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); m_PayloadEntries.push_back(Entry); - } - Blob.LastAccessTime = GcClock::TickCount(); - } - else - { - PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); - m_PayloadEntries.push_back(Entry); - const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); - // we only remove during GC and compact this then... - m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); - m_BlobLookup.insert({BlobHash, NewBlobIndex}); + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + // we only remove during GC and compact this then... + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } } + m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); m_LastAccessTimeUpdateCount++; } @@ -370,7 +410,6 @@ BuildStore::PutMetadatas(std::span BlobHashes, std::span BlobHashes, std::span& ClaimDiskReserveFunc) +ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(uint64_t LogPosition, + bool ResetLog, + const std::function& ClaimDiskReserveFunc) { ZEN_TRACE_CPU("Z$::Bucket::WriteIndexSnapshot"); - if (m_LogFlushPosition == m_SlogFile.GetLogCount()) + if (m_LogFlushPosition == LogPosition) { return; } @@ -877,7 +889,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool ResetLog, const st throw std::system_error(Ec, fmt::format("failed to create new snapshot file in '{}'", m_BucketDir)); } - const uint64_t IndexLogPosition = ResetLog ? 0 : m_SlogFile.GetLogCount(); + const uint64_t IndexLogPosition = ResetLog ? 0 : LogPosition; cache::impl::CacheBucketIndexHeader Header = {.EntryCount = EntryCount, .LogPosition = IndexLogPosition, @@ -930,12 +942,14 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool ResetLog, const st if (IsFile(LogPath)) { + m_SlogFile.Close(); if (!RemoveFile(LogPath, Ec) || Ec) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); } + m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite); } } m_LogFlushPosition = IndexLogPosition; @@ -1149,13 +1163,6 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco } } - if (IsNew || LogEntryCount > 0 || m_LogFlushPosition != 0) - { - WriteIndexSnapshot(IndexLock, /*Flush log*/ true); - } - - m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite); - BlockStore::BlockIndexSet KnownBlocks; for (const auto& Entry : m_Index) { @@ -1173,7 +1180,53 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco KnownBlocks.insert(BlockIndex); } } - m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + BlockStore::BlockIndexSet MissingBlocks = m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite); + + bool RemovedEntries = false; + if (!MissingBlocks.empty()) + { + std::vector MissingEntries; + + for (auto& It : m_Index) + { + BucketPayload& Payload = m_Payloads[It.second]; + DiskLocation Location = Payload.Location; + if (!Location.IsFlagSet(DiskLocation::kStandaloneFile)) + { + if (MissingBlocks.contains(Location.Location.BlockLocation.GetBlockIndex())) + { + RemoveMemCachedData(IndexLock, Payload); + RemoveMetaData(IndexLock, Payload); + } + } + Location.Flags |= DiskLocation::kTombStone; + MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location}); + } + + ZEN_ASSERT(!MissingEntries.empty()); + + for (const DiskIndexEntry& Entry : MissingEntries) + { + m_Index.erase(Entry.Key); + } + m_SlogFile.Append(MissingEntries); + m_SlogFile.Flush(); + { + std::vector Payloads; + std::vector AccessTimes; + std::vector MetaDatas; + std::vector MemCachedPayloads; + IndexMap Index; + CompactState(IndexLock, Payloads, AccessTimes, MetaDatas, MemCachedPayloads, Index); + } + RemovedEntries = true; + } + + if (IsNew || LogEntryCount > 0 || m_LogFlushPosition != 0 || RemovedEntries) + { + WriteIndexSnapshot(IndexLock, m_SlogFile.GetLogCount(), /*Flush log*/ true); + } } void @@ -2024,6 +2077,9 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function& Cl ZEN_TRACE_CPU("Z$::Bucket::SaveSnapshot"); try { + // Be defensive regarding log position as it is written to without acquiring m_LocationMapLock + const uint64_t LogPosition = m_SlogFile.GetLogCount(); + bool UseLegacyScheme = false; IoBuffer Buffer; @@ -2038,7 +2094,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function& Cl { RwLock::SharedLockScope IndexLock(m_IndexLock); - WriteIndexSnapshot(IndexLock, /*Flush log*/ false); + WriteIndexSnapshot(IndexLock, LogPosition, /*Flush log*/ false); // Note: this copy could be eliminated on shutdown to // reduce memory usage and execution time Index = m_Index; @@ -2078,7 +2134,7 @@ ZenCacheDiskLayer::CacheBucket::SaveSnapshot(const std::function& Cl else { RwLock::SharedLockScope IndexLock(m_IndexLock); - WriteIndexSnapshot(IndexLock, /*Flush log*/ false); + WriteIndexSnapshot(IndexLock, LogPosition, /*Flush log*/ false); const uint64_t EntryCount = m_Index.size(); Buffer = ManifestWriter.MakeSidecarManifest(m_BucketId, EntryCount); uint64_t SidecarSize = ManifestWriter.GetSidecarSize(); @@ -2727,7 +2783,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, ZEN_MEMSCOPE(GetCacheDiskTag()); ZEN_TRACE_CPU("Z$::Bucket::UpdateLocation"); DiskLocation Location(BlockStoreLocation, m_Configuration.PayloadAlignment, EntryFlags); - m_SlogFile.Append({.Key = HashKey, .Location = Location}); RwLock::ExclusiveLockScope IndexLock(m_IndexLock); if (m_TrackedCacheKeys) @@ -2757,6 +2812,7 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, m_AccessTimes.emplace_back(GcClock::TickCount()); m_Index.insert_or_assign(HashKey, EntryIndex); } + m_SlogFile.Append({.Key = HashKey, .Location = Location}); }); } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 0c9302ec8..2ab5752ff 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -145,6 +145,16 @@ CasContainerStrategy::CasContainerStrategy(GcManager& Gc) : m_Log(logging::Get(" CasContainerStrategy::~CasContainerStrategy() { + try + { + m_BlockStore.Close(); + m_CasLog.Flush(); + m_CasLog.Close(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~CasContainerStrategy failed with: ", Ex.what()); + } m_Gc.RemoveGcReferenceStore(*this); m_Gc.RemoveGcStorage(this); } @@ -204,12 +214,12 @@ CasContainerStrategy::InsertChunk(const void* ChunkData, size_t ChunkSize, const ZEN_TRACE_CPU("CasContainer::UpdateLocation"); BlockStoreDiskLocation DiskLocation(Location, m_PayloadAlignment); const CasDiskIndexEntry IndexEntry{.Key = ChunkHash, .Location = DiskLocation}; - m_CasLog.Append(IndexEntry); { RwLock::ExclusiveLockScope _(m_LocationMapLock); m_LocationMap.emplace(ChunkHash, m_Locations.size()); m_Locations.push_back(DiskLocation); } + m_CasLog.Append(IndexEntry); }); return CasStore::InsertResult{.New = true}; @@ -273,7 +283,6 @@ CasContainerStrategy::InsertChunks(std::span Chunks, std::span Chunks, std::spansecond]; + BlockStoreDiskLocation& Location = m_CasContainerStrategy.m_Locations[It->second]; + const BlockStoreLocation& OldLocation = BlockCompactState.GetLocation(ChunkIndex); + if (Location.Get(m_CasContainerStrategy.m_PayloadAlignment) != OldLocation) + { + // Someone has moved our chunk so lets just skip the new location we were provided, it will be + // GC:d at a later time + continue; + } MovedEntries.push_back( CasDiskIndexEntry{.Key = Key, .Location = Location, .Flags = CasDiskIndexEntry::kTombstone}); m_CasContainerStrategy.m_LocationMap.erase(It); } } m_CasContainerStrategy.m_CasLog.Append(MovedEntries); + m_CasContainerStrategy.m_CasLog.Flush(); Stats.RemovedDisk += FreedDiskSpace; if (Ctx.IsCancelledFlag.load()) { @@ -984,14 +1002,11 @@ CasContainerStrategy::MakeIndexSnapshot(bool ResetLog) { // Write the current state of the location map to a new index state std::vector Entries; - uint64_t IndexLogPosition = 0; + // Be defensive regarding log position as it is written to without acquiring m_LocationMapLock + const uint64_t IndexLogPosition = ResetLog ? 0 : m_CasLog.GetLogCount(); { RwLock::SharedLockScope ___(m_LocationMapLock); - if (!ResetLog) - { - IndexLogPosition = m_CasLog.GetLogCount(); - } Entries.resize(m_LocationMap.size()); uint64_t EntryIndex = 0; @@ -1036,12 +1051,14 @@ CasContainerStrategy::MakeIndexSnapshot(bool ResetLog) if (IsFile(LogPath)) { + m_CasLog.Close(); if (!RemoveFile(LogPath, Ec) || Ec) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); } + m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } } m_LogFlushPosition = IndexLogPosition; @@ -1154,7 +1171,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski ZEN_WARN("reading full log at '{}', reason: Log position from index snapshot is out of range", LogPath); SkipEntryCount = 0; } - LogEntryCount = EntryCount - SkipEntryCount; + LogEntryCount = SkipEntryCount; CasLog.Replay( [&](const CasDiskIndexEntry& Record) { LogEntryCount++; @@ -1173,7 +1190,6 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski m_Locations.push_back(Record.Location); }, SkipEntryCount); - return LogEntryCount; } return 0; @@ -1229,8 +1245,6 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) } } - m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); - BlockStore::BlockIndexSet KnownBlocks; for (const auto& Entry : m_LocationMap) @@ -1240,9 +1254,39 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) KnownBlocks.insert(BlockIndex); } - m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + BlockStore::BlockIndexSet MissingBlocks = m_BlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + + m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); + + bool RemovedEntries = false; + if (!MissingBlocks.empty()) + { + std::vector MissingEntries; + for (auto& It : m_LocationMap) + { + const uint32_t BlockIndex = m_Locations[It.second].GetBlockIndex(); + if (MissingBlocks.contains(BlockIndex)) + { + MissingEntries.push_back({.Key = It.first, .Location = m_Locations[It.second], .Flags = CasDiskIndexEntry::kTombstone}); + } + } + ZEN_ASSERT(!MissingEntries.empty()); + + for (const CasDiskIndexEntry& Entry : MissingEntries) + { + m_LocationMap.erase(Entry.Key); + } + m_CasLog.Append(MissingEntries); + m_CasLog.Flush(); + + { + RwLock::ExclusiveLockScope IndexLock(m_LocationMapLock); + CompactIndex(IndexLock); + } + RemovedEntries = true; + } - if (IsNewStore || (LogEntryCount > 0)) + if (IsNewStore || (LogEntryCount > 0) || RemovedEntries) { MakeIndexSnapshot(/*ResetLog*/ true); } @@ -1612,6 +1656,236 @@ TEST_CASE("compactcas.threadedinsert") } } +TEST_CASE("compactcas.restart") +{ + uint64_t ExpectedSize = 0; + + auto GenerateChunks = [&](CasContainerStrategy& Cas, size_t ChunkCount, uint64_t ChunkSize, std::vector& Hashes) { + WorkerThreadPool ThreadPool(Max(std::thread::hardware_concurrency() - 1u, 2u), "put"); + + Latch WorkLatch(1); + tsl::robin_set ChunkHashesLookup; + ChunkHashesLookup.reserve(ChunkCount); + RwLock InsertLock; + for (size_t Offset = 0; Offset < ChunkCount;) + { + size_t BatchCount = Min(ChunkCount - Offset, 512u); + WorkLatch.AddCount(1); + ThreadPool.ScheduleWork( + [&WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount, ChunkSize]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + + std::vector BatchBlobs; + std::vector BatchHashes; + BatchBlobs.reserve(BatchCount); + BatchHashes.reserve(BatchCount); + + while (BatchBlobs.size() < BatchCount) + { + IoBuffer Chunk = + CreateSemiRandomBlob(ChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377)); + IoHash Hash = IoHash::HashBuffer(Chunk); + { + RwLock::ExclusiveLockScope __(InsertLock); + if (ChunkHashesLookup.contains(Hash)) + { + continue; + } + ChunkHashesLookup.insert(Hash); + ExpectedSize += Chunk.Size(); + } + + BatchBlobs.emplace_back(CompressedBuffer::Compress(SharedBuffer(Chunk)).GetCompressed().Flatten().AsIoBuffer()); + BatchHashes.push_back(Hash); + } + + Cas.InsertChunks(BatchBlobs, BatchHashes); + { + RwLock::ExclusiveLockScope __(InsertLock); + Hashes.insert(Hashes.end(), BatchHashes.begin(), BatchHashes.end()); + } + }); + Offset += BatchCount; + } + WorkLatch.CountDown(); + WorkLatch.Wait(); + }; + + ScopedTemporaryDirectory TempDir; + std::filesystem::path CasPath = TempDir.Path(); + CreateDirectories(CasPath); + + bool Generate = false; + if (!Generate) + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + } + + const uint64_t kChunkSize = 1048 + 395; + const size_t kChunkCount = 7167; + + std::vector Hashes; + Hashes.reserve(kChunkCount); + + auto ValidateChunks = [&](CasContainerStrategy& Cas, std::span Hashes, bool ShouldExist) { + for (const IoHash& Hash : Hashes) + { + if (ShouldExist) + { + CHECK(Cas.HaveChunk(Hash)); + IoBuffer Buffer = Cas.FindChunk(Hash); + CHECK(Buffer); + IoHash ValidateHash; + uint64_t ValidateRawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Buffer), ValidateHash, ValidateRawSize); + CHECK(Compressed); + CHECK(ValidateHash == Hash); + } + else + { + CHECK(!Cas.HaveChunk(Hash)); + IoBuffer Buffer = Cas.FindChunk(Hash); + CHECK(!Buffer); + } + } + }; + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, true); + GenerateChunks(Cas, kChunkCount, kChunkSize, Hashes); + ValidateChunks(Cas, Hashes, true); + Cas.Flush(); + ValidateChunks(Cas, Hashes, true); + } + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + ValidateChunks(Cas, Hashes, true); + GenerateChunks(Cas, kChunkCount, kChunkSize / 4, Hashes); + ValidateChunks(Cas, Hashes, true); + } + + class GcRefChecker : public GcReferenceChecker + { + public: + explicit GcRefChecker(std::vector&& HashesToKeep) : m_HashesToKeep(std::move(HashesToKeep)) {} + ~GcRefChecker() {} + std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return "test"; + } + void PreCache(GcCtx& Ctx) override { FilterReferences(Ctx, "test", m_HashesToKeep); } + void UpdateLockedState(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); } + std::span GetUnusedReferences(GcCtx& Ctx, std::span IoCids) override + { + ZEN_UNUSED(Ctx); + return KeepUnusedReferences(m_HashesToKeep, IoCids); + } + + private: + std::vector m_HashesToKeep; + }; + + class GcRef : public GcReferencer + { + public: + GcRef(GcManager& Gc, std::span HashesToKeep) : m_Gc(Gc) + { + m_HashesToKeep.insert(m_HashesToKeep.begin(), HashesToKeep.begin(), HashesToKeep.end()); + m_Gc.AddGcReferencer(*this); + } + ~GcRef() { m_Gc.RemoveGcReferencer(*this); } + std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return "test"; + } + GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override + { + ZEN_UNUSED(Ctx, Stats); + return nullptr; + } + std::vector CreateReferenceCheckers(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return {new GcRefChecker(std::move(m_HashesToKeep))}; + } + std::vector CreateReferenceValidators(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return {}; + } + + private: + GcManager& m_Gc; + std::vector m_HashesToKeep; + }; + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + GenerateChunks(Cas, kChunkCount, kChunkSize / 5, Hashes); + } + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + ValidateChunks(Cas, Hashes, true); + GenerateChunks(Cas, kChunkCount, kChunkSize / 2, Hashes); + ValidateChunks(Cas, Hashes, true); + if (true) + { + std::vector DropHashes; + std::vector KeepHashes; + for (size_t Index = 0; Index < Hashes.size(); Index++) + { + if (Index % 5 == 0) + { + KeepHashes.push_back(Hashes[Index]); + } + else + { + DropHashes.push_back(Hashes[Index]); + } + } + // std::span KeepHashes(Hashes); + // ZEN_ASSERT(ExpectedGcCount < Hashes.size()); + // KeepHashes = KeepHashes.subspan(ExpectedGcCount); + GcRef Ref(Gc, KeepHashes); + Gc.CollectGarbage(GcSettings{.CollectSmallObjects = true, .IsDeleteMode = true}); + ValidateChunks(Cas, KeepHashes, true); + ValidateChunks(Cas, DropHashes, false); + Hashes = KeepHashes; + } + GenerateChunks(Cas, kChunkCount, kChunkSize / 3, Hashes); + } + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + ValidateChunks(Cas, Hashes, true); + Cas.Flush(); + ValidateChunks(Cas, Hashes, true); + } + + { + GcManager Gc; + CasContainerStrategy Cas(Gc); + Cas.Initialize(CasPath, "test", 65536 * 128, 8, false); + ValidateChunks(Cas, Hashes, true); + } +} + TEST_CASE("compactcas.iteratechunks") { std::atomic WorkCompleted = 0; diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 539b5e95b..11a266f1c 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -969,14 +969,11 @@ FileCasStrategy::MakeIndexSnapshot(bool ResetLog) { // Write the current state of the location map to a new index state std::vector Entries; - uint64_t IndexLogPosition = 0; + // Be defensive regarding log position as it is written to without acquiring m_LocationMapLock + const uint64_t IndexLogPosition = ResetLog ? 0 : m_CasLog.GetLogCount(); { RwLock::SharedLockScope __(m_Lock); - if (!ResetLog) - { - IndexLogPosition = m_CasLog.GetLogCount(); - } Entries.resize(m_Index.size()); uint64_t EntryIndex = 0; @@ -1019,12 +1016,14 @@ FileCasStrategy::MakeIndexSnapshot(bool ResetLog) if (IsFile(LogPath)) { + m_CasLog.Close(); if (!RemoveFile(LogPath, Ec) || Ec) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); } + m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } } m_LogFlushPosition = IndexLogPosition; diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 8cbcad11b..fce05766f 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -147,10 +147,10 @@ public: typedef tsl::robin_set BlockIndexSet; - // Ask the store to create empty blocks for all locations that does not have a block // Remove any block that is not referenced - void SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks); - BlockEntryCountMap GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent); + // Return a list of blocks that are not present + [[nodiscard]] BlockIndexSet SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks); + BlockEntryCountMap GetBlocksToCompact(const BlockUsageMap& BlockUsage, uint32_t BlockUsageThresholdPercent); void Close(); diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index b67a043df..3cd2d6423 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -409,19 +409,22 @@ public: void SaveSnapshot(const std::function& ClaimDiskReserveFunc = []() { return 0; }); void WriteIndexSnapshot( RwLock::ExclusiveLockScope&, + uint64_t LogPosition, bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }) { - WriteIndexSnapshotLocked(ResetLog, ClaimDiskReserveFunc); + WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc); } void WriteIndexSnapshot( RwLock::SharedLockScope&, + uint64_t LogPosition, bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }) { - WriteIndexSnapshotLocked(ResetLog, ClaimDiskReserveFunc); + WriteIndexSnapshotLocked(LogPosition, ResetLog, ClaimDiskReserveFunc); } void WriteIndexSnapshotLocked( + uint64_t LogPosition, bool ResetLog, const std::function& ClaimDiskReserveFunc = []() { return 0; }); -- cgit v1.2.3 From 98c03a0f5b44dbb0849c76b52b1467daac4023a6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Jun 2025 10:07:46 +0200 Subject: fix namespace picking up wrong argument (#425) --- src/zen/cmds/builds_cmd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1f7dbd91b..426625187 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -10054,7 +10054,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_Host = RemoveQuotes(m_Host); m_OverrideHost = RemoveQuotes(m_OverrideHost); m_Url = RemoveQuotes(m_Url); - m_Namespace = RemoveQuotes(m_Url); + m_Namespace = RemoveQuotes(m_Namespace); m_Bucket = RemoveQuotes(m_Bucket); if (!m_Url.empty()) { -- cgit v1.2.3 From 6f87665745cd432f025215bb259a974dc434964e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Jun 2025 12:00:41 +0200 Subject: use unique tmp name for auth token file (#426) * use Oid to generate unique name since std::tmpnam is not good practice --- src/zenhttp/httpclientauth.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 916b25bff..39efe1d0c 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -94,9 +95,8 @@ namespace zen { namespace httpclientauth { CreateProcOptions ProcOptions; - const std::string AuthTokenPath = ".zen-auth-oidctoken"; - - auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); }); + const std::filesystem::path AuthTokenPath(std::filesystem::temp_directory_path() / fmt::format(".zen-auth-{}", Oid::NewOid())); + auto _ = MakeGuard([AuthTokenPath]() { RemoveFile(AuthTokenPath); }); const std::string ProcArgs = fmt::format("{} --AuthConfigUrl {} --OutFile {} --Unattended={}", OidcExecutablePath, -- cgit v1.2.3 From feb888db2d557066ebe9eb3bb1d7e3b052ae1221 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Jun 2025 12:22:27 +0200 Subject: `--sentry-dsn` option for zen command line and zenserver to control Sentry reporting endpoint (#427) moved sentry database path to temporary directory for zen commandline --- src/zen/zen.cpp | 22 +++++++---- src/zencore/include/zencore/sentryintegration.h | 8 +++- src/zencore/sentryintegration.cpp | 7 +++- src/zenserver/config.cpp | 2 + src/zenserver/config.h | 51 +++++++++++++------------ src/zenserver/main.cpp | 6 ++- 6 files changed, 59 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 119215d40..f0644a4f5 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -872,12 +872,18 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE #if ZEN_USE_SENTRY - bool NoSentry = false; - bool SentryAllowPII = false; - Options.add_options()("no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false")); - Options.add_options()("sentry-allow-personal-info", - "Allow personally identifiable information in sentry crash reports", - cxxopts::value(SentryAllowPII)->default_value("false")); + bool NoSentry = false; + bool SentryAllowPII = false; + std::string SentryDsn; + Options + .add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false"), ""); + Options.add_option("sentry", + "", + "sentry-allow-personal-info", + "Allow personally identifiable information in sentry crash reports", + cxxopts::value(SentryAllowPII)->default_value("false"), + ""); + Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryDsn), ""); #endif Options.parse_positional({"command"}); @@ -926,7 +932,7 @@ main(int argc, char** argv) if (NoSentry == false) { - std::string SentryDatabasePath = (GetRunningExecutablePath().parent_path() / ".sentry-native").string(); + std::string SentryDatabasePath = (std::filesystem::temp_directory_path() / ".zen-sentry-native").string(); ExtendableStringBuilder<512> SB; for (int i = 0; i < argc; ++i) @@ -939,7 +945,7 @@ main(int argc, char** argv) SB.Append(argv[i]); } - Sentry.Initialize(SentryDatabasePath, {}, SentryAllowPII, SB.ToString()); + Sentry.Initialize(SentryDatabasePath, {}, SentryDsn, SentryAllowPII, SB.ToString()); SentryIntegration::ClearCaches(); } diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index 40e22af4e..d14c1c275 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,8 +31,12 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine); - void LogStartupInformation(); + void Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + std::string SentryDsn, + bool AllowPII, + const std::string& CommandLine); + void LogStartupInformation(); static void ClearCaches(); private: diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index d08fb7f1d..520d5162e 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -31,6 +31,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace sentry { +namespace { + static const std::string DefaultDsn("https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); +} + struct SentryAssertImpl : zen::AssertImpl { virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION @@ -194,6 +198,7 @@ SentryIntegration::~SentryIntegration() void SentryIntegration::Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, + std::string SentryDsn, bool AllowPII, const std::string& CommandLine) { @@ -204,7 +209,7 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, SentryDatabasePath = SentryDatabasePath.substr(4); } sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, "https://8ba3441bebc941c1ae24b8cd2fd25d55@o10593.ingest.sentry.io/5919284"); + sentry_options_set_dsn(SentryOptions, SentryDsn.empty() ? sentry::DefaultDsn.c_str() : SentryDsn.c_str()); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); if (!SentryAttachmentsPath.empty()) diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index e097147fc..055376b5c 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -443,6 +443,7 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); @@ -762,6 +763,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", cxxopts::value(ServerOptions.SentryAllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryDsn)); options.add_options()("detach", "Indicate whether zenserver should detach from parent process group", cxxopts::value(ServerOptions.Detach)->default_value("true")); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 1d7d22ce9..1a1793b8d 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -169,31 +169,32 @@ struct ZenServerOptions ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; - bool NoSentry = false; - bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports - bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; + bool NoSentry = false; + bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string SentryDsn; + bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics std::string Loggers[zen::logging::level::LogLevelCount]; diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 0f647cd5c..868126533 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -101,7 +101,11 @@ ZenEntryPoint::Run() std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); - Sentry.Initialize(SentryDatabasePath, SentryAttachmentPath, m_ServerOptions.SentryAllowPII, m_ServerOptions.CommandLine); + Sentry.Initialize(SentryDatabasePath, + SentryAttachmentPath, + m_ServerOptions.SentryDsn, + m_ServerOptions.SentryAllowPII, + m_ServerOptions.CommandLine); } #endif try -- cgit v1.2.3 From 9f64a9c1e45da522745bb67872e6e096838d915e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 10:08:27 +0200 Subject: restore legacy --url option for builds download (#428) --- src/zen/cmds/builds_cmd.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 426625187..133ff795d 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -9646,7 +9646,13 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); - Ops.add_option("cloud build", "", "url", "Cloud Artifact URL", cxxopts::value(m_Url), ""); + Ops.add_option("cloud build", + "", + "url", + "Cloud Builds host url (legacy - use --override-host)", + cxxopts::value(m_OverrideHost), + ""); + Ops.add_option("cloud build", "", "cloud-url", "Cloud Artifact URL", cxxopts::value(m_Url), ""); Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), ""); Ops.add_option("cloud build", "", -- cgit v1.2.3 From f1c6232d73d85870e35e508ba4330865218e63a4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 11:02:17 +0200 Subject: Don't require `--namespace` option when using `zen list-namespaces` command (#429) --- src/zen/cmds/builds_cmd.cpp | 63 +++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 133ff795d..abb4bfe0c 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -10056,7 +10056,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseSystemOptions(); - auto ParseStorageOptions = [&](bool RequireBucket) { + auto ParseStorageOptions = [&](bool RequireNamespace, bool RequireBucket) { m_Host = RemoveQuotes(m_Host); m_OverrideHost = RemoveQuotes(m_OverrideHost); m_Url = RemoveQuotes(m_Url); @@ -10113,7 +10113,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException( fmt::format("host/url/override-host is not compatible with the storage-path option\n{}", SubOption->help())); } - if (m_Namespace.empty()) + if (RequireNamespace && m_Namespace.empty()) { throw zen::OptionParseException(fmt::format("namespace option is required for this storage option\n{}", SubOption->help())); } @@ -10280,8 +10280,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, BuildStorageCache::Statistics& StorageCacheStats, const std::filesystem::path& TempPath, + bool RequireNamespace, bool RequireBucket) -> StorageInstance { - ParseStorageOptions(RequireBucket); + ParseStorageOptions(RequireNamespace, RequireBucket); m_ZenCacheHost = RemoveQuotes(m_ZenCacheHost); @@ -10711,8 +10712,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath), /*RequireBucket*/ false); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(ZenFolderPath), + /*RequriesNamespace*/ false, + /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListNamespaces(m_ListNamespacesRecursive); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -10799,8 +10803,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ false); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ false); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -10854,8 +10861,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); if (m_BuildPartName.empty()) { @@ -10934,8 +10944,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); const Oid BuildId = ParseBuildId(); @@ -11005,8 +11018,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); IoHash BlobHash = ParseBlobHash(); @@ -11050,8 +11066,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); Oid BuildId = ParseBuildId(); @@ -11087,8 +11106,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -11209,8 +11231,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - StorageInstance Storage = - CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(m_ZenFolderPath), /*RequireBucket*/ true); + StorageInstance Storage = CreateBuildStorage(StorageStats, + StorageCacheStats, + ZenTempFolderPath(m_ZenFolderPath), + /*RequriesNamespace*/ true, + /*RequireBucket*/ true); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = m_Path.filename().string(); -- cgit v1.2.3 From 61b4a88fe0b6f0ce43910af4f1afc2fc13f10273 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:05:26 +0200 Subject: add EnvironmentOptions helper --- src/zenutil/environmentoptions.cpp | 84 ++++++++++++++++++++++ src/zenutil/include/zenutil/environmentoptions.h | 92 ++++++++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 src/zenutil/environmentoptions.cpp create mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src') diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp new file mode 100644 index 000000000..fc37b63c6 --- /dev/null +++ b/src/zenutil/environmentoptions.cpp @@ -0,0 +1,84 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +namespace zen { + +EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::StringOption::Parse(std::string_view Value) +{ + RefValue = std::string(Value); +} + +EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::FilePathOption::Parse(std::string_view Value) +{ + RefValue = MakeSafeAbsolutePath(Value); +} + +EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::BoolOption::Parse(std::string_view Value) +{ + const std::string Lower = ToLower(Value); + if (Lower == "true" || Lower == "y" || Lower == "yes") + { + RefValue = true; + } + else if (Lower == "false" || Lower == "n" || Lower == "no") + { + RefValue = false; + } +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::string& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::filesystem::path& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(bool& Value) +{ + return std::make_shared(Value); +} + +EnvironmentOptions::EnvironmentOptions() +{ +} + +void +EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) +{ + for (auto& It : OptionMap) + { + std::string_view EnvName = It.first; + const Option& Opt = It.second; + if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) + { + std::string EnvValue = GetEnvVariable(It.first); + if (EnvValue.empty()) + { + Opt.Value->Parse(EnvValue); + } + } + } +} + +} // namespace zen diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h new file mode 100644 index 000000000..7418608e4 --- /dev/null +++ b/src/zenutil/include/zenutil/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr Value; + }; + + std::shared_ptr MakeOption(std::string& Value); + std::shared_ptr MakeOption(std::filesystem::path& Value); + + template + std::shared_ptr MakeOption(T& Value) + { + return std::make_shared>(Value); + }; + + std::shared_ptr MakeOption(bool& Value); + + template + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map OptionMap; +}; + +} // namespace zen -- cgit v1.2.3 From cadaad632b05e231dadfc977819189d9c486c74b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:06:40 +0200 Subject: add sentry configurations options for debug/environment add env-variable parsing for sentry option --- src/zen/zen.cpp | 40 ++++++++++++++--- src/zencore/include/zencore/sentryintegration.h | 16 ++++--- src/zencore/sentryintegration.cpp | 20 +++++---- src/zenserver/config.cpp | 44 ++++++++++++++++--- src/zenserver/config.h | 57 ++++++++++++++----------- src/zenserver/main.cpp | 12 +++--- src/zenserver/zenserver.cpp | 2 +- 7 files changed, 135 insertions(+), 56 deletions(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index f0644a4f5..d19ff840e 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -872,18 +873,22 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE #if ZEN_USE_SENTRY - bool NoSentry = false; - bool SentryAllowPII = false; - std::string SentryDsn; + + SentryIntegration::Config SentryConfig; + + bool NoSentry = false; + Options .add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false"), ""); Options.add_option("sentry", "", "sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(SentryAllowPII)->default_value("false"), + cxxopts::value(SentryConfig.AllowPII)->default_value("false"), ""); - Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryDsn), ""); + Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryConfig.Dsn), ""); + Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value(SentryConfig.Environment), ""); + Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value(SentryConfig.Debug)->default_value("false")); #endif Options.parse_positional({"command"}); @@ -928,6 +933,27 @@ main(int argc, char** argv) } #if ZEN_USE_SENTRY + + { + EnvironmentOptions EnvOptions; + + EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !NoSentry; + EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv); + + EnvOptions.Parse(ParseResult); + + if (EnvEnableSentry != !NoSentry) + { + NoSentry = !EnvEnableSentry; + } + } + SentryIntegration Sentry; if (NoSentry == false) @@ -945,7 +971,9 @@ main(int argc, char** argv) SB.Append(argv[i]); } - Sentry.Initialize(SentryDatabasePath, {}, SentryDsn, SentryAllowPII, SB.ToString()); + SentryConfig.DatabasePath = SentryDatabasePath; + + Sentry.Initialize(SentryConfig, SB.ToString()); SentryIntegration::ClearCaches(); } diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index d14c1c275..faf1238b7 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,11 +31,17 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine); + struct Config + { + std::string DatabasePath; + std::string AttachmentsPath; + std::string Dsn; + std::string Environment; + bool AllowPII = false; + bool Debug = false; + }; + + void Initialize(const Config& Conf, const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 520d5162e..118c4158a 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -196,22 +196,23 @@ SentryIntegration::~SentryIntegration() } void -SentryIntegration::Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine) +SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine) { - m_AllowPII = AllowPII; + m_AllowPII = Conf.AllowPII; + std::string SentryDatabasePath = Conf.DatabasePath; if (SentryDatabasePath.starts_with("\\\\?\\")) { SentryDatabasePath = SentryDatabasePath.substr(4); } sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, SentryDsn.empty() ? sentry::DefaultDsn.c_str() : SentryDsn.c_str()); + + sentry_options_set_dsn(SentryOptions, Conf.Dsn.empty() ? sentry::DefaultDsn.c_str() : Conf.Dsn.c_str()); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + sentry_options_set_environment(SentryOptions, Conf.Environment.empty() ? "production" : Conf.Environment.c_str()); + + std::string SentryAttachmentsPath = Conf.AttachmentsPath; if (!SentryAttachmentsPath.empty()) { if (SentryAttachmentsPath.starts_with("\\\\?\\")) @@ -222,7 +223,10 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, } sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - // sentry_options_set_debug(SentryOptions, 1); + if (Conf.Debug) + { + sentry_options_set_debug(SentryOptions, 1); + } m_SentryErrorCode = sentry_init(SentryOptions); diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 055376b5c..1f9ae5fb6 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -16,6 +16,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -428,6 +429,29 @@ MakeOption(std::vector>& return std::make_shared(Value); }; +void +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +{ + using namespace std::literals; + + EnvironmentOptions Options; + Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; + Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + + Options.Parse(CmdLineResult); + + if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) + { + ServerOptions.SentryConfig.Disable = !EnvEnableSentry; + } +} + void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, @@ -441,9 +465,11 @@ ParseConfigFile(const std::filesystem::path& Path, ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); - LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); @@ -759,11 +785,15 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", - cxxopts::value(ServerOptions.NoSentry)->default_value("false")); + cxxopts::value(ServerOptions.SentryConfig.Disable)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(ServerOptions.SentryAllowPII)->default_value("false")); - options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryDsn)); + cxxopts::value(ServerOptions.SentryConfig.AllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryConfig.Dsn)); + options.add_options()("sentry-environment", "Sentry environment", cxxopts::value(ServerOptions.SentryConfig.Environment)); + options.add_options()("sentry-debug", + "Enable debug mode for Sentry", + cxxopts::value(ServerOptions.SentryConfig.Debug)->default_value("false")); options.add_options()("detach", "Indicate whether zenserver should detach from parent process group", cxxopts::value(ServerOptions.Detach)->default_value("true")); @@ -1336,6 +1366,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); + ParseEnvVariables(ServerOptions, Result); + if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 1a1793b8d..9753e3ae2 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -157,6 +157,15 @@ struct ZenWorkspacesConfig bool AllowConfigurationChanges = false; }; +struct ZenSentryConfig +{ + bool Disable = false; + bool AllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string Dsn; + std::string Environment; + bool Debug = false; // Enable debug mode for Sentry +}; + struct ZenServerOptions { ZenUpstreamCacheConfig UpstreamCacheConfig; @@ -169,31 +178,29 @@ struct ZenServerOptions ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; - bool NoSentry = false; - bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports - std::string SentryDsn; + ZenSentryConfig SentryConfig; + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 868126533..b0d945814 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -96,15 +96,17 @@ ZenEntryPoint::Run() #if ZEN_USE_SENTRY SentryIntegration Sentry; - if (m_ServerOptions.NoSentry == false) + if (m_ServerOptions.SentryConfig.Disable == false) { std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); - Sentry.Initialize(SentryDatabasePath, - SentryAttachmentPath, - m_ServerOptions.SentryDsn, - m_ServerOptions.SentryAllowPII, + Sentry.Initialize({.DatabasePath = SentryDatabasePath, + .AttachmentsPath = SentryAttachmentPath, + .Dsn = m_ServerOptions.SentryConfig.Dsn, + .Environment = m_ServerOptions.SentryConfig.Environment, + .AllowPII = m_ServerOptions.SentryConfig.AllowPII, + .Debug = m_ServerOptions.SentryConfig.Debug}, m_ServerOptions.CommandLine); } #endif diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 1ad94ed63..27ec4c690 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.NoSentry == false; + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; -- cgit v1.2.3 From f696e52d150ae284e26de5bdd78d1b1edf914314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:08:59 +0200 Subject: revert 61b4a88f and cadaad63 --- src/zen/zen.cpp | 40 ++--------- src/zencore/include/zencore/sentryintegration.h | 16 ++--- src/zencore/sentryintegration.cpp | 20 +++--- src/zenserver/config.cpp | 44 ++---------- src/zenserver/config.h | 57 +++++++-------- src/zenserver/main.cpp | 12 ++-- src/zenserver/zenserver.cpp | 2 +- src/zenutil/environmentoptions.cpp | 84 ---------------------- src/zenutil/include/zenutil/environmentoptions.h | 92 ------------------------ 9 files changed, 56 insertions(+), 311 deletions(-) delete mode 100644 src/zenutil/environmentoptions.cpp delete mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index d19ff840e..f0644a4f5 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include @@ -873,22 +872,18 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE #if ZEN_USE_SENTRY - - SentryIntegration::Config SentryConfig; - - bool NoSentry = false; - + bool NoSentry = false; + bool SentryAllowPII = false; + std::string SentryDsn; Options .add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false"), ""); Options.add_option("sentry", "", "sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(SentryConfig.AllowPII)->default_value("false"), + cxxopts::value(SentryAllowPII)->default_value("false"), ""); - Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryConfig.Dsn), ""); - Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value(SentryConfig.Environment), ""); - Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value(SentryConfig.Debug)->default_value("false")); + Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryDsn), ""); #endif Options.parse_positional({"command"}); @@ -933,27 +928,6 @@ main(int argc, char** argv) } #if ZEN_USE_SENTRY - - { - EnvironmentOptions EnvOptions; - - EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv); - EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv); - EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv); - - bool EnvEnableSentry = !NoSentry; - EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); - - EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv); - - EnvOptions.Parse(ParseResult); - - if (EnvEnableSentry != !NoSentry) - { - NoSentry = !EnvEnableSentry; - } - } - SentryIntegration Sentry; if (NoSentry == false) @@ -971,9 +945,7 @@ main(int argc, char** argv) SB.Append(argv[i]); } - SentryConfig.DatabasePath = SentryDatabasePath; - - Sentry.Initialize(SentryConfig, SB.ToString()); + Sentry.Initialize(SentryDatabasePath, {}, SentryDsn, SentryAllowPII, SB.ToString()); SentryIntegration::ClearCaches(); } diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index faf1238b7..d14c1c275 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,17 +31,11 @@ public: SentryIntegration(); ~SentryIntegration(); - struct Config - { - std::string DatabasePath; - std::string AttachmentsPath; - std::string Dsn; - std::string Environment; - bool AllowPII = false; - bool Debug = false; - }; - - void Initialize(const Config& Conf, const std::string& CommandLine); + void Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + std::string SentryDsn, + bool AllowPII, + const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 118c4158a..520d5162e 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -196,23 +196,22 @@ SentryIntegration::~SentryIntegration() } void -SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine) +SentryIntegration::Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + std::string SentryDsn, + bool AllowPII, + const std::string& CommandLine) { - m_AllowPII = Conf.AllowPII; + m_AllowPII = AllowPII; - std::string SentryDatabasePath = Conf.DatabasePath; if (SentryDatabasePath.starts_with("\\\\?\\")) { SentryDatabasePath = SentryDatabasePath.substr(4); } sentry_options_t* SentryOptions = sentry_options_new(); - - sentry_options_set_dsn(SentryOptions, Conf.Dsn.empty() ? sentry::DefaultDsn.c_str() : Conf.Dsn.c_str()); + sentry_options_set_dsn(SentryOptions, SentryDsn.empty() ? sentry::DefaultDsn.c_str() : SentryDsn.c_str()); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); - sentry_options_set_environment(SentryOptions, Conf.Environment.empty() ? "production" : Conf.Environment.c_str()); - - std::string SentryAttachmentsPath = Conf.AttachmentsPath; if (!SentryAttachmentsPath.empty()) { if (SentryAttachmentsPath.starts_with("\\\\?\\")) @@ -223,10 +222,7 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine } sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - if (Conf.Debug) - { - sentry_options_set_debug(SentryOptions, 1); - } + // sentry_options_set_debug(SentryOptions, 1); m_SentryErrorCode = sentry_init(SentryOptions); diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 1f9ae5fb6..055376b5c 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -16,7 +16,6 @@ #include #include #include -#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -429,29 +428,6 @@ MakeOption(std::vector>& return std::make_shared(Value); }; -void -ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) -{ - using namespace std::literals; - - EnvironmentOptions Options; - Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); - Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); - Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); - - bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; - Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); - - Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); - - Options.Parse(CmdLineResult); - - if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) - { - ServerOptions.SentryConfig.Disable = !EnvEnableSentry; - } -} - void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, @@ -465,11 +441,9 @@ ParseConfigFile(const std::filesystem::path& Path, ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); - LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); - LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); - LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); @@ -785,15 +759,11 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", - cxxopts::value(ServerOptions.SentryConfig.Disable)->default_value("false")); + cxxopts::value(ServerOptions.NoSentry)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(ServerOptions.SentryConfig.AllowPII)->default_value("false")); - options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryConfig.Dsn)); - options.add_options()("sentry-environment", "Sentry environment", cxxopts::value(ServerOptions.SentryConfig.Environment)); - options.add_options()("sentry-debug", - "Enable debug mode for Sentry", - cxxopts::value(ServerOptions.SentryConfig.Debug)->default_value("false")); + cxxopts::value(ServerOptions.SentryAllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryDsn)); options.add_options()("detach", "Indicate whether zenserver should detach from parent process group", cxxopts::value(ServerOptions.Detach)->default_value("true")); @@ -1366,8 +1336,6 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); - ParseEnvVariables(ServerOptions, Result); - if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 9753e3ae2..1a1793b8d 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -157,15 +157,6 @@ struct ZenWorkspacesConfig bool AllowConfigurationChanges = false; }; -struct ZenSentryConfig -{ - bool Disable = false; - bool AllowPII = false; // Allow personally identifiable information in sentry crash reports - std::string Dsn; - std::string Environment; - bool Debug = false; // Enable debug mode for Sentry -}; - struct ZenServerOptions { ZenUpstreamCacheConfig UpstreamCacheConfig; @@ -178,29 +169,31 @@ struct ZenServerOptions ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; - ZenSentryConfig SentryConfig; - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; + bool NoSentry = false; + bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string SentryDsn; bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index b0d945814..868126533 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -96,17 +96,15 @@ ZenEntryPoint::Run() #if ZEN_USE_SENTRY SentryIntegration Sentry; - if (m_ServerOptions.SentryConfig.Disable == false) + if (m_ServerOptions.NoSentry == false) { std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); - Sentry.Initialize({.DatabasePath = SentryDatabasePath, - .AttachmentsPath = SentryAttachmentPath, - .Dsn = m_ServerOptions.SentryConfig.Dsn, - .Environment = m_ServerOptions.SentryConfig.Environment, - .AllowPII = m_ServerOptions.SentryConfig.AllowPII, - .Debug = m_ServerOptions.SentryConfig.Debug}, + Sentry.Initialize(SentryDatabasePath, + SentryAttachmentPath, + m_ServerOptions.SentryDsn, + m_ServerOptions.SentryAllowPII, m_ServerOptions.CommandLine); } #endif diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 27ec4c690..1ad94ed63 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.SentryConfig.Disable == false; + m_UseSentry = ServerOptions.NoSentry == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp deleted file mode 100644 index fc37b63c6..000000000 --- a/src/zenutil/environmentoptions.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include - -namespace zen { - -EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::StringOption::Parse(std::string_view Value) -{ - RefValue = std::string(Value); -} - -EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::FilePathOption::Parse(std::string_view Value) -{ - RefValue = MakeSafeAbsolutePath(Value); -} - -EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::BoolOption::Parse(std::string_view Value) -{ - const std::string Lower = ToLower(Value); - if (Lower == "true" || Lower == "y" || Lower == "yes") - { - RefValue = true; - } - else if (Lower == "false" || Lower == "n" || Lower == "no") - { - RefValue = false; - } -} - -std::shared_ptr -EnvironmentOptions::MakeOption(std::string& Value) -{ - return std::make_shared(Value); -} - -std::shared_ptr -EnvironmentOptions::MakeOption(std::filesystem::path& Value) -{ - return std::make_shared(Value); -} - -std::shared_ptr -EnvironmentOptions::MakeOption(bool& Value) -{ - return std::make_shared(Value); -} - -EnvironmentOptions::EnvironmentOptions() -{ -} - -void -EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) -{ - for (auto& It : OptionMap) - { - std::string_view EnvName = It.first; - const Option& Opt = It.second; - if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) - { - std::string EnvValue = GetEnvVariable(It.first); - if (EnvValue.empty()) - { - Opt.Value->Parse(EnvValue); - } - } - } -} - -} // namespace zen diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h deleted file mode 100644 index 7418608e4..000000000 --- a/src/zenutil/include/zenutil/environmentoptions.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include - -namespace zen { - -class EnvironmentOptions -{ -public: - class OptionValue - { - public: - virtual void Parse(std::string_view Value) = 0; - - virtual ~OptionValue() {} - }; - - class StringOption : public OptionValue - { - public: - explicit StringOption(std::string& Value); - virtual void Parse(std::string_view Value) override; - std::string& RefValue; - }; - - class FilePathOption : public OptionValue - { - public: - explicit FilePathOption(std::filesystem::path& Value); - virtual void Parse(std::string_view Value) override; - std::filesystem::path& RefValue; - }; - - class BoolOption : public OptionValue - { - public: - explicit BoolOption(bool& Value); - virtual void Parse(std::string_view Value); - bool& RefValue; - }; - - template - class NumberOption : public OptionValue - { - public: - explicit NumberOption(T& Value) : RefValue(Value) {} - virtual void Parse(std::string_view Value) override - { - if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) - { - RefValue = OptionalValue.value(); - } - } - T& RefValue; - }; - - struct Option - { - std::string CommandLineOptionName; - std::shared_ptr Value; - }; - - std::shared_ptr MakeOption(std::string& Value); - std::shared_ptr MakeOption(std::filesystem::path& Value); - - template - std::shared_ptr MakeOption(T& Value) - { - return std::make_shared>(Value); - }; - - std::shared_ptr MakeOption(bool& Value); - - template - void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") - { - OptionMap.insert_or_assign(std::string(EnvName), - Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); - }; - - EnvironmentOptions(); - - void Parse(const cxxopts::ParseResult& CmdLineResult); - -private: - std::unordered_map OptionMap; -}; - -} // namespace zen -- cgit v1.2.3 From 1442dae38a54b553d2285176f4969520332a9843 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 14:48:56 +0200 Subject: crash in composite buffer stream (#431) * fix BufferedReadFileStream calculating buffer end wrong --- src/zenhttp/httpclient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index f2b26b922..a2d323b5e 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -333,7 +333,7 @@ namespace detail { m_Buffer = (uint8_t*)Memory::Alloc(gsl::narrow(m_BufferSize)); } m_BufferStart = Begin; - m_BufferEnd = Min(Begin + m_BufferSize, m_FileSize); + m_BufferEnd = Min(Begin + m_BufferSize, m_FileEnd); Read(m_Buffer, m_BufferEnd - m_BufferStart, m_BufferStart); uint64_t Count = Min(m_BufferEnd, End) - m_BufferStart; memcpy(WritePtr + Begin - m_FileOffset, m_Buffer, Count); -- cgit v1.2.3 From c4b2b2d99f8d037c26effd8597925c6327075f44 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 16:37:43 +0200 Subject: make sure we finish progress at 100% if not done (#432) * make sure we finish progress at 100% if not done * fix condition --- src/zen/zen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index f0644a4f5..e84e258b8 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -576,7 +576,7 @@ ProgressBar::ForceLinebreak() void ProgressBar::Finish() { - if (m_LastOutputLength > 0) + if (m_LastOutputLength > 0 || m_State.RemainingCount > 0) { State NewState = m_State; NewState.RemainingCount = 0; -- cgit v1.2.3 From fdad92dddba8047930c4e7496a7f412d760c312e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Jun 2025 09:30:54 +0200 Subject: sentry config (#430) - Feature: Added `--sentry-environment` to `zen` and `zenserver` - Feature: Added `--sentry-debug` to `zen` and `zenserver` - Feature: Added environment variable parsing for the following options: - `UE_ZEN_SENTRY_ENABLED`: `--no-sentry` (inverted) - `UE_ZEN_SENTRY_DEBUG`: `--sentry-debug` - `UE_ZEN_SENTRY_ALLOWPERSONALINFO`: `--sentry-allow-personal-info` - `UE_ZEN_SENTRY_DSN`: `--sentry-dsn` - `UE_ZEN_SENTRY_ENVIRONMENT`: `--sentry-environment` --- src/zen/zen.cpp | 40 +++++++++-- src/zencore/include/zencore/sentryintegration.h | 16 +++-- src/zencore/sentryintegration.cpp | 20 +++--- src/zenserver/config.cpp | 44 ++++++++++-- src/zenserver/config.h | 57 ++++++++------- src/zenserver/main.cpp | 12 ++-- src/zenserver/zenserver.cpp | 2 +- src/zenutil/environmentoptions.cpp | 84 ++++++++++++++++++++++ src/zenutil/include/zenutil/environmentoptions.h | 92 ++++++++++++++++++++++++ 9 files changed, 311 insertions(+), 56 deletions(-) create mode 100644 src/zenutil/environmentoptions.cpp create mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src') diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index e84e258b8..76a866204 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -872,18 +873,22 @@ main(int argc, char** argv) #endif // ZEN_WITH_TRACE #if ZEN_USE_SENTRY - bool NoSentry = false; - bool SentryAllowPII = false; - std::string SentryDsn; + + SentryIntegration::Config SentryConfig; + + bool NoSentry = false; + Options .add_option("sentry", "", "no-sentry", "Disable Sentry crash handler", cxxopts::value(NoSentry)->default_value("false"), ""); Options.add_option("sentry", "", "sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(SentryAllowPII)->default_value("false"), + cxxopts::value(SentryConfig.AllowPII)->default_value("false"), ""); - Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryDsn), ""); + Options.add_option("sentry", "", "sentry-dsn", "Sentry DSN to send events to", cxxopts::value(SentryConfig.Dsn), ""); + Options.add_option("sentry", "", "sentry-environment", "Sentry environment", cxxopts::value(SentryConfig.Environment), ""); + Options.add_options()("sentry-debug", "Enable debug mode for Sentry", cxxopts::value(SentryConfig.Debug)->default_value("false")); #endif Options.parse_positional({"command"}); @@ -928,6 +933,27 @@ main(int argc, char** argv) } #if ZEN_USE_SENTRY + + { + EnvironmentOptions EnvOptions; + + EnvOptions.AddOption("UE_ZEN_SENTRY_DSN"sv, SentryConfig.Dsn, "sentry-dsn"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + EnvOptions.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !NoSentry; + EnvOptions.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + EnvOptions.AddOption("UE_ZEN_SENTRY_DEBUG"sv, SentryConfig.Debug, "sentry-debug"sv); + + EnvOptions.Parse(ParseResult); + + if (EnvEnableSentry != !NoSentry) + { + NoSentry = !EnvEnableSentry; + } + } + SentryIntegration Sentry; if (NoSentry == false) @@ -945,7 +971,9 @@ main(int argc, char** argv) SB.Append(argv[i]); } - Sentry.Initialize(SentryDatabasePath, {}, SentryDsn, SentryAllowPII, SB.ToString()); + SentryConfig.DatabasePath = SentryDatabasePath; + + Sentry.Initialize(SentryConfig, SB.ToString()); SentryIntegration::ClearCaches(); } diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index d14c1c275..faf1238b7 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,11 +31,17 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine); + struct Config + { + std::string DatabasePath; + std::string AttachmentsPath; + std::string Dsn; + std::string Environment; + bool AllowPII = false; + bool Debug = false; + }; + + void Initialize(const Config& Conf, const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 520d5162e..118c4158a 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -196,22 +196,23 @@ SentryIntegration::~SentryIntegration() } void -SentryIntegration::Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine) +SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine) { - m_AllowPII = AllowPII; + m_AllowPII = Conf.AllowPII; + std::string SentryDatabasePath = Conf.DatabasePath; if (SentryDatabasePath.starts_with("\\\\?\\")) { SentryDatabasePath = SentryDatabasePath.substr(4); } sentry_options_t* SentryOptions = sentry_options_new(); - sentry_options_set_dsn(SentryOptions, SentryDsn.empty() ? sentry::DefaultDsn.c_str() : SentryDsn.c_str()); + + sentry_options_set_dsn(SentryOptions, Conf.Dsn.empty() ? sentry::DefaultDsn.c_str() : Conf.Dsn.c_str()); sentry_options_set_database_path(SentryOptions, SentryDatabasePath.c_str()); sentry_options_set_logger(SentryOptions, SentryLogFunction, this); + sentry_options_set_environment(SentryOptions, Conf.Environment.empty() ? "production" : Conf.Environment.c_str()); + + std::string SentryAttachmentsPath = Conf.AttachmentsPath; if (!SentryAttachmentsPath.empty()) { if (SentryAttachmentsPath.starts_with("\\\\?\\")) @@ -222,7 +223,10 @@ SentryIntegration::Initialize(std::string SentryDatabasePath, } sentry_options_set_release(SentryOptions, ZEN_CFG_VERSION); - // sentry_options_set_debug(SentryOptions, 1); + if (Conf.Debug) + { + sentry_options_set_debug(SentryOptions, 1); + } m_SentryErrorCode = sentry_init(SentryOptions); diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 055376b5c..1f9ae5fb6 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -16,6 +16,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -428,6 +429,29 @@ MakeOption(std::vector>& return std::make_shared(Value); }; +void +ParseEnvVariables(ZenServerOptions& ServerOptions, const cxxopts::ParseResult& CmdLineResult) +{ + using namespace std::literals; + + EnvironmentOptions Options; + Options.AddOption("UE_ZEN_SENTRY_ALLOWPERSONALINFO"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + Options.AddOption("UE_ZEN_SENTRY_DSN"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + Options.AddOption("UE_ZEN_SENTRY_ENVIRONMENT"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + + bool EnvEnableSentry = !ServerOptions.SentryConfig.Disable; + Options.AddOption("UE_ZEN_SENTRY_ENABLED"sv, EnvEnableSentry, "no-sentry"sv); + + Options.AddOption("UE_ZEN_SENTRY_DEBUG"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); + + Options.Parse(CmdLineResult); + + if (EnvEnableSentry != !ServerOptions.SentryConfig.Disable) + { + ServerOptions.SentryConfig.Disable = !EnvEnableSentry; + } +} + void ParseConfigFile(const std::filesystem::path& Path, ZenServerOptions& ServerOptions, @@ -441,9 +465,11 @@ ParseConfigFile(const std::filesystem::path& Path, ////// server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); - LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.NoSentry, "no-sentry"sv); - LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryAllowPII, "sentry-allow-personal-info"sv); - LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryDsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); + LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); + LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); + LuaOptions.AddOption("server.sentry.environment"sv, ServerOptions.SentryConfig.Environment, "sentry-environment"sv); + LuaOptions.AddOption("server.sentry.debug"sv, ServerOptions.SentryConfig.Debug, "sentry-debug"sv); LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); @@ -759,11 +785,15 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_options()("write-config", "Path to output Lua config file", cxxopts::value(OutputConfigFile)); options.add_options()("no-sentry", "Disable Sentry crash handler", - cxxopts::value(ServerOptions.NoSentry)->default_value("false")); + cxxopts::value(ServerOptions.SentryConfig.Disable)->default_value("false")); options.add_options()("sentry-allow-personal-info", "Allow personally identifiable information in sentry crash reports", - cxxopts::value(ServerOptions.SentryAllowPII)->default_value("false")); - options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryDsn)); + cxxopts::value(ServerOptions.SentryConfig.AllowPII)->default_value("false")); + options.add_options()("sentry-dsn", "Sentry DSN to send events to", cxxopts::value(ServerOptions.SentryConfig.Dsn)); + options.add_options()("sentry-environment", "Sentry environment", cxxopts::value(ServerOptions.SentryConfig.Environment)); + options.add_options()("sentry-debug", + "Enable debug mode for Sentry", + cxxopts::value(ServerOptions.SentryConfig.Debug)->default_value("false")); options.add_options()("detach", "Indicate whether zenserver should detach from parent process group", cxxopts::value(ServerOptions.Detach)->default_value("true")); @@ -1336,6 +1366,8 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); + ParseEnvVariables(ServerOptions, Result); + if (!ServerOptions.ConfigFile.empty()) { ParseConfigFile(ServerOptions.ConfigFile, ServerOptions, Result, OutputConfigFile); diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 1a1793b8d..9753e3ae2 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -157,6 +157,15 @@ struct ZenWorkspacesConfig bool AllowConfigurationChanges = false; }; +struct ZenSentryConfig +{ + bool Disable = false; + bool AllowPII = false; // Allow personally identifiable information in sentry crash reports + std::string Dsn; + std::string Environment; + bool Debug = false; // Enable debug mode for Sentry +}; + struct ZenServerOptions { ZenUpstreamCacheConfig UpstreamCacheConfig; @@ -169,31 +178,29 @@ struct ZenServerOptions ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path AbsLogFile; // Absolute path to main log file - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path PluginsConfigFile; // Path to plugins config file - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string EncryptionKey; // 256 bit AES encryption key - std::string EncryptionIV; // 128 bit AES initialization vector - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool InstallService = false; // Flag used to initiate service install (temporary) - bool UninstallService = false; // Flag used to initiate service uninstall (temporary) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; - bool NoSentry = false; - bool SentryAllowPII = false; // Allow personally identifiable information in sentry crash reports - std::string SentryDsn; + ZenSentryConfig SentryConfig; + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::string LogId; // Id for tagging log output + std::string EncryptionKey; // 256 bit AES encryption key + std::string EncryptionIV; // 128 bit AES initialization vector + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool InstallService = false; // Flag used to initiate service install (temporary) + bool UninstallService = false; // Flag used to initiate service uninstall (temporary) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) bool ObjectStoreEnabled = false; bool NoConsoleOutput = false; // Control default use of stdout for diagnostics diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 868126533..b0d945814 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -96,15 +96,17 @@ ZenEntryPoint::Run() #if ZEN_USE_SENTRY SentryIntegration Sentry; - if (m_ServerOptions.NoSentry == false) + if (m_ServerOptions.SentryConfig.Disable == false) { std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); - Sentry.Initialize(SentryDatabasePath, - SentryAttachmentPath, - m_ServerOptions.SentryDsn, - m_ServerOptions.SentryAllowPII, + Sentry.Initialize({.DatabasePath = SentryDatabasePath, + .AttachmentsPath = SentryAttachmentPath, + .Dsn = m_ServerOptions.SentryConfig.Dsn, + .Environment = m_ServerOptions.SentryConfig.Environment, + .AllowPII = m_ServerOptions.SentryConfig.AllowPII, + .Debug = m_ServerOptions.SentryConfig.Debug}, m_ServerOptions.CommandLine); } #endif diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 1ad94ed63..27ec4c690 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -137,7 +137,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen return -1; } - m_UseSentry = ServerOptions.NoSentry == false; + m_UseSentry = ServerOptions.SentryConfig.Disable == false; m_ServerEntry = ServerEntry; m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; m_IsPowerCycle = ServerOptions.IsPowerCycle; diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp new file mode 100644 index 000000000..1b7ce8029 --- /dev/null +++ b/src/zenutil/environmentoptions.cpp @@ -0,0 +1,84 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +namespace zen { + +EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::StringOption::Parse(std::string_view Value) +{ + RefValue = std::string(Value); +} + +EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::FilePathOption::Parse(std::string_view Value) +{ + RefValue = MakeSafeAbsolutePath(Value); +} + +EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::BoolOption::Parse(std::string_view Value) +{ + const std::string Lower = ToLower(Value); + if (Lower == "true" || Lower == "y" || Lower == "yes") + { + RefValue = true; + } + else if (Lower == "false" || Lower == "n" || Lower == "no") + { + RefValue = false; + } +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::string& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::filesystem::path& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(bool& Value) +{ + return std::make_shared(Value); +} + +EnvironmentOptions::EnvironmentOptions() +{ +} + +void +EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) +{ + for (auto& It : OptionMap) + { + std::string_view EnvName = It.first; + const Option& Opt = It.second; + if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) + { + std::string EnvValue = GetEnvVariable(It.first); + if (!EnvValue.empty()) + { + Opt.Value->Parse(EnvValue); + } + } + } +} + +} // namespace zen diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h new file mode 100644 index 000000000..7418608e4 --- /dev/null +++ b/src/zenutil/include/zenutil/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr Value; + }; + + std::shared_ptr MakeOption(std::string& Value); + std::shared_ptr MakeOption(std::filesystem::path& Value); + + template + std::shared_ptr MakeOption(T& Value) + { + return std::make_shared>(Value); + }; + + std::shared_ptr MakeOption(bool& Value); + + template + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map OptionMap; +}; + +} // namespace zen -- cgit v1.2.3 From 2dbfa547abd7f9e393cb63f9c70de15d91de2815 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 Jun 2025 13:17:37 +0200 Subject: fix build store range check (#437) * fix range check for blob store fetch * don't try to parse blockdesriptions if empty result is returned * add range to log when fetching blob range fails --- src/zen/cmds/builds_cmd.cpp | 11 +++++++++-- src/zenserver/buildstore/httpbuildstore.cpp | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index abb4bfe0c..706fdf9ba 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3503,7 +3503,11 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount)); + CbObject BlockDescriptionList = Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount); + if (BlockDescriptionList) + { + Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList); + } FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -7440,7 +7444,10 @@ namespace { } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + throw std::runtime_error(fmt::format("Block {} is missing when fetching range {} -> {}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeStart + BlockRange.RangeLength)); } if (!AbortFlag) { diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index 75a333687..bcec74ce6 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -177,7 +177,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) const uint64_t BlobSize = Blob.GetSize(); const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); - if (Range.Start + RangeSize >= BlobSize) + if (Range.Start + RangeSize > BlobSize) { return ServerRequest.WriteResponse(HttpResponseCode::NoContent); } -- cgit v1.2.3 From d000167e12c6dde651ef86be9f67552291ff1b7d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 Jun 2025 13:17:54 +0200 Subject: graceful wait in parallelwork destructor (#438) * exception safety when issuing ParallelWork * add asserts to Latch usage to catch usage errors * extended error messaging and recovery handling in ParallelWork destructor to help find issues --- src/zencore/include/zencore/thread.h | 4 +- src/zenserver/cache/httpstructuredcache.cpp | 11 +- src/zenserver/projectstore/projectstore.cpp | 171 +++++++++++++++++----------- src/zenstore/buildstore/buildstore.cpp | 82 +++++++------ src/zenstore/cache/cachedisklayer.cpp | 80 +++++++------ src/zenstore/compactcas.cpp | 137 +++++++++++----------- src/zenstore/filecas.cpp | 86 ++++++++------ src/zenutil/include/zenutil/parallelwork.h | 3 +- src/zenutil/parallelwork.cpp | 29 ++++- 9 files changed, 359 insertions(+), 244 deletions(-) (limited to 'src') diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index 8fb781571..d9fb5c023 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -183,6 +183,7 @@ public: void CountDown() { std::ptrdiff_t Old = Counter.fetch_sub(1); + ZEN_ASSERT(Old > 0); if (Old == 1) { Complete.Set(); @@ -197,8 +198,7 @@ public: void AddCount(std::ptrdiff_t Count) { std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); - ZEN_UNUSED(Old); - ZEN_ASSERT_SLOW(Old > 0); + ZEN_ASSERT(Old > 0); } bool Wait(int TimeoutMs = -1) diff --git a/src/zenserver/cache/httpstructuredcache.cpp b/src/zenserver/cache/httpstructuredcache.cpp index 9f2e826d6..bb0c55618 100644 --- a/src/zenserver/cache/httpstructuredcache.cpp +++ b/src/zenserver/cache/httpstructuredcache.cpp @@ -1593,10 +1593,19 @@ HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Co ZEN_INFO("Replaying {} requests", RequestCount); for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) { - Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic&) { + if (AbortFlag) + { + break; + } + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic& AbortFlag) { IoBuffer Body; zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); + if (AbortFlag) + { + return; + } + if (Body) { uint32_t AcceptMagic = 0; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 6359b9db9..53e687983 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -1599,20 +1599,36 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) + try { - if (OptionalWorkerPool) - { - Work.ScheduleWork(*OptionalWorkerPool, [&, Index = OpIndex](std::atomic&) { - ZEN_MEMSCOPE(GetProjectstoreTag()); - ValidateOne(Index); - }); - } - else + for (uint32_t OpIndex = 0; !IsCancelledFlag && OpIndex < Result.OpCount; OpIndex++) { - ValidateOne(OpIndex); + if (AbortFlag) + { + break; + } + if (OptionalWorkerPool) + { + Work.ScheduleWork(*OptionalWorkerPool, [&ValidateOne, Index = OpIndex](std::atomic& AbortFlag) { + ZEN_MEMSCOPE(GetProjectstoreTag()); + if (AbortFlag) + { + return; + } + ValidateOne(Index); + }); + } + else + { + ValidateOne(OpIndex); + } } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed validating oplogs in {}. Reason: '{}'", m_BasePath, Ex.what()); + } Work.Wait(); { @@ -2110,67 +2126,81 @@ ProjectStore::Oplog::IterateChunks(std::span ChunkIds, } if (OptionalWorkerPool) { - std::atomic_bool Result = true; std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - - for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) + try { - if (Result.load() == false) + for (size_t ChunkIndex = 0; ChunkIndex < FileChunkIndexes.size(); ChunkIndex++) { - break; - } - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback, &Result]( - std::atomic&) { - if (Result.load() == false) - { - return; - } - size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; - const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex]; - try - { - IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); - if (!Payload) + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ChunkIds, IncludeModTag, ChunkIndex, &FileChunkIndexes, &FileChunkPaths, &AsyncCallback]( + std::atomic& AbortFlag) { + if (AbortFlag) { - ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath); + return; } + size_t FileChunkIndex = FileChunkIndexes[ChunkIndex]; + const std::filesystem::path& FilePath = FileChunkPaths[ChunkIndex]; + try + { + IoBuffer Payload = IoBufferBuilder::MakeFromFile(FilePath); + if (!Payload) + { + ZEN_WARN("Trying to fetch chunk {} using file path {} failed", ChunkIds[FileChunkIndex], FilePath); + } - if (!AsyncCallback(FileChunkIndex, Payload, IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) + if (!AsyncCallback(FileChunkIndex, + Payload, + IncludeModTag ? GetModificationTagFromModificationTime(Payload) : 0)) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) { - Result.store(false); + ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'", + m_OuterProject->Identifier, + m_OplogId, + FileChunkIndex, + FilePath, + Ex.what()); } - } - catch (const std::exception& Ex) - { - ZEN_WARN("oplog '{}/{}': exception caught when iterating file chunk {}, path '{}'. Reason: '{}'", - m_OuterProject->Identifier, - m_OplogId, - FileChunkIndex, - FilePath, - Ex.what()); - } - }); - } + }); + } - if (!CidChunkHashes.empty()) + if (!CidChunkHashes.empty() && !AbortFlag) + { + m_CidStore.IterateChunks( + CidChunkHashes, + [&](size_t Index, const IoBuffer& Payload) { + size_t CidChunkIndex = CidChunkIndexes[Index]; + if (AbortFlag) + { + return false; + } + return AsyncCallback(CidChunkIndex, + Payload, + IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); + }, + OptionalWorkerPool, + LargeSizeLimit); + } + } + catch (const std::exception& Ex) { - m_CidStore.IterateChunks( - CidChunkHashes, - [&](size_t Index, const IoBuffer& Payload) { - size_t CidChunkIndex = CidChunkIndexes[Index]; - return AsyncCallback(CidChunkIndex, Payload, IncludeModTag ? GetModificationTagFromRawHash(CidChunkHashes[Index]) : 0); - }, - OptionalWorkerPool, - LargeSizeLimit); + AbortFlag.store(true); + ZEN_WARN("Failed iterating oplog chunks in {}. Reason: '{}'", m_BasePath, Ex.what()); } Work.Wait(); - return Result.load(); + return !AbortFlag; } else { @@ -3894,19 +3924,26 @@ ProjectStore::Flush() std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - - for (const Ref& Project : Projects) + try { - Work.ScheduleWork(WorkerPool, [this, Project](std::atomic&) { - try - { - Project->Flush(); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what()); - } - }); + for (const Ref& Project : Projects) + { + Work.ScheduleWork(WorkerPool, [this, Project](std::atomic&) { + try + { + Project->Flush(); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Exception while flushing project {}: {}", Project->Identifier, Ex.what()); + } + }); + } + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed projects in {}. Reason: '{}'", m_ProjectBasePath, Ex.what()); } Work.Wait(); diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index c25f762f5..20dc55bca 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -528,43 +528,55 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - m_MetadataBlockStore.IterateChunks( - MetaLocations, - [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( - uint32_t BlockIndex, - std::span ChunkIndexes) -> bool { - ZEN_UNUSED(BlockIndex); - if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) - { - return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); - } - else - { - ZEN_ASSERT(OptionalWorkerPool != nullptr); - std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &Result, &MetaLocations, &MetaLocationResultIndexes, DoOneBlock, ChunkIndexes = std::move(TmpChunkIndexes)]( - std::atomic& AbortFlag) { - if (AbortFlag) - { - return; - } - try - { - if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) + try + { + m_MetadataBlockStore.IterateChunks( + MetaLocations, + [this, OptionalWorkerPool, &Work, &Result, &MetaLocations, &MetaLocationResultIndexes, &ReferencedBlocks, DoOneBlock]( + uint32_t BlockIndex, + std::span ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + { + return DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result); + } + else + { + ZEN_ASSERT(OptionalWorkerPool != nullptr); + std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, + &Result, + &MetaLocations, + &MetaLocationResultIndexes, + DoOneBlock, + ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic& AbortFlag) { + if (AbortFlag) { - AbortFlag.store(true); + return; } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); - } - }); - return !Work.IsAborted(); - } - }); + try + { + if (!DoOneBlock(MetaLocations, MetaLocationResultIndexes, ChunkIndexes, Result)) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + return !Work.IsAborted(); + } + }); + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating block metadata chunks in {}. Reason: '{}'", m_Config.RootDirectory, Ex.what()); + } Work.Wait(); } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 0ee70890c..0d2aef612 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -4036,50 +4036,58 @@ ZenCacheDiskLayer::DiscoverBuckets() std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (auto& BucketPath : FoundBucketDirectories) + try { - Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic&) { - ZEN_MEMSCOPE(GetCacheDiskTag()); - - const std::string BucketName = PathToUtf8(BucketPath.stem()); - try - { - BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; - if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName), - std::hash(), - eastl::equal_to_2()); - It != m_Configuration.BucketConfigMap.end()) - { - BucketConfig = &It->second; - } - - std::unique_ptr NewBucket = - std::make_unique(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig); + for (auto& BucketPath : FoundBucketDirectories) + { + Work.ScheduleWork(Pool, [this, &SyncLock, BucketPath](std::atomic&) { + ZEN_MEMSCOPE(GetCacheDiskTag()); - CacheBucket* Bucket = nullptr; + const std::string BucketName = PathToUtf8(BucketPath.stem()); + try { - RwLock::ExclusiveLockScope __(SyncLock); - auto InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket)); - Bucket = InsertResult.first->second.get(); - } - ZEN_ASSERT(Bucket); + BucketConfiguration* BucketConfig = &m_Configuration.BucketConfig; + if (auto It = m_Configuration.BucketConfigMap.find_as(std::string_view(BucketName), + std::hash(), + eastl::equal_to_2()); + It != m_Configuration.BucketConfigMap.end()) + { + BucketConfig = &It->second; + } - if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false)) - { - ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + std::unique_ptr NewBucket = + std::make_unique(m_Gc, m_TotalMemCachedSize, BucketName, *BucketConfig); + CacheBucket* Bucket = nullptr; { RwLock::ExclusiveLockScope __(SyncLock); - m_Buckets.erase(BucketName); + auto InsertResult = m_Buckets.emplace(BucketName, std::move(NewBucket)); + Bucket = InsertResult.first->second.get(); + } + ZEN_ASSERT(Bucket); + + if (!Bucket->OpenOrCreate(BucketPath, /* AllowCreate */ false)) + { + ZEN_WARN("Found directory '{}' in our base directory '{}' but it is not a valid bucket", BucketName, m_RootDir); + + { + RwLock::ExclusiveLockScope __(SyncLock); + m_Buckets.erase(BucketName); + } } } - } - catch (const std::exception& Err) - { - ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what()); - return; - } - }); + catch (const std::exception& Err) + { + ZEN_ERROR("Opening bucket '{}' in '{}' FAILED, reason: '{}'", BucketName, BucketPath, Err.what()); + return; + } + }); + } + } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed discovering buckets in {}. Reason: '{}'", m_RootDir, Ex.what()); } Work.Wait(); } @@ -4220,8 +4228,10 @@ ZenCacheDiskLayer::Flush() } catch (const std::exception& Ex) { + AbortFlag.store(true); ZEN_ERROR("Failed to flush buckets at '{}'. Reason: '{}'", m_RootDir, Ex.what()); } + Work.Wait(1000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t RemainingWork) { ZEN_UNUSED(IsAborted, IsPaused); ZEN_DEBUG("Waiting for {} buckets at '{}' to flush", RemainingWork, m_RootDir); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2ab5752ff..b00abb2cb 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -396,89 +396,96 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas return m_BlockStore.IterateBlock( FoundChunkLocations, ChunkIndexes, - [AsyncCallback, FoundChunkIndexes, LargeSizeLimit](size_t ChunkIndex, const void* Data, uint64_t Size) { + [AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, const void* Data, uint64_t Size) { if (Data == nullptr) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer()); } return AsyncCallback(FoundChunkIndexes[ChunkIndex], IoBuffer(IoBuffer::Wrap, Data, Size)); }, - [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { + [AsyncCallback, FoundChunkIndexes](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { return AsyncCallback(FoundChunkIndexes[ChunkIndex], File.GetChunk(Offset, Size)); }, LargeSizeLimit); }; - std::atomic AsyncContinue = true; + std::atomic AbortFlag; { - std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - const bool Continue = m_BlockStore.IterateChunks( - FoundChunkLocations, - [this, - &Work, - &AsyncContinue, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - &FoundChunkIndexes, - &FoundChunkLocations, - OptionalWorkerPool](uint32_t BlockIndex, std::span ChunkIndexes) { - if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) - { - std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); - Work.ScheduleWork( - *OptionalWorkerPool, - [this, - &AsyncContinue, - &AsyncCallback, - LargeSizeLimit, - DoOneBlock, - BlockIndex, - &FoundChunkIndexes, - &FoundChunkLocations, - ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic& AbortFlag) { - if (AbortFlag) - { - AsyncContinue.store(false); - } - if (!AsyncContinue) - { - return; - } - try - { - bool Continue = - DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - if (!Continue) - { - AsyncContinue.store(false); - } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", - m_RootDirectory, - BlockIndex, - Ex.what()); - AsyncContinue.store(false); - } - }); - return AsyncContinue.load(); - } - else - { - return DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); - } - }); - if (!Continue) + try { - AsyncContinue.store(false); + const bool Continue = m_BlockStore.IterateChunks( + FoundChunkLocations, + [this, + &Work, + &AbortFlag, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + &FoundChunkIndexes, + &FoundChunkLocations, + OptionalWorkerPool](uint32_t BlockIndex, std::span ChunkIndexes) { + if (OptionalWorkerPool && (ChunkIndexes.size() > 3)) + { + std::vector TmpChunkIndexes(ChunkIndexes.begin(), ChunkIndexes.end()); + Work.ScheduleWork( + *OptionalWorkerPool, + [this, + &AsyncCallback, + LargeSizeLimit, + DoOneBlock, + BlockIndex, + &FoundChunkIndexes, + &FoundChunkLocations, + ChunkIndexes = std::move(TmpChunkIndexes)](std::atomic& AbortFlag) { + if (AbortFlag) + { + return; + } + try + { + bool Continue = + DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes); + if (!Continue) + { + AbortFlag.store(true); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, block {}. Reason: '{}'", + m_RootDirectory, + BlockIndex, + Ex.what()); + AbortFlag.store(true); + } + }); + return !AbortFlag.load(); + } + else + { + if (!DoOneBlock(AsyncCallback, LargeSizeLimit, FoundChunkIndexes, FoundChunkLocations, ChunkIndexes)) + { + AbortFlag.store(true); + } + return !AbortFlag.load(); + } + }); + if (!Continue) + { + AbortFlag.store(true); + } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating chunks for cas root path {}. Reason: '{}'", m_RootDirectory, Ex.what()); + } + Work.Wait(); } - return AsyncContinue.load(); + return !AbortFlag.load(); } void diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 11a266f1c..68644be2d 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -666,52 +666,64 @@ FileCasStrategy::IterateChunks(std::span ChunkHashes, std::atomic AbortFlag; std::atomic PauseFlag; ParallelWork Work(AbortFlag, PauseFlag); - for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) + try { - if (!AsyncContinue) + for (size_t Index = 0; Index < FoundChunkIndexes.size(); Index++) { - break; - } - size_t ChunkIndex = FoundChunkIndexes[Index]; - uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; - if (OptionalWorkerPool) - { - Work.ScheduleWork( - *OptionalWorkerPool, - [this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic& AbortFlag) { - if (AbortFlag) - { - AsyncContinue.store(false); - } - if (!AsyncContinue) - { - return; - } - try - { - if (!ProcessOne(ChunkIndex, ExpectedSize)) + if (AbortFlag) + { + AsyncContinue.store(false); + } + if (!AsyncContinue) + { + break; + } + size_t ChunkIndex = FoundChunkIndexes[Index]; + uint64_t ExpectedSize = FoundChunkExpectedSizes[Index]; + if (OptionalWorkerPool) + { + Work.ScheduleWork( + *OptionalWorkerPool, + [this, &ProcessOne, &ChunkHashes, ChunkIndex, ExpectedSize, &AsyncContinue](std::atomic& AbortFlag) { + if (AbortFlag) { AsyncContinue.store(false); } - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", - m_RootDirectory, - ChunkHashes[ChunkIndex], - Ex.what()); - AsyncContinue.store(false); - } - }); - } - else - { - if (!ProcessOne(ChunkIndex, ExpectedSize)) + if (!AsyncContinue) + { + return; + } + try + { + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed iterating chunks for cas root path {}, chunk {}. Reason: '{}'", + m_RootDirectory, + ChunkHashes[ChunkIndex], + Ex.what()); + AsyncContinue.store(false); + } + }); + } + else { - AsyncContinue.store(false); + if (!ProcessOne(ChunkIndex, ExpectedSize)) + { + AsyncContinue.store(false); + } } } } + catch (const std::exception& Ex) + { + AbortFlag.store(true); + ZEN_WARN("Failed iterating chunks in {}. Reason: '{}'", this->m_RootDirectory, Ex.what()); + } Work.Wait(); } return AsyncContinue.load(); diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h index d7e986551..639c6968c 100644 --- a/src/zenutil/include/zenutil/parallelwork.h +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include @@ -26,6 +27,7 @@ public: try { WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { + auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); }); try { while (m_PauseFlag && !m_AbortFlag) @@ -38,7 +40,6 @@ public: { OnError(std::current_exception(), m_AbortFlag); } - m_PendingWork.CountDown(); }); } catch (const std::exception&) diff --git a/src/zenutil/parallelwork.cpp b/src/zenutil/parallelwork.cpp index 67fc03c04..aa806438b 100644 --- a/src/zenutil/parallelwork.cpp +++ b/src/zenutil/parallelwork.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -34,7 +35,33 @@ ParallelWork::~ParallelWork() m_PendingWork.CountDown(); } m_PendingWork.Wait(); - ZEN_ASSERT(m_PendingWork.Remaining() == 0); + ptrdiff_t RemainingWork = m_PendingWork.Remaining(); + if (RemainingWork != 0) + { + void* Frames[8]; + uint32_t FrameCount = GetCallstack(2, 8, Frames); + CallstackFrames* Callstack = CreateCallstack(FrameCount, Frames); + ZEN_ERROR("ParallelWork destructor waited for outstanding work but pending work count is {} instead of 0\n{}", + RemainingWork, + CallstackToString(Callstack, " ")); + FreeCallstack(Callstack); + + uint32_t WaitedMs = 0; + while (m_PendingWork.Remaining() > 0 && WaitedMs < 2000) + { + Sleep(50); + WaitedMs += 50; + } + RemainingWork = m_PendingWork.Remaining(); + if (RemainingWork != 0) + { + ZEN_WARN("ParallelWork destructor safety wait failed, pending work count at {}", RemainingWork) + } + else + { + ZEN_INFO("ParallelWork destructor safety wait succeeded"); + } + } } catch (const std::exception& Ex) { -- cgit v1.2.3 From ecd3df960ea77df179f9a6fb991536f3f5e6b6d4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 17 Jun 2025 13:24:29 +0200 Subject: accept Cloud urls without the api/v2/builds/ part (#439) --- src/zen/cmds/builds_cmd.cpp | 70 +++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 706fdf9ba..be640fe31 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -806,6 +806,50 @@ namespace { return CleanWipe; } + bool ParseCloudUrl(std::string_view InUrl, + std::string& OutHost, + std::string& OutNamespace, + std::string& OutBucket, + std::string& OutBuildId) + { + std::string Url(RemoveQuotes(InUrl)); + const std::string_view ExtendedApiString = "api/v2/builds/"; + if (auto ApiString = ToLower(Url).find(ExtendedApiString); ApiString != std::string::npos) + { + Url.erase(ApiString, ExtendedApiString.length()); + } + + const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))"; + const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase); + std::match_results MatchResults; + std::string_view UrlToParse(Url); + if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) + { + auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { + ZEN_ASSERT(Index < MatchResults.size()); + + const auto& Match = MatchResults[Index]; + + return std::string_view(&*Match.first, Match.second - Match.first); + }; + + const std::string_view Host = GetMatch(1); + const std::string_view Namespace = GetMatch(2); + const std::string_view Bucket = GetMatch(3); + const std::string_view BuildId = GetMatch(4); + + OutHost = Host; + OutNamespace = Namespace; + OutBucket = Bucket; + OutBuildId = BuildId; + return true; + } + else + { + return false; + } + } + std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { if (!IsFile(Path)) @@ -10083,31 +10127,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("buildid is not compatible with the url option\n{}", SubOption->help())); } - const std::string ArtifactURLRegExString = R"((.*?:\/\/.*?)\/api\/v2\/builds\/(.*?)\/(.*?)\/(.*))"; - const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript); - std::match_results MatchResults; - const std::string_view Url(RemoveQuotes(m_Url)); - if (regex_match(begin(Url), end(Url), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5) - { - auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view { - ZEN_ASSERT(Index < MatchResults.size()); - - const auto& Match = MatchResults[Index]; - - return std::string_view(&*Match.first, Match.second - Match.first); - }; - - const std::string_view Host = GetMatch(1); - const std::string_view Namespace = GetMatch(2); - const std::string_view Bucket = GetMatch(3); - const std::string_view BuildId = GetMatch(4); - - m_Host = Host; - m_Namespace = Namespace; - m_Bucket = Bucket; - m_BuildId = BuildId; - } - else + if (!ParseCloudUrl(m_Url, m_Host, m_Namespace, m_Bucket, m_BuildId)) { throw zen::OptionParseException(fmt::format("url does not match the Cloud Artifact URL format\n{}", SubOption->help())); } -- cgit v1.2.3 From 18f52147774fa36e1ae7d59a8ae10fb2720f6c01 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 18 Jun 2025 12:47:07 +0200 Subject: `--output-path` option added to `zen version` command (#440) * `--output-path` option added to `zen version` command --- src/zen/cmds/version_cmd.cpp | 49 +++++++++++++++++++++++++++++--------------- src/zen/cmds/version_cmd.h | 10 ++++++--- src/zen/zen.cpp | 2 +- 3 files changed, 41 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/version_cmd.cpp b/src/zen/cmds/version_cmd.cpp index 41042533d..7dfa125e4 100644 --- a/src/zen/cmds/version_cmd.cpp +++ b/src/zen/cmds/version_cmd.cpp @@ -2,10 +2,12 @@ #include "version_cmd.h" +#include #include #include #include #include +#include #include #include @@ -17,11 +19,14 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { +using namespace std::literals; + VersionCommand::VersionCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), "[hosturl]"); m_Options.add_option("", "d", "detailed", "Detailed Version", cxxopts::value(m_DetailedVersion), "[detailedversion]"); + m_Options.add_option("", "o", "output-path", "Path for output", cxxopts::value(m_OutputPath), "[outputpath]"); m_Options.parse_positional({"hosturl"}); } @@ -51,28 +56,40 @@ VersionCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - const std::string UrlBase = fmt::format("{}/health", m_HostName); - cpr::Session Session; - std::string VersionRequest = fmt::format("{}/version{}", UrlBase, m_DetailedVersion ? "?detailed=true" : ""); - Session.SetUrl(VersionRequest); - cpr::Response Response = Session.Get(); - if (!zen::IsHttpSuccessCode(Response.status_code)) + if (!m_OutputPath.empty()) { - if (Response.status_code) - { - ZEN_ERROR("{} failed: {}: {} ({})", VersionRequest, Response.status_code, Response.reason, Response.text); - } - else - { - ZEN_ERROR("{} failed: {}", VersionRequest, Response.error.message); - } + ZEN_CONSOLE("Querying host {}", m_HostName); + } + HttpClient Client(m_HostName, HttpClientSettings{.Timeout = std::chrono::milliseconds(5000)}); + + HttpClient::KeyValueMap Parameters; + if (m_DetailedVersion) + { + Parameters.Entries.insert_or_assign("detailed", "true"); + } + const std::string_view VersionRequest("/health/version"sv); + HttpClient::Response Response = Client.Get(VersionRequest, {}, Parameters); + if (!Response.IsSuccess()) + { + ZEN_ERROR("{} failed: {}", VersionRequest, Response.ErrorMessage(""sv)); return 1; } - Version = Response.text; + Version = Response.AsText(); } - ZEN_CONSOLE("{}", Version); + if (m_OutputPath.empty()) + { + ZEN_CONSOLE("{}", Version); + } + else + { + ZEN_CONSOLE("Writing version '{}' to '{}'", Version, m_OutputPath); + + BasicFile OutputFile(m_OutputPath, BasicFile::Mode::kTruncate); + OutputFile.Write(Version.data(), Version.length(), 0); + OutputFile.Close(); + } return 0; } diff --git a/src/zen/cmds/version_cmd.h b/src/zen/cmds/version_cmd.h index f8d16fb96..7a910e463 100644 --- a/src/zen/cmds/version_cmd.h +++ b/src/zen/cmds/version_cmd.h @@ -9,6 +9,9 @@ namespace zen { class VersionCommand : public ZenCmdBase { public: + static constexpr char Name[] = "version"; + static constexpr char Description[] = "Get zen service version"; + VersionCommand(); ~VersionCommand(); @@ -16,9 +19,10 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"version", "Get zen service version"}; - std::string m_HostName; - bool m_DetailedVersion = false; + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + bool m_DetailedVersion = false; + std::filesystem::path m_OutputPath; }; } // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 76a866204..598ef9314 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -736,7 +736,7 @@ main(int argc, char** argv) {"top", &TopCmd, "Monitor zen server activity"}, {"trace", &TraceCmd, "Control zen realtime tracing"}, {"up", &UpCmd, "Bring zen server up"}, - {"version", &VersionCmd, "Get zen server version"}, + {VersionCommand::Name, &VersionCmd, VersionCommand::Description}, {"vfs", &VfsCmd, "Manage virtual file system"}, {"flush", &FlushCmd, "Flush storage"}, {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, -- cgit v1.2.3 From 675c4b4987fb18a3f96b340f24d9530aa59942c2 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 18 Jun 2025 12:48:48 +0200 Subject: Surfaced basic z$ information to self-hosted dashboard (#441) - Namespaces are listed on the start page. - Namespaces can be dropped. - New page to show details of a namespace and list its buckets. - Buckets can be dropped. --- src/zenserver/frontend/html.zip | Bin 157061 -> 161002 bytes src/zenserver/frontend/html/pages/start.js | 54 +++++++++++++++++++++ src/zenserver/frontend/html/pages/zcache.js | 70 ++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 src/zenserver/frontend/html/pages/zcache.js (limited to 'src') diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index eadfcf84c..5778fa3d2 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js index 472bb27ae..d1c13ccc7 100644 --- a/src/zenserver/frontend/html/pages/start.js +++ b/src/zenserver/frontend/html/pages/start.js @@ -41,6 +41,41 @@ export class Page extends ZenPage action_tb.left().add("drop").on_click((x) => this.drop_project(x), project.Id); } + // cache + var section = this.add_section("z$"); + columns = [ + "namespace", + "dir", + "buckets", + "entries", + "size disk", + "size mem", + "actions", + ] + var zcache_info = new Fetcher().resource("/z$/").json(); + const cache_table = section.add_widget(Table, columns, Table.Flag_FitLeft|Table.Flag_PackRight); + for (const namespace of (await zcache_info)["Namespaces"]) + { + new Fetcher().resource(`/z$/${namespace}/`).json().then((data) => { + const row = cache_table.add_row( + "", + data["Configuration"]["RootDir"], + data["Buckets"].length, + data["EntryCount"], + Friendly.kib(data["StorageSize"].DiskSize), + Friendly.kib(data["StorageSize"].MemorySize) + ); + var cell = row.get_cell(0); + cell.tag().text(namespace).on_click(() => this.view_zcache(namespace)); + row.get_cell(1).tag().text(namespace); + + cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view").on_click(() => this.view_zcache(namespace)); + action_tb.left().add("drop").on_click(() => this.drop_zcache(namespace)); + }); + } + // stats section = this.add_section("stats"); columns = [ @@ -92,4 +127,23 @@ export class Page extends ZenPage .option("Yes", () => drop()) .option("No"); } + + view_zcache(namespace) + { + window.location = "?page=zcache&namespace=" + namespace; + } + + drop_zcache(namespace) + { + const drop = async () => { + await new Fetcher().resource("z$", namespace).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop zcache '${namespace}'?`) + .option("Yes", () => drop()) + .option("No"); + } } diff --git a/src/zenserver/frontend/html/pages/zcache.js b/src/zenserver/frontend/html/pages/zcache.js new file mode 100644 index 000000000..974893b21 --- /dev/null +++ b/src/zenserver/frontend/html/pages/zcache.js @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Friendly } from "../util/friendly.js" +import { Modal } from "../util/modal.js" +import { Table, PropTable, Toolbar } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + async main() + { + const namespace = this.get_param("namespace"); + + var info = new Fetcher().resource(`/z$/${namespace}/`).json(); + + this.set_title("cache - " + namespace); + + var section = this.add_section("info"); + var cfg_table = section.add_section("config").add_widget(PropTable); + var storage_table = section.add_section("storage").add_widget(PropTable); + + info = await info; + + cfg_table.add_object(info["Configuration"], true); + + storage_table.add_property("disk", Friendly.kib(info["StorageSize"]["DiskSize"])); + storage_table.add_property("mem", Friendly.kib(info["StorageSize"]["MemorySize"])); + storage_table.add_property("entries", Friendly.sep(info["EntryCount"])); + + var column_names = ["name", "disk", "mem", "entries", "actions"]; + var bucket_table = this.add_section("buckets").add_widget( + Table, + column_names, + Table.Flag_BiasLeft + ); + for (const bucket of info["Buckets"]) + { + const row = bucket_table.add_row(bucket); + new Fetcher().resource(`/z$/${namespace}/${bucket}`).json().then((data) => { + row.get_cell(1).text(Friendly.kib(data["StorageSize"]["DiskSize"])); + row.get_cell(2).text(Friendly.kib(data["StorageSize"]["MemorySize"])); + row.get_cell(3).text(Friendly.sep(data["DiskEntryCount"])); + + const cell = row.get_cell(-1); + const action_tb = new Toolbar(cell, true); + action_tb.left().add("view") + action_tb.left().add("drop").on_click(() => this.drop_bucket(bucket)); + }); + } + } + + drop_bucket(bucket) + { + const drop = async () => { + const namespace = this.get_param("namespace"); + await new Fetcher().resource("z$", namespace, bucket).delete(); + this.reload(); + }; + + new Modal() + .title("Confirmation") + .message(`Drop bucket '${bucket}'?`) + .option("Yes", () => drop()) + .option("No"); + } +} -- cgit v1.2.3 From 583e98fc49a4c3ebe24208d082ce5fe71ccef0ef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 09:29:50 +0200 Subject: make sure we unregister from GC before we drop bucket/namespaces (#443) --- src/zenstore/cache/cachedisklayer.cpp | 4 +++- src/zenstore/cache/structuredcachestore.cpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 0d2aef612..15a1c9650 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2004,6 +2004,8 @@ ZenCacheDiskLayer::CacheBucket::Drop() { ZEN_TRACE_CPU("Z$::Bucket::Drop"); + m_Gc.RemoveGcReferencer(*this); + RwLock::ExclusiveLockScope _(m_IndexLock); std::vector> ShardLocks; @@ -3165,7 +3167,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) return nullptr; } - if (Ctx.Settings.IsDeleteMode) + if (Ctx.Settings.IsDeleteMode && !ExpiredEntries.empty()) { for (const DiskIndexEntry& Entry : ExpiredEntries) { diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 5ce254fac..d956384ca 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -297,6 +297,7 @@ ZenCacheNamespace::EnumerateBucketContents(std::string_view std::function ZenCacheNamespace::Drop() { + m_Gc.RemoveGcStorage(this); return m_DiskLayer.Drop(); } -- cgit v1.2.3 From fe4cac6a733706e6b523e9ec3e9d014571723f72 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 19:20:19 +0200 Subject: add retry for failed block metadata upload (#445) * add retry for failed block metadata upload --- src/zen/cmds/builds_cmd.cpp | 76 +++++++++++++++++++---------- src/zenutil/filebuildstorage.cpp | 3 +- src/zenutil/include/zenutil/buildstorage.h | 6 +-- src/zenutil/jupiter/jupiterbuildstorage.cpp | 7 ++- 4 files changed, 60 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index be640fe31..cd27daa9e 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2779,14 +2779,18 @@ namespace { std::vector({BlockMetaData})); } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - BlockHash, - NiceBytes(BlockMetaData.GetSize())); + bool MetadataSucceeded = + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", + BlockHash, + NiceBytes(BlockMetaData.GetSize())); - OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + } - UploadStats.BlocksBytes += BlockMetaData.GetSize(); UploadStats.BlockCount++; if (UploadStats.BlockCount == NewBlockCount) { @@ -2988,13 +2992,16 @@ namespace { std::vector({BlockHash}), std::vector({BlockMetaData})); } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); + bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + } UploadStats.BlockCount++; - UploadStats.BlocksBytes += BlockMetaData.GetSize(); UploadedBlockCount++; if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) @@ -4204,29 +4211,44 @@ namespace { if (!NewBlocks.BlockDescriptions.empty() && !AbortFlag) { - uint64_t UploadBlockMetadataCount = 0; - std::vector BlockHashes; - BlockHashes.reserve(NewBlocks.BlockDescriptions.size()); + uint64_t UploadBlockMetadataCount = 0; Stopwatch UploadBlockMetadataTimer; - for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) + + uint32_t FailedMetadataUploadCount = 1; + int32_t MetadataUploadRetryCount = 3; + while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0)) { - const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; - if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) + FailedMetadataUploadCount = 0; + for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++) { - const CbObject BlockMetaData = - BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (Storage.BuildCacheStorage) + if (AbortFlag) + { + break; + } + const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash; + if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex]) { - Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); + const CbObject BlockMetaData = + BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + bool MetadataSucceeded = Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (MetadataSucceeded) + { + UploadStats.BlocksBytes += BlockMetaData.GetSize(); + NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; + UploadBlockMetadataCount++; + } + else + { + FailedMetadataUploadCount++; + } } - Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - UploadStats.BlocksBytes += BlockMetaData.GetSize(); - NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; - UploadBlockMetadataCount++; } - BlockHashes.push_back(BlockHash); } if (UploadBlockMetadataCount > 0) { diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index c389d16c5..c2cc5ab3c 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -450,7 +450,7 @@ public: return {}; } - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata"); ZEN_UNUSED(BuildId); @@ -467,6 +467,7 @@ public: m_Stats.TotalBytesWritten += MetaData.GetSize(); WriteAsJson(BlockMetaDataPath, MetaData); SimulateLatency(0, 0); + return true; } virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 5422c837c..03496f0f9 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,9 +55,9 @@ public: std::function&& OnReceive, std::function&& OnComplete) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; - virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; + virtual [[nodiscard]] bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index 53cad78da..9974725ff 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -334,7 +334,7 @@ public: return WorkList; } - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { ZEN_TRACE_CPU("Jupiter::PutBlockMetadata"); @@ -346,9 +346,14 @@ public: AddStatistic(PutMetaResult); if (!PutMetaResult.Success) { + if (PutMetaResult.ErrorCode == int32_t(HttpResponseCode::NotFound)) + { + return false; + } throw std::runtime_error( fmt::format("Failed putting build block metadata: {} ({})", PutMetaResult.Reason, PutMetaResult.ErrorCode)); } + return true; } virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override -- cgit v1.2.3 From 9c23f846f51bbf6288a73880619aa11e7ef18274 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 22:18:59 +0200 Subject: move nodiscard to proper location (#447) --- src/zenutil/include/zenutil/buildstorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 03496f0f9..f49d4b42a 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,7 +55,7 @@ public: std::function&& OnReceive, std::function&& OnComplete) = 0; - virtual [[nodiscard]] bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + [[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; -- cgit v1.2.3