From d39724eb644cab4ec5bbf19a703cb770b34e68c4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 08:58:52 +0100 Subject: improved builds api interface in jupiter (#281) * multipart upload/download iterface in jupiter * review fixes --- .../include/zenutil/jupiter/jupitersession.h | 70 +++++++++++++++------- 1 file changed, 47 insertions(+), 23 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 6a80332f4..075c35b40 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -102,29 +102,48 @@ public: std::vector Filter(std::string_view Namespace, std::string_view BucketId, const std::vector& ChunkHashes); - JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); - JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); - JupiterResult FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); - PutBuildPartResult PutBuildPart(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - std::string_view PartName, - const IoBuffer& Payload); - JupiterResult GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); - JupiterResult PutBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - const IoHash& Hash, - ZenContentType ContentType, - const CompositeBuffer& Payload); - JupiterResult GetBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - const IoHash& Hash, - std::filesystem::path TempFolderPath); + JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); + JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); + JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FinalizeBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + PutBuildPartResult PutBuildPart(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + std::string_view PartName, + const IoBuffer& Payload); + JupiterResult GetBuildPart(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); + JupiterResult PutBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + ZenContentType ContentType, + const CompositeBuffer& Payload); + JupiterResult GetBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + std::filesystem::path TempFolderPath); + + JupiterResult PutMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::vector>& OutWorkItems); + JupiterResult GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& Receiver, + std::vector>& OutWorkItems); JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, @@ -137,6 +156,11 @@ public: const Oid& PartId, const IoHash& RawHash); JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); + JupiterResult GetBlockMetadata(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& PartId, + IoBuffer Payload); private: inline LoggerRef Log() { return m_Log; } -- cgit v1.2.3 From da9179d330a37132488f6deb8d8068783b087256 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 09:02:35 +0100 Subject: moving and small refactor of chunk blocks to prepare for builds api (#282) --- src/zenutil/include/zenutil/chunkblock.h | 32 +++++++++++++++++ src/zenutil/include/zenutil/chunkedfile.h | 58 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 src/zenutil/include/zenutil/chunkblock.h create mode 100644 src/zenutil/include/zenutil/chunkedfile.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h new file mode 100644 index 000000000..9b7414629 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +#include +#include + +namespace zen { + +struct ChunkBlockDescription +{ + IoHash BlockHash; + std::vector ChunkHashes; + std::vector ChunkRawLengths; +}; + +std::vector ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); +ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); +CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); + +typedef std::function(const IoHash& RawHash)> FetchChunkFunc; + +CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); +bool IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor); + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h new file mode 100644 index 000000000..7110ad317 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkedfile.h @@ -0,0 +1,58 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace zen { + +class BasicFile; + +struct ChunkedInfo +{ + uint64_t RawSize = 0; + IoHash RawHash; + std::vector ChunkSequence; + std::vector ChunkHashes; +}; + +struct ChunkSource +{ + uint64_t Offset; // 8 + uint32_t Size; // 4 +}; + +struct ChunkedInfoWithSource +{ + ChunkedInfo Info; + std::vector ChunkSources; +}; + +struct ChunkedParams +{ + bool UseThreshold = true; + size_t MinSize = (2u * 1024u) - 128u; + size_t MaxSize = (16u * 1024u); + size_t AvgSize = (3u * 1024u); +}; + +static const ChunkedParams UShaderByteCodeParams = {.UseThreshold = true, .MinSize = 17280, .MaxSize = 139264, .AvgSize = 36340}; + +ChunkedInfoWithSource ChunkData(BasicFile& RawData, + uint64_t Offset, + uint64_t Size, + ChunkedParams Params = {}, + std::atomic* BytesProcessed = nullptr); +void Reconstruct(const ChunkedInfo& Info, + const std::filesystem::path& TargetPath, + std::function GetChunk); +IoBuffer SerializeChunkedInfo(const ChunkedInfo& Info); +ChunkedInfo DeserializeChunkedInfo(IoBuffer& Buffer); + +void chunkedfile_forcelink(); +} // namespace zen -- cgit v1.2.3 From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 25 Feb 2025 15:48:43 +0100 Subject: improvements and infrastructure for upcoming builds api command line (#284) * add modification tick to filesystem traversal * add ShowDetails option to ProgressBar * log callstack if we terminate process * handle chunking if MaxSize > 1MB * BasicFile write helpers and WriteToTempFile simplifications * bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data revert of earlier optimization * faster compress/decompress for large disk-based files * enable progress feedback in IoHash::HashBuffer * add payload validation in HttpClient::Get * fix range requests (range is including end byte) * remove BuildPartId for blob/block related operations in builds api --- src/zenutil/include/zenutil/jupiter/jupitersession.h | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 075c35b40..852271868 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -116,21 +116,18 @@ public: JupiterResult PutBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, const CompositeBuffer& Payload); JupiterResult GetBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, std::filesystem::path TempFolderPath); JupiterResult PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, ZenContentType ContentType, uint64_t PayloadSize, @@ -139,7 +136,6 @@ public: JupiterResult GetMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, uint64_t ChunkSize, std::function&& Receiver, @@ -147,7 +143,6 @@ public: JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, - const Oid& PartId, const IoHash& Hash, const IoBuffer& Payload); FinalizeBuildPartResult FinalizeBuildPart(std::string_view Namespace, @@ -155,12 +150,8 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const Oid& PartId); - JupiterResult GetBlockMetadata(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const Oid& PartId, - IoBuffer Payload); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); private: inline LoggerRef Log() { return m_Log; } -- cgit v1.2.3 From 7d8fe45af3b49d800f84f0ddce051c0b3b2e837d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Feb 2025 15:10:14 +0100 Subject: builds upload command (#278) - Feature: **EXPERIMENTAL** New `zen builds` command to list, upload and download folders to Cloud Build API - `builds list` list available builds (**INCOMPLETE - FILTERING MISSING**) - `builds upload` upload a folder to Cloud Build API - `--local-path` source folder to upload - `--create-build` creates a new parent build object (using the object id), if omitted a parent build must exist and `--build-id` must be given - `--build-id` an Oid in hex form for the Build identifier to use - omit to have the id auto generated - `--build-part-id` and Oid in hex form for the Build Part identifier for the folder - omit to have the id auto generated - `--build-part-name` name of the build part - if omitted the name of the leaf folder name give in `--local-path` - `--metadata-path` path to a json formatted file with meta data information about the build. Meta-data must be provided if `--create-build` is set - `--metadata` key-value pairs separated by ';' with build meta data for the build. (key1=value1;key2=value2). Meta-data must be provided if `--create-build` is set - `--clean` ignore any existing blocks of chunk data and upload a fresh set of blocks - `--allow-multipart` enable usage of multi-part http upload requests - `--manifest-path` path to text file listing files to include in upload. Exclude to upload everything in `--local-path` - `builds download` download a folder from Cloud Build API (**INCOMPLETE - WILL WIPE UNTRACKED DATA FROM TARGET FOLDER**) - `--local-path` target folder to download to - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` a comma separated list of Oid in hex for the build part identifier(s) to download - mutually exclusive to `--build-part-name` - `--build-part-name` a comma separated list of names for the build part(s) to download - if omitted the name of the leaf folder name give in `--local-path` - `--clean` deletes all data in target folder before downloading (NON-CLEAN IS NOT IMPLEMENTED YET) - `--allow-multipart` enable usage of multi-part http download reqeusts - `builds diff` download a folder from Cloud Build API - `--local-path` target folder to download to - `--compare-path` folder to compare target with - `--only-chunked` compare only files that would be chunked - `builds fetch-blob` fetch and validate a blob from remote store - `--build-id` an Oid in hex form for the Build identifier to use - `--blob-hash` an IoHash in hex form identifying the blob to download - `builds validate part` fetch a build part and validate all referenced attachments - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` an Oid in hex for the build part identifier to validate - mutually exclusive to `--build-part-name` - `--build-part-name` a name for the build part to validate - mutually exclusive to `--build-part-id` - `builds test` a series of operation that uploads, downloads and test various aspects of incremental operations - `--local-path` source folder to upload - Options for Cloud Build API remote store (`list`, `upload`, `download`, `fetch-blob`, `validate-part`) - `--url` Cloud Builds URL - `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake - `--namespace` Builds Storage namespace - `--bucket` Builds Storage bucket - Authentication options for Cloud Build API - Auth token - `--access-token` http auth Cloud Storage access token - `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token - `--access-token-path` path to json file that holds the Http auth Cloud Storage access token - OpenId authentication - `--openid-provider-name` Open ID provider name - `--openid-provider-url` Open ID provider url - `--openid-client-id`Open ID client id - `--openid-refresh-token` Open ID refresh token - `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials - `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials - OAuth authentication - `--oauth-url` OAuth provier url - `--oauth-clientid` OAuth client id - `--oauth-clientsecret` OAuth client secret - Options for file based remote store used for for testing purposes (`list`, `upload`, `download`, `fetch-blob`, `validate-part`, `test`) - `--storage-path` path to folder to store builds data - `--json-metadata` enable json output in store for all compact binary objects (off by default) - Output options for all builds commands - `--plain-progress` use plain line-by-line progress output - `--verbose` --- src/zenutil/include/zenutil/buildstorage.h | 55 +++++ src/zenutil/include/zenutil/chunkblock.h | 17 +- src/zenutil/include/zenutil/chunkedcontent.h | 256 +++++++++++++++++++++ src/zenutil/include/zenutil/chunkingcontroller.h | 55 +++++ src/zenutil/include/zenutil/filebuildstorage.h | 16 ++ .../include/zenutil/jupiter/jupiterbuildstorage.h | 17 ++ src/zenutil/include/zenutil/parallellwork.h | 69 ++++++ 7 files changed, 480 insertions(+), 5 deletions(-) create mode 100644 src/zenutil/include/zenutil/buildstorage.h create mode 100644 src/zenutil/include/zenutil/chunkedcontent.h create mode 100644 src/zenutil/include/zenutil/chunkingcontroller.h create mode 100644 src/zenutil/include/zenutil/filebuildstorage.h create mode 100644 src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h create mode 100644 src/zenutil/include/zenutil/parallellwork.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h new file mode 100644 index 000000000..9c236310f --- /dev/null +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class BuildStorage +{ +public: + struct Statistics + { + std::atomic TotalBytesRead = 0; + std::atomic TotalBytesWritten = 0; + std::atomic TotalRequestCount = 0; + std::atomic TotalRequestTimeUs = 0; + std::atomic TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorage() {} + + virtual CbObject ListBuilds(CbObject Query) = 0; + virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; + virtual CbObject GetBuild(const Oid& BuildId) = 0; + virtual void FinalizeBuild(const Oid& BuildId) = 0; + + virtual std::pair> PutBuildPart(const Oid& BuildId, + const Oid& BuildPartId, + std::string_view PartName, + const CbObject& MetaData) = 0; + virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) = 0; + virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) = 0; + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual std::vector> PutLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + uint64_t PayloadSize, + std::function&& Transmitter, + std::function&& OnSentBytes) = 0; + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; + virtual std::vector> GetLargeBuildBlob( + const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& Receiver) = 0; + + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual std::vector FindBlocks(const Oid& BuildId) = 0; + virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h index 9b7414629..21107fb7c 100644 --- a/src/zenutil/include/zenutil/chunkblock.h +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -12,21 +12,28 @@ namespace zen { -struct ChunkBlockDescription +struct ThinChunkBlockDescription { - IoHash BlockHash; - std::vector ChunkHashes; + IoHash BlockHash; + std::vector ChunkRawHashes; +}; + +struct ChunkBlockDescription : public ThinChunkBlockDescription +{ + uint64_t HeaderSize; std::vector ChunkRawLengths; + std::vector ChunkCompressedLengths; }; std::vector ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); - +ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function(const IoHash& RawHash)> FetchChunkFunc; CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); bool IterateChunkBlock(const SharedBuffer& BlockPayload, - std::function Visitor); + std::function Visitor, + uint64_t& OutHeaderSize); } // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h new file mode 100644 index 000000000..15c687462 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -0,0 +1,256 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CbWriter; +class ChunkingController; +class WorkerThreadPool; + +enum class SourcePlatform +{ + Windows = 0, + Linux = 1, + MacOS = 2, + _Count +}; + +std::string_view ToString(SourcePlatform Platform); +SourcePlatform FromString(std::string_view Platform, SourcePlatform Default); +SourcePlatform GetSourceCurrentPlatform(); + +struct FolderContent +{ + SourcePlatform Platform = GetSourceCurrentPlatform(); + std::vector Paths; + std::vector RawSizes; + std::vector Attributes; + std::vector ModificationTicks; + + bool operator==(const FolderContent& Rhs) const; + + bool AreKnownFilesEqual(const FolderContent& Rhs) const; + void UpdateState(const FolderContent& Rhs, std::vector& PathIndexesOufOfDate); + static bool AreFileAttributesEqual(const uint32_t Lhs, const uint32_t Rhs); +}; + +FolderContent GetUpdatedContent(const FolderContent& Old, + const FolderContent& New, + std::vector& OutDeletedPathIndexes); + +void SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output); +FolderContent LoadFolderContentToCompactBinary(CbObjectView Input); + +struct GetFolderContentStatistics +{ + std::atomic FoundFileCount = 0; + std::atomic FoundFileByteCount = 0; + std::atomic AcceptedFileCount = 0; + std::atomic AcceptedFileByteCount = 0; + uint64_t ElapsedWallTimeUS = 0; +}; + +FolderContent GetFolderContent(GetFolderContentStatistics& Stats, + const std::filesystem::path& RootPath, + std::function&& AcceptDirectory, + std::function&& AcceptFile, + WorkerThreadPool& WorkerPool, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag); + +struct ChunkedContentData +{ + // To describe one asset with a particular RawHash, find the index of the hash in SequenceRawHashes + // ChunkCounts for that index will be the number of indexes in ChunkOrders that describe + // the sequence of chunks required to reconstruct the asset. + // Offset into ChunkOrders is based on how many entries in ChunkOrders the previous [n - 1] SequenceRawHashes uses + std::vector SequenceRawHashes; // Raw hash for Chunk sequence + std::vector ChunkCounts; // Chunk count of ChunkOrder for SequenceRawHashes[n] + std::vector ChunkOrders; // Chunk sequence indexed into ChunkHashes, ChunkCounts[n] indexes per SequenceRawHashes[n] + std::vector ChunkHashes; // Unique chunk hashes + std::vector ChunkRawSizes; // Unique chunk raw size for ChunkHash[n] +}; + +struct ChunkedFolderContent +{ + SourcePlatform Platform = GetSourceCurrentPlatform(); + std::vector Paths; + std::vector RawSizes; + std::vector Attributes; + std::vector RawHashes; + ChunkedContentData ChunkedContent; +}; + +void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output); +ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input); + +ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays); +ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span DeletedPaths); + +struct ChunkingStatistics +{ + std::atomic FilesProcessed = 0; + std::atomic FilesChunked = 0; + std::atomic BytesHashed = 0; + std::atomic UniqueChunksFound = 0; + std::atomic UniqueSequencesFound = 0; + std::atomic UniqueBytesFound = 0; + uint64_t ElapsedWallTimeUS = 0; +}; + +ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateInteralMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag); + +struct ChunkedContentLookup +{ + struct ChunkLocation + { + uint32_t PathIndex; + uint64_t Offset; + }; + tsl::robin_map ChunkHashToChunkIndex; + tsl::robin_map RawHashToSequenceRawHashIndex; + std::vector SequenceRawHashIndexChunkOrderOffset; + std::vector ChunkLocations; + std::vector ChunkLocationOffset; // ChunkLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkLocationCounts; // ChunkLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex +}; + +ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); + +inline std::pair +GetChunkLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +{ + return std::make_pair(Lookup.ChunkLocationOffset[ChunkIndex], Lookup.ChunkLocationCounts[ChunkIndex]); +} + +inline std::span +GetChunkLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +{ + std::pair Range = GetChunkLocationRange(Lookup, ChunkIndex); + return std::span(Lookup.ChunkLocations).subspan(Range.first, Range.second); +} + +namespace compactbinary_helpers { + template + void WriteArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const Type Value : Values) + { + Output << Value; + } + Output.EndArray(); + } + + template + void WriteArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + template<> + inline void WriteArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const std::filesystem::path& Path : Values) + { + Output.AddString((const char*)Path.generic_u8string().c_str()); + } + Output.EndArray(); + } + + template<> + inline void WriteArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + inline void WriteBinaryAttachmentArray(std::span Values, std::string_view ArrayName, CbWriter& Output) + { + Output.BeginArray(ArrayName); + for (const IoHash& Hash : Values) + { + Output.AddBinaryAttachment(Hash); + } + Output.EndArray(); + } + + inline void WriteBinaryAttachmentArray(const std::vector& Values, std::string_view ArrayName, CbWriter& Output) + { + WriteArray(std::span(Values), ArrayName, Output); + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsUInt32()); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsUInt64()); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + std::u8string_view U8Path = ItemView.AsU8String(); + Result.push_back(std::filesystem::path(U8Path)); + } + } + + inline void ReadArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsHash()); + } + } + + inline void ReadBinaryAttachmentArray(std::string_view ArrayName, CbObjectView Input, std::vector& Result) + { + CbArrayView Array = Input[ArrayName].AsArrayView(); + Result.reserve(Array.Num()); + for (CbFieldView ItemView : Array) + { + Result.push_back(ItemView.AsBinaryAttachment()); + } + } + +} // namespace compactbinary_helpers + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h new file mode 100644 index 000000000..fe4fc1bb5 --- /dev/null +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include + +namespace zen { + +const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; + +const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, + .MaxSize = 128u * 1024u, + .AvgSize = ((8u * 4u) * 1024u) + 128u}; + +const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize; + +const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u; + +struct ChunkedInfoWithSource; + +class ChunkingController +{ +public: + virtual ~ChunkingController() {} + + // Return true if the input file was processed. If true is returned OutChunked will contain the chunked info + virtual bool ProcessFile(const std::filesystem::path& InputPath, + uint64_t RawSize, + ChunkedInfoWithSource& OutChunked, + std::atomic& BytesProcessed) const = 0; + virtual std::string_view GetName() const = 0; + virtual CbObject GetParameters() const = 0; +}; + +std::unique_ptr CreateBasicChunkingController( + std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, + const ChunkedParams& ChunkingParams = DefaultChunkedParams); +std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); + +std::unique_ptr CreateChunkingControllerWithFixedChunking( + std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, + const ChunkedParams& ChunkingParams = DefaultChunkedParams, + uint32_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize); +std::unique_ptr CreateChunkingControllerWithFixedChunking(CbObjectView Parameters); + +std::unique_ptr CreateChunkingController(std::string_view Name, CbObjectView Parameters); + +} // namespace zen diff --git a/src/zenutil/include/zenutil/filebuildstorage.h b/src/zenutil/include/zenutil/filebuildstorage.h new file mode 100644 index 000000000..c95fb32e6 --- /dev/null +++ b/src/zenutil/include/zenutil/filebuildstorage.h @@ -0,0 +1,16 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { +class HttpClient; + +std::unique_ptr CreateFileBuildStorage(const std::filesystem::path& StoragePath, + BuildStorage::Statistics& Stats, + bool EnableJsonOutput, + double LatencySec = 0.0, + double DelayPerKBSec = 0.0); +} // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h new file mode 100644 index 000000000..89fc70140 --- /dev/null +++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { +class HttpClient; + +std::unique_ptr CreateJupiterBuildStorage(LoggerRef InLog, + HttpClient& InHttpClient, + BuildStorage::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath); +} // namespace zen diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h new file mode 100644 index 000000000..7a8218c51 --- /dev/null +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -0,0 +1,69 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen { + +class ParallellWork +{ +public: + ParallellWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {} + + ~ParallellWork() + { + // Make sure to call Wait before destroying + ZEN_ASSERT(m_PendingWork.Remaining() == 0); + } + + void ScheduleWork(WorkerThreadPool& WorkerPool, + std::function& AbortFlag)>&& Work, + std::function& AbortFlag)>&& OnError) + { + m_PendingWork.AddCount(1); + try + { + WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] { + try + { + Work(m_AbortFlag); + } + catch (const std::exception& Ex) + { + OnError(Ex, m_AbortFlag); + } + m_PendingWork.CountDown(); + }); + } + catch (const std::exception&) + { + m_PendingWork.CountDown(); + throw; + } + } + + void Abort() { m_AbortFlag = true; } + + bool IsAborted() const { return m_AbortFlag.load(); } + + void Wait(int32_t UpdateInteralMS, std::function&& UpdateCallback) + { + ZEN_ASSERT(m_PendingWork.Remaining() > 0); + m_PendingWork.CountDown(); + while (!m_PendingWork.Wait(UpdateInteralMS)) + { + UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); + } + } + Latch& PendingWork() { return m_PendingWork; } + +private: + std::atomic& m_AbortFlag; + Latch m_PendingWork; +}; + +} // namespace zen -- cgit v1.2.3 From 38a58059214bacc18c8a6406acf7f46c57f51e86 Mon Sep 17 00:00:00 2001 From: Zousar Shaker Date: Thu, 27 Feb 2025 01:27:34 -0700 Subject: Zs/auth bad function fix (#287) * Describe fix in changelog * remove JupiterClient::m_TokenProvider --------- Co-authored-by: zousar <2936246+zousar@users.noreply.github.com> Co-authored-by: Dan Engelbrecht --- src/zenutil/include/zenutil/jupiter/jupiterclient.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/jupiter/jupiterclient.h b/src/zenutil/include/zenutil/jupiter/jupiterclient.h index defe50edc..8a51bd60a 100644 --- a/src/zenutil/include/zenutil/jupiter/jupiterclient.h +++ b/src/zenutil/include/zenutil/jupiter/jupiterclient.h @@ -44,12 +44,11 @@ public: HttpClient& Client() { return m_HttpClient; } private: - LoggerRef m_Log; - const std::string m_DefaultDdcNamespace; - const std::string m_DefaultBlobStoreNamespace; - const std::string m_ComputeCluster; - std::function m_TokenProvider; - HttpClient m_HttpClient; + LoggerRef m_Log; + const std::string m_DefaultDdcNamespace; + const std::string m_DefaultBlobStoreNamespace; + const std::string m_ComputeCluster; + HttpClient m_HttpClient; friend class JupiterSession; }; -- cgit v1.2.3 From 5791f51cccea1d4e5365456c8da89dbac0dd3ec0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 28 Feb 2025 12:39:48 +0100 Subject: improve error handling (#289) * clearer errors * quicker abort * handle deleted local files * simplify parallellwork error handling * don't finish progress on destructor - gives wrong impression * graceful ctrl-c handling --- src/zenutil/include/zenutil/chunkedfile.h | 3 +- src/zenutil/include/zenutil/chunkingcontroller.h | 7 ++-- src/zenutil/include/zenutil/parallellwork.h | 48 ++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkedfile.h b/src/zenutil/include/zenutil/chunkedfile.h index 7110ad317..4cec80fdb 100644 --- a/src/zenutil/include/zenutil/chunkedfile.h +++ b/src/zenutil/include/zenutil/chunkedfile.h @@ -47,7 +47,8 @@ ChunkedInfoWithSource ChunkData(BasicFile& RawData, uint64_t Offset, uint64_t Size, ChunkedParams Params = {}, - std::atomic* BytesProcessed = nullptr); + std::atomic* BytesProcessed = nullptr, + std::atomic* AbortFlag = nullptr); void Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function GetChunk); diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index fe4fc1bb5..ebc80e207 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -32,9 +32,10 @@ public: virtual bool ProcessFile(const std::filesystem::path& InputPath, uint64_t RawSize, ChunkedInfoWithSource& OutChunked, - std::atomic& BytesProcessed) const = 0; - virtual std::string_view GetName() const = 0; - virtual CbObject GetParameters() const = 0; + std::atomic& BytesProcessed, + std::atomic& AbortFlag) const = 0; + virtual std::string_view GetName() const = 0; + virtual CbObject GetParameters() const = 0; }; std::unique_ptr CreateBasicChunkingController( diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h index 7a8218c51..79798fc8d 100644 --- a/src/zenutil/include/zenutil/parallellwork.h +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -2,6 +2,8 @@ #pragma once +#include +#include #include #include @@ -20,6 +22,14 @@ public: ZEN_ASSERT(m_PendingWork.Remaining() == 0); } + std::function& AbortFlag)> DefaultErrorFunction() + { + return [&](const std::exception& Ex, std::atomic& AbortFlag) { + m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); }); + AbortFlag = true; + }; + } + void ScheduleWork(WorkerThreadPool& WorkerPool, std::function& AbortFlag)>&& Work, std::function& AbortFlag)>&& OnError) @@ -32,6 +42,27 @@ public: { Work(m_AbortFlag); } + catch (const AssertException& AssertEx) + { + OnError( + std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())), + m_AbortFlag); + } + catch (const std::system_error& SystemError) + { + if (IsOOM(SystemError.code())) + { + OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag); + } + else if (IsOOD(SystemError.code())) + { + OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag); + } + else + { + OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag); + } + } catch (const std::exception& Ex) { OnError(Ex, m_AbortFlag); @@ -58,12 +89,29 @@ public: { UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); } + if (m_Errors.size() == 1) + { + throw std::runtime_error(m_Errors.front()); + } + else if (m_Errors.size() > 1) + { + ExtendableStringBuilder<128> SB; + SB.Append("Multiple errors:"); + for (const std::string& Error : m_Errors) + { + SB.Append(fmt::format("\n {}", Error)); + } + throw std::runtime_error(SB.ToString()); + } } Latch& PendingWork() { return m_PendingWork; } private: std::atomic& m_AbortFlag; Latch m_PendingWork; + + RwLock m_ErrorLock; + std::vector m_Errors; }; } // namespace zen -- cgit v1.2.3 From 1270bfeffbc81b1e4940c5c454ee6acde43e696a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 3 Mar 2025 17:53:11 +0100 Subject: refactor use chunk sequence download (#291) * work on chunk sequences on download, not paths * write chunksequences to .tmp file and move when complete * cleanup * Added on the fly validation `zen builds download` of files built from smaller chunks as each file is completed Added `--verify` option to `zen builds upload` to verify all uploaded data once entire upload is complete Added `--verify` option to `zen builds download` to verify all files in target folder once entire download is complete Fixed/improved progress updated Multithreaded part validation * added rates to Write Chunks task * b/s -> bits/s * dont validate partial content as complete payload * handle legacy c# builds --- src/zenutil/include/zenutil/chunkedcontent.h | 53 +++++++++++++++++++++------- 1 file changed, 40 insertions(+), 13 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 15c687462..309341550 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -122,32 +122,59 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, struct ChunkedContentLookup { - struct ChunkLocation + struct ChunkSequenceLocation { - uint32_t PathIndex; + uint32_t SequenceIndex; uint64_t Offset; }; tsl::robin_map ChunkHashToChunkIndex; - tsl::robin_map RawHashToSequenceRawHashIndex; - std::vector SequenceRawHashIndexChunkOrderOffset; - std::vector ChunkLocations; - std::vector ChunkLocationOffset; // ChunkLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex - std::vector ChunkLocationCounts; // ChunkLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + tsl::robin_map RawHashToSequenceIndex; + std::vector SequenceIndexChunkOrderOffset; + std::vector ChunkSequenceLocations; + std::vector + ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash }; ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); inline std::pair -GetChunkLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +GetChunkSequenceLocationRange(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) { - return std::make_pair(Lookup.ChunkLocationOffset[ChunkIndex], Lookup.ChunkLocationCounts[ChunkIndex]); + return std::make_pair(Lookup.ChunkSequenceLocationOffset[ChunkIndex], Lookup.ChunkSequenceLocationCounts[ChunkIndex]); } -inline std::span -GetChunkLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) +inline std::span +GetChunkSequenceLocations(const ChunkedContentLookup& Lookup, uint32_t ChunkIndex) { - std::pair Range = GetChunkLocationRange(Lookup, ChunkIndex); - return std::span(Lookup.ChunkLocations).subspan(Range.first, Range.second); + std::pair Range = GetChunkSequenceLocationRange(Lookup, ChunkIndex); + return std::span(Lookup.ChunkSequenceLocations).subspan(Range.first, Range.second); +} + +inline uint32_t +GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + return Lookup.RawHashToSequenceIndex.at(RawHash); +} + +inline uint32_t +GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + return Lookup.RawHashToSequenceIndex.at(RawHash); +} + +inline uint32_t +GetFirstPathIndexForSeqeuenceIndex(const ChunkedContentLookup& Lookup, const uint32_t SequenceIndex) +{ + return Lookup.SequenceIndexFirstPathIndex[SequenceIndex]; +} + +inline uint32_t +GetFirstPathIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) +{ + const uint32_t SequenceIndex = GetSequenceIndexForRawHash(Lookup, RawHash); + return GetFirstPathIndexForSeqeuenceIndex(Lookup, SequenceIndex); } namespace compactbinary_helpers { -- cgit v1.2.3 From 920120bbcec9f91df3336f62970b3e010a4fa6c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 6 Mar 2025 16:18:32 +0100 Subject: reduced memory churn using fixed_xxx containers (#236) * Added EASTL to help with eliminating memory allocations * Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F) --- src/zenutil/include/zenutil/cache/cachekey.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/cache/cachekey.h b/src/zenutil/include/zenutil/cache/cachekey.h index 741375946..0ab05f4f1 100644 --- a/src/zenutil/include/zenutil/cache/cachekey.h +++ b/src/zenutil/include/zenutil/cache/cachekey.h @@ -17,6 +17,12 @@ struct CacheKey static CacheKey Create(std::string_view Bucket, const IoHash& Hash) { return {.Bucket = ToLower(Bucket), .Hash = Hash}; } + // This should be used whenever the bucket name has already been validated to avoid redundant ToLower calls + static CacheKey CreateValidated(std::string&& BucketValidated, const IoHash& Hash) + { + return {.Bucket = std::move(BucketValidated), .Hash = Hash}; + } + auto operator<=>(const CacheKey& that) const { if (auto b = caseSensitiveCompareStrings(Bucket, that.Bucket); b != std::strong_ordering::equal) -- cgit v1.2.3 From 9b24647facccc9c7848a52f1f4c5e32055bf2f01 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 7 Mar 2025 09:58:20 +0100 Subject: partial block fetch (#298) - Improvement: Do partial requests of blocks if not all of the block is needed - Improvement: Better progress/statistics on download - Bugfix: Ensure that temporary folder for Jupiter downloads exists during verify phase --- src/zenutil/include/zenutil/buildstorage.h | 5 ++++- src/zenutil/include/zenutil/jupiter/jupitersession.h | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 9c236310f..9d2bab170 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -40,7 +40,10 @@ public: std::function&& Transmitter, std::function&& OnSentBytes) = 0; - virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; virtual std::vector> GetLargeBuildBlob( const Oid& BuildId, const IoHash& RawHash, diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 852271868..2c5fc73b8 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -123,7 +123,9 @@ public: std::string_view BucketId, const Oid& BuildId, const IoHash& Hash, - std::filesystem::path TempFolderPath); + std::filesystem::path TempFolderPath, + uint64_t Offset = 0, + uint64_t Size = (uint64_t)-1); JupiterResult PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, -- cgit v1.2.3 From 7de3d4218ee5969af6147f9ab20bda538a136d9a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 10 Mar 2025 18:33:24 +0100 Subject: pick up existing cache (#299) - Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run - Improvement: Faster abort during stream compression - Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write - Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails --- src/zenutil/include/zenutil/chunkblock.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkblock.h b/src/zenutil/include/zenutil/chunkblock.h index 21107fb7c..277580c74 100644 --- a/src/zenutil/include/zenutil/chunkblock.h +++ b/src/zenutil/include/zenutil/chunkblock.h @@ -31,9 +31,10 @@ CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function(const IoHash& RawHash)> FetchChunkFunc; -CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); -bool IterateChunkBlock(const SharedBuffer& BlockPayload, - std::function Visitor, - uint64_t& OutHeaderSize); +CompressedBuffer GenerateChunkBlock(std::vector>&& FetchChunks, ChunkBlockDescription& OutBlock); +bool IterateChunkBlock(const SharedBuffer& BlockPayload, + std::function Visitor, + uint64_t& OutHeaderSize); +std::vector ReadChunkBlockHeader(const MemoryView BlockView, uint64_t& OutHeaderSize); } // namespace zen -- cgit v1.2.3 From 90db3ced033d4e06da2739e5d97cdeff2b0ba3b9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 11 Mar 2025 09:58:07 +0100 Subject: Build command tweaks (#301) - Improvement: Don't chunk up .mp4 files as they generally won't benefit from deduplication or partial in-place-updates - Improvement: Emit build name to console output when downloading a build - Improvement: Added some debug logging - Bugfix: Logging setup would previously not function correctly when not logging to file --- src/zenutil/include/zenutil/chunkingcontroller.h | 2 +- src/zenutil/include/zenutil/logging.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index ebc80e207..246f4498a 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -11,7 +11,7 @@ namespace zen { -const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self"}; +const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h index ebf6372fc..d64eef207 100644 --- a/src/zenutil/include/zenutil/logging.h +++ b/src/zenutil/include/zenutil/logging.h @@ -32,6 +32,7 @@ struct LoggingOptions bool IsDebug = false; bool IsVerbose = false; bool IsTest = false; + bool AllowAsync = true; bool NoConsoleOutput = false; std::filesystem::path AbsLogFile; // Absolute path to main log file std::string LogId; -- cgit v1.2.3 From fb09d861fd76e459ac86bec388bd406aaca8e681 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 10:51:57 +0100 Subject: improved block gen logic (#302) - Improvement: Reduced memory usage during upload and part upload validation - Improvement: Reduced I/O usage during upload and download - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild - Improvement: More trace scopes for build upload operations - Bugfix: Fixed crash during download when trying to write outside a file range --- src/zenutil/include/zenutil/chunkedcontent.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 309341550..57b55cb8e 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -124,8 +124,8 @@ struct ChunkedContentLookup { struct ChunkSequenceLocation { - uint32_t SequenceIndex; - uint64_t Offset; + uint32_t SequenceIndex = (uint32_t)-1; + uint64_t Offset = (uint64_t)-1; }; tsl::robin_map ChunkHashToChunkIndex; tsl::robin_map RawHashToSequenceIndex; -- cgit v1.2.3 From 7046fc9dc202307ba92d05a6386bfb52e9db0ab9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 12 Mar 2025 13:54:16 +0100 Subject: fixes for log timestamps (#304) * add GetTimeSinceProcessStart returning time since process start. implemented using https://github.com/maxliani/GetTimeSinceProcessStart/tree/main * fix fractions when using epoch mode. Previously it would show the fraction from the absolute time stamp and not relative to epoch * used GetTimeSinceProcessStart to offset the epoch so that it represents the process spawn time --- src/zenutil/include/zenutil/logging/fullformatter.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h index 07ad408fa..0326870e5 100644 --- a/src/zenutil/include/zenutil/logging/fullformatter.h +++ b/src/zenutil/include/zenutil/logging/fullformatter.h @@ -45,6 +45,8 @@ public: std::chrono::seconds TimestampSeconds; + std::chrono::milliseconds millis; + if (m_UseFullDate) { TimestampSeconds = std::chrono::duration_cast(msg.time.time_since_epoch()); @@ -69,6 +71,8 @@ public: spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime); m_CachedDatetime.push_back('.'); } + + millis = spdlog::details::fmt_helper::time_fraction(msg.time); } else { @@ -97,6 +101,8 @@ public: spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime); m_CachedDatetime.push_back('.'); } + + millis = std::chrono::duration_cast(ElapsedTime - TimestampSeconds); } { @@ -104,7 +110,6 @@ public: OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); } - auto millis = spdlog::details::fmt_helper::time_fraction(msg.time); spdlog::details::fmt_helper::pad3(static_cast(millis.count()), OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); -- cgit v1.2.3 From 4dc972bdaf752676426550b2eed500a9b7dcc4f5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 10:22:52 +0100 Subject: improved reporting on async error (#312) * report async errors as individual errors --- src/zenutil/include/zenutil/parallellwork.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h index 79798fc8d..8ea77c65d 100644 --- a/src/zenutil/include/zenutil/parallellwork.h +++ b/src/zenutil/include/zenutil/parallellwork.h @@ -9,6 +9,14 @@ #include +class ParallellWorkException : public std::runtime_error +{ +public: + explicit ParallellWorkException(std::vector&& Errors) : std::runtime_error(Errors.front()), m_Errors(std::move(Errors)) {} + + const std::vector m_Errors; +}; + namespace zen { class ParallellWork @@ -95,13 +103,7 @@ public: } else if (m_Errors.size() > 1) { - ExtendableStringBuilder<128> SB; - SB.Append("Multiple errors:"); - for (const std::string& Error : m_Errors) - { - SB.Append(fmt::format("\n {}", Error)); - } - throw std::runtime_error(SB.ToString()); + throw ParallellWorkException(std::move(m_Errors)); } } Latch& PendingWork() { return m_PendingWork; } -- cgit v1.2.3 From b625980599880e1e7d1149a51b9562e79c0f2994 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 19 Mar 2025 16:12:38 +0100 Subject: jupiter builds stats upload (#315) - Improvement: At end of build upload we post statistics to the Jupiter build stats endpoint: - `totalSize` - `reusedRatio` - `reusedBlockCount` - `reusedBlockByteCount` - `newBlockCount` - `newBlockByteCount` - `uploadedCount` - `uploadedByteCount` - `elapsedTimeSec` - `uploadedBytesPerSec` --- src/zenutil/include/zenutil/buildstorage.h | 6 ++++++ src/zenutil/include/zenutil/jupiter/jupitersession.h | 6 ++++++ 2 files changed, 12 insertions(+) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 9d2bab170..2ebd65a00 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -5,6 +5,10 @@ #include #include +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + namespace zen { class BuildStorage @@ -53,6 +57,8 @@ public: virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual std::vector FindBlocks(const Oid& BuildId) = 0; virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; + + virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 2c5fc73b8..fda4a7bfe 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -155,6 +155,12 @@ public: JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); + JupiterResult PutBuildPartStats(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const Oid& BuildPartId, + IoBuffer Payload); + private: inline LoggerRef Log() { return m_Log; } -- cgit v1.2.3 From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Mar 2025 17:06:23 +0100 Subject: zen build cache service (#318) - **EXPERIMENTAL** `zen builds` - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata - Feature: New `/builds` endpoint for caching build blobs and blob metadata - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs --- src/zenutil/include/zenutil/buildstorage.h | 6 +-- src/zenutil/include/zenutil/buildstoragecache.h | 52 ++++++++++++++++++++++ .../include/zenutil/logging/rotatingfilesink.h | 1 - 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 src/zenutil/include/zenutil/buildstoragecache.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 2ebd65a00..f8c7c012c 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -54,9 +54,9 @@ public: uint64_t ChunkSize, std::function&& Receiver) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual std::vector FindBlocks(const Oid& BuildId) = 0; - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h new file mode 100644 index 000000000..08c936bf5 --- /dev/null +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include + +namespace zen { + +class HttpClient; + +class BuildStorageCache +{ +public: + struct Statistics + { + std::atomic TotalBytesRead = 0; + std::atomic TotalBytesWritten = 0; + std::atomic TotalRequestCount = 0; + std::atomic TotalRequestTimeUs = 0; + std::atomic TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorageCache() {} + + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) = 0; + virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) = 0; + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; +}; + +std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath); +} // namespace zen diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 758722156..cd28bdcb2 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -27,7 +27,6 @@ public: { ZEN_MEMSCOPE(ELLMTag::Logging); - ZEN_MEMSCOPE(ELLMTag::Logging); std::error_code Ec; if (RotateOnOpen) { -- cgit v1.2.3 From a0a0dba13317533f882a85b7f4087588cfa09066 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 14:09:01 +0100 Subject: optional compress of block chunks (#326) - Feature: zenserver: Add command line option `--gc-buildstore-duration-seconds` to control GC life time of build store data - Improvement: ELF and MachO executable files are no longer chunked - Improvement: Compress chunks in blocks that encloses a full file (such as small executables) - Bugfix: Strip path delimiter at end of string in StringToPath --- src/zenutil/include/zenutil/chunkingcontroller.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 246f4498a..970917fb0 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -12,6 +12,8 @@ namespace zen { const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -40,6 +42,8 @@ public: std::unique_ptr CreateBasicChunkingController( std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, const ChunkedParams& ChunkingParams = DefaultChunkedParams); std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); -- cgit v1.2.3 From 013ac818cd09c1d31bf9411e00b2bbbf02defa3f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 16:08:47 +0100 Subject: build cache prime (#327) - Feature: zen `--boost-workers` option to builds `upload`, `download` and `validate-part` that will increase the number of worker threads, may cause computer to be less responsive - Feature: zen `--cache-prime-only` that uploads referenced data from a part to `--zen-cache-host` if it is not already present. Target folder will be untouched. --- src/zenutil/include/zenutil/buildstoragecache.h | 7 ++++++- src/zenutil/include/zenutil/workerpools.h | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index 08c936bf5..cab35328d 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -42,11 +42,16 @@ public: }; virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; + + virtual void Flush( + int32_t UpdateInteralMS, + std::function&& UpdateCallback = [](intptr_t) { return true; }) = 0; }; std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath); + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount); } // namespace zen diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h index 9683ad720..df2033bca 100644 --- a/src/zenutil/include/zenutil/workerpools.h +++ b/src/zenutil/include/zenutil/workerpools.h @@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType); // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType); +// Worker pool with minimum number of worker threads, but at least one thread +WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType); + // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread // This is useful for debugging when multiple async thread can make stepping in debugger complicated WorkerThreadPool& GetSyncWorkerPool(); -- cgit v1.2.3 From e244e2a07a0f3bf3b11d6d875d6c2c8bb169efc5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 15:46:05 +0200 Subject: added --find-max-block-count option to builds upload (#337) --- src/zenutil/include/zenutil/buildstorage.h | 2 +- src/zenutil/include/zenutil/jupiter/jupitersession.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index f8c7c012c..b0665dbf8 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,7 +55,7 @@ public: std::function&& Receiver) = 0; virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index fda4a7bfe..417ed7384 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -152,7 +152,7 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); JupiterResult PutBuildPartStats(std::string_view Namespace, -- cgit v1.2.3 From dec3f27c488a1dda8a2f1133361e2fda9315e0d2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Apr 2025 19:22:21 +0200 Subject: fix race condition in multipart download (#358) --- src/zenutil/include/zenutil/buildstorage.h | 12 ++++++------ src/zenutil/include/zenutil/jupiter/jupitersession.h | 15 ++++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index b0665dbf8..05e3ca22d 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -47,12 +47,12 @@ public: virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset = 0, - uint64_t RangeBytes = (uint64_t)-1) = 0; - virtual std::vector> GetLargeBuildBlob( - const Oid& BuildId, - const IoHash& RawHash, - uint64_t ChunkSize, - std::function&& Receiver) = 0; + uint64_t RangeBytes = (uint64_t)-1) = 0; + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete) = 0; virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 417ed7384..c2886ca4c 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -135,13 +135,14 @@ public: uint64_t PayloadSize, std::function&& Transmitter, std::vector>& OutWorkItems); - JupiterResult GetMultipartBuildBlob(std::string_view Namespace, - std::string_view BucketId, - const Oid& BuildId, - const IoHash& Hash, - uint64_t ChunkSize, - std::function&& Receiver, - std::vector>& OutWorkItems); + JupiterResult GetMultipartBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete, + std::vector>& OutWorkItems); JupiterResult PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, -- cgit v1.2.3 From 9cdfac225f0dc986ce449cbe8ef72faa39025971 Mon Sep 17 00:00:00 2001 From: zousar Date: Mon, 14 Apr 2025 21:51:39 -0600 Subject: Add a list-container subcommand to zen builds command --- src/zenutil/include/zenutil/buildstorage.h | 1 + src/zenutil/include/zenutil/jupiter/jupitersession.h | 2 ++ 2 files changed, 3 insertions(+) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 05e3ca22d..0785acd62 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -25,6 +25,7 @@ public: virtual ~BuildStorage() {} + virtual CbObject ListContainers() = 0; virtual CbObject ListBuilds(CbObject Query) = 0; virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; virtual CbObject GetBuild(const Oid& BuildId) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index c2886ca4c..32bfd50f4 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -102,6 +102,8 @@ public: std::vector Filter(std::string_view Namespace, std::string_view BucketId, const std::vector& ChunkHashes); + JupiterResult ListBuildNamespaces(); + JupiterResult ListBuildBuckets(std::string_view Namespace); JupiterResult ListBuilds(std::string_view Namespace, std::string_view BucketId, const IoBuffer& Payload); JupiterResult PutBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, const IoBuffer& Payload); JupiterResult GetBuild(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); -- cgit v1.2.3 From 732a1cb1e78abbabaa0d926e9b1e58a36538dc1b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 22 Apr 2025 16:28:08 +0200 Subject: add cxxopts overload for parsing file paths from command line (#362) --- src/zenutil/include/zenutil/commandlineoptions.h | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/zenutil/include/zenutil/commandlineoptions.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h new file mode 100644 index 000000000..3afbac854 --- /dev/null +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START + +namespace cxxopts::values { +// We declare this specialization before including cxxopts to make it stick +void parse_value(const std::string& text, std::filesystem::path& value); +} // namespace cxxopts::values + +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); +void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); +std::filesystem::path StringToPath(const std::string_view& Path); + +void commandlineoptions_forcelink(); // internal + +} // namespace zen -- cgit v1.2.3 From 6646a12bbffd349ad234a4265e339797e815493b Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 15:57:02 -0600 Subject: Replace container with namespaces --- src/zenutil/include/zenutil/buildstorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 0785acd62..5422c837c 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -25,7 +25,7 @@ public: virtual ~BuildStorage() {} - virtual CbObject ListContainers() = 0; + virtual CbObject ListNamespaces(bool bRecursive = false) = 0; virtual CbObject ListBuilds(CbObject Query) = 0; virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) = 0; virtual CbObject GetBuild(const Oid& BuildId) = 0; -- cgit v1.2.3 From 37d3a1b76ef256431f3883853fd625dc33b7a991 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 23 Apr 2025 14:14:27 +0200 Subject: parse system dir for builds (#365) * make sure we always parse system options for zen builds command * make MakeSafeAbsolutePath nodiscard --- src/zenutil/include/zenutil/commandlineoptions.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h index 3afbac854..b7581f6cd 100644 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -17,11 +17,11 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -std::vector ParseCommandLine(std::string_view CommandLine); -std::vector StripCommandlineQuotes(std::vector& InOutArgs); -void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); -std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); -std::filesystem::path StringToPath(const std::string_view& Path); +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); +void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +[[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); +std::filesystem::path StringToPath(const std::string_view& Path); void commandlineoptions_forcelink(); // internal -- cgit v1.2.3 From 2a727175c7eaf369593a4ac5039ecd7f3da5aa66 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 5 May 2025 19:27:42 +0200 Subject: builds allow redirect option (#379) * add --allow-redirect to zen builds upload/download --- src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h | 1 + src/zenutil/include/zenutil/jupiter/jupitersession.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h index 89fc70140..bbf070993 100644 --- a/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h +++ b/src/zenutil/include/zenutil/jupiter/jupiterbuildstorage.h @@ -13,5 +13,6 @@ std::unique_ptr CreateJupiterBuildStorage(LoggerRef InLog, BuildStorage::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, + bool AllowRedirect, const std::filesystem::path& TempFolderPath); } // namespace zen diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index 32bfd50f4..b79790f25 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -65,7 +65,7 @@ struct FinalizeBuildPartResult : JupiterResult class JupiterSession { public: - JupiterSession(LoggerRef InLog, HttpClient& InHttpClient); + JupiterSession(LoggerRef InLog, HttpClient& InHttpClient, bool AllowRedirect); ~JupiterSession(); JupiterResult Authenticate(); @@ -173,6 +173,7 @@ private: LoggerRef m_Log; HttpClient& m_HttpClient; + const bool m_AllowRedirect = false; }; } // namespace zen -- cgit v1.2.3 From 4e2efa1051e3eb86ab48d92b3f6ad5896cda5d81 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 16 May 2025 19:51:36 +0200 Subject: parallel work handle dispatch exception (#400) - Bugfix: Wait for async threads if dispatching of work using ParallellWork throws exception --- src/zenutil/include/zenutil/buildstoragecache.h | 2 +- src/zenutil/include/zenutil/chunkedcontent.h | 4 +- src/zenutil/include/zenutil/parallellwork.h | 119 ------------------------ src/zenutil/include/zenutil/parallelwork.h | 71 ++++++++++++++ 4 files changed, 74 insertions(+), 122 deletions(-) delete mode 100644 src/zenutil/include/zenutil/parallellwork.h create mode 100644 src/zenutil/include/zenutil/parallelwork.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index cab35328d..e1fb73fd4 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -44,7 +44,7 @@ public: virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; virtual void Flush( - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback = [](intptr_t) { return true; }) = 0; }; diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 57b55cb8e..d33869be2 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -67,7 +67,7 @@ FolderContent GetFolderContent(GetFolderContentStatistics& Stats, std::function&& AcceptDirectory, std::function&& AcceptFile, WorkerThreadPool& WorkerPool, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag); @@ -116,7 +116,7 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, const std::filesystem::path& RootPath, const FolderContent& Content, const ChunkingController& InChunkingController, - int32_t UpdateInteralMS, + int32_t UpdateIntervalMS, std::function&& UpdateCallback, std::atomic& AbortFlag); diff --git a/src/zenutil/include/zenutil/parallellwork.h b/src/zenutil/include/zenutil/parallellwork.h deleted file mode 100644 index 8ea77c65d..000000000 --- a/src/zenutil/include/zenutil/parallellwork.h +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include - -#include - -class ParallellWorkException : public std::runtime_error -{ -public: - explicit ParallellWorkException(std::vector&& Errors) : std::runtime_error(Errors.front()), m_Errors(std::move(Errors)) {} - - const std::vector m_Errors; -}; - -namespace zen { - -class ParallellWork -{ -public: - ParallellWork(std::atomic& AbortFlag) : m_AbortFlag(AbortFlag), m_PendingWork(1) {} - - ~ParallellWork() - { - // Make sure to call Wait before destroying - ZEN_ASSERT(m_PendingWork.Remaining() == 0); - } - - std::function& AbortFlag)> DefaultErrorFunction() - { - return [&](const std::exception& Ex, std::atomic& AbortFlag) { - m_ErrorLock.WithExclusiveLock([&]() { m_Errors.push_back(Ex.what()); }); - AbortFlag = true; - }; - } - - void ScheduleWork(WorkerThreadPool& WorkerPool, - std::function& AbortFlag)>&& Work, - std::function& AbortFlag)>&& OnError) - { - m_PendingWork.AddCount(1); - try - { - WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = std::move(OnError)] { - try - { - Work(m_AbortFlag); - } - catch (const AssertException& AssertEx) - { - OnError( - std::runtime_error(fmt::format("Caught assert exception while handling request: {}", AssertEx.FullDescription())), - m_AbortFlag); - } - catch (const std::system_error& SystemError) - { - if (IsOOM(SystemError.code())) - { - OnError(std::runtime_error(fmt::format("Out of memory. Reason: {}", SystemError.what())), m_AbortFlag); - } - else if (IsOOD(SystemError.code())) - { - OnError(std::runtime_error(fmt::format("Out of disk. Reason: {}", SystemError.what())), m_AbortFlag); - } - else - { - OnError(std::runtime_error(fmt::format("System error. Reason: {}", SystemError.what())), m_AbortFlag); - } - } - catch (const std::exception& Ex) - { - OnError(Ex, m_AbortFlag); - } - m_PendingWork.CountDown(); - }); - } - catch (const std::exception&) - { - m_PendingWork.CountDown(); - throw; - } - } - - void Abort() { m_AbortFlag = true; } - - bool IsAborted() const { return m_AbortFlag.load(); } - - void Wait(int32_t UpdateInteralMS, std::function&& UpdateCallback) - { - ZEN_ASSERT(m_PendingWork.Remaining() > 0); - m_PendingWork.CountDown(); - while (!m_PendingWork.Wait(UpdateInteralMS)) - { - UpdateCallback(m_AbortFlag.load(), m_PendingWork.Remaining()); - } - if (m_Errors.size() == 1) - { - throw std::runtime_error(m_Errors.front()); - } - else if (m_Errors.size() > 1) - { - throw ParallellWorkException(std::move(m_Errors)); - } - } - Latch& PendingWork() { return m_PendingWork; } - -private: - std::atomic& m_AbortFlag; - Latch m_PendingWork; - - RwLock m_ErrorLock; - std::vector m_Errors; -}; - -} // namespace zen diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h new file mode 100644 index 000000000..08e730b28 --- /dev/null +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen { + +class ParallelWork +{ +public: + ParallelWork(std::atomic& AbortFlag); + + ~ParallelWork(); + + typedef std::function& AbortFlag)> WorkCallback; + typedef std::function& AbortFlag)> ExceptionCallback; + typedef std::function UpdateCallback; + + void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {}) + { + m_PendingWork.AddCount(1); + try + { + WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { + try + { + Work(m_AbortFlag); + } + catch (...) + { + OnError(std::current_exception(), m_AbortFlag); + } + m_PendingWork.CountDown(); + }); + } + catch (const std::exception&) + { + m_PendingWork.CountDown(); + throw; + } + } + + void Abort() { m_AbortFlag = true; } + + bool IsAborted() const { return m_AbortFlag.load(); } + + void Wait(int32_t UpdateIntervalMS, UpdateCallback&& UpdateCallback); + + void Wait(); + + Latch& PendingWork() { return m_PendingWork; } + +private: + ExceptionCallback DefaultErrorFunction(); + void RethrowErrors(); + + std::atomic& m_AbortFlag; + bool m_DispatchComplete = false; + Latch m_PendingWork; + + RwLock m_ErrorLock; + std::vector m_Errors; +}; + +void parallellwork_forcelink(); + +} // namespace zen -- cgit v1.2.3 From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- .../include/zenutil/bufferedwritefilecache.h | 106 +++++++++++++++++++++ src/zenutil/include/zenutil/chunkedcontent.h | 1 + src/zenutil/include/zenutil/chunkingcontroller.h | 45 ++++++--- 3 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 src/zenutil/include/zenutil/bufferedwritefilecache.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/bufferedwritefilecache.h b/src/zenutil/include/zenutil/bufferedwritefilecache.h new file mode 100644 index 000000000..68d6c375e --- /dev/null +++ b/src/zenutil/include/zenutil/bufferedwritefilecache.h @@ -0,0 +1,106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +class CompositeBuffer; + +class BufferedWriteFileCache +{ +public: + BufferedWriteFileCache(const BufferedWriteFileCache&) = delete; + BufferedWriteFileCache& operator=(const BufferedWriteFileCache&) = delete; + + BufferedWriteFileCache(); + + ~BufferedWriteFileCache(); + + std::unique_ptr Get(uint32_t FileIndex); + + void Put(uint32_t FileIndex, std::unique_ptr&& Writer); + + void Close(std::span FileIndexes); + + class Local + { + public: + struct Writer + { + std::unique_ptr File; + std::unique_ptr Writer; + + inline void Write(const CompositeBuffer& Chunk, uint64_t FileOffset) + { + if (Writer) + { + Writer->Write(Chunk, FileOffset); + } + else + { + File->Write(Chunk, FileOffset); + } + } + }; + + Local(const Local&) = delete; + Local& operator=(const Local&) = delete; + + explicit Local(BufferedWriteFileCache& Cache); + ~Local(); + + Writer* GetWriter(uint32_t FileIndex); + Writer* PutWriter(uint32_t FileIndex, std::unique_ptr Writer); + + private: + tsl::robin_map m_FileIndexToWriterIndex; + std::vector> m_ChunkWriters; + BufferedWriteFileCache& m_Cache; + }; + +private: + static constexpr size_t MaxHandlesPerPath = 7; + static constexpr size_t MaxBufferedCount = 1024; + struct TOpenHandles + { + BasicFile* Files[MaxHandlesPerPath]; + uint64_t Size = 0; + inline BasicFile* Pop() + { + if (Size > 0) + { + return Files[--Size]; + } + else + { + return nullptr; + } + } + inline bool Push(BasicFile* File) + { + if (Size < MaxHandlesPerPath) + { + Files[Size++] = File; + return true; + } + return false; + } + }; + static_assert(sizeof(TOpenHandles) == 64); + + RwLock m_WriterLock; + tsl::robin_map m_ChunkWriters; + std::vector m_OpenFiles; + std::atomic m_CacheHitCount; + std::atomic m_CacheMissCount; + std::atomic m_OpenHandleCount; + std::atomic m_DroppedHandleCount; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index d33869be2..03f52e5f6 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -135,6 +135,7 @@ struct ChunkedContentLookup ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash + std::vector PathExtensionHash; }; ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 970917fb0..315502265 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -11,9 +11,11 @@ namespace zen { -const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; -const bool DefaultChunkingExcludeElfFiles = true; -const bool DefaultChunkingExcludeMachOFiles = true; +const std::vector DefaultChunkingExcludeExtensions = + {".exe", ".dll", ".pdb", ".self", ".mp4", ".zip", ".7z", ".bzip", ".rar", ".gzip"}; +const std::vector DefaultFixedChunkingExtensions = {".apk", ".nsp", ".xvc", ".pkg", ".dmg", ".ipa"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -21,7 +23,8 @@ const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128 const size_t DefaultChunkingFileSizeLimit = DefaultChunkedParams.MaxSize; -const uint32_t DefaultFixedChunkingChunkSize = 16u * 1024u * 1024u; +const uint64_t DefaultFixedChunkingChunkSize = 32u * 1024u * 1024u; +const uint64_t DefaultMinSizeForFixedChunking = DefaultFixedChunkingChunkSize * 8u; struct ChunkedInfoWithSource; @@ -40,19 +43,31 @@ public: virtual CbObject GetParameters() const = 0; }; -std::unique_ptr CreateBasicChunkingController( - std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, - bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, - bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, - uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, - const ChunkedParams& ChunkingParams = DefaultChunkedParams); +struct BasicChunkingControllerSettings +{ + std::vector ExcludeExtensions = DefaultChunkingExcludeExtensions; + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles; + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles; + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit; + ChunkedParams ChunkingParams = DefaultChunkedParams; +}; + +std::unique_ptr CreateBasicChunkingController(const BasicChunkingControllerSettings& Settings); std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); -std::unique_ptr CreateChunkingControllerWithFixedChunking( - std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, - uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, - const ChunkedParams& ChunkingParams = DefaultChunkedParams, - uint32_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize); +struct ChunkingControllerWithFixedChunkingSettings +{ + std::vector FixedChunkingExtensions = DefaultFixedChunkingExtensions; + std::vector ExcludeExtensions = DefaultChunkingExcludeExtensions; + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles; + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles; + uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit; + ChunkedParams ChunkingParams = DefaultChunkedParams; + uint64_t FixedChunkingChunkSize = DefaultFixedChunkingChunkSize; + uint64_t MinSizeForFixedChunking = DefaultMinSizeForFixedChunking; +}; + +std::unique_ptr CreateChunkingControllerWithFixedChunking(const ChunkingControllerWithFixedChunkingSettings& Setting); std::unique_ptr CreateChunkingControllerWithFixedChunking(CbObjectView Parameters); std::unique_ptr CreateChunkingController(std::string_view Name, CbObjectView Parameters); -- cgit v1.2.3 From 937510356143f83ecd15d0a9f58b611c7418ed61 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Jun 2025 08:59:44 +0200 Subject: faster scavenge (#417) - Improvement: Multithreaded scavenge pass for zen builds download - Improvement: Optimized check for modified files when verifying state of scavenged paths --- src/zenutil/include/zenutil/chunkedcontent.h | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 03f52e5f6..225b1a3a5 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -94,10 +94,31 @@ struct ChunkedFolderContent ChunkedContentData ChunkedContent; }; +struct ChunkedContentLookup +{ + struct ChunkSequenceLocation + { + uint32_t SequenceIndex = (uint32_t)-1; + uint64_t Offset = (uint64_t)-1; + }; + tsl::robin_map ChunkHashToChunkIndex; + tsl::robin_map RawHashToSequenceIndex; + std::vector SequenceIndexChunkOrderOffset; + std::vector ChunkSequenceLocations; + std::vector + ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex + std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex + std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash + std::vector PathExtensionHash; +}; + void SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output); ChunkedFolderContent LoadChunkedFolderContentToCompactBinary(CbObjectView Input); ChunkedFolderContent MergeChunkedFolderContents(const ChunkedFolderContent& Base, std::span Overlays); +ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, + const ChunkedContentLookup& BaseContentLookup, + std::span DeletedPaths); ChunkedFolderContent DeletePathsFromChunkedContent(const ChunkedFolderContent& Base, std::span DeletedPaths); struct ChunkingStatistics @@ -120,24 +141,6 @@ ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, std::function&& UpdateCallback, std::atomic& AbortFlag); -struct ChunkedContentLookup -{ - struct ChunkSequenceLocation - { - uint32_t SequenceIndex = (uint32_t)-1; - uint64_t Offset = (uint64_t)-1; - }; - tsl::robin_map ChunkHashToChunkIndex; - tsl::robin_map RawHashToSequenceIndex; - std::vector SequenceIndexChunkOrderOffset; - std::vector ChunkSequenceLocations; - std::vector - ChunkSequenceLocationOffset; // ChunkSequenceLocations[ChunkLocationOffset[ChunkIndex]] -> start of sources for ChunkIndex - std::vector ChunkSequenceLocationCounts; // ChunkSequenceLocationCounts[ChunkIndex] count of chunk locations for ChunkIndex - std::vector SequenceIndexFirstPathIndex; // SequenceIndexFirstPathIndex[SequenceIndex] -> first path index with that RawHash - std::vector PathExtensionHash; -}; - ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); inline std::pair -- cgit v1.2.3 From c44b9f7151a047cde5edd369f9adb09518a0bc6f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Jun 2025 14:03:55 +0200 Subject: builds download url (#419) * RemoveQuotes helper * `--url` option for `zen builds` command has been reworked to accept a "Cloud Artifact URL", removing the need to specify "host", "namespace" and "bucket" separately --- src/zenutil/include/zenutil/commandlineoptions.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h index b7581f6cd..f927d41e5 100644 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ b/src/zenutil/include/zenutil/commandlineoptions.h @@ -22,6 +22,7 @@ std::vector StripCommandlineQuotes(std::vector& InOutArg void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); std::filesystem::path StringToPath(const std::string_view& Path); +std::string_view RemoveQuotes(const std::string_view& Arg); void commandlineoptions_forcelink(); // internal -- cgit v1.2.3 From 40b9386054de3c23f77da74eefaa743240d164fd Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Jun 2025 14:40:02 +0200 Subject: pause, resume and abort running builds cmd (#421) - Feature: `zen builds pause`, `zen builds resume` and `zen builds abort` commands to control a running `zen builds` command - `--process-id` the process id to control, if omitted it tries to find a running process using the same executable as itself - Improvement: Process report now indicates if it is pausing or aborting --- src/zenutil/include/zenutil/chunkedcontent.h | 17 +++++++++-------- src/zenutil/include/zenutil/parallelwork.h | 13 +++++++++---- 2 files changed, 18 insertions(+), 12 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h index 225b1a3a5..306a5d990 100644 --- a/src/zenutil/include/zenutil/chunkedcontent.h +++ b/src/zenutil/include/zenutil/chunkedcontent.h @@ -132,14 +132,15 @@ struct ChunkingStatistics uint64_t ElapsedWallTimeUS = 0; }; -ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, - WorkerThreadPool& WorkerPool, - const std::filesystem::path& RootPath, - const FolderContent& Content, - const ChunkingController& InChunkingController, - int32_t UpdateIntervalMS, - std::function&& UpdateCallback, - std::atomic& AbortFlag); +ChunkedFolderContent ChunkFolderContent(ChunkingStatistics& Stats, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& RootPath, + const FolderContent& Content, + const ChunkingController& InChunkingController, + int32_t UpdateIntervalMS, + std::function&& UpdateCallback, + std::atomic& AbortFlag, + std::atomic& PauseFlag); ChunkedContentLookup BuildChunkedContentLookup(const ChunkedFolderContent& Content); diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h index 08e730b28..d7e986551 100644 --- a/src/zenutil/include/zenutil/parallelwork.h +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -12,13 +12,13 @@ namespace zen { class ParallelWork { public: - ParallelWork(std::atomic& AbortFlag); + ParallelWork(std::atomic& AbortFlag, std::atomic& PauseFlag); ~ParallelWork(); - typedef std::function& AbortFlag)> WorkCallback; - typedef std::function& AbortFlag)> ExceptionCallback; - typedef std::function UpdateCallback; + typedef std::function& AbortFlag)> WorkCallback; + typedef std::function& AbortFlag)> ExceptionCallback; + typedef std::function UpdateCallback; void ScheduleWork(WorkerThreadPool& WorkerPool, WorkCallback&& Work, ExceptionCallback&& OnError = {}) { @@ -28,6 +28,10 @@ public: WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { try { + while (m_PauseFlag && !m_AbortFlag) + { + Sleep(2000); + } Work(m_AbortFlag); } catch (...) @@ -59,6 +63,7 @@ private: void RethrowErrors(); std::atomic& m_AbortFlag; + std::atomic& m_PauseFlag; bool m_DispatchComplete = false; Latch m_PendingWork; -- cgit v1.2.3 From 61b4a88fe0b6f0ce43910af4f1afc2fc13f10273 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:05:26 +0200 Subject: add EnvironmentOptions helper --- src/zenutil/include/zenutil/environmentoptions.h | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h new file mode 100644 index 000000000..7418608e4 --- /dev/null +++ b/src/zenutil/include/zenutil/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr Value; + }; + + std::shared_ptr MakeOption(std::string& Value); + std::shared_ptr MakeOption(std::filesystem::path& Value); + + template + std::shared_ptr MakeOption(T& Value) + { + return std::make_shared>(Value); + }; + + std::shared_ptr MakeOption(bool& Value); + + template + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map OptionMap; +}; + +} // namespace zen -- cgit v1.2.3 From f696e52d150ae284e26de5bdd78d1b1edf914314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:08:59 +0200 Subject: revert 61b4a88f and cadaad63 --- src/zenutil/include/zenutil/environmentoptions.h | 92 ------------------------ 1 file changed, 92 deletions(-) delete mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h deleted file mode 100644 index 7418608e4..000000000 --- a/src/zenutil/include/zenutil/environmentoptions.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include - -namespace zen { - -class EnvironmentOptions -{ -public: - class OptionValue - { - public: - virtual void Parse(std::string_view Value) = 0; - - virtual ~OptionValue() {} - }; - - class StringOption : public OptionValue - { - public: - explicit StringOption(std::string& Value); - virtual void Parse(std::string_view Value) override; - std::string& RefValue; - }; - - class FilePathOption : public OptionValue - { - public: - explicit FilePathOption(std::filesystem::path& Value); - virtual void Parse(std::string_view Value) override; - std::filesystem::path& RefValue; - }; - - class BoolOption : public OptionValue - { - public: - explicit BoolOption(bool& Value); - virtual void Parse(std::string_view Value); - bool& RefValue; - }; - - template - class NumberOption : public OptionValue - { - public: - explicit NumberOption(T& Value) : RefValue(Value) {} - virtual void Parse(std::string_view Value) override - { - if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) - { - RefValue = OptionalValue.value(); - } - } - T& RefValue; - }; - - struct Option - { - std::string CommandLineOptionName; - std::shared_ptr Value; - }; - - std::shared_ptr MakeOption(std::string& Value); - std::shared_ptr MakeOption(std::filesystem::path& Value); - - template - std::shared_ptr MakeOption(T& Value) - { - return std::make_shared>(Value); - }; - - std::shared_ptr MakeOption(bool& Value); - - template - void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") - { - OptionMap.insert_or_assign(std::string(EnvName), - Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); - }; - - EnvironmentOptions(); - - void Parse(const cxxopts::ParseResult& CmdLineResult); - -private: - std::unordered_map OptionMap; -}; - -} // namespace zen -- cgit v1.2.3 From fdad92dddba8047930c4e7496a7f412d760c312e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Jun 2025 09:30:54 +0200 Subject: sentry config (#430) - Feature: Added `--sentry-environment` to `zen` and `zenserver` - Feature: Added `--sentry-debug` to `zen` and `zenserver` - Feature: Added environment variable parsing for the following options: - `UE_ZEN_SENTRY_ENABLED`: `--no-sentry` (inverted) - `UE_ZEN_SENTRY_DEBUG`: `--sentry-debug` - `UE_ZEN_SENTRY_ALLOWPERSONALINFO`: `--sentry-allow-personal-info` - `UE_ZEN_SENTRY_DSN`: `--sentry-dsn` - `UE_ZEN_SENTRY_ENVIRONMENT`: `--sentry-environment` --- src/zenutil/include/zenutil/environmentoptions.h | 92 ++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 src/zenutil/include/zenutil/environmentoptions.h (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h new file mode 100644 index 000000000..7418608e4 --- /dev/null +++ b/src/zenutil/include/zenutil/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr Value; + }; + + std::shared_ptr MakeOption(std::string& Value); + std::shared_ptr MakeOption(std::filesystem::path& Value); + + template + std::shared_ptr MakeOption(T& Value) + { + return std::make_shared>(Value); + }; + + std::shared_ptr MakeOption(bool& Value); + + template + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map OptionMap; +}; + +} // namespace zen -- cgit v1.2.3 From d000167e12c6dde651ef86be9f67552291ff1b7d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 Jun 2025 13:17:54 +0200 Subject: graceful wait in parallelwork destructor (#438) * exception safety when issuing ParallelWork * add asserts to Latch usage to catch usage errors * extended error messaging and recovery handling in ParallelWork destructor to help find issues --- src/zenutil/include/zenutil/parallelwork.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/parallelwork.h b/src/zenutil/include/zenutil/parallelwork.h index d7e986551..639c6968c 100644 --- a/src/zenutil/include/zenutil/parallelwork.h +++ b/src/zenutil/include/zenutil/parallelwork.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include @@ -26,6 +27,7 @@ public: try { WorkerPool.ScheduleWork([this, Work = std::move(Work), OnError = OnError ? std::move(OnError) : DefaultErrorFunction()] { + auto _ = MakeGuard([this]() { m_PendingWork.CountDown(); }); try { while (m_PauseFlag && !m_AbortFlag) @@ -38,7 +40,6 @@ public: { OnError(std::current_exception(), m_AbortFlag); } - m_PendingWork.CountDown(); }); } catch (const std::exception&) -- cgit v1.2.3 From fe4cac6a733706e6b523e9ec3e9d014571723f72 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 19:20:19 +0200 Subject: add retry for failed block metadata upload (#445) * add retry for failed block metadata upload --- src/zenutil/include/zenutil/buildstorage.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 5422c837c..03496f0f9 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,9 +55,9 @@ public: std::function&& OnReceive, std::function&& OnComplete) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; - virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; + virtual [[nodiscard]] bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; -- cgit v1.2.3 From 9c23f846f51bbf6288a73880619aa11e7ef18274 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 22:18:59 +0200 Subject: move nodiscard to proper location (#447) --- src/zenutil/include/zenutil/buildstorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/zenutil/include') diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 03496f0f9..f49d4b42a 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,7 +55,7 @@ public: std::function&& OnReceive, std::function&& OnComplete) = 0; - virtual [[nodiscard]] bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + [[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; -- cgit v1.2.3