From 800305f7f2afe01e9ca038a71a68e9318c52ee77 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Feb 2025 09:05:07 +0100 Subject: move WriteToTempFile to basicfile.h (#283) add helper constructors to BasicFile --- src/zencore/include/zencore/basicfile.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 03c5605df..7edd40c9c 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -46,6 +46,10 @@ public: kPreventWrite = 0x2000'0000, // Do not open with write sharing mode (prevent other processes from writing to file while open) }; + BasicFile(const std::filesystem::path& FileName, Mode Mode); + BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec); + BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback); + void Open(const std::filesystem::path& FileName, Mode Mode); void Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec); void Open(const std::filesystem::path& FileName, Mode Mode, std::function&& RetryCallback); @@ -56,7 +60,7 @@ public: void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(MemoryView Data, uint64_t FileOffset); void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec); - uint64_t Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec); + uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec); void Flush(); @@ -180,6 +184,10 @@ private: uint64_t m_BufferEnd; }; +IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, + const std::filesystem::path& Path, + std::function&& RetryCallback); + ZENCORE_API void basicfile_forcelink(); } // namespace zen -- cgit v1.2.3 From 5bc5b0dd59c0f02afe553e5074dfe57951b19044 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 25 Feb 2025 15:48:43 +0100 Subject: improvements and infrastructure for upcoming builds api command line (#284) * add modification tick to filesystem traversal * add ShowDetails option to ProgressBar * log callstack if we terminate process * handle chunking if MaxSize > 1MB * BasicFile write helpers and WriteToTempFile simplifications * bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data revert of earlier optimization * faster compress/decompress for large disk-based files * enable progress feedback in IoHash::HashBuffer * add payload validation in HttpClient::Get * fix range requests (range is including end byte) * remove BuildPartId for blob/block related operations in builds api --- src/zencore/include/zencore/basicfile.h | 6 +++--- src/zencore/include/zencore/filesystem.h | 20 ++++++++++++-------- src/zencore/include/zencore/iohash.h | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 7edd40c9c..a78132879 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -60,6 +60,7 @@ public: void StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function&& ChunkFun); void Write(MemoryView Data, uint64_t FileOffset); void Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec); + uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset); uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); void Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec); @@ -174,6 +175,7 @@ public: ~BasicFileWriter(); void Write(const void* Data, uint64_t Size, uint64_t FileOffset); + void Write(const CompositeBuffer& Data, uint64_t FileOffset); void Flush(); private: @@ -184,9 +186,7 @@ private: uint64_t m_BufferEnd; }; -IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, - const std::filesystem::path& Path, - std::function&& RetryCallback); +IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path); ZENCORE_API void basicfile_forcelink(); diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index ca8682cd7..250745e86 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -203,7 +203,8 @@ public: virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize, - uint32_t NativeModeOrAttributes) = 0; + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) = 0; // This should return true if we should recurse into the directory virtual bool VisitDirectory(const std::filesystem::path& Parent, @@ -216,13 +217,14 @@ public: enum class DirectoryContentFlags : uint8_t { - None = 0, - IncludeDirs = 1u << 0, - IncludeFiles = 1u << 1, - Recursive = 1u << 2, - IncludeFileSizes = 1u << 3, - IncludeAttributes = 1u << 4, - IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive + None = 0, + IncludeDirs = 1u << 0, + IncludeFiles = 1u << 1, + Recursive = 1u << 2, + IncludeFileSizes = 1u << 3, + IncludeAttributes = 1u << 4, + IncludeModificationTick = 1u << 5, + IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive }; ENUM_CLASS_FLAGS(DirectoryContentFlags) @@ -232,6 +234,7 @@ struct DirectoryContent std::vector Files; std::vector FileSizes; std::vector FileAttributes; + std::vector FileModificationTicks; std::vector Directories; std::vector DirectoryAttributes; }; @@ -246,6 +249,7 @@ public: std::vector FileNames; std::vector FileSizes; std::vector FileAttributes; + std::vector FileModificationTicks; std::vector DirectoryNames; std::vector DirectoryAttributes; }; diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h index 8871a5895..7443e17b7 100644 --- a/src/zencore/include/zencore/iohash.h +++ b/src/zencore/include/zencore/iohash.h @@ -47,8 +47,8 @@ struct IoHash static IoHash HashBuffer(const void* data, size_t byteCount); static IoHash HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); } - static IoHash HashBuffer(const CompositeBuffer& Buffer); - static IoHash HashBuffer(const IoBuffer& Buffer); + static IoHash HashBuffer(const CompositeBuffer& Buffer, std::atomic* ProcessedBytes = nullptr); + static IoHash HashBuffer(const IoBuffer& Buffer, std::atomic* ProcessedBytes = nullptr); static IoHash FromHexString(const char* string); static IoHash FromHexString(const std::string_view string); static bool TryParse(std::string_view Str, IoHash& Hash); -- cgit v1.2.3 From 7d8fe45af3b49d800f84f0ddce051c0b3b2e837d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Feb 2025 15:10:14 +0100 Subject: builds upload command (#278) - Feature: **EXPERIMENTAL** New `zen builds` command to list, upload and download folders to Cloud Build API - `builds list` list available builds (**INCOMPLETE - FILTERING MISSING**) - `builds upload` upload a folder to Cloud Build API - `--local-path` source folder to upload - `--create-build` creates a new parent build object (using the object id), if omitted a parent build must exist and `--build-id` must be given - `--build-id` an Oid in hex form for the Build identifier to use - omit to have the id auto generated - `--build-part-id` and Oid in hex form for the Build Part identifier for the folder - omit to have the id auto generated - `--build-part-name` name of the build part - if omitted the name of the leaf folder name give in `--local-path` - `--metadata-path` path to a json formatted file with meta data information about the build. Meta-data must be provided if `--create-build` is set - `--metadata` key-value pairs separated by ';' with build meta data for the build. (key1=value1;key2=value2). Meta-data must be provided if `--create-build` is set - `--clean` ignore any existing blocks of chunk data and upload a fresh set of blocks - `--allow-multipart` enable usage of multi-part http upload requests - `--manifest-path` path to text file listing files to include in upload. Exclude to upload everything in `--local-path` - `builds download` download a folder from Cloud Build API (**INCOMPLETE - WILL WIPE UNTRACKED DATA FROM TARGET FOLDER**) - `--local-path` target folder to download to - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` a comma separated list of Oid in hex for the build part identifier(s) to download - mutually exclusive to `--build-part-name` - `--build-part-name` a comma separated list of names for the build part(s) to download - if omitted the name of the leaf folder name give in `--local-path` - `--clean` deletes all data in target folder before downloading (NON-CLEAN IS NOT IMPLEMENTED YET) - `--allow-multipart` enable usage of multi-part http download reqeusts - `builds diff` download a folder from Cloud Build API - `--local-path` target folder to download to - `--compare-path` folder to compare target with - `--only-chunked` compare only files that would be chunked - `builds fetch-blob` fetch and validate a blob from remote store - `--build-id` an Oid in hex form for the Build identifier to use - `--blob-hash` an IoHash in hex form identifying the blob to download - `builds validate part` fetch a build part and validate all referenced attachments - `--build-id` an Oid in hex form for the Build identifier to use - `--build-part-id` an Oid in hex for the build part identifier to validate - mutually exclusive to `--build-part-name` - `--build-part-name` a name for the build part to validate - mutually exclusive to `--build-part-id` - `builds test` a series of operation that uploads, downloads and test various aspects of incremental operations - `--local-path` source folder to upload - Options for Cloud Build API remote store (`list`, `upload`, `download`, `fetch-blob`, `validate-part`) - `--url` Cloud Builds URL - `--assume-http2` assume that the builds endpoint is a HTTP/2 endpoint skipping HTTP/1.1 upgrade handshake - `--namespace` Builds Storage namespace - `--bucket` Builds Storage bucket - Authentication options for Cloud Build API - Auth token - `--access-token` http auth Cloud Storage access token - `--access-token-env` name of environment variable that holds the Http auth Cloud Storage access token - `--access-token-path` path to json file that holds the Http auth Cloud Storage access token - OpenId authentication - `--openid-provider-name` Open ID provider name - `--openid-provider-url` Open ID provider url - `--openid-client-id`Open ID client id - `--openid-refresh-token` Open ID refresh token - `--encryption-aes-key` 256 bit AES encryption key for storing OpenID credentials - `--encryption-aes-iv` 128 bit AES encryption initialization vector for storing OpenID credentials - OAuth authentication - `--oauth-url` OAuth provier url - `--oauth-clientid` OAuth client id - `--oauth-clientsecret` OAuth client secret - Options for file based remote store used for for testing purposes (`list`, `upload`, `download`, `fetch-blob`, `validate-part`, `test`) - `--storage-path` path to folder to store builds data - `--json-metadata` enable json output in store for all compact binary objects (off by default) - Output options for all builds commands - `--plain-progress` use plain line-by-line progress output - `--verbose` --- src/zencore/include/zencore/filesystem.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 250745e86..20f6dc56c 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -52,6 +52,10 @@ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); */ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +/** Get a native time tick of last modification time + */ +ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -271,6 +275,23 @@ std::error_code RotateDirectories(const std::filesystem::path& DirectoryName, st std::filesystem::path PickDefaultSystemRootDirectory(); +#if ZEN_PLATFORM_WINDOWS +uint32_t GetFileAttributes(const std::filesystem::path& Filename); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +uint32_t GetFileMode(const std::filesystem::path& Filename); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + +bool IsFileAttributeReadOnly(uint32_t FileAttributes); +bool IsFileModeReadOnly(uint32_t FileMode); +uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); +uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); + +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal -- cgit v1.2.3 From 19b3c492dcc0fc3f8879ecb60124ca64dea9b7ef Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sat, 1 Mar 2025 10:10:53 +0100 Subject: builds download incremental (#290) * incremental download * merge rebuild state and output state building * fix writing when > 1 zero size file --- src/zencore/include/zencore/filesystem.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 20f6dc56c..e020668fc 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -97,11 +97,11 @@ struct CopyFileOptions bool MustClone = false; }; -ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); -ZENCORE_API void CopyFile(std::filesystem::path FromPath, - std::filesystem::path ToPath, - const CopyFileOptions& Options, - std::error_code& OutError); +ZENCORE_API bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options); +ZENCORE_API void CopyFile(const std::filesystem::path& FromPath, + const std::filesystem::path& ToPath, + const CopyFileOptions& Options, + std::error_code& OutError); ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options); ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); -- cgit v1.2.3 From 2232eb28256ec54beaf3dbe06f5176698c7245a0 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 4 Mar 2025 09:38:13 +0100 Subject: limit and validate responses before logging the text (#292) Improvement: When logging HTTP responses, the body is now sanity checked to ensure it is human readable, and the length of the output is capped to prevent inadvertent log bloat --- src/zencore/include/zencore/string.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index e2ef1c1a0..68129b691 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -522,6 +522,9 @@ public: ////////////////////////////////////////////////////////////////////////// +bool IsValidUtf8(const std::string_view& str); +std::string_view::const_iterator FindFirstInvalidUtf8Byte(const std::string_view& str); + void Utf8ToWide(const char8_t* str, WideStringBuilderBase& out); void Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out); void Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out); -- cgit v1.2.3 From 7cb1de5a966aa23e0c098d2bbf9009d0f82a6477 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 4 Mar 2025 10:22:53 +0100 Subject: stream decompress (#293) * clean up latency parameters and slow down rate updates * add DecompressToStream --- src/zencore/include/zencore/compress.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 5e761ceef..c056d7561 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -196,6 +196,16 @@ public: */ [[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const; + /** + * Decompress into and call callback for ranges of decompressed data. + * The buffer in the callback will be overwritten when the callback returns. + * + * @return True if the buffer is valid and can be decompressed. + */ + [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const; + /** A null compressed buffer. */ static const CompressedBuffer Null; -- cgit v1.2.3 From 4b981aed0281ac995c326e3d03dc482353f66405 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 5 Mar 2025 11:27:54 +0100 Subject: streaming compress (#295) - Improvement: Validate hash of decompressed data inline with streaming decompression - Improvement: Do streaming compression of large blobs to improve memory and I/O performance --- src/zencore/include/zencore/compress.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index c056d7561..3969e9dbd 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -74,6 +74,11 @@ public: OodleCompressor Compressor = OodleCompressor::Mermaid, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, uint64_t BlockSize = 0); + [[nodiscard]] ZENCORE_API static bool CompressToStream(const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); /** * Construct from a compressed buffer previously created by Compress(). -- cgit v1.2.3 From 920120bbcec9f91df3336f62970b3e010a4fa6c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 6 Mar 2025 16:18:32 +0100 Subject: reduced memory churn using fixed_xxx containers (#236) * Added EASTL to help with eliminating memory allocations * Applied EASTL to eliminate memory allocations, primarily by using `fixed_vector` et al to use stack allocations / inline struct allocations Reduces memory events in traces by close to a factor of 10 in test scenario (starting editor for project F) --- src/zencore/include/zencore/compactbinarybuilder.h | 8 +++-- src/zencore/include/zencore/compactbinarypackage.h | 15 ++++++--- src/zencore/include/zencore/compositebuffer.h | 38 +++++++++++++++++++--- src/zencore/include/zencore/eastlutil.h | 20 ++++++++++++ src/zencore/include/zencore/memory/newdelete.h | 26 +++++++++++++++ 5 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 src/zencore/include/zencore/eastlutil.h (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h index 1c625cacc..f11717453 100644 --- a/src/zencore/include/zencore/compactbinarybuilder.h +++ b/src/zencore/include/zencore/compactbinarybuilder.h @@ -18,6 +18,8 @@ #include #include +#include + #include namespace zen { @@ -367,6 +369,8 @@ public: /** Private flags that are public to work with ENUM_CLASS_FLAGS. */ enum class StateFlags : uint8_t; + typedef eastl::fixed_vector CbWriterData_t; + protected: /** Reserve the specified size up front until the format is optimized. */ ZENCORE_API explicit CbWriter(int64_t InitialSize); @@ -409,8 +413,8 @@ private: // provided externally, such as on the stack. That format will store the offsets that require // object or array sizes to be inserted and field types to be removed, and will perform those // operations only when saving to a buffer. - std::vector Data; - std::vector States; + eastl::fixed_vector States; + CbWriterData_t Data; }; /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h index 12fcc41b7..9ec12cb0f 100644 --- a/src/zencore/include/zencore/compactbinarypackage.h +++ b/src/zencore/include/zencore/compactbinarypackage.h @@ -12,6 +12,8 @@ #include #include +#include + #ifdef GetObject # error "windows.h pollution" # undef GetObject @@ -265,7 +267,10 @@ public: } /** Returns the attachments in this package. */ - inline std::span GetAttachments() const { return Attachments; } + inline std::span GetAttachments() const + { + return std::span(begin(Attachments), end(Attachments)); + } /** * Find an attachment by its hash. @@ -286,6 +291,8 @@ public: void AddAttachments(std::span Attachments); + void ReserveAttachments(size_t Count); + /** * Remove an attachment by hash. * @@ -324,9 +331,9 @@ private: void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver); /** Attachments ordered by their hash. */ - std::vector Attachments; - CbObject Object; - IoHash ObjectHash; + eastl::fixed_vector Attachments; + CbObject Object; + IoHash ObjectHash; }; namespace legacy { diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h index b435c5e74..1e1611de9 100644 --- a/src/zencore/include/zencore/compositebuffer.h +++ b/src/zencore/include/zencore/compositebuffer.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include @@ -9,6 +10,8 @@ #include #include +#include + namespace zen { /** @@ -35,7 +38,7 @@ public: { m_Segments.reserve((GetBufferCount(std::forward(Buffers)) + ...)); (AppendBuffers(std::forward(Buffers)), ...); - std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); + erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); }); } } @@ -46,7 +49,10 @@ public: [[nodiscard]] ZENCORE_API uint64_t GetSize() const; /** Returns the segments that the buffer is composed from. */ - [[nodiscard]] inline std::span GetSegments() const { return std::span{m_Segments}; } + [[nodiscard]] inline std::span GetSegments() const + { + return std::span{begin(m_Segments), end(m_Segments)}; + } /** Returns true if the composite buffer is not null. */ [[nodiscard]] inline explicit operator bool() const { return !IsNull(); } @@ -120,6 +126,8 @@ public: static const CompositeBuffer Null; private: + typedef eastl::fixed_vector SharedBufferVector_t; + static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); } inline void AppendBuffers(const CompositeBuffer& Buffer) { @@ -134,12 +142,25 @@ private: inline void AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); } inline void AppendBuffers(IoBuffer&& Buffer) { m_Segments.push_back(SharedBuffer(std::move(Buffer))); } + static inline size_t GetBufferCount(std::span&& Container) { return Container.size(); } + inline void AppendBuffers(std::span&& Container) + { + m_Segments.reserve(m_Segments.size() + Container.size()); + for (IoBuffer& Buffer : Container) + { + m_Segments.emplace_back(SharedBuffer(std::move(Buffer))); + } + } + static inline size_t GetBufferCount(std::vector&& Container) { return Container.size(); } static inline size_t GetBufferCount(std::vector&& Container) { return Container.size(); } inline void AppendBuffers(std::vector&& Container) { m_Segments.reserve(m_Segments.size() + Container.size()); - m_Segments.insert(m_Segments.end(), std::make_move_iterator(Container.begin()), std::make_move_iterator(Container.end())); + for (SharedBuffer& Buffer : Container) + { + m_Segments.emplace_back(std::move(Buffer)); + } } inline void AppendBuffers(std::vector&& Container) { @@ -150,8 +171,17 @@ private: } } + inline void AppendBuffers(SharedBufferVector_t&& Container) + { + m_Segments.reserve(m_Segments.size() + Container.size()); + for (SharedBuffer& Buffer : Container) + { + m_Segments.emplace_back(std::move(Buffer)); + } + } + private: - std::vector m_Segments; + SharedBufferVector_t m_Segments; }; void compositebuffer_forcelink(); // internal diff --git a/src/zencore/include/zencore/eastlutil.h b/src/zencore/include/zencore/eastlutil.h new file mode 100644 index 000000000..642321dae --- /dev/null +++ b/src/zencore/include/zencore/eastlutil.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +size_t +erase_if(auto& _Cont, auto Predicate) +{ + auto _First = _Cont.begin(); + const auto _Last = _Cont.end(); + const auto _Old_size = _Cont.size(); + _First = std::remove_if(_First, _Last, Predicate); + _Cont.erase(_First, _Last); + return _Old_size - _Cont.size(); +} + +} // namespace zen diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h index d22c8604f..059f1d5ea 100644 --- a/src/zencore/include/zencore/memory/newdelete.h +++ b/src/zencore/include/zencore/memory/newdelete.h @@ -153,3 +153,29 @@ operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexce return zen_new_aligned_nothrow(n, static_cast(al)); } #endif + +// EASTL operator new + +void* +operator new[](size_t size, const char* pName, int flags, unsigned debugFlags, const char* file, int line) +{ + ZEN_UNUSED(pName, flags, debugFlags, file, line); + return zen_new(size); +} + +void* +operator new[](size_t size, + size_t alignment, + size_t alignmentOffset, + const char* pName, + int flags, + unsigned debugFlags, + const char* file, + int line) +{ + ZEN_UNUSED(alignmentOffset, pName, flags, debugFlags, file, line); + + ZEN_ASSERT_SLOW(alignmentOffset == 0); // currently not supported + + return zen_new_aligned(size, alignment); +} -- cgit v1.2.3 From 7de3d4218ee5969af6147f9ab20bda538a136d9a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 10 Mar 2025 18:33:24 +0100 Subject: pick up existing cache (#299) - Improvement: Scavenge .zen temp folders for existing data (downloaded, decompressed or written) from previous failed run - Improvement: Faster abort during stream compression - Improvement: Try to move downloaded blobs with rename if possible avoiding an extra disk write - Improvement: Only clean temp folders on successful or cancelled build - keep it if download fails --- src/zencore/include/zencore/basicfile.h | 2 +- src/zencore/include/zencore/compress.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index a78132879..57798b6f4 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -186,7 +186,7 @@ private: uint64_t m_BufferEnd; }; -IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path); +IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path); ZENCORE_API void basicfile_forcelink(); diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 3969e9dbd..74fd5f767 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -209,7 +209,7 @@ public: */ [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, uint64_t RawSize, - std::function&& Callback) const; + std::function&& Callback) const; /** A null compressed buffer. */ static const CompressedBuffer Null; -- cgit v1.2.3 From 90db3ced033d4e06da2739e5d97cdeff2b0ba3b9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 11 Mar 2025 09:58:07 +0100 Subject: Build command tweaks (#301) - Improvement: Don't chunk up .mp4 files as they generally won't benefit from deduplication or partial in-place-updates - Improvement: Emit build name to console output when downloading a build - Improvement: Added some debug logging - Bugfix: Logging setup would previously not function correctly when not logging to file --- src/zencore/include/zencore/compactbinaryfmt.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/zencore/include/zencore/compactbinaryfmt.h (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h new file mode 100644 index 000000000..ae0c3eb42 --- /dev/null +++ b/src/zencore/include/zencore/compactbinaryfmt.h @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include + +template +requires DerivedFrom +struct fmt::formatter : fmt::formatter +{ + auto format(const zen::CbObject& a, format_context& ctx) const + { + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(a, ObjStr); + return fmt::formatter::format(ObjStr.ToView(), ctx); + } +}; -- cgit v1.2.3 From 7046fc9dc202307ba92d05a6386bfb52e9db0ab9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 12 Mar 2025 13:54:16 +0100 Subject: fixes for log timestamps (#304) * add GetTimeSinceProcessStart returning time since process start. implemented using https://github.com/maxliani/GetTimeSinceProcessStart/tree/main * fix fractions when using epoch mode. Previously it would show the fraction from the absolute time stamp and not relative to epoch * used GetTimeSinceProcessStart to offset the epoch so that it represents the process spawn time --- src/zencore/include/zencore/timer.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h index e4ddc3505..767dc4314 100644 --- a/src/zencore/include/zencore/timer.h +++ b/src/zencore/include/zencore/timer.h @@ -21,6 +21,10 @@ ZENCORE_API uint64_t GetHifreqTimerFrequency(); ZENCORE_API double GetHifreqTimerToSeconds(); ZENCORE_API uint64_t GetHifreqTimerFrequencySafe(); // May be used during static init +// Query time since process was spawned (returns time in ms) + +uint64_t GetTimeSinceProcessStart(); + class Stopwatch { public: -- cgit v1.2.3 From 908f99b749fbbf9754f9485d680914792034334c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 12 Mar 2025 18:58:24 +0100 Subject: fix quoted command lines arguments (#306) Handling of quotes and quotes with leading backslash for command line parsing - UE-231677 --- src/zencore/include/zencore/filesystem.h | 2 ++ src/zencore/include/zencore/process.h | 3 +++ 2 files changed, 5 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index e020668fc..9a2b15d1d 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -292,6 +292,8 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); +std::filesystem::path StringToPath(const std::string_view& Path); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index d1394cd9a..0c5931ba0 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -100,6 +100,9 @@ int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); + void process_forcelink(); // internal } // namespace zen -- cgit v1.2.3