From fa4ef162b1dd53cbad135850a8f9cf8fb532f395 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 18 Mar 2025 08:56:40 +0100 Subject: improved post upload/download summary (#308) * added ValidateStatistics and improved post upload summary * improved download statistics * smoother stats update when compressing * better feedback during stream compresss/decompress * don't capture TotalPartWriteCount by reference * disk stats cleanup * multi-test-download overall timer --- src/zencore/include/zencore/compress.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h index 74fd5f767..09fa6249d 100644 --- a/src/zencore/include/zencore/compress.h +++ b/src/zencore/include/zencore/compress.h @@ -74,11 +74,12 @@ public: OodleCompressor Compressor = OodleCompressor::Mermaid, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, uint64_t BlockSize = 0); - [[nodiscard]] ZENCORE_API static bool CompressToStream(const CompositeBuffer& RawData, - std::function&& Callback, - OodleCompressor Compressor = OodleCompressor::Mermaid, - OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, - uint64_t BlockSize = 0); + [[nodiscard]] ZENCORE_API static bool CompressToStream( + const CompositeBuffer& RawData, + std::function&& Callback, + OodleCompressor Compressor = OodleCompressor::Mermaid, + OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, + uint64_t BlockSize = 0); /** * Construct from a compressed buffer previously created by Compress(). @@ -207,9 +208,10 @@ public: * * @return True if the buffer is valid and can be decompressed. */ - [[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t RawOffset, - uint64_t RawSize, - std::function&& Callback) const; + [[nodiscard]] ZENCORE_API bool DecompressToStream( + uint64_t RawOffset, + uint64_t RawSize, + std::function&& Callback) const; /** A null compressed buffer. */ static const CompressedBuffer Null; -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zencore/include/zencore/filesystem.h | 85 ++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 9a2b15d1d..c23f16d03 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -20,21 +20,35 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists. + + Will also create any required parent direCleanDirectoryctories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ @@ -44,6 +58,46 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_ */ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); @@ -56,6 +110,22 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -277,12 +347,16 @@ std::filesystem::path PickDefaultSystemRootDirectory(); #if ZEN_PLATFORM_WINDOWS uint32_t GetFileAttributes(const std::filesystem::path& Filename); +uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec); void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC uint32_t GetFileMode(const std::filesystem::path& Filename); -void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool IsFileAttributeReadOnly(uint32_t FileAttributes); @@ -290,6 +364,7 @@ bool IsFileModeReadOnly(uint32_t FileMode); uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); std::filesystem::path StringToPath(const std::string_view& Path); -- cgit v1.2.3 From ebe13120c030f8d24c5f05c068d79b2f72fc3c0b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 11:30:28 +0200 Subject: multithreaded clean (#331) - Improvement: Faster cleaning of directories - Improvement: Faster initial scanning of local state --- src/zencore/include/zencore/filesystem.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index c23f16d03..66deffa6f 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -110,6 +110,11 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes); + /** Move a file, if the files are not on the same drive the function will fail */ ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -- cgit v1.2.3 From f535a53cdda32b2b4950e32901dcd333bea82c23 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 20:35:53 +0200 Subject: use oidctoken executable to generate auth (#336) - Feature: `zen builds` auth option `--oidctoken-exe-path` to let zen run the OidcToken executable to get and refresh authentication token --- src/zencore/include/zencore/fmtutils.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h index 8482157fb..10dfa5393 100644 --- a/src/zencore/include/zencore/fmtutils.h +++ b/src/zencore/include/zencore/fmtutils.h @@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END +#include #include // Custom formatting for some zencore types @@ -102,3 +103,16 @@ struct fmt::formatter : fmt::formatter return fmt::formatter::format(a.ToView(), ctx); } }; + +template<> +struct fmt::formatter : formatter +{ + template + auto format(const std::chrono::system_clock::time_point& TimePoint, FormatContext& ctx) const + { + std::time_t Time = std::chrono::system_clock::to_time_t(TimePoint); + char TimeString[std::size("yyyy-mm-ddThh:mm:ss")]; + std::strftime(std::data(TimeString), std::size(TimeString), "%FT%T", std::localtime(&Time)); + return fmt::format_to(ctx.out(), "{}", TimeString); + } +}; -- cgit v1.2.3 From 732a1cb1e78abbabaa0d926e9b1e58a36538dc1b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 22 Apr 2025 16:28:08 +0200 Subject: add cxxopts overload for parsing file paths from command line (#362) --- src/zencore/include/zencore/filesystem.h | 2 -- src/zencore/include/zencore/process.h | 3 --- 2 files changed, 5 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 66deffa6f..1bc3943df 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -372,8 +372,6 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -std::filesystem::path StringToPath(const std::string_view& Path); - ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 0c5931ba0..d1394cd9a 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -100,9 +100,6 @@ int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); -std::vector ParseCommandLine(std::string_view CommandLine); -std::vector StripCommandlineQuotes(std::vector& InOutArgs); - void process_forcelink(); // internal } // namespace zen -- cgit v1.2.3 From 6d9ff7e404a22ed1cc7e529cfa77ef7d593d9547 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 6 May 2025 16:50:57 +0200 Subject: add sentry for zen command (#373) * refactor sentry integration and add to zen command line tool * move add_ldflags("-framework Security") --- src/zencore/include/zencore/sentryintegration.h | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/zencore/include/zencore/sentryintegration.h (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h new file mode 100644 index 000000000..40e22af4e --- /dev/null +++ b/src/zencore/include/zencore/sentryintegration.h @@ -0,0 +1,50 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#if !defined(ZEN_USE_SENTRY) +# define ZEN_USE_SENTRY 1 +#endif + +#if ZEN_USE_SENTRY + +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace sentry { + +struct SentryAssertImpl; + +} // namespace sentry + +namespace zen { + +class SentryIntegration +{ +public: + SentryIntegration(); + ~SentryIntegration(); + + void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine); + void LogStartupInformation(); + static void ClearCaches(); + +private: + int m_SentryErrorCode = 0; + bool m_IsInitialized = false; + bool m_AllowPII = false; + std::unique_ptr m_SentryAssert; + std::string m_SentryUserName; + std::string m_SentryHostName; + std::string m_SentryId; + std::shared_ptr m_SentryLogger; +}; + +} // namespace zen +#endif -- cgit v1.2.3 From 68938614c95635045a394ff0a52786b82f01ffc4 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 7 May 2025 10:23:42 +0200 Subject: optimize block store CompactBlocks (#384) - Improvement: Optimize block compact reducing memcpy operations - Improvement: Handle padding of block store blocks when compacting to avoid excessive flusing of write buffer - Improvement: Handle padding when writing oplog index snapshot to avoid unnecessary flushing of write buffer --- src/zencore/include/zencore/basicfile.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h index 57798b6f4..465499d2b 100644 --- a/src/zencore/include/zencore/basicfile.h +++ b/src/zencore/include/zencore/basicfile.h @@ -174,9 +174,11 @@ public: BasicFileWriter(BasicFile& Base, uint64_t BufferSize); ~BasicFileWriter(); - void Write(const void* Data, uint64_t Size, uint64_t FileOffset); - void Write(const CompositeBuffer& Data, uint64_t FileOffset); - void Flush(); + void Write(const void* Data, uint64_t Size, uint64_t FileOffset); + void Write(const CompositeBuffer& Data, uint64_t FileOffset); + void AddPadding(uint64_t Padding); + uint64_t AlignTo(uint64_t Alignment); + void Flush(); private: BasicFile& m_Base; -- cgit v1.2.3 From f3d794f2a8f8ae96760bcab4880d34c589250b6a Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 16 May 2025 12:03:21 +0200 Subject: sparse win file write (#398) * Added `--use-sparse-files` option to `zen builds` command improving write performance of large files. Enabled by default. --- src/zencore/include/zencore/filesystem.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 1bc3943df..e62170eba 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -137,6 +137,8 @@ ZENCORE_API std::filesystem::path GetRunningExecutablePath(); */ ZENCORE_API void MaximizeOpenFileCount(); +ZENCORE_API bool PrepareFileForScatteredWrite(void* FileHandle, uint64_t FinalSize); + struct FileContents { std::vector Data; -- cgit v1.2.3 From 459a8c7a46d1392f04007bf30315dfc86cba7abb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 26 May 2025 22:34:32 +0200 Subject: made fmt use of format_context more consistent (#409) fixes compilation issues on Linux in some cases --- src/zencore/include/zencore/compactbinaryfmt.h | 3 ++- src/zencore/include/zencore/fmtutils.h | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h index ae0c3eb42..b03683db4 100644 --- a/src/zencore/include/zencore/compactbinaryfmt.h +++ b/src/zencore/include/zencore/compactbinaryfmt.h @@ -14,7 +14,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::CbObject& a, format_context& ctx) const + template + auto format(const zen::CbObject& a, FormatContext& ctx) const { zen::ExtendableStringBuilder<1024> ObjStr; zen::CompactBinaryToJson(a, ObjStr); diff --git a/src/zencore/include/zencore/fmtutils.h b/src/zencore/include/zencore/fmtutils.h index 10dfa5393..404e570fd 100644 --- a/src/zencore/include/zencore/fmtutils.h +++ b/src/zencore/include/zencore/fmtutils.h @@ -21,7 +21,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::StringBuilderBase& a, format_context& ctx) const + template + auto format(const zen::StringBuilderBase& a, FormatContext& ctx) const { return fmt::formatter::format(a.ToView(), ctx); } @@ -31,7 +32,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::NiceBase& a, format_context& ctx) const + template + auto format(const zen::NiceBase& a, FormatContext& ctx) const { return fmt::formatter::format(std::string_view(a), ctx); } @@ -98,7 +100,8 @@ template requires DerivedFrom struct fmt::formatter : fmt::formatter { - auto format(const zen::PathBuilderBase& a, format_context& ctx) const + template + auto format(const zen::PathBuilderBase& a, FormatContext& ctx) const { return fmt::formatter::format(a.ToView(), ctx); } -- cgit v1.2.3 From 33d443f5361d007f4971bf0d98585b81ee691437 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 2 Jun 2025 19:14:52 +0200 Subject: http client streaming upload (#413) - Improvement: Add streaming upload from HttpClient to reduce I/O caused by excessive MMap usage --- src/zencore/include/zencore/filesystem.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index e62170eba..dfd0eedc9 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -102,6 +102,10 @@ ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::er */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); +/** Query file size from native file handle + */ +ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); + /** Get a native time tick of last modification time */ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); @@ -167,6 +171,13 @@ ZENCORE_API void ScanFile(void* NativeHandle, uint64_t Size, uint64_t ChunkSize, std::function&& ProcessFunc); +ZENCORE_API void WriteFile(void* NativeHandle, + const void* Data, + uint64_t Size, + uint64_t FileOffset, + uint64_t ChunkSize, + std::error_code& Ec); +ZENCORE_API void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); struct CopyFileOptions { -- cgit v1.2.3 From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- src/zencore/include/zencore/blake3.h | 1 + src/zencore/include/zencore/iohash.h | 6 ++++++ 2 files changed, 7 insertions(+) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/blake3.h b/src/zencore/include/zencore/blake3.h index 28bb348c0..f01e45266 100644 --- a/src/zencore/include/zencore/blake3.h +++ b/src/zencore/include/zencore/blake3.h @@ -53,6 +53,7 @@ struct BLAKE3Stream void Reset(); // Begin streaming hash compute (not needed on freshly constructed instance) BLAKE3Stream& Append(const void* data, size_t byteCount); // Append another chunk BLAKE3Stream& Append(MemoryView DataView) { return Append(DataView.GetData(), DataView.GetSize()); } // Append another chunk + BLAKE3Stream& Append(const IoBuffer& Buffer); // Append another chunk BLAKE3 GetHash(); // Obtain final hash. If you wish to reuse the instance call reset() private: diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h index 7443e17b7..a619b0053 100644 --- a/src/zencore/include/zencore/iohash.h +++ b/src/zencore/include/zencore/iohash.h @@ -102,6 +102,12 @@ struct IoHashStream return *this; } + IoHashStream& Append(const IoBuffer& Buffer) + { + m_Blake3Stream.Append(Buffer); + return *this; + } + /// Append another chunk IoHashStream& Append(MemoryView Data) { -- cgit v1.2.3 From 40b9386054de3c23f77da74eefaa743240d164fd Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Jun 2025 14:40:02 +0200 Subject: pause, resume and abort running builds cmd (#421) - Feature: `zen builds pause`, `zen builds resume` and `zen builds abort` commands to control a running `zen builds` command - `--process-id` the process id to control, if omitted it tries to find a running process using the same executable as itself - Improvement: Process report now indicates if it is pausing or aborting --- src/zencore/include/zencore/filesystem.h | 10 ++++++++++ src/zencore/include/zencore/process.h | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index dfd0eedc9..36d4d1b68 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -385,6 +385,16 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); +class SharedMemory +{ +public: + virtual ~SharedMemory() {} + virtual void* GetData() = 0; +}; + +std::unique_ptr OpenSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal); +std::unique_ptr CreateSharedMemory(std::string_view Name, size_t Size, bool SystemGlobal); + ////////////////////////////////////////////////////////////////////////// void filesystem_forcelink(); // internal diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index d1394cd9a..d3e1de703 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -98,7 +98,7 @@ ZENCORE_API int GetCurrentProcessId(); int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); -std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle); +std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); void process_forcelink(); // internal -- cgit v1.2.3 From feb888db2d557066ebe9eb3bb1d7e3b052ae1221 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Jun 2025 12:22:27 +0200 Subject: `--sentry-dsn` option for zen command line and zenserver to control Sentry reporting endpoint (#427) moved sentry database path to temporary directory for zen commandline --- src/zencore/include/zencore/sentryintegration.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index 40e22af4e..d14c1c275 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,8 +31,12 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, std::string SentryAttachmentsPath, bool AllowPII, const std::string& CommandLine); - void LogStartupInformation(); + void Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + std::string SentryDsn, + bool AllowPII, + const std::string& CommandLine); + void LogStartupInformation(); static void ClearCaches(); private: -- cgit v1.2.3 From cadaad632b05e231dadfc977819189d9c486c74b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:06:40 +0200 Subject: add sentry configurations options for debug/environment add env-variable parsing for sentry option --- src/zencore/include/zencore/sentryintegration.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index d14c1c275..faf1238b7 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,11 +31,17 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine); + struct Config + { + std::string DatabasePath; + std::string AttachmentsPath; + std::string Dsn; + std::string Environment; + bool AllowPII = false; + bool Debug = false; + }; + + void Initialize(const Config& Conf, const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); -- cgit v1.2.3 From f696e52d150ae284e26de5bdd78d1b1edf914314 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 10 Jun 2025 13:08:59 +0200 Subject: revert 61b4a88f and cadaad63 --- src/zencore/include/zencore/sentryintegration.h | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index faf1238b7..d14c1c275 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,17 +31,11 @@ public: SentryIntegration(); ~SentryIntegration(); - struct Config - { - std::string DatabasePath; - std::string AttachmentsPath; - std::string Dsn; - std::string Environment; - bool AllowPII = false; - bool Debug = false; - }; - - void Initialize(const Config& Conf, const std::string& CommandLine); + void Initialize(std::string SentryDatabasePath, + std::string SentryAttachmentsPath, + std::string SentryDsn, + bool AllowPII, + const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); -- cgit v1.2.3 From fdad92dddba8047930c4e7496a7f412d760c312e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Jun 2025 09:30:54 +0200 Subject: sentry config (#430) - Feature: Added `--sentry-environment` to `zen` and `zenserver` - Feature: Added `--sentry-debug` to `zen` and `zenserver` - Feature: Added environment variable parsing for the following options: - `UE_ZEN_SENTRY_ENABLED`: `--no-sentry` (inverted) - `UE_ZEN_SENTRY_DEBUG`: `--sentry-debug` - `UE_ZEN_SENTRY_ALLOWPERSONALINFO`: `--sentry-allow-personal-info` - `UE_ZEN_SENTRY_DSN`: `--sentry-dsn` - `UE_ZEN_SENTRY_ENVIRONMENT`: `--sentry-environment` --- src/zencore/include/zencore/sentryintegration.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index d14c1c275..faf1238b7 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -31,11 +31,17 @@ public: SentryIntegration(); ~SentryIntegration(); - void Initialize(std::string SentryDatabasePath, - std::string SentryAttachmentsPath, - std::string SentryDsn, - bool AllowPII, - const std::string& CommandLine); + struct Config + { + std::string DatabasePath; + std::string AttachmentsPath; + std::string Dsn; + std::string Environment; + bool AllowPII = false; + bool Debug = false; + }; + + void Initialize(const Config& Conf, const std::string& CommandLine); void LogStartupInformation(); static void ClearCaches(); -- cgit v1.2.3 From d000167e12c6dde651ef86be9f67552291ff1b7d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 16 Jun 2025 13:17:54 +0200 Subject: graceful wait in parallelwork destructor (#438) * exception safety when issuing ParallelWork * add asserts to Latch usage to catch usage errors * extended error messaging and recovery handling in ParallelWork destructor to help find issues --- src/zencore/include/zencore/thread.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/zencore/include') diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index 8fb781571..d9fb5c023 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -183,6 +183,7 @@ public: void CountDown() { std::ptrdiff_t Old = Counter.fetch_sub(1); + ZEN_ASSERT(Old > 0); if (Old == 1) { Complete.Set(); @@ -197,8 +198,7 @@ public: void AddCount(std::ptrdiff_t Count) { std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); - ZEN_UNUSED(Old); - ZEN_ASSERT_SLOW(Old > 0); + ZEN_ASSERT(Old > 0); } bool Wait(int TimeoutMs = -1) -- cgit v1.2.3