Merge remote-tracking branch 'origin/main' into de/zen-service-command

13 files changed, 196 insertions, 26 deletions

diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index 03c5605df..57798b6f4 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -46,6 +46,10 @@ public:
 		kPreventWrite  = 0x2000'0000,  // Do not open with write sharing mode (prevent other processes from writing to file while open)
 	};
 
+	BasicFile(const std::filesystem::path& FileName, Mode Mode);
+	BasicFile(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
+	BasicFile(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
+
 	void	 Open(const std::filesystem::path& FileName, Mode Mode);
 	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::error_code& Ec);
 	void	 Open(const std::filesystem::path& FileName, Mode Mode, std::function<bool(std::error_code& Ec)>&& RetryCallback);
@@ -56,7 +60,8 @@ public:
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	void	 Write(MemoryView Data, uint64_t FileOffset);
 	void	 Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
-	uint64_t Write(CompositeBuffer Data, uint64_t FileOffset, std::error_code& Ec);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
 	void	 Flush();
@@ -170,6 +175,7 @@ public:
 	~BasicFileWriter();
 
 	void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	void Flush();
 
 private:
@@ -180,6 +186,8 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
+IoBuffer WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path);
+
 ZENCORE_API void basicfile_forcelink();
 
 }  // namespace zen
diff --git a/src/zencore/include/zencore/compactbinarybuilder.h b/src/zencore/include/zencore/compactbinarybuilder.h
index 1c625cacc..f11717453 100644
--- a/src/zencore/include/zencore/compactbinarybuilder.h
+++ b/src/zencore/include/zencore/compactbinarybuilder.h
@@ -18,6 +18,8 @@
 #include <type_traits>
 #include <vector>
 
+#include <EASTL/fixed_vector.h>
+
 #include <gsl/gsl-lite.hpp>
 
 namespace zen {
@@ -367,6 +369,8 @@ public:
 	/** Private flags that are public to work with ENUM_CLASS_FLAGS. */
 	enum class StateFlags : uint8_t;
 
+	typedef eastl::fixed_vector<uint8_t, 2048> CbWriterData_t;
+
 protected:
 	/** Reserve the specified size up front until the format is optimized. */
 	ZENCORE_API explicit CbWriter(int64_t InitialSize);
@@ -409,8 +413,8 @@ private:
 	// provided externally, such as on the stack. That format will store the offsets that require
 	// object or array sizes to be inserted and field types to be removed, and will perform those
 	// operations only when saving to a buffer.
-	std::vector<uint8_t>	 Data;
-	std::vector<WriterState> States;
+	eastl::fixed_vector<WriterState, 4> States;
+	CbWriterData_t						Data;
 };
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/src/zencore/include/zencore/compactbinaryfmt.h b/src/zencore/include/zencore/compactbinaryfmt.h
new file mode 100644
index 000000000..ae0c3eb42
--- /dev/null
+++ b/src/zencore/include/zencore/compactbinaryfmt.h
@@ -0,0 +1,23 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <zencore/compactbinary.h>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <fmt/format.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+#include <string_view>
+
+template<typename T>
+requires DerivedFrom<T, zen::CbObjectView>
+struct fmt::formatter<T> : fmt::formatter<std::string_view>
+{
+	auto format(const zen::CbObject& a, format_context& ctx) const
+	{
+		zen::ExtendableStringBuilder<1024> ObjStr;
+		zen::CompactBinaryToJson(a, ObjStr);
+		return fmt::formatter<std::string_view>::format(ObjStr.ToView(), ctx);
+	}
+};
diff --git a/src/zencore/include/zencore/compactbinarypackage.h b/src/zencore/include/zencore/compactbinarypackage.h
index 12fcc41b7..9ec12cb0f 100644
--- a/src/zencore/include/zencore/compactbinarypackage.h
+++ b/src/zencore/include/zencore/compactbinarypackage.h
@@ -12,6 +12,8 @@
 #include <span>
 #include <variant>
 
+#include <EASTL/fixed_vector.h>
+
 #ifdef GetObject
 #	error "windows.h pollution"
 #	undef GetObject
@@ -265,7 +267,10 @@ public:
 	}
 
 	/** Returns the attachments in this package. */
-	inline std::span<const CbAttachment> GetAttachments() const { return Attachments; }
+	inline std::span<const CbAttachment> GetAttachments() const
+	{
+		return std::span<const CbAttachment>(begin(Attachments), end(Attachments));
+	}
 
 	/**
 	 * Find an attachment by its hash.
@@ -286,6 +291,8 @@ public:
 
 	void AddAttachments(std::span<const CbAttachment> Attachments);
 
+	void ReserveAttachments(size_t Count);
+
 	/**
 	 * Remove an attachment by hash.
 	 *
@@ -324,9 +331,9 @@ private:
 	void GatherAttachments(const CbObject& Object, AttachmentResolver Resolver);
 
 	/** Attachments ordered by their hash. */
-	std::vector<CbAttachment> Attachments;
-	CbObject				  Object;
-	IoHash					  ObjectHash;
+	eastl::fixed_vector<CbAttachment, 32> Attachments;
+	CbObject							  Object;
+	IoHash								  ObjectHash;
 };
 
 namespace legacy {
diff --git a/src/zencore/include/zencore/compositebuffer.h b/src/zencore/include/zencore/compositebuffer.h
index b435c5e74..1e1611de9 100644
--- a/src/zencore/include/zencore/compositebuffer.h
+++ b/src/zencore/include/zencore/compositebuffer.h
@@ -2,6 +2,7 @@
 
 #pragma once
 
+#include <zencore/eastlutil.h>
 #include <zencore/sharedbuffer.h>
 #include <zencore/zencore.h>
 
@@ -9,6 +10,8 @@
 #include <span>
 #include <vector>
 
+#include <EASTL/fixed_vector.h>
+
 namespace zen {
 
 /**
@@ -35,7 +38,7 @@ public:
 		{
 			m_Segments.reserve((GetBufferCount(std::forward<BufferTypes>(Buffers)) + ...));
 			(AppendBuffers(std::forward<BufferTypes>(Buffers)), ...);
-			std::erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); });
+			erase_if(m_Segments, [](const SharedBuffer& It) { return It.IsNull(); });
 		}
 	}
 
@@ -46,7 +49,10 @@ public:
 	[[nodiscard]] ZENCORE_API uint64_t GetSize() const;
 
 	/** Returns the segments that the buffer is composed from. */
-	[[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const { return std::span<const SharedBuffer>{m_Segments}; }
+	[[nodiscard]] inline std::span<const SharedBuffer> GetSegments() const
+	{
+		return std::span<const SharedBuffer>{begin(m_Segments), end(m_Segments)};
+	}
 
 	/** Returns true if the composite buffer is not null. */
 	[[nodiscard]] inline explicit operator bool() const { return !IsNull(); }
@@ -120,6 +126,8 @@ public:
 	static const CompositeBuffer Null;
 
 private:
+	typedef eastl::fixed_vector<SharedBuffer, 4> SharedBufferVector_t;
+
 	static inline size_t GetBufferCount(const CompositeBuffer& Buffer) { return Buffer.m_Segments.size(); }
 	inline void			 AppendBuffers(const CompositeBuffer& Buffer)
 	{
@@ -134,12 +142,25 @@ private:
 	inline void			 AppendBuffers(SharedBuffer&& Buffer) { m_Segments.push_back(std::move(Buffer)); }
 	inline void			 AppendBuffers(IoBuffer&& Buffer) { m_Segments.push_back(SharedBuffer(std::move(Buffer))); }
 
+	static inline size_t GetBufferCount(std::span<IoBuffer>&& Container) { return Container.size(); }
+	inline void			 AppendBuffers(std::span<IoBuffer>&& Container)
+	{
+		m_Segments.reserve(m_Segments.size() + Container.size());
+		for (IoBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(SharedBuffer(std::move(Buffer)));
+		}
+	}
+
 	static inline size_t GetBufferCount(std::vector<SharedBuffer>&& Container) { return Container.size(); }
 	static inline size_t GetBufferCount(std::vector<IoBuffer>&& Container) { return Container.size(); }
 	inline void			 AppendBuffers(std::vector<SharedBuffer>&& Container)
 	{
 		m_Segments.reserve(m_Segments.size() + Container.size());
-		m_Segments.insert(m_Segments.end(), std::make_move_iterator(Container.begin()), std::make_move_iterator(Container.end()));
+		for (SharedBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(std::move(Buffer));
+		}
 	}
 	inline void AppendBuffers(std::vector<IoBuffer>&& Container)
 	{
@@ -150,8 +171,17 @@ private:
 		}
 	}
 
+	inline void AppendBuffers(SharedBufferVector_t&& Container)
+	{
+		m_Segments.reserve(m_Segments.size() + Container.size());
+		for (SharedBuffer& Buffer : Container)
+		{
+			m_Segments.emplace_back(std::move(Buffer));
+		}
+	}
+
 private:
-	std::vector<SharedBuffer> m_Segments;
+	SharedBufferVector_t m_Segments;
 };
 
 void compositebuffer_forcelink();  // internal
diff --git a/src/zencore/include/zencore/compress.h b/src/zencore/include/zencore/compress.h
index 5e761ceef..74fd5f767 100644
--- a/src/zencore/include/zencore/compress.h
+++ b/src/zencore/include/zencore/compress.h
@@ -74,6 +74,11 @@ public:
 															   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
 															   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
 															   uint64_t				 BlockSize		  = 0);
+	[[nodiscard]] ZENCORE_API static bool			  CompressToStream(const CompositeBuffer&												RawData,
+																	   std::function<void(uint64_t Offset, const CompositeBuffer& Range)>&& Callback,
+																	   OodleCompressor		 Compressor		  = OodleCompressor::Mermaid,
+																	   OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast,
+																	   uint64_t				 BlockSize		  = 0);
 
 	/**
 	 * Construct from a compressed buffer previously created by Compress().
@@ -196,6 +201,16 @@ public:
 	 */
 	[[nodiscard]] ZENCORE_API CompositeBuffer DecompressToComposite() const;
 
+	/**
+	 * Decompress into and call callback for ranges of decompressed data.
+	 * The buffer in the callback will be overwritten when the callback returns.
+	 *
+	 * @return True if the buffer is valid and can be decompressed.
+	 */
+	[[nodiscard]] ZENCORE_API bool DecompressToStream(uint64_t															   RawOffset,
+													  uint64_t															   RawSize,
+													  std::function<bool(uint64_t Offset, const CompositeBuffer& Range)>&& Callback) const;
+
 	/** A null compressed buffer. */
 	static const CompressedBuffer Null;
 
diff --git a/src/zencore/include/zencore/eastlutil.h b/src/zencore/include/zencore/eastlutil.h
new file mode 100644
index 000000000..642321dae
--- /dev/null
+++ b/src/zencore/include/zencore/eastlutil.h
@@ -0,0 +1,20 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include <algorithm>
+
+namespace zen {
+
+size_t
+erase_if(auto& _Cont, auto Predicate)
+{
+	auto	   _First	 = _Cont.begin();
+	const auto _Last	 = _Cont.end();
+	const auto _Old_size = _Cont.size();
+	_First				 = std::remove_if(_First, _Last, Predicate);
+	_Cont.erase(_First, _Last);
+	return _Old_size - _Cont.size();
+}
+
+}  // namespace zen
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index ca8682cd7..9a2b15d1d 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -52,6 +52,10 @@ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle);
  */
 ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec);
 
+/** Get a native time tick of last modification time
+ */
+ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename);
+
 ZENCORE_API std::filesystem::path GetRunningExecutablePath();
 
 /** Set the max open file handle count to max allowed for the current process on Linux and MacOS
@@ -93,11 +97,11 @@ struct CopyFileOptions
 	bool MustClone	 = false;
 };
 
-ZENCORE_API bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
-ZENCORE_API void CopyFile(std::filesystem::path	 FromPath,
-						  std::filesystem::path	 ToPath,
-						  const CopyFileOptions& Options,
-						  std::error_code&		 OutError);
+ZENCORE_API bool CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToPath, const CopyFileOptions& Options);
+ZENCORE_API void CopyFile(const std::filesystem::path& FromPath,
+						  const std::filesystem::path& ToPath,
+						  const CopyFileOptions&	   Options,
+						  std::error_code&			   OutError);
 ZENCORE_API void CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options);
 ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path);
 
@@ -203,7 +207,8 @@ public:
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) = 0;
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) = 0;
 
 		// This should return true if we should recurse into the directory
 		virtual bool VisitDirectory(const std::filesystem::path& Parent,
@@ -216,13 +221,14 @@ public:
 
 enum class DirectoryContentFlags : uint8_t
 {
-	None			  = 0,
-	IncludeDirs		  = 1u << 0,
-	IncludeFiles	  = 1u << 1,
-	Recursive		  = 1u << 2,
-	IncludeFileSizes  = 1u << 3,
-	IncludeAttributes = 1u << 4,
-	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+	None					= 0,
+	IncludeDirs				= 1u << 0,
+	IncludeFiles			= 1u << 1,
+	Recursive				= 1u << 2,
+	IncludeFileSizes		= 1u << 3,
+	IncludeAttributes		= 1u << 4,
+	IncludeModificationTick = 1u << 5,
+	IncludeAllEntries		= IncludeDirs | IncludeFiles | Recursive
 };
 
 ENUM_CLASS_FLAGS(DirectoryContentFlags)
@@ -232,6 +238,7 @@ struct DirectoryContent
 	std::vector<std::filesystem::path> Files;
 	std::vector<uint64_t>			   FileSizes;
 	std::vector<uint32_t>			   FileAttributes;
+	std::vector<uint64_t>			   FileModificationTicks;
 	std::vector<std::filesystem::path> Directories;
 	std::vector<uint32_t>			   DirectoryAttributes;
 };
@@ -246,6 +253,7 @@ public:
 		std::vector<std::filesystem::path> FileNames;
 		std::vector<uint64_t>			   FileSizes;
 		std::vector<uint32_t>			   FileAttributes;
+		std::vector<uint64_t>			   FileModificationTicks;
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
@@ -267,6 +275,25 @@ std::error_code RotateDirectories(const std::filesystem::path& DirectoryName, st
 
 std::filesystem::path PickDefaultSystemRootDirectory();
 
+#if ZEN_PLATFORM_WINDOWS
+uint32_t GetFileAttributes(const std::filesystem::path& Filename);
+void	 SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes);
+#endif	// ZEN_PLATFORM_WINDOWS
+
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+uint32_t GetFileMode(const std::filesystem::path& Filename);
+void	 SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes);
+#endif	// ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+
+bool	 IsFileAttributeReadOnly(uint32_t FileAttributes);
+bool	 IsFileModeReadOnly(uint32_t FileMode);
+uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly);
+uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly);
+
+bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly);
+
+std::filesystem::path StringToPath(const std::string_view& Path);
+
 //////////////////////////////////////////////////////////////////////////
 
 void filesystem_forcelink();  // internal
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
index 8871a5895..7443e17b7 100644
--- a/src/zencore/include/zencore/iohash.h
+++ b/src/zencore/include/zencore/iohash.h
@@ -47,8 +47,8 @@ struct IoHash
 
 	static IoHash	   HashBuffer(const void* data, size_t byteCount);
 	static IoHash	   HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); }
-	static IoHash	   HashBuffer(const CompositeBuffer& Buffer);
-	static IoHash	   HashBuffer(const IoBuffer& Buffer);
+	static IoHash	   HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
+	static IoHash	   HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
 	static IoHash	   FromHexString(const char* string);
 	static IoHash	   FromHexString(const std::string_view string);
 	static bool		   TryParse(std::string_view Str, IoHash& Hash);
diff --git a/src/zencore/include/zencore/memory/newdelete.h b/src/zencore/include/zencore/memory/newdelete.h
index d22c8604f..059f1d5ea 100644
--- a/src/zencore/include/zencore/memory/newdelete.h
+++ b/src/zencore/include/zencore/memory/newdelete.h
@@ -153,3 +153,29 @@ operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexce
 	return zen_new_aligned_nothrow(n, static_cast<size_t>(al));
 }
 #endif
+
+// EASTL operator new
+
+void*
+operator new[](size_t size, const char* pName, int flags, unsigned debugFlags, const char* file, int line)
+{
+	ZEN_UNUSED(pName, flags, debugFlags, file, line);
+	return zen_new(size);
+}
+
+void*
+operator new[](size_t	   size,
+			   size_t	   alignment,
+			   size_t	   alignmentOffset,
+			   const char* pName,
+			   int		   flags,
+			   unsigned	   debugFlags,
+			   const char* file,
+			   int		   line)
+{
+	ZEN_UNUSED(alignmentOffset, pName, flags, debugFlags, file, line);
+
+	ZEN_ASSERT_SLOW(alignmentOffset == 0);	// currently not supported
+
+	return zen_new_aligned(size, alignment);
+}
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 36c2a2481..335e3d909 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -101,6 +101,9 @@ int				 GetProcessId(CreateProcResult ProcId);
 std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc);
 std::error_code		  FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle);
 
+std::vector<std::string> ParseCommandLine(std::string_view CommandLine);
+std::vector<char*>		 StripCommandlineQuotes(std::vector<std::string>& InOutArgs);
+
 #if ZEN_PLATFORM_LINUX
 void IgnoreChildSignals();
 #endif
diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h
index e2ef1c1a0..68129b691 100644
--- a/src/zencore/include/zencore/string.h
+++ b/src/zencore/include/zencore/string.h
@@ -522,6 +522,9 @@ public:
 
 //////////////////////////////////////////////////////////////////////////
 
+bool							 IsValidUtf8(const std::string_view& str);
+std::string_view::const_iterator FindFirstInvalidUtf8Byte(const std::string_view& str);
+
 void		 Utf8ToWide(const char8_t* str, WideStringBuilderBase& out);
 void		 Utf8ToWide(const std::u8string_view& wstr, WideStringBuilderBase& out);
 void		 Utf8ToWide(const std::string_view& wstr, WideStringBuilderBase& out);
diff --git a/src/zencore/include/zencore/timer.h b/src/zencore/include/zencore/timer.h
index e4ddc3505..767dc4314 100644
--- a/src/zencore/include/zencore/timer.h
+++ b/src/zencore/include/zencore/timer.h
@@ -21,6 +21,10 @@ ZENCORE_API uint64_t GetHifreqTimerFrequency();
 ZENCORE_API double	 GetHifreqTimerToSeconds();
 ZENCORE_API uint64_t GetHifreqTimerFrequencySafe();	 // May be used during static init
 
+// Query time since process was spawned (returns time in ms)
+
+uint64_t GetTimeSinceProcessStart();
+
 class Stopwatch
 {
 public: