improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

3 files changed, 17 insertions, 13 deletions

diff --git a/src/zencore/include/zencore/basicfile.h b/src/zencore/include/zencore/basicfile.h
index 7edd40c9c..a78132879 100644
--- a/src/zencore/include/zencore/basicfile.h
+++ b/src/zencore/include/zencore/basicfile.h
@@ -60,6 +60,7 @@ public:
 	void	 StreamByteRange(uint64_t FileOffset, uint64_t Size, std::function<void(const void* Data, uint64_t Size)>&& ChunkFun);
 	void	 Write(MemoryView Data, uint64_t FileOffset);
 	void	 Write(MemoryView Data, uint64_t FileOffset, std::error_code& Ec);
+	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	uint64_t Write(const CompositeBuffer& Data, uint64_t FileOffset, std::error_code& Ec);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset);
 	void	 Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::error_code& Ec);
@@ -174,6 +175,7 @@ public:
 	~BasicFileWriter();
 
 	void Write(const void* Data, uint64_t Size, uint64_t FileOffset);
+	void Write(const CompositeBuffer& Data, uint64_t FileOffset);
 	void Flush();
 
 private:
@@ -184,9 +186,7 @@ private:
 	uint64_t	   m_BufferEnd;
 };
 
-IoBuffer WriteToTempFile(CompositeBuffer&&							Buffer,
-						 const std::filesystem::path&				Path,
-						 std::function<bool(std::error_code& Ec)>&& RetryCallback);
+IoBuffer WriteToTempFile(const CompositeBuffer& Buffer, const std::filesystem::path& Path);
 
 ZENCORE_API void basicfile_forcelink();
 
diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h
index ca8682cd7..250745e86 100644
--- a/src/zencore/include/zencore/filesystem.h
+++ b/src/zencore/include/zencore/filesystem.h
@@ -203,7 +203,8 @@ public:
 		virtual void VisitFile(const std::filesystem::path& Parent,
 							   const path_view&				File,
 							   uint64_t						FileSize,
-							   uint32_t						NativeModeOrAttributes) = 0;
+							   uint32_t						NativeModeOrAttributes,
+							   uint64_t						NativeModificationTick) = 0;
 
 		// This should return true if we should recurse into the directory
 		virtual bool VisitDirectory(const std::filesystem::path& Parent,
@@ -216,13 +217,14 @@ public:
 
 enum class DirectoryContentFlags : uint8_t
 {
-	None			  = 0,
-	IncludeDirs		  = 1u << 0,
-	IncludeFiles	  = 1u << 1,
-	Recursive		  = 1u << 2,
-	IncludeFileSizes  = 1u << 3,
-	IncludeAttributes = 1u << 4,
-	IncludeAllEntries = IncludeDirs | IncludeFiles | Recursive
+	None					= 0,
+	IncludeDirs				= 1u << 0,
+	IncludeFiles			= 1u << 1,
+	Recursive				= 1u << 2,
+	IncludeFileSizes		= 1u << 3,
+	IncludeAttributes		= 1u << 4,
+	IncludeModificationTick = 1u << 5,
+	IncludeAllEntries		= IncludeDirs | IncludeFiles | Recursive
 };
 
 ENUM_CLASS_FLAGS(DirectoryContentFlags)
@@ -232,6 +234,7 @@ struct DirectoryContent
 	std::vector<std::filesystem::path> Files;
 	std::vector<uint64_t>			   FileSizes;
 	std::vector<uint32_t>			   FileAttributes;
+	std::vector<uint64_t>			   FileModificationTicks;
 	std::vector<std::filesystem::path> Directories;
 	std::vector<uint32_t>			   DirectoryAttributes;
 };
@@ -246,6 +249,7 @@ public:
 		std::vector<std::filesystem::path> FileNames;
 		std::vector<uint64_t>			   FileSizes;
 		std::vector<uint32_t>			   FileAttributes;
+		std::vector<uint64_t>			   FileModificationTicks;
 		std::vector<std::filesystem::path> DirectoryNames;
 		std::vector<uint32_t>			   DirectoryAttributes;
 	};
diff --git a/src/zencore/include/zencore/iohash.h b/src/zencore/include/zencore/iohash.h
index 8871a5895..7443e17b7 100644
--- a/src/zencore/include/zencore/iohash.h
+++ b/src/zencore/include/zencore/iohash.h
@@ -47,8 +47,8 @@ struct IoHash
 
 	static IoHash	   HashBuffer(const void* data, size_t byteCount);
 	static IoHash	   HashBuffer(MemoryView Data) { return HashBuffer(Data.GetData(), Data.GetSize()); }
-	static IoHash	   HashBuffer(const CompositeBuffer& Buffer);
-	static IoHash	   HashBuffer(const IoBuffer& Buffer);
+	static IoHash	   HashBuffer(const CompositeBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
+	static IoHash	   HashBuffer(const IoBuffer& Buffer, std::atomic<uint64_t>* ProcessedBytes = nullptr);
 	static IoHash	   FromHexString(const char* string);
 	static IoHash	   FromHexString(const std::string_view string);
 	static bool		   TryParse(std::string_view Str, IoHash& Hash);