improvements and infrastructure for upcoming builds api command line (#284)

* add modification tick to filesystem traversal
* add ShowDetails option to ProgressBar
* log callstack if we terminate process
* handle chunking if MaxSize > 1MB
* BasicFile write helpers and WriteToTempFile simplifications
* bugfix for CompositeBuffer::IterateRange when using DecompressToComposite for actually comrpessed data
revert of earlier optimization
* faster compress/decompress for large disk-based files
* enable progress feedback in IoHash::HashBuffer
* add payload validation in HttpClient::Get
* fix range requests (range is including end byte)
* remove BuildPartId for blob/block related operations in builds api

1 files changed, 161 insertions, 53 deletions

diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp
index 29c1d9256..0e2ce2b54 100644
--- a/src/zencore/compress.cpp
+++ b/src/zencore/compress.cpp
@@ -2,6 +2,7 @@
 
 #include <zencore/compress.h>
 
+#include <zencore/basicfile.h>
 #include <zencore/blake3.h>
 #include <zencore/compositebuffer.h>
 #include <zencore/crc32.h>
@@ -314,37 +315,77 @@ BlockEncoder::Compress(const CompositeBuffer& RawData, const uint64_t BlockSize)
 	CompressedBlockSizes.reserve(BlockCount);
 	uint64_t CompressedSize = 0;
 	{
-		UniqueBuffer	  RawBlockCopy;
 		MutableMemoryView CompressedBlocksView = CompressedData.GetMutableView() + sizeof(BufferHeader) + MetaSize;
 
-		CompositeBuffer::Iterator It = RawData.GetIterator(0);
-
-		for (uint64_t RawOffset = 0; RawOffset < RawSize;)
+		IoBufferFileReference FileRef = {nullptr, 0, 0};
+		if ((RawData.GetSegments().size() == 1) && RawData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 		{
-			const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
-			const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
-			RawHash.Append(RawBlock);
-
-			MutableMemoryView CompressedBlock = CompressedBlocksView;
-			if (!CompressBlock(CompressedBlock, RawBlock))
+			ZEN_ASSERT(FileRef.FileHandle != nullptr);
+			UniqueBuffer RawBlockCopy = UniqueBuffer::Alloc(BlockSize);
+			BasicFile	 Source;
+			Source.Attach(FileRef.FileHandle);
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				return CompositeBuffer();
-			}
+				const uint64_t RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				Source.Read(RawBlockCopy.GetData(), RawBlockSize, FileRef.FileChunkOffset + RawOffset);
+				const MemoryView RawBlock = RawBlockCopy.GetView().Left(RawBlockSize);
+				RawHash.Append(RawBlock);
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					Source.Detach();
+					return CompositeBuffer();
+				}
 
-			uint64_t CompressedBlockSize = CompressedBlock.GetSize();
-			if (RawBlockSize <= CompressedBlockSize)
-			{
-				CompressedBlockSize	 = RawBlockSize;
-				CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
 			}
-			else
+			Source.Detach();
+		}
+		else
+		{
+			UniqueBuffer			  RawBlockCopy;
+			CompositeBuffer::Iterator It = RawData.GetIterator(0);
+
+			for (uint64_t RawOffset = 0; RawOffset < RawSize;)
 			{
-				CompressedBlocksView += CompressedBlockSize;
-			}
+				const uint64_t	 RawBlockSize = zen::Min(RawSize - RawOffset, BlockSize);
+				const MemoryView RawBlock	  = RawData.ViewOrCopyRange(It, RawBlockSize, RawBlockCopy);
+				RawHash.Append(RawBlock);
 
-			CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
-			CompressedSize += CompressedBlockSize;
-			RawOffset += RawBlockSize;
+				MutableMemoryView CompressedBlock = CompressedBlocksView;
+				if (!CompressBlock(CompressedBlock, RawBlock))
+				{
+					return CompositeBuffer();
+				}
+
+				uint64_t CompressedBlockSize = CompressedBlock.GetSize();
+				if (RawBlockSize <= CompressedBlockSize)
+				{
+					CompressedBlockSize	 = RawBlockSize;
+					CompressedBlocksView = CompressedBlocksView.CopyFrom(RawBlock);
+				}
+				else
+				{
+					CompressedBlocksView += CompressedBlockSize;
+				}
+
+				CompressedBlockSizes.push_back(static_cast<uint32_t>(CompressedBlockSize));
+				CompressedSize += CompressedBlockSize;
+				RawOffset += RawBlockSize;
+			}
 		}
 	}
 
@@ -560,51 +601,118 @@ BlockDecoder::TryDecompressTo(const BufferHeader&	 Header,
 		CompressedOffset += CompressedBlockSize;
 	}
 
-	for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+	IoBufferFileReference FileRef = {nullptr, 0, 0};
+	if ((CompressedData.GetSegments().size() == 1) && CompressedData.GetSegments()[0].AsIoBuffer().GetFileReference(FileRef))
 	{
-		const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
-		const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
-		const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
+		ZEN_ASSERT(FileRef.FileHandle != nullptr);
+		BasicFile Source;
+		Source.Attach(FileRef.FileHandle);
 
-		const uint64_t BytesToUncompress = OffsetInFirstBlock > 0 ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
-																  : zen::Min(RemainingRawSize, BlockSize);
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
+		{
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
 
-		if (IsCompressed)
-		{
-			MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+			if (CompressedBlockCopy.GetSize() < CompressedBlockSize)
+			{
+				CompressedBlockCopy = UniqueBuffer::Alloc(CompressedBlockSize);
+			}
+			Source.Read(CompressedBlockCopy.GetData(), CompressedBlockSize, FileRef.FileChunkOffset + CompressedOffset);
 
-			const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
-			if (!IsAligned)
+			MemoryView CompressedBlock = CompressedBlockCopy.GetView().Left(CompressedBlockSize);
+
+			if (IsCompressed)
 			{
-				// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
-				if (UncompressedBlockCopy.IsNull())
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
 				{
-					UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
 				}
-				UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
-			}
 
-			if (!DecompressBlock(UncompressedBlock, CompressedBlock))
-			{
-				return false;
-			}
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					Source.Detach();
+					return false;
+				}
 
-			if (!IsAligned)
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
 			{
-				RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
 			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
 		}
-		else
+		Source.Detach();
+	}
+	else
+	{
+		for (size_t BlockIndex = FirstBlockIndex; BlockIndex <= LastBlockIndex; BlockIndex++)
 		{
-			RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
-		}
+			const uint64_t UncompressedBlockSize = BlockIndex == Header.BlockCount - 1 ? LastBlockSize : BlockSize;
+			const uint32_t CompressedBlockSize	 = ByteSwap(CompressedBlockSizes[BlockIndex]);
+			const bool	   IsCompressed			 = CompressedBlockSize < UncompressedBlockSize;
 
-		OffsetInFirstBlock = 0;
-		RemainingRawSize -= BytesToUncompress;
-		CompressedOffset += CompressedBlockSize;
-		RawView += BytesToUncompress;
+			const uint64_t BytesToUncompress = OffsetInFirstBlock > 0
+												   ? zen::Min(RawView.GetSize(), UncompressedBlockSize - OffsetInFirstBlock)
+												   : zen::Min(RemainingRawSize, BlockSize);
+
+			MemoryView CompressedBlock = CompressedData.ViewOrCopyRange(CompressedOffset, CompressedBlockSize, CompressedBlockCopy);
+
+			if (IsCompressed)
+			{
+				MutableMemoryView UncompressedBlock = RawView.Left(BytesToUncompress);
+
+				const bool IsAligned = BytesToUncompress == UncompressedBlockSize;
+				if (!IsAligned)
+				{
+					// Decompress to a temporary buffer when the first or the last block reads are not aligned with the block boundaries.
+					if (UncompressedBlockCopy.IsNull())
+					{
+						UncompressedBlockCopy = UniqueBuffer::Alloc(BlockSize);
+					}
+					UncompressedBlock = UncompressedBlockCopy.GetMutableView().Mid(0, UncompressedBlockSize);
+				}
+
+				if (!DecompressBlock(UncompressedBlock, CompressedBlock))
+				{
+					return false;
+				}
+
+				if (!IsAligned)
+				{
+					RawView.CopyFrom(UncompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+				}
+			}
+			else
+			{
+				RawView.CopyFrom(CompressedBlock.Mid(OffsetInFirstBlock, BytesToUncompress));
+			}
+
+			OffsetInFirstBlock = 0;
+			RemainingRawSize -= BytesToUncompress;
+			CompressedOffset += CompressedBlockSize;
+			RawView += BytesToUncompress;
+		}
 	}
 
 	return RemainingRawSize == 0;