improved block gen logic (#302)

 - Improvement: Reduced memory usage during upload and part upload validation
 - Improvement: Reduced I/O usage during upload and download
 - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild
 - Improvement: More trace scopes for build upload operations
 - Bugfix: Fixed crash during download when trying to write outside a file range

1 files changed, 6 insertions, 0 deletions

diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp
index 4f9344039..a2c041ffd 100644
--- a/src/zenutil/chunkedfile.cpp
+++ b/src/zenutil/chunkedfile.cpp
@@ -3,6 +3,7 @@
 #include <zenutil/chunkedfile.h>
 
 #include <zencore/basicfile.h>
+#include <zencore/trace.h>
 
 #include "chunking.h"
 
@@ -33,6 +34,7 @@ namespace {
 IoBuffer
 SerializeChunkedInfo(const ChunkedInfo& Info)
 {
+	ZEN_TRACE_CPU("SerializeChunkedInfo");
 	size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) +
 						RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16);
 	IoBuffer HeaderData(HeaderSize);
@@ -65,6 +67,7 @@ SerializeChunkedInfo(const ChunkedInfo& Info)
 ChunkedInfo
 DeserializeChunkedInfo(IoBuffer& Buffer)
 {
+	ZEN_TRACE_CPU("DeserializeChunkedInfo");
 	MemoryView	  View = Buffer.GetView();
 	ChunkedHeader Header;
 	{
@@ -99,6 +102,7 @@ DeserializeChunkedInfo(IoBuffer& Buffer)
 void
 Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk)
 {
+	ZEN_TRACE_CPU("Reconstruct");
 	BasicFile Reconstructed;
 	Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate);
 	BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024);
@@ -119,6 +123,8 @@ ChunkData(BasicFile&			 RawData,
 		  std::atomic<uint64_t>* BytesProcessed,
 		  std::atomic<bool>*	 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkData");
+
 	ChunkedInfoWithSource							 Result;
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> FoundChunks;