improved block gen logic (#302)

 - Improvement: Reduced memory usage during upload and part upload validation
 - Improvement: Reduced I/O usage during upload and download
 - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild
 - Improvement: More trace scopes for build upload operations
 - Bugfix: Fixed crash during download when trying to write outside a file range

5 files changed, 42 insertions, 4 deletions

diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index 4ca89d996..bb1ee5183 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -96,6 +96,8 @@ namespace {
 					   uint32_t											 PathIndex,
 					   std::atomic<bool>&								 AbortFlag)
 	{
+		ZEN_TRACE_CPU("ChunkFolderContent");
+
 		const uint64_t				 RawSize = OutChunkedContent.RawSizes[PathIndex];
 		const std::filesystem::path& Path	 = OutChunkedContent.Paths[PathIndex];
 
@@ -136,6 +138,8 @@ namespace {
 			}
 			else
 			{
+				ZEN_TRACE_CPU("HashOnly");
+
 				IoBuffer	 Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
 				const IoHash Hash	= IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
 
@@ -228,6 +232,7 @@ FolderContent::operator==(const FolderContent& Rhs) const
 bool
 FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
 {
+	ZEN_TRACE_CPU("FolderContent::AreKnownFilesEqual");
 	tsl::robin_map<std::string, size_t> RhsPathToIndex;
 	const size_t						RhsPathCount = Rhs.Paths.size();
 	RhsPathToIndex.reserve(RhsPathCount);
@@ -259,6 +264,7 @@ FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
 void
 FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutPathIndexesOufOfDate)
 {
+	ZEN_TRACE_CPU("FolderContent::UpdateState");
 	tsl::robin_map<std::string, uint32_t> RhsPathToIndex;
 	const uint32_t						  RhsPathCount = gsl::narrow<uint32_t>(Rhs.Paths.size());
 	RhsPathToIndex.reserve(RhsPathCount);
@@ -297,6 +303,7 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutP
 FolderContent
 GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes)
 {
+	ZEN_TRACE_CPU("FolderContent::GetUpdatedContent");
 	FolderContent						  Result = {.Platform = Old.Platform};
 	tsl::robin_map<std::string, uint32_t> NewPathToIndex;
 	const uint32_t						  NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
@@ -332,6 +339,7 @@ GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vecto
 void
 SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
 {
+	ZEN_TRACE_CPU("SaveFolderContentToCompactBinary");
 	Output.AddString("platform"sv, ToString(Content.Platform));
 	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
 	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
@@ -342,6 +350,7 @@ SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
 FolderContent
 LoadFolderContentToCompactBinary(CbObjectView Input)
 {
+	ZEN_TRACE_CPU("LoadFolderContentToCompactBinary");
 	FolderContent Content;
 	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
 	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
@@ -494,6 +503,7 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 void
 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output)
 {
+	ZEN_TRACE_CPU("SaveChunkedFolderContentToCompactBinary");
 	Output.AddString("platform"sv, ToString(Content.Platform));
 	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
 	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
@@ -512,6 +522,7 @@ SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbW
 ChunkedFolderContent
 LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 {
+	ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary");
 	ChunkedFolderContent Content;
 	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
 	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
@@ -788,7 +799,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 
 	struct ChunkLocationReference
 	{
-		uint32_t									ChunkIndex;
+		uint32_t									ChunkIndex = (uint32_t)-1;
 		ChunkedContentLookup::ChunkSequenceLocation Location;
 	};
 
@@ -853,7 +864,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 	{
 		Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex});
 		uint32_t Count = 0;
-		while (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex)
+		while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex))
 		{
 			Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location);
 			Count++;
diff --git a/src/zenutil/chunkedfile.cpp b/src/zenutil/chunkedfile.cpp
index 4f9344039..a2c041ffd 100644
--- a/src/zenutil/chunkedfile.cpp
+++ b/src/zenutil/chunkedfile.cpp
@@ -3,6 +3,7 @@
 #include <zenutil/chunkedfile.h>
 
 #include <zencore/basicfile.h>
+#include <zencore/trace.h>
 
 #include "chunking.h"
 
@@ -33,6 +34,7 @@ namespace {
 IoBuffer
 SerializeChunkedInfo(const ChunkedInfo& Info)
 {
+	ZEN_TRACE_CPU("SerializeChunkedInfo");
 	size_t HeaderSize = RoundUp(sizeof(ChunkedHeader), 16) + RoundUp(sizeof(uint32_t) * Info.ChunkSequence.size(), 16) +
 						RoundUp(sizeof(IoHash) * Info.ChunkHashes.size(), 16);
 	IoBuffer HeaderData(HeaderSize);
@@ -65,6 +67,7 @@ SerializeChunkedInfo(const ChunkedInfo& Info)
 ChunkedInfo
 DeserializeChunkedInfo(IoBuffer& Buffer)
 {
+	ZEN_TRACE_CPU("DeserializeChunkedInfo");
 	MemoryView	  View = Buffer.GetView();
 	ChunkedHeader Header;
 	{
@@ -99,6 +102,7 @@ DeserializeChunkedInfo(IoBuffer& Buffer)
 void
 Reconstruct(const ChunkedInfo& Info, const std::filesystem::path& TargetPath, std::function<IoBuffer(const IoHash& ChunkHash)> GetChunk)
 {
+	ZEN_TRACE_CPU("Reconstruct");
 	BasicFile Reconstructed;
 	Reconstructed.Open(TargetPath, BasicFile::Mode::kTruncate);
 	BasicFileWriter ReconstructedWriter(Reconstructed, 64 * 1024);
@@ -119,6 +123,8 @@ ChunkData(BasicFile&			 RawData,
 		  std::atomic<uint64_t>* BytesProcessed,
 		  std::atomic<bool>*	 AbortFlag)
 {
+	ZEN_TRACE_CPU("ChunkData");
+
 	ChunkedInfoWithSource							 Result;
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> FoundChunks;
 
diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp
index 017d12433..2a7057a46 100644
--- a/src/zenutil/chunkingcontroller.cpp
+++ b/src/zenutil/chunkingcontroller.cpp
@@ -4,6 +4,7 @@
 
 #include <zencore/basicfile.h>
 #include <zencore/compactbinarybuilder.h>
+#include <zencore/trace.h>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <tsl/robin_map.h>
@@ -61,6 +62,7 @@ public:
 							 std::atomic<uint64_t>&		  BytesProcessed,
 							 std::atomic<bool>&			  AbortFlag) const override
 	{
+		ZEN_TRACE_CPU("BasicChunkingController::ProcessFile");
 		const bool ExcludeFromChunking =
 			std::find(m_ChunkExcludeExtensions.begin(), m_ChunkExcludeExtensions.end(), InputPath.extension()) !=
 			m_ChunkExcludeExtensions.end();
@@ -136,6 +138,7 @@ public:
 							 std::atomic<uint64_t>&		  BytesProcessed,
 							 std::atomic<bool>&			  AbortFlag) const override
 	{
+		ZEN_TRACE_CPU("ChunkingControllerWithFixedChunking::ProcessFile");
 		if (RawSize < m_ChunkFileSizeLimit)
 		{
 			return false;
@@ -145,6 +148,7 @@ public:
 
 		if (FixedChunking)
 		{
+			ZEN_TRACE_CPU("FixedChunking");
 			IoHashStream									 FullHash;
 			IoBuffer										 Source = IoBufferBuilder::MakeFromFile(InputPath);
 			uint64_t										 Offset = 0;
diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp
index e57109006..47a4e1cc4 100644
--- a/src/zenutil/filebuildstorage.cpp
+++ b/src/zenutil/filebuildstorage.cpp
@@ -8,6 +8,7 @@
 #include <zencore/fmtutils.h>
 #include <zencore/scopeguard.h>
 #include <zencore/timer.h>
+#include <zencore/trace.h>
 
 namespace zen {
 
@@ -36,6 +37,7 @@ public:
 
 	virtual CbObject ListBuilds(CbObject Query) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::ListBuilds");
 		ZEN_UNUSED(Query);
 
 		SimulateLatency(Query.GetSize(), 0);
@@ -72,6 +74,7 @@ public:
 
 	virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuild");
 		SimulateLatency(MetaData.GetSize(), 0);
 
 		Stopwatch ExecutionTimer;
@@ -93,6 +96,7 @@ public:
 
 	virtual CbObject GetBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuild");
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -105,6 +109,7 @@ public:
 
 	virtual void FinalizeBuild(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuild");
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -119,6 +124,7 @@ public:
 																std::string_view PartName,
 																const CbObject&	 MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildPart");
 		SimulateLatency(MetaData.GetSize(), 0);
 		Stopwatch ExecutionTimer;
 		auto	  _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
@@ -164,6 +170,7 @@ public:
 
 	virtual CbObject GetBuildPart(const Oid& BuildId, const Oid& BuildPartId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildPart");
 		SimulateLatency(0, 0);
 
 		Stopwatch ExecutionTimer;
@@ -186,6 +193,7 @@ public:
 
 	virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FinalizeBuildPart");
 		SimulateLatency(0, 0);
 
 		Stopwatch ExecutionTimer;
@@ -215,6 +223,7 @@ public:
 							  ZenContentType		 ContentType,
 							  const CompositeBuffer& Payload) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBuildBlob");
 		ZEN_UNUSED(BuildId);
 		ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
 		SimulateLatency(Payload.GetSize(), 0);
@@ -242,6 +251,7 @@ public:
 																 std::function<IoBuffer(uint64_t Offset, uint64_t Size)>&& Transmitter,
 																 std::function<void(uint64_t, bool)>&& OnSentBytes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob");
 		ZEN_UNUSED(BuildId);
 		ZEN_UNUSED(ContentType);
 		SimulateLatency(0, 0);
@@ -281,6 +291,7 @@ public:
 				uint64_t Size = Min(32u * 1024u * 1024u, PayloadSize - Offset);
 
 				WorkItems.push_back([this, RawHash, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::PutLargeBuildBlob_Work");
 					IoBuffer PartPayload = Workload->Transmitter(Offset, Size);
 					SimulateLatency(PartPayload.GetSize(), 0);
 
@@ -327,6 +338,7 @@ public:
 
 	virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlob");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -363,6 +375,7 @@ public:
 		uint64_t																			   ChunkSize,
 		std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -392,6 +405,7 @@ public:
 			{
 				uint64_t Size = Min(ChunkSize, BlobSize - Offset);
 				WorkItems.push_back([this, BlockPath, Workload, Offset, Size]() {
+					ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob_Work");
 					SimulateLatency(0, 0);
 					IoBuffer PartPayload(Size);
 					Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset);
@@ -411,6 +425,7 @@ public:
 
 	virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata");
 		ZEN_UNUSED(BuildId);
 
 		SimulateLatency(MetaData.GetSize(), 0);
@@ -429,6 +444,7 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::FindBlocks");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
@@ -461,6 +477,7 @@ public:
 
 	virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override
 	{
+		ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata");
 		ZEN_UNUSED(BuildId);
 		SimulateLatency(0, 0);
 		Stopwatch ExecutionTimer;
diff --git a/src/zenutil/include/zenutil/chunkedcontent.h b/src/zenutil/include/zenutil/chunkedcontent.h
index 309341550..57b55cb8e 100644
--- a/src/zenutil/include/zenutil/chunkedcontent.h
+++ b/src/zenutil/include/zenutil/chunkedcontent.h
@@ -124,8 +124,8 @@ struct ChunkedContentLookup
 {
 	struct ChunkSequenceLocation
 	{
-		uint32_t SequenceIndex;
-		uint64_t Offset;
+		uint32_t SequenceIndex = (uint32_t)-1;
+		uint64_t Offset		   = (uint64_t)-1;
 	};
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex;
 	tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceIndex;