improved block gen logic (#302)

 - Improvement: Reduced memory usage during upload and part upload validation
 - Improvement: Reduced I/O usage during upload and download
 - Improvement: Faster block regeneration when uploading in response to PutBuild/FinalizeBuild
 - Improvement: More trace scopes for build upload operations
 - Bugfix: Fixed crash during download when trying to write outside a file range

1 files changed, 13 insertions, 2 deletions

diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp
index 4ca89d996..bb1ee5183 100644
--- a/src/zenutil/chunkedcontent.cpp
+++ b/src/zenutil/chunkedcontent.cpp
@@ -96,6 +96,8 @@ namespace {
 					   uint32_t											 PathIndex,
 					   std::atomic<bool>&								 AbortFlag)
 	{
+		ZEN_TRACE_CPU("ChunkFolderContent");
+
 		const uint64_t				 RawSize = OutChunkedContent.RawSizes[PathIndex];
 		const std::filesystem::path& Path	 = OutChunkedContent.Paths[PathIndex];
 
@@ -136,6 +138,8 @@ namespace {
 			}
 			else
 			{
+				ZEN_TRACE_CPU("HashOnly");
+
 				IoBuffer	 Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred());
 				const IoHash Hash	= IoHash::HashBuffer(Buffer, &Stats.BytesHashed);
 
@@ -228,6 +232,7 @@ FolderContent::operator==(const FolderContent& Rhs) const
 bool
 FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
 {
+	ZEN_TRACE_CPU("FolderContent::AreKnownFilesEqual");
 	tsl::robin_map<std::string, size_t> RhsPathToIndex;
 	const size_t						RhsPathCount = Rhs.Paths.size();
 	RhsPathToIndex.reserve(RhsPathCount);
@@ -259,6 +264,7 @@ FolderContent::AreKnownFilesEqual(const FolderContent& Rhs) const
 void
 FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutPathIndexesOufOfDate)
 {
+	ZEN_TRACE_CPU("FolderContent::UpdateState");
 	tsl::robin_map<std::string, uint32_t> RhsPathToIndex;
 	const uint32_t						  RhsPathCount = gsl::narrow<uint32_t>(Rhs.Paths.size());
 	RhsPathToIndex.reserve(RhsPathCount);
@@ -297,6 +303,7 @@ FolderContent::UpdateState(const FolderContent& Rhs, std::vector<uint32_t>& OutP
 FolderContent
 GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes)
 {
+	ZEN_TRACE_CPU("FolderContent::GetUpdatedContent");
 	FolderContent						  Result = {.Platform = Old.Platform};
 	tsl::robin_map<std::string, uint32_t> NewPathToIndex;
 	const uint32_t						  NewPathCount = gsl::narrow<uint32_t>(New.Paths.size());
@@ -332,6 +339,7 @@ GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vecto
 void
 SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
 {
+	ZEN_TRACE_CPU("SaveFolderContentToCompactBinary");
 	Output.AddString("platform"sv, ToString(Content.Platform));
 	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
 	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
@@ -342,6 +350,7 @@ SaveFolderContentToCompactBinary(const FolderContent& Content, CbWriter& Output)
 FolderContent
 LoadFolderContentToCompactBinary(CbObjectView Input)
 {
+	ZEN_TRACE_CPU("LoadFolderContentToCompactBinary");
 	FolderContent Content;
 	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
 	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
@@ -494,6 +503,7 @@ GetFolderContent(GetFolderContentStatistics&															  Stats,
 void
 SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbWriter& Output)
 {
+	ZEN_TRACE_CPU("SaveChunkedFolderContentToCompactBinary");
 	Output.AddString("platform"sv, ToString(Content.Platform));
 	compactbinary_helpers::WriteArray(Content.Paths, "paths"sv, Output);
 	compactbinary_helpers::WriteArray(Content.RawSizes, "rawSizes"sv, Output);
@@ -512,6 +522,7 @@ SaveChunkedFolderContentToCompactBinary(const ChunkedFolderContent& Content, CbW
 ChunkedFolderContent
 LoadChunkedFolderContentToCompactBinary(CbObjectView Input)
 {
+	ZEN_TRACE_CPU("LoadChunkedFolderContentToCompactBinary");
 	ChunkedFolderContent Content;
 	Content.Platform = FromString(Input["platform"sv].AsString(), GetSourceCurrentPlatform());
 	compactbinary_helpers::ReadArray("paths"sv, Input, Content.Paths);
@@ -788,7 +799,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 
 	struct ChunkLocationReference
 	{
-		uint32_t									ChunkIndex;
+		uint32_t									ChunkIndex = (uint32_t)-1;
 		ChunkedContentLookup::ChunkSequenceLocation Location;
 	};
 
@@ -853,7 +864,7 @@ BuildChunkedContentLookup(const ChunkedFolderContent& Content)
 	{
 		Result.ChunkHashToChunkIndex.insert({Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkIndex});
 		uint32_t Count = 0;
-		while (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex)
+		while ((RangeOffset + Count < Locations.size()) && (Locations[RangeOffset + Count].ChunkIndex == ChunkIndex))
 		{
 			Result.ChunkSequenceLocations.push_back(Locations[RangeOffset + Count].Location);
 			Count++;