prime temp file sizes and order request based on first write positionde/disk-io-optimizations

3 files changed, 143 insertions, 8 deletions

diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index f7f9e3abb..302c2ce4a 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -5825,12 +5825,63 @@ namespace {
 
 			Stopwatch WriteTimer;
 
-			FilteredRate FilteredDownloadedBytesPerSecond;
-			FilteredRate FilteredWrittenBytesPerSecond;
-
 			WorkerThreadPool& NetworkPool = GetNetworkPool();
 			WorkerThreadPool& WritePool	  = GetIOWorkerPool();
 
+			if (!PrimeCacheOnly)
+			{
+				ProgressBar	  PrepProgressBar(UsePlainProgress);
+				ParallellWork PrepareWork(AbortFlag);
+
+				std::atomic<uint64_t> CompletedCount = 0;
+
+				for (uint32_t SequenceIndex = 0; SequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++)
+				{
+					if (!AbortFlag)
+					{
+						PrepareWork.ScheduleWork(
+							WritePool,
+							[&, SequenceIndex](std::atomic<bool>&) {
+								if (!AbortFlag)
+								{
+									const IoHash& SequenceHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex];
+									// const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+									const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+									const uint64_t RawSize	 = RemoteContent.RawSizes[PathIndex];
+									if (SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 1 && RawSize > 1u * 1024u * 1024u)
+									{
+										const std::filesystem::path TempFilePath =
+											GetTempChunkedSequenceFileName(CacheFolderPath, SequenceHash);
+
+										std::error_code Ec;
+										BasicFile		TmpCacheFile(TempFilePath, BasicFile::Mode::kWrite, Ec);
+										if (!Ec)
+										{
+											TmpCacheFile.Write("\0", 1, RawSize - 1, Ec);
+										}
+									}
+									CompletedCount++;
+								}
+							},
+							PrepareWork.DefaultErrorFunction());
+					}
+				}
+
+				PrepareWork.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
+					ZEN_UNUSED(IsAborted, PendingWork);
+					PrepProgressBar.UpdateState(
+						{.Task		 = "Allocating space ",
+						 .Details	 = fmt::format("{}/{}", CompletedCount.load(), RemoteContent.ChunkedContent.SequenceRawHashes.size()),
+						 .TotalCount = RemoteContent.ChunkedContent.SequenceRawHashes.size(),
+						 .RemainingCount = RemoteContent.ChunkedContent.SequenceRawHashes.size() - CompletedCount},
+						false);
+				});
+				PrepProgressBar.Finish();
+			}
+
+			FilteredRate FilteredDownloadedBytesPerSecond;
+			FilteredRate FilteredWrittenBytesPerSecond;
+
 			ProgressBar	  WriteProgressBar(UsePlainProgress);
 			ParallellWork Work(AbortFlag);
 
@@ -6783,7 +6834,53 @@ namespace {
 					Work.DefaultErrorFunction());
 			}
 
-			for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
+			std::vector<uint32_t> BlockRangeOrder;
+			{
+				BlockRangeOrder.reserve(BlockRangeWorks.size());
+
+				tsl::robin_map<uint32_t, uint64_t> BlockRangeFirstChunkWriteOffset;
+				BlockRangeFirstChunkWriteOffset.reserve(BlockRangeWorks.size());
+				for (uint32_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
+				{
+					BlockRangeOrder.push_back(BlockRangeIndex);
+
+					uint64_t					MinOffset  = (uint64_t)-1;
+					const BlockRangeDescriptor& BlockRange = BlockRangeWorks[BlockRangeIndex];
+
+					const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockRange.BlockIndex];
+					for (uint32_t BlockChunkIndex = BlockRange.ChunkBlockIndexStart;
+						 BlockChunkIndex < BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount && MinOffset > 0;
+						 BlockChunkIndex++)
+					{
+						const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[BlockChunkIndex];
+						if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end())
+						{
+							const uint32_t ChunkIndex = It->second;
+
+							std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources =
+								GetChunkSequenceLocations(RemoteLookup, ChunkIndex);
+							for (const ChunkedContentLookup::ChunkSequenceLocation& Location : ChunkSources)
+							{
+								MinOffset = Min(MinOffset, Location.Offset);
+							}
+						}
+					}
+					ZEN_ASSERT(MinOffset != (uint64_t)-1);
+					BlockRangeFirstChunkWriteOffset.insert({BlockRangeIndex, MinOffset});
+				}
+
+				std::sort(BlockRangeOrder.begin(), BlockRangeOrder.end(), [&](const uint32_t Lhs, const uint32_t Rhs) {
+					const uint64_t LhsFirstOffset = BlockRangeFirstChunkWriteOffset.at(Lhs);
+					const uint64_t RhsFirstOffset = BlockRangeFirstChunkWriteOffset.at(Rhs);
+					if (LhsFirstOffset == RhsFirstOffset)
+					{
+						return Lhs < Rhs;
+					}
+					return LhsFirstOffset < RhsFirstOffset;
+				});
+			}
+
+			for (size_t BlockRangeIndex : BlockRangeOrder)
 			{
 				ZEN_ASSERT(!PrimeCacheOnly);
 				if (AbortFlag)
@@ -6949,6 +7046,44 @@ namespace {
 					Work.DefaultErrorFunction());
 			}
 
+			{
+				tsl::robin_map<uint32_t, uint64_t> BlockFirstChunkWriteOffset;
+				BlockFirstChunkWriteOffset.reserve(FullBlockWorks.size());
+				for (uint32_t BlockIndex : FullBlockWorks)
+				{
+					uint64_t					 MinOffset		  = (uint64_t)-1;
+					const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+					for (uint32_t BlockChunkIndex = 0; BlockChunkIndex < BlockDescription.ChunkRawHashes.size() && MinOffset > 0;
+						 BlockChunkIndex++)
+					{
+						const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[BlockChunkIndex];
+						if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end())
+						{
+							const uint32_t ChunkIndex = It->second;
+
+							std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources =
+								GetChunkSequenceLocations(RemoteLookup, ChunkIndex);
+							for (const ChunkedContentLookup::ChunkSequenceLocation& Location : ChunkSources)
+							{
+								MinOffset = Min(MinOffset, Location.Offset);
+							}
+						}
+					}
+					ZEN_ASSERT(MinOffset != (uint64_t)-1);
+					BlockFirstChunkWriteOffset.insert({BlockIndex, MinOffset});
+				}
+
+				std::sort(FullBlockWorks.begin(), FullBlockWorks.end(), [&](const uint32_t Lhs, const uint32_t Rhs) {
+					const uint64_t LhsFirstOffset = BlockFirstChunkWriteOffset.at(Lhs);
+					const uint64_t RhsFirstOffset = BlockFirstChunkWriteOffset.at(Rhs);
+					if (LhsFirstOffset == RhsFirstOffset)
+					{
+						return Lhs < Rhs;
+					}
+					return LhsFirstOffset < RhsFirstOffset;
+				});
+			}
+
 			for (uint32_t BlockIndex : FullBlockWorks)
 			{
 				if (AbortFlag)
diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index ea526399c..effc5bf26 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -181,7 +181,7 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount)
 void
 BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
 {
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024;
 
 	while (BytesToRead)
 	{
@@ -327,7 +327,7 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro
 
 	Ec.clear();
 
-	const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+	const uint64_t MaxChunkSize = 2u * 1024 * 1024;
 
 	while (Size)
 	{
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 763f3262a..6a067ff50 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -220,7 +220,7 @@ namespace detail {
 		{
 			ZEN_TRACE_CPU("TempPayloadFile::AppendData");
 			ZEN_ASSERT(m_FileHandle != nullptr);
-			const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+			const uint64_t MaxChunkSize = 2u * 1024u * 1024u;
 
 			while (Size)
 			{
@@ -1206,7 +1206,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
 						   std::optional<size_t> ContentLength = ParseInt<size_t>(Header.second);
 						   if (ContentLength.has_value())
 						   {
-							   if (ContentLength.value() > 1024 * 1024)
+							   if (ContentLength.value() > 4u * 1024u * 1024u)
 							   {
 								   PayloadFile = std::make_unique<detail::TempPayloadFile>();
 								   std::error_code Ec = PayloadFile->Open(TempFolderPath);