aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/zen/cmds/builds_cmd.cpp143
-rw-r--r--src/zencore/basicfile.cpp4
-rw-r--r--src/zenhttp/httpclient.cpp4
3 files changed, 143 insertions, 8 deletions
diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp
index f7f9e3abb..302c2ce4a 100644
--- a/src/zen/cmds/builds_cmd.cpp
+++ b/src/zen/cmds/builds_cmd.cpp
@@ -5825,12 +5825,63 @@ namespace {
Stopwatch WriteTimer;
- FilteredRate FilteredDownloadedBytesPerSecond;
- FilteredRate FilteredWrittenBytesPerSecond;
-
WorkerThreadPool& NetworkPool = GetNetworkPool();
WorkerThreadPool& WritePool = GetIOWorkerPool();
+ if (!PrimeCacheOnly)
+ {
+ ProgressBar PrepProgressBar(UsePlainProgress);
+ ParallellWork PrepareWork(AbortFlag);
+
+ std::atomic<uint64_t> CompletedCount = 0;
+
+ for (uint32_t SequenceIndex = 0; SequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); SequenceIndex++)
+ {
+ if (!AbortFlag)
+ {
+ PrepareWork.ScheduleWork(
+ WritePool,
+ [&, SequenceIndex](std::atomic<bool>&) {
+ if (!AbortFlag)
+ {
+ const IoHash& SequenceHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex];
+ // const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+ const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const uint64_t RawSize = RemoteContent.RawSizes[PathIndex];
+ if (SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 1 && RawSize > 1u * 1024u * 1024u)
+ {
+ const std::filesystem::path TempFilePath =
+ GetTempChunkedSequenceFileName(CacheFolderPath, SequenceHash);
+
+ std::error_code Ec;
+ BasicFile TmpCacheFile(TempFilePath, BasicFile::Mode::kWrite, Ec);
+ if (!Ec)
+ {
+ TmpCacheFile.Write("\0", 1, RawSize - 1, Ec);
+ }
+ }
+ CompletedCount++;
+ }
+ },
+ PrepareWork.DefaultErrorFunction());
+ }
+ }
+
+ PrepareWork.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(IsAborted, PendingWork);
+ PrepProgressBar.UpdateState(
+ {.Task = "Allocating space ",
+ .Details = fmt::format("{}/{}", CompletedCount.load(), RemoteContent.ChunkedContent.SequenceRawHashes.size()),
+ .TotalCount = RemoteContent.ChunkedContent.SequenceRawHashes.size(),
+ .RemainingCount = RemoteContent.ChunkedContent.SequenceRawHashes.size() - CompletedCount},
+ false);
+ });
+ PrepProgressBar.Finish();
+ }
+
+ FilteredRate FilteredDownloadedBytesPerSecond;
+ FilteredRate FilteredWrittenBytesPerSecond;
+
ProgressBar WriteProgressBar(UsePlainProgress);
ParallellWork Work(AbortFlag);
@@ -6783,7 +6834,53 @@ namespace {
Work.DefaultErrorFunction());
}
- for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
+ std::vector<uint32_t> BlockRangeOrder;
+ {
+ BlockRangeOrder.reserve(BlockRangeWorks.size());
+
+ tsl::robin_map<uint32_t, uint64_t> BlockRangeFirstChunkWriteOffset;
+ BlockRangeFirstChunkWriteOffset.reserve(BlockRangeWorks.size());
+ for (uint32_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++)
+ {
+ BlockRangeOrder.push_back(BlockRangeIndex);
+
+ uint64_t MinOffset = (uint64_t)-1;
+ const BlockRangeDescriptor& BlockRange = BlockRangeWorks[BlockRangeIndex];
+
+ const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockRange.BlockIndex];
+ for (uint32_t BlockChunkIndex = BlockRange.ChunkBlockIndexStart;
+ BlockChunkIndex < BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount && MinOffset > 0;
+ BlockChunkIndex++)
+ {
+ const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[BlockChunkIndex];
+ if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t ChunkIndex = It->second;
+
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources =
+ GetChunkSequenceLocations(RemoteLookup, ChunkIndex);
+ for (const ChunkedContentLookup::ChunkSequenceLocation& Location : ChunkSources)
+ {
+ MinOffset = Min(MinOffset, Location.Offset);
+ }
+ }
+ }
+ ZEN_ASSERT(MinOffset != (uint64_t)-1);
+ BlockRangeFirstChunkWriteOffset.insert({BlockRangeIndex, MinOffset});
+ }
+
+ std::sort(BlockRangeOrder.begin(), BlockRangeOrder.end(), [&](const uint32_t Lhs, const uint32_t Rhs) {
+ const uint64_t LhsFirstOffset = BlockRangeFirstChunkWriteOffset.at(Lhs);
+ const uint64_t RhsFirstOffset = BlockRangeFirstChunkWriteOffset.at(Rhs);
+ if (LhsFirstOffset == RhsFirstOffset)
+ {
+ return Lhs < Rhs;
+ }
+ return LhsFirstOffset < RhsFirstOffset;
+ });
+ }
+
+ for (size_t BlockRangeIndex : BlockRangeOrder)
{
ZEN_ASSERT(!PrimeCacheOnly);
if (AbortFlag)
@@ -6949,6 +7046,44 @@ namespace {
Work.DefaultErrorFunction());
}
+ {
+ tsl::robin_map<uint32_t, uint64_t> BlockFirstChunkWriteOffset;
+ BlockFirstChunkWriteOffset.reserve(FullBlockWorks.size());
+ for (uint32_t BlockIndex : FullBlockWorks)
+ {
+ uint64_t MinOffset = (uint64_t)-1;
+ const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex];
+ for (uint32_t BlockChunkIndex = 0; BlockChunkIndex < BlockDescription.ChunkRawHashes.size() && MinOffset > 0;
+ BlockChunkIndex++)
+ {
+ const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[BlockChunkIndex];
+ if (auto It = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != RemoteLookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t ChunkIndex = It->second;
+
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources =
+ GetChunkSequenceLocations(RemoteLookup, ChunkIndex);
+ for (const ChunkedContentLookup::ChunkSequenceLocation& Location : ChunkSources)
+ {
+ MinOffset = Min(MinOffset, Location.Offset);
+ }
+ }
+ }
+ ZEN_ASSERT(MinOffset != (uint64_t)-1);
+ BlockFirstChunkWriteOffset.insert({BlockIndex, MinOffset});
+ }
+
+ std::sort(FullBlockWorks.begin(), FullBlockWorks.end(), [&](const uint32_t Lhs, const uint32_t Rhs) {
+ const uint64_t LhsFirstOffset = BlockFirstChunkWriteOffset.at(Lhs);
+ const uint64_t RhsFirstOffset = BlockFirstChunkWriteOffset.at(Rhs);
+ if (LhsFirstOffset == RhsFirstOffset)
+ {
+ return Lhs < Rhs;
+ }
+ return LhsFirstOffset < RhsFirstOffset;
+ });
+ }
+
for (uint32_t BlockIndex : FullBlockWorks)
{
if (AbortFlag)
diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp
index ea526399c..effc5bf26 100644
--- a/src/zencore/basicfile.cpp
+++ b/src/zencore/basicfile.cpp
@@ -181,7 +181,7 @@ BasicFile::ReadRange(uint64_t FileOffset, uint64_t ByteCount)
void
BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset)
{
- const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+ const uint64_t MaxChunkSize = 2u * 1024 * 1024;
while (BytesToRead)
{
@@ -327,7 +327,7 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro
Ec.clear();
- const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+ const uint64_t MaxChunkSize = 2u * 1024 * 1024;
while (Size)
{
diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp
index 763f3262a..6a067ff50 100644
--- a/src/zenhttp/httpclient.cpp
+++ b/src/zenhttp/httpclient.cpp
@@ -220,7 +220,7 @@ namespace detail {
{
ZEN_TRACE_CPU("TempPayloadFile::AppendData");
ZEN_ASSERT(m_FileHandle != nullptr);
- const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024;
+ const uint64_t MaxChunkSize = 2u * 1024u * 1024u;
while (Size)
{
@@ -1206,7 +1206,7 @@ HttpClient::Download(std::string_view Url, const std::filesystem::path& TempFold
std::optional<size_t> ContentLength = ParseInt<size_t>(Header.second);
if (ContentLength.has_value())
{
- if (ContentLength.value() > 1024 * 1024)
+ if (ContentLength.value() > 4u * 1024u * 1024u)
{
PayloadFile = std::make_unique<detail::TempPayloadFile>();
std::error_code Ec = PayloadFile->Open(TempFolderPath);
generated by cgit v1.2.3 (git 2.25.1) at 2026-03-11 03:53:06 +0000