aboutsummaryrefslogtreecommitdiff
path: root/src/zenremotestore/builds
diff options
context:
space:
mode:
Diffstat (limited to 'src/zenremotestore/builds')
-rw-r--r--src/zenremotestore/builds/buildinspect.cpp464
-rw-r--r--src/zenremotestore/builds/buildmanifest.cpp4
-rw-r--r--src/zenremotestore/builds/buildoperations-tests.cpp454
-rw-r--r--src/zenremotestore/builds/buildprimecache.cpp350
-rw-r--r--src/zenremotestore/builds/buildsavedstate.cpp12
-rw-r--r--src/zenremotestore/builds/buildstoragecache.cpp353
-rw-r--r--src/zenremotestore/builds/buildstorageoperations.cpp7943
-rw-r--r--src/zenremotestore/builds/buildstorageresolve.cpp249
-rw-r--r--src/zenremotestore/builds/buildstorageutil.cpp1792
-rw-r--r--src/zenremotestore/builds/buildupdatefolder.cpp4948
-rw-r--r--src/zenremotestore/builds/builduploadfolder.cpp2635
-rw-r--r--src/zenremotestore/builds/buildvalidatebuildpart.cpp374
-rw-r--r--src/zenremotestore/builds/filebuildstorage.cpp39
-rw-r--r--src/zenremotestore/builds/jupiterbuildstorage.cpp233
14 files changed, 11576 insertions, 8274 deletions
diff --git a/src/zenremotestore/builds/buildinspect.cpp b/src/zenremotestore/builds/buildinspect.cpp
new file mode 100644
index 000000000..86f2cfd75
--- /dev/null
+++ b/src/zenremotestore/builds/buildinspect.cpp
@@ -0,0 +1,464 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildinspect.h>
+
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/buildcontent.h>
+#include <zenremotestore/builds/buildmanifest.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildupdatefolder.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/chunking/chunkingcache.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
+#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/filteredrate.h>
+#include <zenutil/progress.h>
+#include <zenutil/wildcard.h>
+
+#include <numeric>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_map.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+ChunkedFolderContent
+ScanAndChunkFolder(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ bool IsQuiet,
+ TransferThreadWorkers& Workers,
+ GetFolderContentStatistics& GetFolderContentStats,
+ ChunkingStatistics& ChunkingStats,
+ const std::filesystem::path& Path,
+ std::function<bool(const std::string_view& RelativePath)>&& IsAcceptedFolder,
+ std::function<bool(std::string_view RelativePath, uint64_t Size, uint32_t Attributes)>&& IsAcceptedFile,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache)
+{
+ Stopwatch Timer;
+
+ ZEN_TRACE_CPU("ScanAndChunkFolder");
+
+ FolderContent Content = GetFolderContent(
+ GetFolderContentStats,
+ Path,
+ std::move(IsAcceptedFolder),
+ std::move(IsAcceptedFile),
+ Workers.GetIOWorkerPool(),
+ Progress.GetProgressUpdateDelayMS(),
+ [](bool, std::ptrdiff_t) {},
+ AbortFlag);
+ if (AbortFlag)
+ {
+ return {};
+ }
+
+ BuildState LocalContent = GetLocalContent(Progress,
+ AbortFlag,
+ PauseFlag,
+ IsQuiet,
+ Workers,
+ GetFolderContentStats,
+ ChunkingStats,
+ Path,
+ ZenStateFilePath(Path / ZenFolderName),
+ ChunkController,
+ ChunkCache)
+ .State;
+
+ std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalContent.ChunkedContent.Paths, Content.Paths);
+
+ BuildState UntrackedLocalContent = GetLocalStateFromPaths(Progress,
+ AbortFlag,
+ PauseFlag,
+ Workers,
+ GetFolderContentStats,
+ ChunkingStats,
+ Path,
+ ChunkController,
+ ChunkCache,
+ UntrackedPaths)
+ .State;
+
+ ChunkedFolderContent Result =
+ MergeChunkedFolderContents(LocalContent.ChunkedContent, std::vector<ChunkedFolderContent>{UntrackedLocalContent.ChunkedContent});
+
+ const uint64_t TotalRawSize = std::accumulate(Result.RawSizes.begin(), Result.RawSizes.end(), std::uint64_t(0));
+ const uint64_t ChunkedRawSize =
+ std::accumulate(Result.ChunkedContent.ChunkRawSizes.begin(), Result.ChunkedContent.ChunkRawSizes.end(), std::uint64_t(0));
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Result.Paths.size(),
+ NiceBytes(TotalRawSize),
+ Result.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(ChunkedRawSize),
+ Path,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
+ return Result;
+};
+
+void
+ListBuild(bool IsQuiet,
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ CbObjectWriter* OptionalStructuredOutput)
+{
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, IsQuiet);
+
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->AddObjectId("buildId"sv, BuildId);
+ OptionalStructuredOutput->AddObject("build"sv, BuildObject);
+ }
+
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+
+ if (!AllBuildParts.empty())
+ {
+ Stopwatch GetBuildPartTimer;
+
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->BeginArray("parts"sv);
+ }
+
+ for (size_t BuildPartIndex = 0; BuildPartIndex < AllBuildParts.size(); BuildPartIndex++)
+ {
+ const Oid BuildPartId = AllBuildParts[BuildPartIndex].first;
+ const std::string_view BuildPartName = AllBuildParts[BuildPartIndex].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->BeginObject();
+ OptionalStructuredOutput->AddObjectId("id"sv, BuildPartId);
+ OptionalStructuredOutput->AddString("partName"sv, BuildPartName);
+ }
+ {
+ if (OptionalStructuredOutput != nullptr)
+ {
+ }
+ else if (!IsQuiet)
+ {
+ ZEN_CONSOLE("{}Part: {} ('{}'):\n",
+ BuildPartIndex > 0 ? "\n" : "",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ }
+
+ std::vector<std::filesystem::path> Paths;
+ std::vector<IoHash> RawHashes;
+ std::vector<uint64_t> RawSizes;
+ std::vector<uint32_t> Attributes;
+
+ SourcePlatform Platform;
+ std::vector<IoHash> SequenceRawHashes;
+ std::vector<uint32_t> ChunkCounts;
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<IoHash> LooseChunkHashes;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ Platform,
+ Paths,
+ RawHashes,
+ RawSizes,
+ Attributes,
+ SequenceRawHashes,
+ ChunkCounts,
+ AbsoluteChunkOrders,
+ LooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ std::vector<size_t> Order(Paths.size());
+ std::iota(Order.begin(), Order.end(), 0);
+
+ std::sort(Order.begin(), Order.end(), [&](size_t Lhs, size_t Rhs) {
+ const std::filesystem::path& LhsPath = Paths[Lhs];
+ const std::filesystem::path& RhsPath = Paths[Rhs];
+ return LhsPath < RhsPath;
+ });
+
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->BeginArray("files"sv);
+ }
+ {
+ for (size_t Index : Order)
+ {
+ const std::filesystem::path& Path = Paths[Index];
+ if (IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(Path.generic_string()), /*CaseSensitive*/ true))
+ {
+ const IoHash& RawHash = RawHashes[Index];
+ const uint64_t RawSize = RawSizes[Index];
+ const uint32_t Attribute = Attributes[Index];
+
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->BeginObject();
+ {
+ OptionalStructuredOutput->AddString("path"sv, fmt::format("{}", Path));
+ OptionalStructuredOutput->AddInteger("rawSize"sv, RawSize);
+ OptionalStructuredOutput->AddHash("rawHash"sv, RawHash);
+ switch (Platform)
+ {
+ case SourcePlatform::Windows:
+ OptionalStructuredOutput->AddInteger("attributes"sv, Attribute);
+ break;
+ case SourcePlatform::MacOS:
+ case SourcePlatform::Linux:
+ OptionalStructuredOutput->AddString("chmod"sv, fmt::format("{:#04o}", Attribute));
+ break;
+ default:
+ throw std::runtime_error(fmt::format("Unsupported platform: {}", (int)Platform));
+ }
+ }
+ OptionalStructuredOutput->EndObject();
+ }
+ else
+ {
+ ZEN_CONSOLE("{}\t{}\t{}", Path, RawSize, RawHash);
+ }
+ }
+ }
+ }
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->EndArray(); // "files"
+ }
+ }
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->EndObject();
+ }
+ }
+ if (OptionalStructuredOutput != nullptr)
+ {
+ OptionalStructuredOutput->EndArray(); // parts
+ }
+ }
+}
+
+void
+DiffFolders(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ bool IsQuiet,
+ TransferThreadWorkers& Workers,
+ const std::filesystem::path& BasePath,
+ const std::filesystem::path& ComparePath,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ const std::vector<std::string>& ExcludeFolders,
+ const std::vector<std::string>& ExcludeExtensions)
+{
+ ZEN_TRACE_CPU("DiffFolders");
+
+ Progress.SetLogOperationName("Diff Folders");
+
+ enum TaskSteps : uint32_t
+ {
+ CheckBase,
+ CheckCompare,
+ Diff,
+ Cleanup,
+ StepCount
+ };
+
+ auto EndProgress = MakeGuard([&]() { Progress.SetLogOperationProgress(TaskSteps::StepCount, TaskSteps::StepCount); });
+
+ ChunkedFolderContent BaseFolderContent;
+ ChunkedFolderContent CompareFolderContent;
+
+ {
+ auto IsAcceptedFolder = [ExcludeFolders](const std::string_view& RelativePath) -> bool {
+ for (const std::string& ExcludeFolder : ExcludeFolders)
+ {
+ if (StrCaseStartsWith(RelativePath, ExcludeFolder))
+ {
+ if (RelativePath.length() == ExcludeFolder.length())
+ {
+ return false;
+ }
+ else if (RelativePath[ExcludeFolder.length()] == '/')
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+ };
+
+ auto IsAcceptedFile = [ExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool {
+ for (const std::string& ExcludeExtension : ExcludeExtensions)
+ {
+ if (StrCaseEndsWith(RelativePath, ExcludeExtension))
+ {
+ return false;
+ }
+ }
+ return true;
+ };
+
+ Progress.SetLogOperationProgress(TaskSteps::CheckBase, TaskSteps::StepCount);
+
+ GetFolderContentStatistics BaseGetFolderContentStats;
+ ChunkingStatistics BaseChunkingStats;
+ BaseFolderContent = ScanAndChunkFolder(Progress,
+ AbortFlag,
+ PauseFlag,
+ IsQuiet,
+ Workers,
+ BaseGetFolderContentStats,
+ BaseChunkingStats,
+ BasePath,
+ IsAcceptedFolder,
+ IsAcceptedFile,
+ ChunkController,
+ ChunkCache);
+ if (AbortFlag)
+ {
+ return;
+ }
+
+ Progress.SetLogOperationProgress(TaskSteps::CheckCompare, TaskSteps::StepCount);
+
+ GetFolderContentStatistics CompareGetFolderContentStats;
+ ChunkingStatistics CompareChunkingStats;
+ CompareFolderContent = ScanAndChunkFolder(Progress,
+ AbortFlag,
+ PauseFlag,
+ IsQuiet,
+ Workers,
+ CompareGetFolderContentStats,
+ CompareChunkingStats,
+ ComparePath,
+ IsAcceptedFolder,
+ IsAcceptedFile,
+ ChunkController,
+ ChunkCache);
+
+ if (AbortFlag)
+ {
+ return;
+ }
+ }
+
+ Progress.SetLogOperationProgress(TaskSteps::Diff, TaskSteps::StepCount);
+
+ std::vector<IoHash> AddedHashes;
+ std::vector<IoHash> RemovedHashes;
+ uint64_t RemovedSize = 0;
+ uint64_t AddedSize = 0;
+
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> BaseRawHashLookup;
+ for (size_t PathIndex = 0; PathIndex < BaseFolderContent.RawHashes.size(); PathIndex++)
+ {
+ const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
+ BaseRawHashLookup.insert_or_assign(RawHash, PathIndex);
+ }
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CompareRawHashLookup;
+ for (size_t PathIndex = 0; PathIndex < CompareFolderContent.RawHashes.size(); PathIndex++)
+ {
+ const IoHash& RawHash = CompareFolderContent.RawHashes[PathIndex];
+ if (!BaseRawHashLookup.contains(RawHash))
+ {
+ AddedHashes.push_back(RawHash);
+ AddedSize += CompareFolderContent.RawSizes[PathIndex];
+ }
+ CompareRawHashLookup.insert_or_assign(RawHash, PathIndex);
+ }
+ for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
+ {
+ const IoHash& RawHash = BaseFolderContent.RawHashes[PathIndex];
+ if (!CompareRawHashLookup.contains(RawHash))
+ {
+ RemovedHashes.push_back(RawHash);
+ RemovedSize += BaseFolderContent.RawSizes[PathIndex];
+ }
+ }
+
+ uint64_t BaseTotalRawSize = 0;
+ for (uint32_t PathIndex = 0; PathIndex < BaseFolderContent.Paths.size(); PathIndex++)
+ {
+ BaseTotalRawSize += BaseFolderContent.RawSizes[PathIndex];
+ }
+
+ double KeptPercent = BaseTotalRawSize > 0 ? (100.0 * (BaseTotalRawSize - RemovedSize)) / BaseTotalRawSize : 0;
+
+ ZEN_CONSOLE("File diff : {} ({}) removed, {} ({}) added, {} ({} {:.1f}%) kept",
+ RemovedHashes.size(),
+ NiceBytes(RemovedSize),
+ AddedHashes.size(),
+ NiceBytes(AddedSize),
+ BaseFolderContent.Paths.size() - RemovedHashes.size(),
+ NiceBytes(BaseTotalRawSize - RemovedSize),
+ KeptPercent);
+
+ uint64_t CompareTotalRawSize = 0;
+
+ uint64_t FoundChunkCount = 0;
+ uint64_t FoundChunkSize = 0;
+ uint64_t NewChunkCount = 0;
+ uint64_t NewChunkSize = 0;
+ const ChunkedContentLookup BaseFolderLookup = BuildChunkedContentLookup(BaseFolderContent);
+ for (uint32_t ChunkIndex = 0; ChunkIndex < CompareFolderContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
+ {
+ const IoHash& ChunkHash = CompareFolderContent.ChunkedContent.ChunkHashes[ChunkIndex];
+ if (BaseFolderLookup.ChunkHashToChunkIndex.contains(ChunkHash))
+ {
+ FoundChunkCount++;
+ FoundChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+ else
+ {
+ NewChunkCount++;
+ NewChunkSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+ CompareTotalRawSize += CompareFolderContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+
+ double FoundPercent = CompareTotalRawSize > 0 ? (100.0 * FoundChunkSize) / CompareTotalRawSize : 0;
+ double NewPercent = CompareTotalRawSize > 0 ? (100.0 * NewChunkSize) / CompareTotalRawSize : 0;
+
+ ZEN_CONSOLE("Chunk diff: {} ({} {:.1f}%) out of {} ({}) chunks in {} ({}) base chunks. Added {} ({} {:.1f}%) chunks.",
+ FoundChunkCount,
+ NiceBytes(FoundChunkSize),
+ FoundPercent,
+ CompareFolderContent.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(CompareTotalRawSize),
+ BaseFolderContent.ChunkedContent.ChunkHashes.size(),
+ NiceBytes(BaseTotalRawSize),
+ NewChunkCount,
+ NiceBytes(NewChunkSize),
+ NewPercent);
+
+ Progress.SetLogOperationProgress(TaskSteps::Cleanup, TaskSteps::StepCount);
+}
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/buildmanifest.cpp b/src/zenremotestore/builds/buildmanifest.cpp
index 051436e96..738e4b33b 100644
--- a/src/zenremotestore/builds/buildmanifest.cpp
+++ b/src/zenremotestore/builds/buildmanifest.cpp
@@ -97,6 +97,8 @@ ParseBuildManifest(const std::filesystem::path& ManifestPath)
}
#if ZEN_WITH_TESTS
+TEST_SUITE_BEGIN("remotestore.buildmanifest");
+
TEST_CASE("buildmanifest.unstructured")
{
ScopedTemporaryDirectory Root;
@@ -163,6 +165,8 @@ TEST_CASE("buildmanifest.structured")
CHECK_EQ(Manifest.Parts[1].Files[0].generic_string(), "baz.pdb");
}
+TEST_SUITE_END();
+
void
buildmanifest_forcelink()
{
diff --git a/src/zenremotestore/builds/buildoperations-tests.cpp b/src/zenremotestore/builds/buildoperations-tests.cpp
new file mode 100644
index 000000000..b1c856193
--- /dev/null
+++ b/src/zenremotestore/builds/buildoperations-tests.cpp
@@ -0,0 +1,454 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+// Round-trip integration tests for BuildsOperationUploadFolder / BuildsOperationUpdateFolder.
+// Runs in-process against CreateFileBuildStorage so no HTTP server is needed.
+
+#include <zenremotestore/builds/buildupdatefolder.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/builds/filebuildstorage.h>
+#include <zenremotestore/chunking/chunkingcache.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
+#include <zenremotestore/transferthreadworkers.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/iohash.h>
+#include <zencore/logging.h>
+#include <zencore/scopeguard.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
+#include <zencore/workthreadpool.h>
+#include <zenutil/progress.h>
+
+#include <algorithm>
+#include <atomic>
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace zen {
+
+void
+buildoperations_tests_forcelink()
+{
+}
+
+#if ZEN_WITH_TESTS
+
+namespace buildops_test {
+
+ using namespace std::literals;
+
+ struct FolderSpec
+ {
+ uint64_t Seed = 1;
+ uint32_t SmallFileCount = 40;
+ uint32_t MediumFileCount = 10;
+ uint32_t LargeFileCount = 2;
+ uint32_t DuplicateFileCount = 6;
+ };
+
+ static IoBuffer MakeBlob(uint64_t Seed, size_t Size)
+ {
+ FastRandom Rnd{.Seed = Seed};
+ IoBuffer Blob(Size);
+ uint8_t* Data = static_cast<uint8_t*>(Blob.MutableData());
+ size_t Offset = 0;
+ while (Offset < Size)
+ {
+ uint64_t Word = Rnd.Next();
+ size_t Chunk = std::min<size_t>(sizeof(Word), Size - Offset);
+ std::memcpy(Data + Offset, &Word, Chunk);
+ Offset += Chunk;
+ }
+ return Blob;
+ }
+
+ static void WriteTestFile(const std::filesystem::path& Path, const IoBuffer& Blob)
+ {
+ CreateDirectories(Path.parent_path());
+ zen::WriteFile(Path, Blob);
+ }
+
+ static std::vector<std::filesystem::path> MakeTestFolder(const std::filesystem::path& Root, const FolderSpec& Spec)
+ {
+ CreateDirectories(Root);
+ FastRandom Rnd{.Seed = Spec.Seed};
+
+ std::vector<std::filesystem::path> Written;
+ Written.reserve(Spec.SmallFileCount + Spec.MediumFileCount + Spec.LargeFileCount + Spec.DuplicateFileCount);
+
+ auto Emit = [&](std::string_view SubDir, uint32_t Index, size_t Size) {
+ std::filesystem::path Rel = std::filesystem::path(std::string(SubDir)) / fmt::format("f_{:05}.bin", Index);
+ WriteTestFile(Root / Rel, MakeBlob(Spec.Seed * 7919ull + Index, Size));
+ Written.push_back(Rel);
+ };
+
+ for (uint32_t I = 0; I < Spec.SmallFileCount; ++I)
+ {
+ Emit("small", I, 1024u + static_cast<size_t>(Rnd.Next() & 0xFFFu));
+ }
+ for (uint32_t I = 0; I < Spec.MediumFileCount; ++I)
+ {
+ Emit("medium", I, 60u * 1024u + static_cast<size_t>(Rnd.Next() & 0x3FFFu));
+ }
+ for (uint32_t I = 0; I < Spec.LargeFileCount; ++I)
+ {
+ Emit("large", I, 900u * 1024u + static_cast<size_t>(Rnd.Next() & 0x1FFFFu));
+ }
+
+ // Duplicates of previously-written small files so upload can re-use blocks / chunks.
+ for (uint32_t I = 0; I < Spec.DuplicateFileCount && Spec.SmallFileCount > 0; ++I)
+ {
+ std::filesystem::path Source = Root / Written[I % Spec.SmallFileCount];
+ std::filesystem::path Rel = std::filesystem::path("dupes") / fmt::format("d_{:05}.bin", I);
+ CreateDirectories((Root / Rel).parent_path());
+ std::error_code Ec;
+ std::filesystem::copy_file(Source, Root / Rel, std::filesystem::copy_options::overwrite_existing, Ec);
+ if (!Ec)
+ {
+ Written.push_back(Rel);
+ }
+ }
+
+ return Written;
+ }
+
+ static void CopyTreeExcludingZen(const std::filesystem::path& Src, const std::filesystem::path& Dst)
+ {
+ CreateDirectories(Dst);
+ std::error_code Ec;
+ for (auto It = std::filesystem::recursive_directory_iterator(Src, Ec); !Ec && It != std::filesystem::recursive_directory_iterator();
+ It.increment(Ec))
+ {
+ const std::filesystem::path Rel = std::filesystem::relative(It->path(), Src);
+ if (!Rel.empty() && Rel.begin()->string() == ".zen")
+ {
+ It.disable_recursion_pending();
+ continue;
+ }
+ if (It->is_directory())
+ {
+ CreateDirectories(Dst / Rel);
+ }
+ else if (It->is_regular_file())
+ {
+ CreateDirectories((Dst / Rel).parent_path());
+ std::error_code CopyEc;
+ std::filesystem::copy_file(It->path(), Dst / Rel, std::filesystem::copy_options::overwrite_existing, CopyEc);
+ }
+ }
+ }
+
+ static std::vector<std::filesystem::path> ListRelative(const std::filesystem::path& Root)
+ {
+ std::vector<std::filesystem::path> Paths;
+ std::error_code Ec;
+ for (auto It = std::filesystem::recursive_directory_iterator(Root, Ec);
+ !Ec && It != std::filesystem::recursive_directory_iterator();
+ It.increment(Ec))
+ {
+ const std::filesystem::path Rel = std::filesystem::relative(It->path(), Root);
+ if (!Rel.empty() && Rel.begin()->string() == ".zen")
+ {
+ It.disable_recursion_pending();
+ continue;
+ }
+ if (It->is_regular_file())
+ {
+ Paths.push_back(Rel);
+ }
+ }
+ std::sort(Paths.begin(), Paths.end());
+ return Paths;
+ }
+
+ static bool FoldersEquivalent(const std::filesystem::path& A, const std::filesystem::path& B)
+ {
+ const auto AFiles = ListRelative(A);
+ const auto BFiles = ListRelative(B);
+ if (AFiles != BFiles)
+ {
+ return false;
+ }
+ for (const std::filesystem::path& Rel : AFiles)
+ {
+ const IoHash HA = IoHash::HashBuffer(ReadFile(A / Rel).Flatten());
+ const IoHash HB = IoHash::HashBuffer(ReadFile(B / Rel).Flatten());
+ if (HA != HB)
+ {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ struct TestHarness
+ {
+ TestHarness() : Workers(/*BoostWorkers*/ false, /*SingleThreaded*/ false), Progress(CreateStandardProgress(zen::logging::Default()))
+ {
+ }
+
+ StorageInstance MakeStorage(const std::filesystem::path& StoragePath)
+ {
+ StorageInstance SI;
+ SI.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, /*EnableJsonOutput*/ false);
+ return SI;
+ }
+
+ std::pair<Oid, Oid> UploadOnce(StorageInstance& Storage,
+ const std::filesystem::path& SourceFolder,
+ const std::filesystem::path& TempDir)
+ {
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+
+ auto ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
+ auto ChunkCache = CreateNullChunkingCache();
+
+ CreateDirectories(TempDir);
+
+ UploadFolderOptions Options{};
+ Options.TempDir = TempDir;
+ Options.FindBlockMaxCount = 10000;
+ Options.BlockReuseMinPercentLimit = 85;
+ Options.AllowMultiparts = true;
+ Options.CreateBuild = true;
+ Options.IgnoreExistingBlocks = false;
+ Options.UploadToZenCache = false;
+ Options.IsQuiet = true;
+
+ const CbObject MetaData;
+
+ UploadFolder(zen::logging::Default(),
+ *Progress,
+ Workers,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ BuildId,
+ BuildPartId,
+ /*BuildPartName*/ "default"sv,
+ SourceFolder,
+ /*ManifestPath*/ {},
+ MetaData,
+ *ChunkController,
+ *ChunkCache,
+ Options);
+
+ return {BuildId, BuildPartId};
+ }
+
+ void DownloadOnce(StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::filesystem::path& TargetFolder,
+ const std::filesystem::path& ZenFolderPath,
+ const std::filesystem::path& SystemRootDir,
+ const DownloadOptions* OverrideOptions = nullptr)
+ {
+ CreateDirectories(TargetFolder);
+ CreateDirectories(ZenFolderPath);
+ CreateDirectories(SystemRootDir);
+
+ DownloadOptions Options;
+ if (OverrideOptions)
+ {
+ Options = *OverrideOptions;
+ }
+ Options.ZenFolderPath = ZenFolderPath;
+ Options.SystemRootDir = SystemRootDir;
+ Options.IsQuiet = true;
+
+ const std::vector<Oid> BuildPartIds;
+ const std::vector<std::string> BuildPartNames;
+
+ DownloadFolder(zen::logging::Default(),
+ *Progress,
+ Workers,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ StorageCacheStats,
+ BuildId,
+ BuildPartIds,
+ BuildPartNames,
+ /*DownloadSpecPath*/ {},
+ TargetFolder,
+ Options);
+ }
+
+ std::atomic<bool> AbortFlag{false};
+ std::atomic<bool> PauseFlag{false};
+ TransferThreadWorkers Workers;
+ std::unique_ptr<ProgressBase> Progress;
+ BuildStorageBase::Statistics StorageStats;
+ BuildStorageCache::Statistics StorageCacheStats;
+ };
+
+} // namespace buildops_test
+
+TEST_SUITE_BEGIN("remotestore.buildoperations");
+
+// Flagship case: one upload + reupload + multiple download variants against
+// the same in-process storage. Exercises scavenge, local-chunk copy,
+// cached-block reuse, partial-block fetch, and full-block download.
+TEST_CASE("buildoperations.roundtrip.full_variations")
+{
+ using namespace buildops_test;
+
+ ScopedTemporaryDirectory Root;
+ TestHarness H;
+
+ const std::filesystem::path FolderA = Root.Path() / "src_a";
+ const std::filesystem::path FolderB = Root.Path() / "src_b";
+ const std::filesystem::path StoragePath = Root.Path() / "storage";
+ const std::filesystem::path UploadTemp = Root.Path() / "upload_tmp";
+ const std::filesystem::path SystemRoot = Root.Path() / "sys";
+
+ MakeTestFolder(FolderA, FolderSpec{.Seed = 1});
+ MakeTestFolder(FolderB, FolderSpec{.Seed = 1, .DuplicateFileCount = 20});
+
+ CreateDirectories(StoragePath);
+ StorageInstance Storage = H.MakeStorage(StoragePath);
+
+ const auto [BuildIdA, PartIdA] = H.UploadOnce(Storage, FolderA, UploadTemp);
+ CHECK(BuildIdA != Oid::Zero);
+
+ // Re-upload A: should round-trip without error and still produce a
+ // usable build (we verify via a subsequent download).
+ const auto [BuildIdA2, PartIdA2] = H.UploadOnce(Storage, FolderA, UploadTemp);
+ CHECK(BuildIdA2 != Oid::Zero);
+
+ // Upload B (shares content with A).
+ const auto [BuildIdB, PartIdB] = H.UploadOnce(Storage, FolderB, UploadTemp);
+ CHECK(BuildIdB != Oid::Zero);
+
+ // Download A into an empty target. Exercises ScheduleFullBlockDownloads +
+ // ScheduleLooseChunkWrites.
+ {
+ const std::filesystem::path Target = Root.Path() / "dl_empty";
+ const std::filesystem::path ZenState = Target / ".zen";
+ H.DownloadOnce(Storage, BuildIdA, Target, ZenState, SystemRoot);
+ CHECK(FoldersEquivalent(FolderA, Target));
+ }
+
+ // Re-download A after removing some files but keeping the .zen state
+ // dir. Exercises ScheduleCachedBlockWrites.
+ {
+ const std::filesystem::path Target = Root.Path() / "dl_cached";
+ const std::filesystem::path ZenState = Target / ".zen";
+ H.DownloadOnce(Storage, BuildIdA, Target, ZenState, SystemRoot);
+
+ int Deleted = 0;
+ for (auto& E : std::filesystem::recursive_directory_iterator(Target))
+ {
+ if (Deleted >= 5)
+ break;
+ if (E.is_regular_file())
+ {
+ const std::filesystem::path Rel = std::filesystem::relative(E.path(), Target);
+ if (!Rel.empty() && Rel.begin()->string() == ".zen")
+ continue;
+ std::error_code Ec;
+ std::filesystem::remove(E.path(), Ec);
+ if (!Ec)
+ ++Deleted;
+ }
+ }
+ CHECK(Deleted > 0);
+
+ H.DownloadOnce(Storage, BuildIdA, Target, ZenState, SystemRoot);
+ CHECK(FoldersEquivalent(FolderA, Target));
+ }
+
+ // Download B into a target pre-seeded with A's content. Exercises
+ // ScheduleLocalChunkCopies and ScheduleScavengedSequenceWrites (the two
+ // span-capture sites that were fixed).
+ {
+ const std::filesystem::path Target = Root.Path() / "dl_scavenge";
+ const std::filesystem::path ZenState = Target / ".zen";
+ CopyTreeExcludingZen(FolderA, Target);
+
+ DownloadOptions Opts;
+ Opts.EnableTargetFolderScavenging = true;
+ Opts.EnableOtherDownloadsScavenging = true;
+ H.DownloadOnce(Storage, BuildIdB, Target, ZenState, SystemRoot, &Opts);
+
+ CHECK(FoldersEquivalent(FolderB, Target));
+ }
+
+ // Partial-block mode.
+ {
+ const std::filesystem::path Target = Root.Path() / "dl_partial";
+ const std::filesystem::path ZenState = Target / ".zen";
+
+ DownloadOptions Opts;
+ Opts.PartialBlockRequestMode = EPartialBlockRequestMode::All;
+ H.DownloadOnce(Storage, BuildIdB, Target, ZenState, SystemRoot, &Opts);
+
+ CHECK(FoldersEquivalent(FolderB, Target));
+ }
+}
+
+// Abort the download before it can do meaningful work. Expected to unwind
+// cleanly, not crash or assert.
+TEST_CASE("buildoperations.download.abort_midway")
+{
+ using namespace buildops_test;
+
+ ScopedTemporaryDirectory Root;
+ TestHarness H;
+
+ const std::filesystem::path Folder = Root.Path() / "src";
+ const std::filesystem::path StoragePath = Root.Path() / "storage";
+ const std::filesystem::path UploadTemp = Root.Path() / "upload_tmp";
+ const std::filesystem::path SystemRoot = Root.Path() / "sys";
+
+ MakeTestFolder(Folder, FolderSpec{.Seed = 42});
+ CreateDirectories(StoragePath);
+ StorageInstance Storage = H.MakeStorage(StoragePath);
+
+ const auto [BuildId, PartId] = H.UploadOnce(Storage, Folder, UploadTemp);
+
+ const std::filesystem::path Target = Root.Path() / "dl_abort";
+ const std::filesystem::path ZenState = Target / ".zen";
+
+ H.AbortFlag.store(true);
+ CHECK_NOTHROW(H.DownloadOnce(Storage, BuildId, Target, ZenState, SystemRoot));
+}
+
+// Empty source folder round-trip: must not crash, must produce an empty
+// download target.
+TEST_CASE("buildoperations.roundtrip.empty_folder")
+{
+ using namespace buildops_test;
+
+ ScopedTemporaryDirectory Root;
+ TestHarness H;
+
+ const std::filesystem::path Folder = Root.Path() / "empty";
+ const std::filesystem::path StoragePath = Root.Path() / "storage";
+ const std::filesystem::path UploadTemp = Root.Path() / "upload_tmp";
+ const std::filesystem::path SystemRoot = Root.Path() / "sys";
+
+ CreateDirectories(Folder);
+ CreateDirectories(StoragePath);
+ StorageInstance Storage = H.MakeStorage(StoragePath);
+
+ const auto [BuildId, PartId] = H.UploadOnce(Storage, Folder, UploadTemp);
+
+ const std::filesystem::path Target = Root.Path() / "dl_empty";
+ const std::filesystem::path ZenState = Target / ".zen";
+ H.DownloadOnce(Storage, BuildId, Target, ZenState, SystemRoot);
+
+ CHECK(ListRelative(Target).empty());
+}
+
+TEST_SUITE_END();
+
+#endif // ZEN_WITH_TESTS
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/buildprimecache.cpp b/src/zenremotestore/builds/buildprimecache.cpp
new file mode 100644
index 000000000..12791f718
--- /dev/null
+++ b/src/zenremotestore/builds/buildprimecache.cpp
@@ -0,0 +1,350 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildprimecache.h>
+
+#include <zencore/compactbinaryutil.h>
+#include <zencore/filesystem.h>
+#include <zencore/fmtutils.h>
+#include <zencore/parallelwork.h>
+#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/buildstorageutil.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenutil/filteredrate.h>
+#include <zenutil/progress.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+BuildsOperationPrimeCache::BuildsOperationPrimeCache(LoggerRef Log,
+ ProgressBase& Progress,
+ StorageInstance& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ WorkerThreadPool& NetworkPool,
+ const Oid& BuildId,
+ std::span<const Oid> BuildPartIds,
+ const Options& Options,
+ BuildStorageCache::Statistics& StorageCacheStats)
+: m_Log(Log)
+, m_Progress(Progress)
+, m_Storage(Storage)
+, m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_NetworkPool(NetworkPool)
+, m_BuildId(BuildId)
+, m_BuildPartIds(BuildPartIds.begin(), BuildPartIds.end())
+, m_Options(Options)
+, m_StorageCacheStats(StorageCacheStats)
+{
+ m_TempPath = m_Options.ZenFolderPath / "tmp";
+ CreateDirectories(m_TempPath);
+}
+
+void
+BuildsOperationPrimeCache::Execute()
+{
+ ZEN_TRACE_CPU("BuildsOperationPrimeCache::Execute");
+
+ Stopwatch PrimeTimer;
+
+ tsl::robin_map<IoHash, uint64_t, IoHash::Hasher> LooseChunkRawSizes;
+ tsl::robin_set<IoHash, IoHash::Hasher> BuildBlobs;
+ CollectReferencedBlobs(BuildBlobs, LooseChunkRawSizes);
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Found {} referenced blobs", BuildBlobs.size());
+ }
+
+ if (BuildBlobs.empty())
+ {
+ return;
+ }
+
+ std::vector<IoHash> BlobsToDownload = FilterAlreadyCachedBlobs(BuildBlobs);
+
+ if (BlobsToDownload.empty())
+ {
+ return;
+ }
+
+ std::atomic<uint64_t> MultipartAttachmentCount;
+ std::atomic<size_t> CompletedDownloadCount;
+ FilteredRate FilteredDownloadedBytesPerSecond;
+
+ ScheduleBlobDownloads(BlobsToDownload,
+ LooseChunkRawSizes,
+ MultipartAttachmentCount,
+ CompletedDownloadCount,
+ FilteredDownloadedBytesPerSecond);
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ if (m_Storage.CacheStorage)
+ {
+ m_Storage.CacheStorage->Flush(m_Progress.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool {
+ ZEN_UNUSED(Remaining);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheHost.Name);
+ }
+ return !m_AbortFlag;
+ });
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load();
+ ZEN_INFO("Downloaded {} ({}bits/s) in {}. {} as multipart. Completed in {}",
+ NiceBytes(DownloadedBytes),
+ NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
+ NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
+ MultipartAttachmentCount.load(),
+ NiceTimeSpanMs(PrimeTimer.GetElapsedTimeMs()));
+ }
+}
+
+void
+BuildsOperationPrimeCache::CollectReferencedBlobs(tsl::robin_set<IoHash, IoHash::Hasher>& OutBuildBlobs,
+ tsl::robin_map<IoHash, uint64_t, IoHash::Hasher>& OutLooseChunkRawSizes)
+{
+ for (const Oid& BuildPartId : m_BuildPartIds)
+ {
+ CbObject BuildPart = m_Storage.BuildStorage->GetBuildPart(m_BuildId, BuildPartId);
+
+ CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView();
+ std::vector<IoHash> BlockAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlockAttachmentsView);
+
+ CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView();
+ std::vector<IoHash> ChunkAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView);
+ std::vector<uint64_t> ChunkRawSizes = compactbinary_helpers::ReadArray<uint64_t>("chunkRawSizes"sv, ChunkAttachmentsView);
+ if (ChunkAttachments.size() != ChunkRawSizes.size())
+ {
+ throw std::runtime_error(fmt::format("Mismatch of loose chunk raw size array, expected {}, found {}",
+ ChunkAttachments.size(),
+ ChunkRawSizes.size()));
+ }
+
+ OutBuildBlobs.reserve(ChunkAttachments.size() + BlockAttachments.size());
+ OutBuildBlobs.insert(BlockAttachments.begin(), BlockAttachments.end());
+ OutBuildBlobs.insert(ChunkAttachments.begin(), ChunkAttachments.end());
+
+ for (size_t ChunkAttachmentIndex = 0; ChunkAttachmentIndex < ChunkAttachments.size(); ChunkAttachmentIndex++)
+ {
+ OutLooseChunkRawSizes.insert_or_assign(ChunkAttachments[ChunkAttachmentIndex], ChunkRawSizes[ChunkAttachmentIndex]);
+ }
+ }
+}
+
+std::vector<IoHash>
+BuildsOperationPrimeCache::FilterAlreadyCachedBlobs(const tsl::robin_set<IoHash, IoHash::Hasher>& BuildBlobs)
+{
+ std::vector<IoHash> BlobsToDownload;
+ BlobsToDownload.reserve(BuildBlobs.size());
+
+ if (m_Storage.CacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload)
+ {
+ ZEN_TRACE_CPU("BlobCacheExistCheck");
+ Stopwatch Timer;
+
+ const std::vector<IoHash> BlobHashes(BuildBlobs.begin(), BuildBlobs.end());
+ const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
+ m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes);
+
+ if (CacheExistsResult.size() == BlobHashes.size())
+ {
+ for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++)
+ {
+ if (!CacheExistsResult[BlobIndex].HasBody)
+ {
+ BlobsToDownload.push_back(BlobHashes[BlobIndex]);
+ }
+ }
+ size_t FoundCount = BuildBlobs.size() - BlobsToDownload.size();
+
+ if (FoundCount > 0 && !m_Options.IsQuiet)
+ {
+ ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}",
+ FoundCount,
+ BuildBlobs.size(),
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
+ }
+ }
+ else
+ {
+ BlobsToDownload.insert(BlobsToDownload.end(), BuildBlobs.begin(), BuildBlobs.end());
+ }
+ return BlobsToDownload;
+}
+
+void
+BuildsOperationPrimeCache::ScheduleBlobDownloads(std::span<const IoHash> BlobsToDownload,
+ const tsl::robin_map<IoHash, uint64_t, IoHash::Hasher>& LooseChunkRawSizes,
+ std::atomic<uint64_t>& MultipartAttachmentCount,
+ std::atomic<size_t>& CompletedDownloadCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond)
+{
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Downloading");
+
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ const size_t BlobCount = BlobsToDownload.size();
+
+ for (size_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++)
+ {
+ Work.ScheduleWork(
+ m_NetworkPool,
+ [this,
+ &Work,
+ BlobsToDownload,
+ BlobCount,
+ &LooseChunkRawSizes,
+ &CompletedDownloadCount,
+ &FilteredDownloadedBytesPerSecond,
+ &MultipartAttachmentCount,
+ BlobIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ const IoHash& BlobHash = BlobsToDownload[BlobIndex];
+ bool IsLargeBlob = false;
+ if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end())
+ {
+ IsLargeBlob = It->second >= m_Options.LargeAttachmentSize;
+ }
+
+ FilteredDownloadedBytesPerSecond.Start();
+
+ if (IsLargeBlob)
+ {
+ DownloadLargeBlobForCache(Work,
+ BlobHash,
+ BlobCount,
+ CompletedDownloadCount,
+ MultipartAttachmentCount,
+ FilteredDownloadedBytesPerSecond);
+ }
+ else
+ {
+ DownloadSingleBlobForCache(BlobHash, BlobCount, CompletedDownloadCount, FilteredDownloadedBytesPerSecond);
+ }
+ }
+ });
+ }
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+
+ uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load();
+ FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
+
+ std::string DownloadRateString = (CompletedDownloadCount == BlobCount)
+ ? ""
+ : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
+ std::string UploadDetails = m_Storage.CacheStorage ? fmt::format(" {} ({}) uploaded.",
+ m_StorageCacheStats.PutBlobCount.load(),
+ NiceBytes(m_StorageCacheStats.PutBlobByteCount.load()))
+ : "";
+
+ std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
+ CompletedDownloadCount.load(),
+ BlobCount,
+ NiceBytes(DownloadedBytes),
+ DownloadRateString,
+ UploadDetails);
+ ProgressBar->UpdateState({.Task = "Downloading",
+ .Details = Details,
+ .TotalCount = BlobCount,
+ .RemainingCount = BlobCount - CompletedDownloadCount.load(),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+
+ FilteredDownloadedBytesPerSecond.Stop();
+ ProgressBar->Finish();
+}
+
+void
+BuildsOperationPrimeCache::DownloadLargeBlobForCache(ParallelWork& Work,
+ const IoHash& BlobHash,
+ size_t BlobCount,
+ std::atomic<size_t>& CompletedDownloadCount,
+ std::atomic<uint64_t>& MultipartAttachmentCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond)
+{
+ DownloadLargeBlob(*m_Storage.BuildStorage,
+ m_TempPath,
+ m_BuildId,
+ BlobHash,
+ m_Options.PreferredMultipartChunkSize,
+ Work,
+ m_NetworkPool,
+ m_DownloadStats.DownloadedChunkByteCount,
+ MultipartAttachmentCount,
+ [this, BlobCount, BlobHash, &FilteredDownloadedBytesPerSecond, &CompletedDownloadCount](IoBuffer&& Payload) {
+ m_DownloadStats.DownloadedChunkCount++;
+ m_DownloadStats.RequestsCompleteCount++;
+
+ if (!m_AbortFlag)
+ {
+ if (Payload && m_Storage.CacheStorage)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId,
+ BlobHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(Payload)));
+ }
+ }
+ if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+ });
+}
+
+void
+BuildsOperationPrimeCache::DownloadSingleBlobForCache(const IoHash& BlobHash,
+ size_t BlobCount,
+ std::atomic<size_t>& CompletedDownloadCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond)
+{
+ IoBuffer Payload;
+ try
+ {
+ Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash);
+
+ m_DownloadStats.DownloadedBlockCount++;
+ m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize();
+ m_DownloadStats.RequestsCompleteCount++;
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+
+ if (!m_AbortFlag)
+ {
+ if (Payload && m_Storage.CacheStorage)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId,
+ BlobHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(std::move(Payload))));
+ }
+ if (CompletedDownloadCount.fetch_add(1) + 1 == BlobCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+ }
+}
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/buildsavedstate.cpp b/src/zenremotestore/builds/buildsavedstate.cpp
index 1d1f4605f..dfc565d4a 100644
--- a/src/zenremotestore/builds/buildsavedstate.cpp
+++ b/src/zenremotestore/builds/buildsavedstate.cpp
@@ -163,6 +163,8 @@ BuildSaveState::Write(const BuildSaveState& SaveState, CbWriter& Output)
{
ZEN_ASSERT(!SaveState.LocalPath.empty());
+ Output.AddInteger("version", SaveState.Version);
+
Output.AddString("path", (const char*)SaveState.LocalPath.u8string().c_str());
BuildsSelection::Write(SaveState.State.Selection, Output);
@@ -182,6 +184,7 @@ BuildSaveState::Write(const BuildSaveState& SaveState, CbWriter& Output)
BuildSaveState
BuildSaveState::Read(CbObjectView& Input)
{
+ uint32_t Version = Input["version"].AsUInt32(BuildSaveState::NoVersion);
BuildState State = BuildState::Read(Input);
CbObjectView LocalFolderStateObject = Input["localFolderState"sv].AsObjectView();
FolderContent FolderState = LoadFolderContentToCompactBinary(LocalFolderStateObject);
@@ -191,7 +194,10 @@ BuildSaveState::Read(CbObjectView& Input)
throw std::runtime_error("BuildSaveState is invalid, 'path' field is empty");
}
- return BuildSaveState{.State = std::move(State), .FolderState = std::move(FolderState), .LocalPath = std::move(LocalPath)};
+ return BuildSaveState{.Version = Version,
+ .State = std::move(State),
+ .FolderState = std::move(FolderState),
+ .LocalPath = std::move(LocalPath)};
}
CbObject
@@ -588,6 +594,8 @@ namespace buildsavestate_test {
}
} // namespace buildsavestate_test
+TEST_SUITE_BEGIN("remotestore.buildsavedstate");
+
TEST_CASE("buildsavestate.BuildsSelection")
{
using namespace buildsavestate_test;
@@ -696,6 +704,8 @@ TEST_CASE("buildsavestate.DownloadedPaths")
}
}
+TEST_SUITE_END();
+
#endif // ZEN_WITH_TESTS
} // namespace zen
diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp
index faa85f81b..8fd31a326 100644
--- a/src/zenremotestore/builds/buildstoragecache.cpp
+++ b/src/zenremotestore/builds/buildstoragecache.cpp
@@ -96,7 +96,8 @@ public:
ZEN_ASSERT(!IsFlushed);
ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary);
- // Move all segments in Payload to be file handle based so if Payload is materialized it does not affect buffers in queue
+ // Move all segments in Payload to be file handle based unless they are very small so if Payload is materialized it does not affect
+ // buffers in queue
std::vector<SharedBuffer> FileBasedSegments;
std::span<const SharedBuffer> Segments = Payload.GetSegments();
FileBasedSegments.reserve(Segments.size());
@@ -104,42 +105,56 @@ public:
tsl::robin_map<void*, std::filesystem::path> HandleToPath;
for (const SharedBuffer& Segment : Segments)
{
- std::filesystem::path FilePath;
- IoBufferFileReference Ref;
- if (Segment.AsIoBuffer().GetFileReference(Ref))
+ const uint64_t SegmentSize = Segment.GetSize();
+ if (SegmentSize < 16u * 1024u)
{
- if (auto It = HandleToPath.find(Ref.FileHandle); It != HandleToPath.end())
- {
- FilePath = It->second;
- }
- else
+ FileBasedSegments.push_back(Segment);
+ }
+ else
+ {
+ std::filesystem::path FilePath;
+ IoBufferFileReference Ref;
+ if (Segment.AsIoBuffer().GetFileReference(Ref))
{
- std::error_code Ec;
- std::filesystem::path Path = PathFromHandle(Ref.FileHandle, Ec);
- if (!Ec && !Path.empty())
+ if (auto It = HandleToPath.find(Ref.FileHandle); It != HandleToPath.end())
+ {
+ FilePath = It->second;
+ }
+ else
{
- HandleToPath.insert_or_assign(Ref.FileHandle, Path);
- FilePath = std::move(Path);
+ std::error_code Ec;
+ std::filesystem::path Path = PathFromHandle(Ref.FileHandle, Ec);
+ if (!Ec && !Path.empty())
+ {
+ HandleToPath.insert_or_assign(Ref.FileHandle, Path);
+ FilePath = std::move(Path);
+ }
+ else
+ {
+ ZEN_WARN("Failed getting path for chunk to upload to cache. Skipping upload.");
+ return;
+ }
}
}
- }
- if (!FilePath.empty())
- {
- IoBuffer BufferFromFile = IoBufferBuilder::MakeFromFile(FilePath, Ref.FileChunkOffset, Ref.FileChunkSize);
- if (BufferFromFile)
+ if (!FilePath.empty())
{
- FileBasedSegments.push_back(SharedBuffer(std::move(BufferFromFile)));
+ IoBuffer BufferFromFile = IoBufferBuilder::MakeFromFile(FilePath, Ref.FileChunkOffset, Ref.FileChunkSize);
+ if (BufferFromFile)
+ {
+ FileBasedSegments.push_back(SharedBuffer(std::move(BufferFromFile)));
+ }
+ else
+ {
+ ZEN_WARN("Failed opening file '{}' to upload to cache. Skipping upload.", FilePath);
+ return;
+ }
}
else
{
FileBasedSegments.push_back(Segment);
}
}
- else
- {
- FileBasedSegments.push_back(Segment);
- }
}
}
@@ -151,7 +166,7 @@ public:
auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
HttpClient::Response CacheResponse =
- m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+ m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash),
Payload,
ContentType);
@@ -178,9 +193,12 @@ public:
{
Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)});
}
- CreateDirectories(m_TempFolderPath);
+ if (!m_TempFolderPath.empty())
+ {
+ CreateDirectories(m_TempFolderPath);
+ }
HttpClient::Response CacheResponse =
- m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()),
+ m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash),
m_TempFolderPath,
Headers);
AddStatistic(CacheResponse);
@@ -191,6 +209,78 @@ public:
return {};
}
+ virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId,
+ const IoHash& RawHash,
+ std::span<const std::pair<uint64_t, uint64_t>> Ranges) override
+ {
+ ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlobRanges");
+
+ Stopwatch ExecutionTimer;
+ auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+
+ CbObjectWriter Writer;
+ Writer.BeginArray("ranges"sv);
+ {
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ Writer.BeginObject();
+ {
+ Writer.AddInteger("offset"sv, Range.first);
+ Writer.AddInteger("length"sv, Range.second);
+ }
+ Writer.EndObject();
+ }
+ }
+ Writer.EndArray(); // ranges
+
+ if (!m_TempFolderPath.empty())
+ {
+ CreateDirectories(m_TempFolderPath);
+ }
+ HttpClient::Response CacheResponse =
+ m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash),
+ Writer.Save(),
+ HttpClient::Accept(ZenContentType::kCbPackage),
+ m_TempFolderPath);
+ AddStatistic(CacheResponse);
+ if (CacheResponse.IsSuccess())
+ {
+ CbPackage ResponsePackage = ParsePackageMessage(CacheResponse.ResponsePayload);
+ CbObjectView ResponseObject = ResponsePackage.GetObject();
+
+ CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView();
+
+ std::vector<std::pair<uint64_t, uint64_t>> ReceivedRanges;
+ ReceivedRanges.reserve(RangeArray.Num());
+
+ uint64_t OffsetInPayloadRanges = 0;
+
+ for (CbFieldView View : RangeArray)
+ {
+ CbObjectView RangeView = View.AsObjectView();
+ uint64_t Offset = RangeView["offset"sv].AsUInt64();
+ uint64_t Length = RangeView["length"sv].AsUInt64();
+
+ const std::pair<uint64_t, uint64_t>& Range = Ranges[ReceivedRanges.size()];
+
+ if (Offset != Range.first || Length != Range.second)
+ {
+ return {};
+ }
+ ReceivedRanges.push_back(std::make_pair(OffsetInPayloadRanges, Length));
+ OffsetInPayloadRanges += Length;
+ }
+
+ const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash);
+ if (DataAttachment)
+ {
+ SharedBuffer PayloadRanges = DataAttachment->AsBinary();
+ return BuildBlobRanges{.PayloadBuffer = PayloadRanges.AsIoBuffer(), .Ranges = std::move(ReceivedRanges)};
+ }
+ }
+ return {};
+ }
+
virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) override
{
ZEN_ASSERT(!IsFlushed);
@@ -460,6 +550,192 @@ CreateZenBuildStorageCache(HttpClient& HttpClient,
return std::make_unique<ZenBuildStorageCache>(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BackgroundWorkerPool);
}
+#if ZEN_WITH_TESTS
+
+class InMemoryBuildStorageCache : public BuildStorageCache
+{
+public:
+ // MaxRangeSupported == 0 : no range requests are accepted, always return full blob
+ // MaxRangeSupported == 1 : single range is supported, multi range returns full blob
+ // MaxRangeSupported > 1 : multirange is supported up to MaxRangeSupported, more ranges returns empty blob (bad request)
+ explicit InMemoryBuildStorageCache(uint64_t MaxRangeSupported,
+ BuildStorageCache::Statistics& Stats,
+ double LatencySec = 0.0,
+ double DelayPerKBSec = 0.0)
+ : m_MaxRangeSupported(MaxRangeSupported)
+ , m_Stats(Stats)
+ , m_LatencySec(LatencySec)
+ , m_DelayPerKBSec(DelayPerKBSec)
+ {
+ }
+ void PutBuildBlob(const Oid&, const IoHash& RawHash, ZenContentType, const CompositeBuffer& Payload) override
+ {
+ IoBuffer Buf = Payload.Flatten().AsIoBuffer();
+ Buf.MakeOwned();
+ const uint64_t SentBytes = Buf.Size();
+ uint64_t ReceivedBytes = 0;
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); });
+ {
+ std::lock_guard Lock(m_Mutex);
+ m_Entries[RawHash] = std::move(Buf);
+ }
+ m_Stats.PutBlobCount.fetch_add(1);
+ m_Stats.PutBlobByteCount.fetch_add(SentBytes);
+ }
+
+ IoBuffer GetBuildBlob(const Oid&, const IoHash& RawHash, uint64_t RangeOffset = 0, uint64_t RangeBytes = (uint64_t)-1) override
+ {
+ uint64_t SentBytes = 0;
+ uint64_t ReceivedBytes = 0;
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); });
+ IoBuffer FullPayload;
+ {
+ std::lock_guard Lock(m_Mutex);
+ auto It = m_Entries.find(RawHash);
+ if (It == m_Entries.end())
+ {
+ return {};
+ }
+ FullPayload = It->second;
+ }
+
+ if (RangeOffset != 0 || RangeBytes != (uint64_t)-1)
+ {
+ if (m_MaxRangeSupported == 0)
+ {
+ ReceivedBytes = FullPayload.Size();
+ return FullPayload;
+ }
+ else
+ {
+ ReceivedBytes = (RangeBytes == (uint64_t)-1) ? FullPayload.Size() - RangeOffset : RangeBytes;
+ return IoBuffer(FullPayload, RangeOffset, RangeBytes);
+ }
+ }
+ else
+ {
+ ReceivedBytes = FullPayload.Size();
+ return FullPayload;
+ }
+ }
+
+ BuildBlobRanges GetBuildBlobRanges(const Oid&, const IoHash& RawHash, std::span<const std::pair<uint64_t, uint64_t>> Ranges) override
+ {
+ ZEN_ASSERT(!Ranges.empty());
+ uint64_t SentBytes = 0;
+ uint64_t ReceivedBytes = 0;
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); });
+ if (m_MaxRangeSupported > 1 && Ranges.size() > m_MaxRangeSupported)
+ {
+ return {};
+ }
+ IoBuffer FullPayload;
+ {
+ std::lock_guard Lock(m_Mutex);
+ auto It = m_Entries.find(RawHash);
+ if (It == m_Entries.end())
+ {
+ return {};
+ }
+ FullPayload = It->second;
+ }
+
+ if (Ranges.size() > m_MaxRangeSupported)
+ {
+ // An empty Ranges signals to the caller: "full buffer given, use it for all requested ranges".
+ ReceivedBytes = FullPayload.Size();
+ return {.PayloadBuffer = FullPayload};
+ }
+ else
+ {
+ uint64_t PayloadStart = Ranges.front().first;
+ uint64_t PayloadSize = Ranges.back().first + Ranges.back().second - PayloadStart;
+ IoBuffer RangeBuffer = IoBuffer(FullPayload, PayloadStart, PayloadSize);
+ std::vector<std::pair<uint64_t, uint64_t>> PayloadRanges;
+ PayloadRanges.reserve(Ranges.size());
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ PayloadRanges.push_back(std::make_pair(Range.first - PayloadStart, Range.second));
+ }
+ ReceivedBytes = PayloadSize;
+ return {.PayloadBuffer = RangeBuffer, .Ranges = std::move(PayloadRanges)};
+ }
+ }
+
+ void PutBlobMetadatas(const Oid&, std::span<const IoHash>, std::span<const CbObject>) override {}
+
+ std::vector<CbObject> GetBlobMetadatas(const Oid&, std::span<const IoHash> Hashes) override
+ {
+ return std::vector<CbObject>(Hashes.size());
+ }
+
+ std::vector<BlobExistsResult> BlobsExists(const Oid&, std::span<const IoHash> Hashes) override
+ {
+ std::lock_guard Lock(m_Mutex);
+ std::vector<BlobExistsResult> Result;
+ Result.reserve(Hashes.size());
+ for (const IoHash& Hash : Hashes)
+ {
+ auto It = m_Entries.find(Hash);
+ Result.push_back({.HasBody = (It != m_Entries.end() && It->second)});
+ }
+ return Result;
+ }
+
+ void Flush(int32_t, std::function<bool(intptr_t)>&&) override {}
+
+private:
+ void AddStatistic(uint64_t ElapsedTimeUs, uint64_t ReceivedBytes, uint64_t SentBytes)
+ {
+ m_Stats.TotalBytesWritten += SentBytes;
+ m_Stats.TotalBytesRead += ReceivedBytes;
+ m_Stats.TotalExecutionTimeUs += ElapsedTimeUs;
+ m_Stats.TotalRequestCount++;
+ SetAtomicMax(m_Stats.PeakSentBytes, SentBytes);
+ SetAtomicMax(m_Stats.PeakReceivedBytes, ReceivedBytes);
+ if (ElapsedTimeUs > 0)
+ {
+ SetAtomicMax(m_Stats.PeakBytesPerSec, (ReceivedBytes + SentBytes) * 1000000 / ElapsedTimeUs);
+ }
+ }
+
+ void SimulateLatency(uint64_t SendBytes, uint64_t ReceiveBytes)
+ {
+ double SleepSec = m_LatencySec;
+ if (m_DelayPerKBSec > 0.0)
+ {
+ SleepSec += m_DelayPerKBSec * (double(SendBytes + ReceiveBytes) / 1024u);
+ }
+ if (SleepSec > 0)
+ {
+ Sleep(int(SleepSec * 1000));
+ }
+ }
+
+ uint64_t m_MaxRangeSupported = 0;
+ BuildStorageCache::Statistics& m_Stats;
+ const double m_LatencySec = 0.0;
+ const double m_DelayPerKBSec = 0.0;
+ std::mutex m_Mutex;
+ std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> m_Entries;
+};
+
+std::unique_ptr<BuildStorageCache>
+CreateInMemoryBuildStorageCache(uint64_t MaxRangeSupported, BuildStorageCache::Statistics& Stats, double LatencySec, double DelayPerKBSec)
+{
+ return std::make_unique<InMemoryBuildStorageCache>(MaxRangeSupported, Stats, LatencySec, DelayPerKBSec);
+}
+#endif // ZEN_WITH_TESTS
+
ZenCacheEndpointTestResult
TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose)
{
@@ -474,15 +750,28 @@ TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const boo
HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds");
if (TestResponse.IsSuccess())
{
- LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health");
+ uint64_t MaxRangeCountPerRequest = 1;
+ CbObject StatusResponse = TestResponse.AsObject();
+ if (StatusResponse["ok"].AsBool())
+ {
+ MaxRangeCountPerRequest = StatusResponse["capabilities"].AsObjectView()["maxrangecountperrequest"].AsUInt64(1);
+
+ LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health");
+
+ if (!LatencyResult.Success)
+ {
+ return {.Success = false, .FailureReason = LatencyResult.FailureReason};
+ }
- if (!LatencyResult.Success)
+ return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds, .MaxRangeCountPerRequest = MaxRangeCountPerRequest};
+ }
+ else
{
- return {.Success = false, .FailureReason = LatencyResult.FailureReason};
+ return {.Success = false,
+ .FailureReason = fmt::format("ZenCache endpoint {}/status/builds did not respond with \"ok\"", BaseUrl)};
}
- return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds};
}
return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")};
-};
+}
} // namespace zen
diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp
deleted file mode 100644
index 08a896f37..000000000
--- a/src/zenremotestore/builds/buildstorageoperations.cpp
+++ /dev/null
@@ -1,7943 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include <zenremotestore/builds/buildstorageoperations.h>
-
-#include <zenremotestore/builds/buildcontent.h>
-#include <zenremotestore/builds/buildmanifest.h>
-#include <zenremotestore/builds/buildsavedstate.h>
-#include <zenremotestore/builds/buildstorage.h>
-#include <zenremotestore/builds/buildstoragecache.h>
-#include <zenremotestore/builds/buildstorageutil.h>
-#include <zenremotestore/chunking/chunkblock.h>
-#include <zenremotestore/chunking/chunkingcache.h>
-#include <zenremotestore/chunking/chunkingcontroller.h>
-#include <zenremotestore/filesystemutils.h>
-#include <zenremotestore/operationlogoutput.h>
-
-#include <zencore/basicfile.h>
-#include <zencore/compactbinary.h>
-#include <zencore/compactbinaryfile.h>
-#include <zencore/compactbinaryutil.h>
-#include <zencore/compactbinaryvalue.h>
-#include <zencore/filesystem.h>
-#include <zencore/fmtutils.h>
-#include <zencore/parallelwork.h>
-#include <zencore/scopeguard.h>
-#include <zencore/string.h>
-#include <zencore/timer.h>
-#include <zencore/trace.h>
-#include <zenutil/wildcard.h>
-
-#include <numeric>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#include <tsl/robin_map.h>
-#include <tsl/robin_set.h>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-#if ZEN_WITH_TESTS
-# include <zencore/testing.h>
-# include <zencore/testutils.h>
-# include <zenremotestore/builds/filebuildstorage.h>
-#endif // ZEN_WITH_TESTS
-
-namespace zen {
-
-using namespace std::literals;
-
-namespace {
- std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath)
- {
- return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences
- }
- std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath)
- {
- return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks
- }
- std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath)
- {
- return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks
- }
-
- uint64_t GetBytesPerSecond(uint64_t ElapsedWallTimeUS, uint64_t Count)
- {
- if (ElapsedWallTimeUS == 0)
- {
- return 0;
- }
- return Count * 1000000 / ElapsedWallTimeUS;
- }
-
- std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
- {
- return CacheFolderPath / (RawHash.ToHexString() + ".tmp");
- }
-
- std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
- {
- return CacheFolderPath / RawHash.ToHexString();
- }
-
- bool CleanDirectory(OperationLogOutput& OperationLogOutput,
- WorkerThreadPool& IOWorkerPool,
- std::atomic<bool>& AbortFlag,
- std::atomic<bool>& PauseFlag,
- bool IsQuiet,
- const std::filesystem::path& Path,
- std::span<const std::string> ExcludeDirectories)
- {
- ZEN_TRACE_CPU("CleanDirectory");
- Stopwatch Timer;
-
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(OperationLogOutput.CreateProgressBar("Clean Folder"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- CleanDirectoryResult Result = CleanDirectory(
- IOWorkerPool,
- AbortFlag,
- PauseFlag,
- Path,
- ExcludeDirectories,
- [&](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) {
- Progress.UpdateState({.Task = "Cleaning folder ",
- .Details = std::string(Details),
- .TotalCount = TotalCount,
- .RemainingCount = RemainingCount,
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- OperationLogOutput.GetProgressUpdateDelayMS());
-
- Progress.Finish();
-
- if (AbortFlag)
- {
- return false;
- }
-
- uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
-
- if (!Result.FailedRemovePaths.empty())
- {
- ExtendableStringBuilder<512> SB;
- for (size_t FailedPathIndex = 0; FailedPathIndex < Result.FailedRemovePaths.size(); FailedPathIndex++)
- {
- SB << fmt::format("\n '{}': ({}) {}",
- Result.FailedRemovePaths[FailedPathIndex].first,
- Result.FailedRemovePaths[FailedPathIndex].second.value(),
- Result.FailedRemovePaths[FailedPathIndex].second.message());
- }
- ZEN_OPERATION_LOG_WARN(OperationLogOutput, "Clean failed to remove files from '{}': {}", Path, SB.ToView());
- }
-
- if (ElapsedTimeMs >= 200 && !IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(OperationLogOutput,
- "Wiped folder '{}' {} ({}) in {}",
- Path,
- Result.FoundCount,
- NiceBytes(Result.DeletedByteCount),
- NiceTimeSpanMs(ElapsedTimeMs));
- }
-
- return Result.FailedRemovePaths.empty();
- }
-
- bool IsExtensionHashCompressable(const tsl::robin_set<uint32_t>& NonCompressableExtensionHashes, const uint32_t PathHash)
- {
- return !NonCompressableExtensionHashes.contains(PathHash);
- }
-
- bool IsChunkCompressable(const tsl::robin_set<uint32_t>& NonCompressableExtensionHashes,
- const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- uint32_t ChunkIndex)
- {
- ZEN_UNUSED(Content);
- const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex];
- if (ChunkLocationCount == 0)
- {
- return false;
- }
- const size_t ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex];
- const uint32_t SequenceIndex = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex;
- const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const uint32_t ExtensionHash = Lookup.PathExtensionHash[PathIndex];
-
- const bool IsCompressable = IsExtensionHashCompressable(NonCompressableExtensionHashes, ExtensionHash);
- return IsCompressable;
- }
-
- template<typename T>
- std::string FormatArray(std::span<const T> Items, std::string_view Prefix)
- {
- ExtendableStringBuilder<512> SB;
- for (const T& Item : Items)
- {
- SB.Append(fmt::format("{}{}", Prefix, Item));
- }
- return SB.ToString();
- }
-
- void DownloadLargeBlob(BuildStorageBase& Storage,
- const std::filesystem::path& DownloadFolder,
- const Oid& BuildId,
- const IoHash& ChunkHash,
- const std::uint64_t PreferredMultipartChunkSize,
- ParallelWork& Work,
- WorkerThreadPool& NetworkPool,
- std::atomic<uint64_t>& DownloadedChunkByteCount,
- std::atomic<uint64_t>& MultipartAttachmentCount,
- std::function<void(IoBuffer&& Payload)>&& OnDownloadComplete)
- {
- ZEN_TRACE_CPU("DownloadLargeBlob");
-
- struct WorkloadData
- {
- TemporaryFile TempFile;
- };
- std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
-
- std::error_code Ec;
- Workload->TempFile.CreateTemporary(DownloadFolder, Ec);
- if (Ec)
- {
- throw std::runtime_error(
- fmt::format("Failed opening temporary file '{}', reason: ({}) {}", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
- }
- std::vector<std::function<void()>> WorkItems = Storage.GetLargeBuildBlob(
- BuildId,
- ChunkHash,
- PreferredMultipartChunkSize,
- [&Work, Workload, &DownloadedChunkByteCount](uint64_t Offset, const IoBuffer& Chunk) {
- DownloadedChunkByteCount += Chunk.GetSize();
-
- if (!Work.IsAborted())
- {
- ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnReceive");
- Workload->TempFile.Write(Chunk.GetView(), Offset);
- }
- },
- [&Work, Workload, &DownloadedChunkByteCount, OnDownloadComplete = std::move(OnDownloadComplete)]() {
- if (!Work.IsAborted())
- {
- ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnComplete");
-
- uint64_t PayloadSize = Workload->TempFile.FileSize();
- void* FileHandle = Workload->TempFile.Detach();
- ZEN_ASSERT(FileHandle != nullptr);
- IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true);
- Payload.SetDeleteOnClose(true);
- OnDownloadComplete(std::move(Payload));
- }
- });
- if (!WorkItems.empty())
- {
- MultipartAttachmentCount++;
- }
- for (auto& WorkItem : WorkItems)
- {
- Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic<bool>& AbortFlag) {
- if (!AbortFlag)
- {
- ZEN_TRACE_CPU("Async_DownloadLargeBlob_Work");
-
- WorkItem();
- }
- });
- }
- }
-
- CompositeBuffer ValidateBlob(std::atomic<bool>& AbortFlag,
- IoBuffer&& Payload,
- const IoHash& BlobHash,
- uint64_t& OutCompressedSize,
- uint64_t& OutDecompressedSize)
- {
- ZEN_TRACE_CPU("ValidateBlob");
-
- if (Payload.GetContentType() != ZenContentType::kCompressedBinary)
- {
- throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'",
- BlobHash,
- Payload.GetSize(),
- ToString(Payload.GetContentType())));
- }
- IoHash RawHash;
- uint64_t RawSize;
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
- if (!Compressed)
- {
- throw std::runtime_error(fmt::format("Blob {} ({} bytes) compressed header is invalid", BlobHash, Payload.GetSize()));
- }
- if (RawHash != BlobHash)
- {
- throw std::runtime_error(
- fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash));
- }
-
- IoHashStream Hash;
- bool CouldDecompress = Compressed.DecompressToStream(
- 0,
- RawSize,
- [&AbortFlag, &Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
- ZEN_UNUSED(SourceOffset, SourceSize, Offset);
- if (!AbortFlag)
- {
- for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
- {
- Hash.Append(Segment.GetView());
- }
- return true;
- }
- return false;
- });
-
- if (AbortFlag)
- {
- return CompositeBuffer{};
- }
-
- if (!CouldDecompress)
- {
- throw std::runtime_error(
- fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize()));
- }
- IoHash ValidateRawHash = Hash.GetHash();
- if (ValidateRawHash != BlobHash)
- {
- throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information",
- BlobHash,
- Payload.GetSize(),
- ValidateRawHash));
- }
- OodleCompressor Compressor;
- OodleCompressionLevel CompressionLevel;
- uint64_t BlockSize;
- if (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize))
- {
- throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to get compression details", BlobHash, Payload.GetSize()));
- }
- OutCompressedSize = Payload.GetSize();
- OutDecompressedSize = RawSize;
- if (CompressionLevel == OodleCompressionLevel::None)
- {
- // Only decompress to composite if we need it for block verification
- CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite();
- if (!DecompressedComposite)
- {
- throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize()));
- }
- return DecompressedComposite;
- }
- return CompositeBuffer{};
- }
-
-} // namespace
-
-bool
-IsSingleFileChunk(const ChunkedFolderContent& RemoteContent,
- const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations)
-{
- if (Locations.size() == 1)
- {
- const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex;
- if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1)
- {
- ZEN_ASSERT_SLOW(Locations[0]->Offset == 0);
- return true;
- }
- }
- return false;
-}
-
-IoBuffer
-MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer)
-{
- ZEN_TRACE_CPU("MakeBufferMemoryBased");
- IoBuffer BlockMemoryBuffer;
- std::span<const SharedBuffer> Segments = PartialBlockBuffer.GetSegments();
- if (Segments.size() == 1)
- {
- IoBufferFileReference FileRef = {};
- if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef))
- {
- BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer();
- BasicFile Reader;
- Reader.Attach(FileRef.FileHandle);
- auto _ = MakeGuard([&Reader]() { Reader.Detach(); });
- MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
- Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
- return BlockMemoryBuffer;
- }
- else
- {
- return PartialBlockBuffer.GetSegments().front().AsIoBuffer();
- }
- }
- else
- {
- // Not a homogenous memory buffer, read all to memory
-
- BlockMemoryBuffer = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer();
- MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
- for (const SharedBuffer& Segment : Segments)
- {
- IoBufferFileReference FileRef = {};
- if (Segment.AsIoBuffer().GetFileReference(FileRef))
- {
- BasicFile Reader;
- Reader.Attach(FileRef.FileHandle);
- auto _ = MakeGuard([&Reader]() { Reader.Detach(); });
- Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
- ReadMem = ReadMem.Mid(FileRef.FileChunkSize);
- }
- else
- {
- ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView());
- }
- }
- return BlockMemoryBuffer;
- }
-}
-
-class FilteredRate
-{
-public:
- FilteredRate() {}
-
- void Start()
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- if (StartTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs()))
- {
- LastTimeUS = StartTimeUS.load();
- }
- }
- }
- void Stop()
- {
- if (EndTimeUS == (uint64_t)-1)
- {
- uint64_t Expected = (uint64_t)-1;
- EndTimeUS.compare_exchange_weak(Expected, Timer.GetElapsedTimeUs());
- }
- }
-
- void Update(uint64_t Count)
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return;
- }
- uint64_t TimeUS = Timer.GetElapsedTimeUs();
- uint64_t TimeDeltaUS = TimeUS - LastTimeUS;
- if (TimeDeltaUS >= 2000000)
- {
- uint64_t Delta = Count - LastCount;
- uint64_t PerSecond = (Delta * 1000000) / TimeDeltaUS;
-
- LastPerSecond = PerSecond;
-
- LastCount = Count;
-
- FilteredPerSecond = (PerSecond + (LastPerSecond * 7)) / 8;
-
- LastTimeUS = TimeUS;
- }
- }
-
- uint64_t GetCurrent() const // If Stopped - return total count / total time
- {
- if (LastTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- return FilteredPerSecond;
- }
-
- uint64_t GetElapsedTimeUS() const
- {
- if (StartTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- if (EndTimeUS == (uint64_t)-1)
- {
- return 0;
- }
- uint64_t TimeDeltaUS = EndTimeUS - StartTimeUS;
- return TimeDeltaUS;
- }
-
- bool IsActive() const { return (StartTimeUS != (uint64_t)-1) && (EndTimeUS == (uint64_t)-1); }
-
-private:
- Stopwatch Timer;
- std::atomic<uint64_t> StartTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> EndTimeUS = (uint64_t)-1;
- std::atomic<uint64_t> LastTimeUS = (uint64_t)-1;
- uint64_t LastCount = 0;
- uint64_t LastPerSecond = 0;
- uint64_t FilteredPerSecond = 0;
-};
-
-std::filesystem::path
-ZenStateFilePath(const std::filesystem::path& ZenFolderPath)
-{
- return ZenFolderPath / "current_state.cbo";
-}
-std::filesystem::path
-ZenTempFolderPath(const std::filesystem::path& ZenFolderPath)
-{
- return ZenFolderPath / "tmp";
-}
-
-////////////////////// BuildsOperationUpdateFolder
-
-BuildsOperationUpdateFolder::BuildsOperationUpdateFolder(OperationLogOutput& OperationLogOutput,
- StorageInstance& Storage,
- std::atomic<bool>& AbortFlag,
- std::atomic<bool>& PauseFlag,
- WorkerThreadPool& IOWorkerPool,
- WorkerThreadPool& NetworkPool,
- const Oid& BuildId,
- const std::filesystem::path& Path,
- const ChunkedFolderContent& LocalContent,
- const ChunkedContentLookup& LocalLookup,
- const ChunkedFolderContent& RemoteContent,
- const ChunkedContentLookup& RemoteLookup,
- const std::vector<ChunkBlockDescription>& BlockDescriptions,
- const std::vector<IoHash>& LooseChunkHashes,
- const Options& Options)
-: m_LogOutput(OperationLogOutput)
-, m_Storage(Storage)
-, m_AbortFlag(AbortFlag)
-, m_PauseFlag(PauseFlag)
-, m_IOWorkerPool(IOWorkerPool)
-, m_NetworkPool(NetworkPool)
-, m_BuildId(BuildId)
-, m_Path(Path)
-, m_LocalContent(LocalContent)
-, m_LocalLookup(LocalLookup)
-, m_RemoteContent(RemoteContent)
-, m_RemoteLookup(RemoteLookup)
-, m_BlockDescriptions(BlockDescriptions)
-, m_LooseChunkHashes(LooseChunkHashes)
-, m_Options(Options)
-, m_CacheFolderPath(ZenTempCacheFolderPath(m_Options.ZenFolderPath))
-, m_TempDownloadFolderPath(ZenTempDownloadFolderPath(m_Options.ZenFolderPath))
-, m_TempBlockFolderPath(ZenTempBlockFolderPath(m_Options.ZenFolderPath))
-{
-}
-
-void
-BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
-{
- ZEN_TRACE_CPU("BuildsOperationUpdateFolder::Execute");
- try
- {
- enum class TaskSteps : uint32_t
- {
- ScanExistingData,
- WriteChunks,
- PrepareTarget,
- FinalizeTarget,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); });
-
- ZEN_ASSERT((!m_Options.PrimeCacheOnly) ||
- (m_Options.PrimeCacheOnly && (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off)));
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::ScanExistingData, (uint32_t)TaskSteps::StepCount);
-
- CreateDirectories(m_CacheFolderPath);
- CreateDirectories(m_TempDownloadFolderPath);
- CreateDirectories(m_TempBlockFolderPath);
-
- Stopwatch CacheMappingTimer;
-
- std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(m_RemoteContent.ChunkedContent.SequenceRawHashes.size());
- std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size());
- std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size());
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound;
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound;
- if (!m_Options.PrimeCacheOnly)
- {
- ScanCacheFolder(CachedChunkHashesFound, CachedSequenceHashesFound);
- }
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound;
- if (!m_Options.PrimeCacheOnly)
- {
- ScanTempBlocksFolder(CachedBlocksFound);
- }
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex;
-
- if (!m_Options.PrimeCacheOnly && m_Options.EnableTargetFolderScavenging)
- {
- // Pick up all whole files we can use from current local state
- ZEN_TRACE_CPU("GetLocalSequences");
-
- Stopwatch LocalTimer;
-
- std::vector<uint32_t> MissingSequenceIndexes = ScanTargetFolder(CachedChunkHashesFound, CachedSequenceHashesFound);
-
- for (uint32_t RemoteSequenceIndex : MissingSequenceIndexes)
- {
- // We must write the sequence
- const uint32_t ChunkCount = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
- const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
- SequenceIndexesLeftToFindToRemoteIndex.insert({RemoteSequenceRawHash, RemoteSequenceIndex});
- }
- }
- else
- {
- for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size();
- RemoteSequenceIndex++)
- {
- const uint32_t ChunkCount = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount;
- }
- }
-
- std::vector<ChunkedFolderContent> ScavengedContents;
- std::vector<ChunkedContentLookup> ScavengedLookups;
- std::vector<std::filesystem::path> ScavengedPaths;
-
- std::vector<ScavengedSequenceCopyOperation> ScavengedSequenceCopyOperations;
- uint64_t ScavengedPathsCount = 0;
-
- if (!m_Options.PrimeCacheOnly && m_Options.EnableOtherDownloadsScavenging)
- {
- ZEN_TRACE_CPU("GetScavengedSequences");
-
- Stopwatch ScavengeTimer;
-
- if (!SequenceIndexesLeftToFindToRemoteIndex.empty())
- {
- std::vector<ScavengeSource> ScavengeSources = FindScavengeSources();
-
- const size_t ScavengePathCount = ScavengeSources.size();
-
- ScavengedContents.resize(ScavengePathCount);
- ScavengedLookups.resize(ScavengePathCount);
- ScavengedPaths.resize(ScavengePathCount);
-
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scavenging"));
- OperationLogOutput::ProgressBar& ScavengeProgressBar(*ProgressBarPtr);
-
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- std::atomic<uint64_t> PathsFound(0);
- std::atomic<uint64_t> ChunksFound(0);
- std::atomic<uint64_t> PathsScavenged(0);
-
- for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++)
- {
- Work.ScheduleWork(m_IOWorkerPool,
- [this,
- &ScavengeSources,
- &ScavengedContents,
- &ScavengedPaths,
- &ScavengedLookups,
- &PathsFound,
- &ChunksFound,
- &PathsScavenged,
- ScavengeIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_FindScavengeContent");
-
- const ScavengeSource& Source = ScavengeSources[ScavengeIndex];
- ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengeIndex];
- ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengeIndex];
-
- if (FindScavengeContent(Source, ScavengedLocalContent, ScavengedLookup))
- {
- ScavengedPaths[ScavengeIndex] = Source.Path;
- PathsFound += ScavengedLocalContent.Paths.size();
- ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size();
- }
- else
- {
- ScavengedPaths[ScavengeIndex].clear();
- }
- PathsScavenged++;
- }
- });
- }
- {
- ZEN_TRACE_CPU("ScavengeScan_Wait");
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging",
- PathsScavenged.load(),
- ScavengePathCount,
- PathsFound.load(),
- ChunksFound.load());
- ScavengeProgressBar.UpdateState(
- {.Task = "Scavenging ",
- .Details = Details,
- .TotalCount = ScavengePathCount,
- .RemainingCount = ScavengePathCount - PathsScavenged.load(),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- }
-
- ScavengeProgressBar.Finish();
- if (m_AbortFlag)
- {
- return;
- }
-
- for (uint32_t ScavengedContentIndex = 0;
- ScavengedContentIndex < ScavengedContents.size() && (!SequenceIndexesLeftToFindToRemoteIndex.empty());
- ScavengedContentIndex++)
- {
- const std::filesystem::path& ScavengePath = ScavengedPaths[ScavengedContentIndex];
- if (!ScavengePath.empty())
- {
- const ChunkedFolderContent& ScavengedLocalContent = ScavengedContents[ScavengedContentIndex];
- const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex];
-
- for (uint32_t ScavengedSequenceIndex = 0;
- ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size();
- ScavengedSequenceIndex++)
- {
- const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex];
- if (auto It = SequenceIndexesLeftToFindToRemoteIndex.find(SequenceRawHash);
- It != SequenceIndexesLeftToFindToRemoteIndex.end())
- {
- const uint32_t RemoteSequenceIndex = It->second;
- const uint64_t RawSize =
- m_RemoteContent.RawSizes[m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]];
- ZEN_ASSERT(RawSize > 0);
-
- const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex];
- ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred()));
-
- ScavengedSequenceCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex,
- .ScavengedPathIndex = ScavengedPathIndex,
- .RemoteSequenceIndex = RemoteSequenceIndex,
- .RawSize = RawSize});
-
- SequenceIndexesLeftToFindToRemoteIndex.erase(SequenceRawHash);
- SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = 0;
-
- m_CacheMappingStats.ScavengedPathsMatchingSequencesCount++;
- m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize;
- }
- }
- ScavengedPathsCount++;
- }
- }
- }
- m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
- }
-
- uint32_t RemainingChunkCount = 0;
- for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
- {
- uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
- if (ChunkWriteCount > 0)
- {
- RemainingChunkCount++;
- }
- }
-
- // Pick up all chunks in current local state
- tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCopyChunkDataIndex;
- std::vector<CopyChunkData> CopyChunkDatas;
-
- if (!m_Options.PrimeCacheOnly && m_Options.EnableTargetFolderScavenging)
- {
- ZEN_TRACE_CPU("GetLocalChunks");
-
- Stopwatch LocalTimer;
-
- ScavengeSourceForChunks(RemainingChunkCount,
- RemoteChunkIndexNeedsCopyFromLocalFileFlags,
- RawHashToCopyChunkDataIndex,
- SequenceIndexChunksLeftToWriteCounters,
- m_LocalContent,
- m_LocalLookup,
- CopyChunkDatas,
- uint32_t(-1),
- m_CacheMappingStats.LocalChunkMatchingRemoteCount,
- m_CacheMappingStats.LocalChunkMatchingRemoteByteCount);
-
- m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
- }
-
- if (!m_Options.PrimeCacheOnly && m_Options.EnableOtherDownloadsScavenging)
- {
- ZEN_TRACE_CPU("GetScavengeChunks");
-
- Stopwatch ScavengeTimer;
-
- for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0);
- ScavengedContentIndex++)
- {
- const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex];
- const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex];
-
- ScavengeSourceForChunks(RemainingChunkCount,
- RemoteChunkIndexNeedsCopyFromLocalFileFlags,
- RawHashToCopyChunkDataIndex,
- SequenceIndexChunksLeftToWriteCounters,
- ScavengedContent,
- ScavengedLookup,
- CopyChunkDatas,
- ScavengedContentIndex,
- m_CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount);
- }
- m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
- }
-
- if (!m_Options.IsQuiet)
- {
- if (m_CacheMappingStats.CacheSequenceHashesCount > 0 || m_CacheMappingStats.CacheChunkCount > 0 ||
- m_CacheMappingStats.CacheBlockCount > 0)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
- m_CacheMappingStats.CacheSequenceHashesCount,
- NiceBytes(m_CacheMappingStats.CacheSequenceHashesByteCount),
- m_CacheMappingStats.CacheChunkCount,
- NiceBytes(m_CacheMappingStats.CacheChunkByteCount),
- m_CacheMappingStats.CacheBlockCount,
- NiceBytes(m_CacheMappingStats.CacheBlocksByteCount),
- NiceTimeSpanMs(m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
- }
-
- if (m_CacheMappingStats.LocalPathsMatchingSequencesCount > 0 || m_CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
- m_CacheMappingStats.LocalPathsMatchingSequencesCount,
- NiceBytes(m_CacheMappingStats.LocalPathsMatchingSequencesByteCount),
- m_CacheMappingStats.LocalChunkMatchingRemoteCount,
- NiceBytes(m_CacheMappingStats.LocalChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
- }
- if (m_CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || m_CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
- ScavengedPathsCount,
- m_CacheMappingStats.ScavengedPathsMatchingSequencesCount,
- NiceBytes(m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
- m_CacheMappingStats.ScavengedChunkMatchingRemoteCount,
- NiceBytes(m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
- NiceTimeSpanMs(m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
- }
- }
-
- uint64_t BytesToWrite = 0;
-
- for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
- {
- uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
- if (ChunkWriteCount > 0)
- {
- BytesToWrite += m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount;
- if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
- {
- RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true;
- }
- }
- }
-
- for (const ScavengedSequenceCopyOperation& ScavengeCopyOp : ScavengedSequenceCopyOperations)
- {
- BytesToWrite += ScavengeCopyOp.RawSize;
- }
-
- uint64_t BytesToValidate = m_Options.ValidateCompletedSequences ? BytesToWrite : 0;
-
- uint64_t TotalRequestCount = 0;
- uint64_t TotalPartWriteCount = 0;
- std::atomic<uint64_t> WritePartsComplete = 0;
-
- tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex;
- RemotePathToRemoteIndex.reserve(m_RemoteContent.Paths.size());
- for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
- {
- RemotePathToRemoteIndex.insert({m_RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
- }
-
- CheckRequiredDiskSpace(RemotePathToRemoteIndex);
-
- BlobsExistsResult ExistsResult;
- {
- ChunkBlockAnalyser BlockAnalyser(m_LogOutput,
- m_BlockDescriptions,
- ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet,
- .IsVerbose = m_Options.IsVerbose,
- .HostLatencySec = m_Storage.BuildStorageLatencySec,
- .HostHighSpeedLatencySec = m_Storage.CacheLatencySec});
-
- std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = BlockAnalyser.GetNeeded(
- m_RemoteLookup.ChunkHashToChunkIndex,
- [&](uint32_t RemoteChunkIndex) -> bool { return RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]; });
-
- std::vector<uint32_t> FetchBlockIndexes;
- std::vector<uint32_t> CachedChunkBlockIndexes;
-
- {
- ZEN_TRACE_CPU("BlockCacheFileExists");
- for (const ChunkBlockAnalyser::NeededBlock& NeededBlock : NeededBlocks)
- {
- if (m_Options.PrimeCacheOnly)
- {
- FetchBlockIndexes.push_back(NeededBlock.BlockIndex);
- }
- else
- {
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex];
- bool UsingCachedBlock = false;
- if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end())
- {
- TotalPartWriteCount++;
-
- std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
- if (IsFile(BlockPath))
- {
- CachedChunkBlockIndexes.push_back(NeededBlock.BlockIndex);
- UsingCachedBlock = true;
- }
- }
- if (!UsingCachedBlock)
- {
- FetchBlockIndexes.push_back(NeededBlock.BlockIndex);
- }
- }
- }
- }
-
- std::vector<uint32_t> NeededLooseChunkIndexes;
-
- {
- NeededLooseChunkIndexes.reserve(m_LooseChunkHashes.size());
- for (uint32_t LooseChunkIndex = 0; LooseChunkIndex < m_LooseChunkHashes.size(); LooseChunkIndex++)
- {
- const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex];
- auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
- ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
- const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
-
- if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Skipping chunk {} due to cache reuse",
- m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]);
- }
- continue;
- }
-
- bool NeedsCopy = true;
- if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false))
- {
- uint64_t WriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
- if (WriteCount == 0)
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Skipping chunk {} due to cache reuse",
- m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]);
- }
- }
- else
- {
- NeededLooseChunkIndexes.push_back(LooseChunkIndex);
- }
- }
- }
- }
-
- if (m_Storage.BuildCacheStorage)
- {
- ZEN_TRACE_CPU("BlobCacheExistCheck");
- Stopwatch Timer;
-
- std::vector<IoHash> BlobHashes;
- BlobHashes.reserve(NeededLooseChunkIndexes.size() + FetchBlockIndexes.size());
-
- for (const uint32_t LooseChunkIndex : NeededLooseChunkIndexes)
- {
- BlobHashes.push_back(m_LooseChunkHashes[LooseChunkIndex]);
- }
-
- for (uint32_t BlockIndex : FetchBlockIndexes)
- {
- BlobHashes.push_back(m_BlockDescriptions[BlockIndex].BlockHash);
- }
-
- const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
- m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes);
-
- if (CacheExistsResult.size() == BlobHashes.size())
- {
- ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size());
- for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++)
- {
- if (CacheExistsResult[BlobIndex].HasBody)
- {
- ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]);
- }
- }
- }
- ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- if (!ExistsResult.ExistingBlobs.empty() && !m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Remote cache : Found {} out of {} needed blobs in {}",
- ExistsResult.ExistingBlobs.size(),
- BlobHashes.size(),
- NiceTimeSpanMs(ExistsResult.ElapsedTimeMs));
- }
- }
-
- std::vector<ChunkBlockAnalyser::EPartialBlockDownloadMode> BlockPartialDownloadModes;
- if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off)
- {
- BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off);
- }
- else
- {
- BlockPartialDownloadModes.reserve(m_BlockDescriptions.size());
- for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++)
- {
- const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash);
- if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All)
- {
- BlockPartialDownloadModes.push_back(BlockExistInCache
- ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed
- : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange);
- }
- else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly)
- {
- BlockPartialDownloadModes.push_back(BlockExistInCache
- ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed
- : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off);
- }
- else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed)
- {
- BlockPartialDownloadModes.push_back(BlockExistInCache
- ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed
- : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange);
- }
- }
- }
- ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size());
-
- ChunkBlockAnalyser::BlockResult PartialBlocks =
- BlockAnalyser.CalculatePartialBlockDownloads(NeededBlocks, BlockPartialDownloadModes);
-
- struct LooseChunkHashWorkData
- {
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs;
- uint32_t RemoteChunkIndex = (uint32_t)-1;
- };
-
- TotalRequestCount += NeededLooseChunkIndexes.size();
- TotalPartWriteCount += NeededLooseChunkIndexes.size();
- TotalRequestCount += PartialBlocks.BlockRanges.size();
- TotalPartWriteCount += PartialBlocks.BlockRanges.size();
- TotalRequestCount += PartialBlocks.FullBlockIndexes.size();
- TotalPartWriteCount += PartialBlocks.FullBlockIndexes.size();
-
- std::vector<LooseChunkHashWorkData> LooseChunkHashWorks;
- for (uint32_t LooseChunkIndex : NeededLooseChunkIndexes)
- {
- const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex];
- auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
- ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
- const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
-
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
-
- ZEN_ASSERT(!ChunkTargetPtrs.empty());
- LooseChunkHashWorks.push_back(
- LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex});
- }
-
- ZEN_TRACE_CPU("WriteChunks");
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount);
-
- Stopwatch WriteTimer;
-
- FilteredRate FilteredDownloadedBytesPerSecond;
- FilteredRate FilteredWrittenBytesPerSecond;
-
- std::unique_ptr<OperationLogOutput::ProgressBar> WriteProgressBarPtr(
- m_LogOutput.CreateProgressBar(m_Options.PrimeCacheOnly ? "Downloading" : "Writing"));
- OperationLogOutput::ProgressBar& WriteProgressBar(*WriteProgressBarPtr);
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- TotalPartWriteCount += CopyChunkDatas.size();
- TotalPartWriteCount += ScavengedSequenceCopyOperations.size();
-
- BufferedWriteFileCache WriteCache;
-
- for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < ScavengedSequenceCopyOperations.size(); ScavengeOpIndex++)
- {
- if (m_AbortFlag)
- {
- break;
- }
- if (!m_Options.PrimeCacheOnly)
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &ScavengedPaths,
- &ScavengedSequenceCopyOperations,
- &ScavengedContents,
- &FilteredWrittenBytesPerSecond,
- ScavengeOpIndex,
- &WritePartsComplete,
- TotalPartWriteCount](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WriteScavenged");
-
- FilteredWrittenBytesPerSecond.Start();
-
- const ScavengedSequenceCopyOperation& ScavengeOp = ScavengedSequenceCopyOperations[ScavengeOpIndex];
- const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex];
- const std::filesystem::path& ScavengeRootPath = ScavengedPaths[ScavengeOp.ScavengedContentIndex];
-
- WriteScavengedSequenceToCache(ScavengeRootPath, ScavengedContent, ScavengeOp);
-
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
- }
- });
- }
- }
-
- for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++)
- {
- if (m_AbortFlag)
- {
- break;
- }
-
- if (m_Options.PrimeCacheOnly)
- {
- const uint32_t RemoteChunkIndex = LooseChunkHashWorks[LooseChunkHashWorkIndex].RemoteChunkIndex;
- if (ExistsResult.ExistingBlobs.contains(m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]))
- {
- m_DownloadStats.RequestsCompleteCount++;
- continue;
- }
- }
-
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &SequenceIndexChunksLeftToWriteCounters,
- &Work,
- &ExistsResult,
- &WritePartsComplete,
- &LooseChunkHashWorks,
- LooseChunkHashWorkIndex,
- TotalRequestCount,
- TotalPartWriteCount,
- &WriteCache,
- &FilteredDownloadedBytesPerSecond,
- &FilteredWrittenBytesPerSecond](std::atomic<bool>&) mutable {
- ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk");
- if (!m_AbortFlag)
- {
- LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex];
- const uint32_t RemoteChunkIndex = LooseChunkHashWorks[LooseChunkHashWorkIndex].RemoteChunkIndex;
- WriteLooseChunk(RemoteChunkIndex,
- ExistsResult,
- SequenceIndexChunksLeftToWriteCounters,
- WritePartsComplete,
- std::move(LooseChunkHashWork.ChunkTargetPtrs),
- WriteCache,
- Work,
- TotalRequestCount,
- TotalPartWriteCount,
- FilteredDownloadedBytesPerSecond,
- FilteredWrittenBytesPerSecond);
- }
- },
- WorkerThreadPool::EMode::EnableBacklog);
- }
-
- std::unique_ptr<CloneQueryInterface> CloneQuery;
- if (m_Options.AllowFileClone)
- {
- CloneQuery = GetCloneQueryInterface(m_CacheFolderPath);
- }
-
- for (size_t CopyDataIndex = 0; CopyDataIndex < CopyChunkDatas.size(); CopyDataIndex++)
- {
- ZEN_ASSERT(!m_Options.PrimeCacheOnly);
- if (m_AbortFlag)
- {
- break;
- }
-
- Work.ScheduleWork(m_IOWorkerPool,
- [this,
- &CloneQuery,
- &SequenceIndexChunksLeftToWriteCounters,
- &WriteCache,
- &Work,
- &FilteredWrittenBytesPerSecond,
- &CopyChunkDatas,
- &ScavengedContents,
- &ScavengedLookups,
- &ScavengedPaths,
- &WritePartsComplete,
- TotalPartWriteCount,
- CopyDataIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_CopyLocal");
-
- FilteredWrittenBytesPerSecond.Start();
- const CopyChunkData& CopyData = CopyChunkDatas[CopyDataIndex];
-
- std::vector<uint32_t> WrittenSequenceIndexes = WriteLocalChunkToCache(CloneQuery.get(),
- CopyData,
- ScavengedContents,
- ScavengedLookups,
- ScavengedPaths,
- WriteCache);
- WritePartsComplete++;
- if (!m_AbortFlag)
- {
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
-
- // Write tracking, updating this must be done without any files open
- std::vector<uint32_t> CompletedChunkSequences;
- for (uint32_t RemoteSequenceIndex : WrittenSequenceIndexes)
- {
- if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
- {
- CompletedChunkSequences.push_back(RemoteSequenceIndex);
- }
- }
- WriteCache.Close(CompletedChunkSequences);
- VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Work);
- }
- }
- });
- }
-
- for (uint32_t BlockIndex : CachedChunkBlockIndexes)
- {
- ZEN_ASSERT(!m_Options.PrimeCacheOnly);
- if (m_AbortFlag)
- {
- break;
- }
-
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &WriteCache,
- &Work,
- &FilteredWrittenBytesPerSecond,
- &WritePartsComplete,
- TotalPartWriteCount,
- BlockIndex](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WriteCachedBlock");
-
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
- FilteredWrittenBytesPerSecond.Start();
-
- std::filesystem::path BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
- IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
- if (!BlockBuffer)
- {
- throw std::runtime_error(
- fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath));
- }
-
- if (!m_AbortFlag)
- {
- if (!WriteChunksBlockToCache(BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
- Work,
- CompositeBuffer(std::move(BlockBuffer)),
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- WriteCache))
- {
- std::error_code DummyEc;
- RemoveFile(BlockChunkPath, DummyEc);
- throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
- }
-
- std::error_code Ec = TryRemoveFile(BlockChunkPath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- BlockChunkPath,
- Ec.value(),
- Ec.message());
- }
-
- WritePartsComplete++;
-
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
- }
- }
- });
- }
-
- for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocks.BlockRanges.size();)
- {
- ZEN_ASSERT(!m_Options.PrimeCacheOnly);
- if (m_AbortFlag)
- {
- break;
- }
-
- size_t RangeCount = 1;
- size_t RangesLeft = PartialBlocks.BlockRanges.size() - BlockRangeIndex;
- const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocks.BlockRanges[BlockRangeIndex];
- while (RangeCount < RangesLeft &&
- CurrentBlockRange.BlockIndex == PartialBlocks.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex)
- {
- RangeCount++;
- }
-
- Work.ScheduleWork(
- m_NetworkPool,
- [this,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &ExistsResult,
- &WriteCache,
- &FilteredDownloadedBytesPerSecond,
- TotalRequestCount,
- &WritePartsComplete,
- TotalPartWriteCount,
- &FilteredWrittenBytesPerSecond,
- &Work,
- &PartialBlocks,
- BlockRangeStartIndex = BlockRangeIndex,
- RangeCount](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_GetPartialBlockRanges");
-
- FilteredDownloadedBytesPerSecond.Start();
-
- for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + RangeCount;
- BlockRangeIndex++)
- {
- ZEN_TRACE_CPU("GetPartialBlock");
-
- const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = PartialBlocks.BlockRanges[BlockRangeIndex];
-
- DownloadPartialBlock(
- BlockRange,
- ExistsResult,
- [this,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &WritePartsComplete,
- &WriteCache,
- &Work,
- TotalRequestCount,
- TotalPartWriteCount,
- &FilteredDownloadedBytesPerSecond,
- &FilteredWrittenBytesPerSecond,
- &BlockRange](IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath) {
- if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
-
- if (!m_AbortFlag)
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &WritePartsComplete,
- &WriteCache,
- &Work,
- TotalPartWriteCount,
- &FilteredWrittenBytesPerSecond,
- &BlockRange,
- BlockChunkPath = std::filesystem::path(OnDiskPath),
- BlockPartialBuffer = std::move(InMemoryBuffer)](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WritePartialBlock");
-
- const uint32_t BlockIndex = BlockRange.BlockIndex;
-
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
-
- if (BlockChunkPath.empty())
- {
- ZEN_ASSERT(BlockPartialBuffer);
- }
- else
- {
- ZEN_ASSERT(!BlockPartialBuffer);
- BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
- if (!BlockPartialBuffer)
- {
- throw std::runtime_error(
- fmt::format("Could not open downloaded block {} from {}",
- BlockDescription.BlockHash,
- BlockChunkPath));
- }
- }
-
- FilteredWrittenBytesPerSecond.Start();
-
- if (!WritePartialBlockChunksToCache(
- BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
- Work,
- CompositeBuffer(std::move(BlockPartialBuffer)),
- BlockRange.ChunkBlockIndexStart,
- BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- WriteCache))
- {
- std::error_code DummyEc;
- RemoveFile(BlockChunkPath, DummyEc);
- throw std::runtime_error(
- fmt::format("Partial block {} is malformed", BlockDescription.BlockHash));
- }
-
- std::error_code Ec = TryRemoveFile(BlockChunkPath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- BlockChunkPath,
- Ec.value(),
- Ec.message());
- }
-
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
- }
- },
- OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog
- : WorkerThreadPool::EMode::EnableBacklog);
- }
- });
- }
- }
- });
- BlockRangeIndex += RangeCount;
- }
-
- for (uint32_t BlockIndex : PartialBlocks.FullBlockIndexes)
- {
- if (m_AbortFlag)
- {
- break;
- }
-
- if (m_Options.PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash))
- {
- m_DownloadStats.RequestsCompleteCount++;
- continue;
- }
-
- Work.ScheduleWork(
- m_NetworkPool,
- [this,
- &WritePartsComplete,
- TotalPartWriteCount,
- &FilteredWrittenBytesPerSecond,
- &ExistsResult,
- &Work,
- &WriteCache,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- &FilteredDownloadedBytesPerSecond,
- TotalRequestCount,
- BlockIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_GetFullBlock");
-
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
-
- FilteredDownloadedBytesPerSecond.Start();
-
- IoBuffer BlockBuffer;
- const bool ExistsInCache =
- m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash);
- if (ExistsInCache)
- {
- BlockBuffer = m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash);
- }
- if (!BlockBuffer)
- {
- BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash);
- if (BlockBuffer && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- BlockDescription.BlockHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(BlockBuffer)));
- }
- }
- if (!BlockBuffer)
- {
- throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash));
- }
- if (!m_AbortFlag)
- {
- uint64_t BlockSize = BlockBuffer.GetSize();
- m_DownloadStats.DownloadedBlockCount++;
- m_DownloadStats.DownloadedBlockByteCount += BlockSize;
- m_DownloadStats.RequestsCompleteCount++;
- if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
-
- if (!m_Options.PrimeCacheOnly)
- {
- std::filesystem::path BlockChunkPath;
-
- // Check if the dowloaded block is file based and we can move it directly without rewriting it
- {
- IoBufferFileReference FileRef;
- if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) &&
- (FileRef.FileChunkSize == BlockSize))
- {
- ZEN_TRACE_CPU("MoveTempFullBlock");
- std::error_code Ec;
- std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
- if (!Ec)
- {
- BlockBuffer.SetDeleteOnClose(false);
- BlockBuffer = {};
- BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
- RenameFile(TempBlobPath, BlockChunkPath, Ec);
- if (Ec)
- {
- BlockChunkPath = std::filesystem::path{};
-
- // Re-open the temp file again
- BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
- BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
- BlockBuffer.SetDeleteOnClose(true);
- }
- }
- }
- }
-
- if (BlockChunkPath.empty() && (BlockSize > m_Options.MaximumInMemoryPayloadSize))
- {
- ZEN_TRACE_CPU("WriteTempFullBlock");
- // Could not be moved and rather large, lets store it on disk
- BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
- TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
- BlockBuffer = {};
- }
-
- if (!m_AbortFlag)
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &Work,
- &RemoteChunkIndexNeedsCopyFromSourceFlags,
- &SequenceIndexChunksLeftToWriteCounters,
- BlockIndex,
- &WriteCache,
- &WritePartsComplete,
- TotalPartWriteCount,
- &FilteredWrittenBytesPerSecond,
- BlockChunkPath,
- BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WriteFullBlock");
-
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
-
- if (BlockChunkPath.empty())
- {
- ZEN_ASSERT(BlockBuffer);
- }
- else
- {
- ZEN_ASSERT(!BlockBuffer);
- BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
- if (!BlockBuffer)
- {
- throw std::runtime_error(
- fmt::format("Could not open dowloaded block {} from {}",
- BlockDescription.BlockHash,
- BlockChunkPath));
- }
- }
-
- FilteredWrittenBytesPerSecond.Start();
- if (!WriteChunksBlockToCache(BlockDescription,
- SequenceIndexChunksLeftToWriteCounters,
- Work,
- CompositeBuffer(std::move(BlockBuffer)),
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- WriteCache))
- {
- std::error_code DummyEc;
- RemoveFile(BlockChunkPath, DummyEc);
- throw std::runtime_error(
- fmt::format("Block {} is malformed", BlockDescription.BlockHash));
- }
-
- if (!BlockChunkPath.empty())
- {
- std::error_code Ec = TryRemoveFile(BlockChunkPath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- BlockChunkPath,
- Ec.value(),
- Ec.message());
- }
- }
-
- WritePartsComplete++;
-
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
- }
- },
- BlockChunkPath.empty() ? WorkerThreadPool::EMode::DisableBacklog
- : WorkerThreadPool::EMode::EnableBacklog);
- }
- }
- }
- }
- });
- }
-
- {
- ZEN_TRACE_CPU("WriteChunks_Wait");
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() +
- m_DownloadStats.DownloadedBlockByteCount.load() +
- +m_DownloadStats.DownloadedPartialBlockByteCount.load();
- FilteredWrittenBytesPerSecond.Update(m_DiskStats.WriteByteCount.load());
- FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
- std::string DownloadRateString =
- (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
- ? ""
- : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
- std::string CloneDetails;
- if (m_DiskStats.CloneCount.load() > 0)
- {
- CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load()));
- }
- std::string WriteDetails = m_Options.PrimeCacheOnly ? ""
- : fmt::format(" {}/{} ({}B/s) written{}",
- NiceBytes(m_WrittenChunkByteCount.load()),
- NiceBytes(BytesToWrite),
- NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()),
- CloneDetails);
-
- std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
- m_DownloadStats.RequestsCompleteCount.load(),
- TotalRequestCount,
- NiceBytes(DownloadedBytes),
- DownloadRateString,
- WriteDetails);
-
- std::string Task;
- if (m_Options.PrimeCacheOnly)
- {
- Task = "Downloading ";
- }
- else if ((m_WrittenChunkByteCount < BytesToWrite) || (BytesToValidate == 0))
- {
- Task = "Writing chunks ";
- }
- else
- {
- Task = "Verifying chunks ";
- }
-
- WriteProgressBar.UpdateState(
- {.Task = Task,
- .Details = Details,
- .TotalCount = m_Options.PrimeCacheOnly ? TotalRequestCount : (BytesToWrite + BytesToValidate),
- .RemainingCount = m_Options.PrimeCacheOnly ? (TotalRequestCount - m_DownloadStats.RequestsCompleteCount.load())
- : ((BytesToWrite + BytesToValidate) -
- (m_WrittenChunkByteCount.load() + m_ValidatedChunkByteCount.load())),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- }
-
- CloneQuery.reset();
-
- FilteredWrittenBytesPerSecond.Stop();
- FilteredDownloadedBytesPerSecond.Stop();
-
- WriteProgressBar.Finish();
- if (m_AbortFlag)
- {
- return;
- }
-
- if (!m_Options.PrimeCacheOnly)
- {
- uint32_t RawSequencesMissingWriteCount = 0;
- for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++)
- {
- const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex];
- if (SequenceIndexChunksLeftToWriteCounter.load() != 0)
- {
- RawSequencesMissingWriteCount++;
- const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const std::filesystem::path& IncompletePath = m_RemoteContent.Paths[PathIndex];
- ZEN_ASSERT(!IncompletePath.empty());
- const uint32_t ExpectedSequenceCount = m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "{}: Max count {}, Current count {}",
- IncompletePath,
- ExpectedSequenceCount,
- SequenceIndexChunksLeftToWriteCounter.load());
- }
- ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount);
- }
- }
- ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
- ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite);
- ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate);
- }
-
- const uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() +
- m_DownloadStats.DownloadedBlockByteCount.load() +
- m_DownloadStats.DownloadedPartialBlockByteCount.load();
- if (!m_Options.IsQuiet)
- {
- std::string CloneDetails;
- if (m_DiskStats.CloneCount.load() > 0)
- {
- CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load()));
- }
- ZEN_OPERATION_LOG_INFO(
- m_LogOutput,
- "Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s){} in {}. Completed in {}",
- NiceBytes(DownloadedBytes),
- NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
- NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceBytes(m_WrittenChunkByteCount.load()),
- NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), m_DiskStats.WriteByteCount.load())),
- CloneDetails,
- NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
- NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
- }
-
- m_WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
- m_WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
- m_WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
- }
-
- if (m_Options.PrimeCacheOnly)
- {
- return;
- }
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::PrepareTarget, (uint32_t)TaskSteps::StepCount);
-
- tsl::robin_map<uint32_t, uint32_t> RemotePathIndexToLocalPathIndex;
- RemotePathIndexToLocalPathIndex.reserve(m_RemoteContent.Paths.size());
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex;
- std::vector<uint32_t> RemoveLocalPathIndexes;
-
- if (m_AbortFlag)
- {
- return;
- }
-
- {
- ZEN_TRACE_CPU("PrepareTarget");
-
- tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences;
-
- std::vector<uint32_t> FilesToCache;
-
- uint64_t MatchCount = 0;
- uint64_t PathMismatchCount = 0;
- uint64_t HashMismatchCount = 0;
- std::atomic<uint64_t> CachedCount = 0;
- std::atomic<uint64_t> CachedByteCount = 0;
- uint64_t SkippedCount = 0;
- uint64_t DeleteCount = 0;
- for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++)
- {
- if (m_AbortFlag)
- {
- break;
- }
- const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
- const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
-
- ZEN_ASSERT_SLOW(IsFile((m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred()));
-
- if (m_Options.EnableTargetFolderScavenging)
- {
- if (!m_Options.WipeTargetFolder)
- {
- // Check if it is already in the correct place
- if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string());
- RemotePathIt != RemotePathToRemoteIndex.end())
- {
- const uint32_t RemotePathIndex = RemotePathIt->second;
- if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash)
- {
- // It is already in it's correct place
- RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex;
- SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex});
- MatchCount++;
- continue;
- }
- else
- {
- HashMismatchCount++;
- }
- }
- else
- {
- PathMismatchCount++;
- }
- }
-
- // Do we need it?
- if (m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
- {
- if (!CachedRemoteSequences.contains(RawHash))
- {
- // We need it, make sure we move it to the cache
- FilesToCache.push_back(LocalPathIndex);
- CachedRemoteSequences.insert(RawHash);
- continue;
- }
- else
- {
- SkippedCount++;
- }
- }
- }
-
- if (!m_Options.WipeTargetFolder)
- {
- // Explicitly delete the unneeded local file
- RemoveLocalPathIndexes.push_back(LocalPathIndex);
- DeleteCount++;
- }
- }
-
- if (m_AbortFlag)
- {
- return;
- }
-
- {
- ZEN_TRACE_CPU("CopyToCache");
-
- Stopwatch Timer;
-
- std::unique_ptr<OperationLogOutput::ProgressBar> CacheLocalProgressBarPtr(
- m_LogOutput.CreateProgressBar("Cache Local Data"));
- OperationLogOutput::ProgressBar& CacheLocalProgressBar(*CacheLocalProgressBarPtr);
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- for (uint32_t LocalPathIndex : FilesToCache)
- {
- if (m_AbortFlag)
- {
- break;
- }
- Work.ScheduleWork(m_IOWorkerPool, [this, &CachedCount, &CachedByteCount, LocalPathIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_CopyToCache");
-
- const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
- const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
- ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
- const std::filesystem::path LocalFilePath = (m_Path / LocalPath).make_preferred();
-
- std::error_code Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
- LocalFilePath,
- CacheFilePath,
- Ec.value(),
- Ec.message());
- Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
- if (Ec)
- {
- throw std::system_error(std::error_code(Ec.value(), std::system_category()),
- fmt::format("Failed to file from '{}' to '{}', reason: ({}) {}",
- LocalFilePath,
- CacheFilePath,
- Ec.value(),
- Ec.message()));
- }
- }
-
- CachedCount++;
- CachedByteCount += m_LocalContent.RawSizes[LocalPathIndex];
- }
- });
- }
-
- {
- ZEN_TRACE_CPU("CopyToCache_Wait");
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- const uint64_t WorkTotal = FilesToCache.size();
- const uint64_t WorkComplete = CachedCount.load();
- std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount));
- CacheLocalProgressBar.UpdateState(
- {.Task = "Caching local ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(WorkTotal),
- .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- }
-
- CacheLocalProgressBar.Finish();
- if (m_AbortFlag)
- {
- return;
- }
-
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, "
- "Delete: {}",
- MatchCount,
- PathMismatchCount,
- HashMismatchCount,
- CachedCount.load(),
- NiceBytes(CachedByteCount.load()),
- SkippedCount,
- DeleteCount);
- }
- }
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeTarget, (uint32_t)TaskSteps::StepCount);
-
- if (m_Options.WipeTargetFolder)
- {
- ZEN_TRACE_CPU("WipeTarget");
- Stopwatch Timer;
-
- // Clean target folder
- if (!CleanDirectory(m_LogOutput, m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.IsQuiet, m_Path, m_Options.ExcludeFolders))
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput, "Some files in {} could not be removed", m_Path);
- }
- m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
- }
-
- if (m_AbortFlag)
- {
- return;
- }
-
- {
- ZEN_TRACE_CPU("FinalizeTree");
-
- Stopwatch Timer;
-
- std::unique_ptr<OperationLogOutput::ProgressBar> RebuildProgressBarPtr(m_LogOutput.CreateProgressBar("Rebuild State"));
- OperationLogOutput::ProgressBar& RebuildProgressBar(*RebuildProgressBarPtr);
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- OutLocalFolderState.Paths.resize(m_RemoteContent.Paths.size());
- OutLocalFolderState.RawSizes.resize(m_RemoteContent.Paths.size());
- OutLocalFolderState.Attributes.resize(m_RemoteContent.Paths.size());
- OutLocalFolderState.ModificationTicks.resize(m_RemoteContent.Paths.size());
-
- std::atomic<uint64_t> DeletedCount = 0;
-
- for (uint32_t LocalPathIndex : RemoveLocalPathIndexes)
- {
- if (m_AbortFlag)
- {
- break;
- }
- Work.ScheduleWork(m_IOWorkerPool, [this, &DeletedCount, LocalPathIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_RemoveFile");
-
- const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
- SetFileReadOnlyWithRetry(LocalFilePath, false);
- RemoveFileWithRetry(LocalFilePath);
- DeletedCount++;
- }
- });
- }
-
- std::atomic<uint64_t> TargetsComplete = 0;
-
- struct FinalizeTarget
- {
- IoHash RawHash;
- uint32_t RemotePathIndex;
- };
-
- std::vector<FinalizeTarget> Targets;
- Targets.reserve(m_RemoteContent.Paths.size());
- for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
- {
- Targets.push_back(
- FinalizeTarget{.RawHash = m_RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex});
- }
- std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) {
- if (Lhs.RawHash < Rhs.RawHash)
- {
- return true;
- }
- else if (Lhs.RawHash > Rhs.RawHash)
- {
- return false;
- }
- return Lhs.RemotePathIndex < Rhs.RemotePathIndex;
- });
-
- size_t TargetOffset = 0;
- while (TargetOffset < Targets.size())
- {
- if (m_AbortFlag)
- {
- break;
- }
-
- size_t TargetCount = 1;
- while ((TargetOffset + TargetCount) < Targets.size() &&
- (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash))
- {
- TargetCount++;
- }
-
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &SequenceHashToLocalPathIndex,
- &Targets,
- &RemotePathIndexToLocalPathIndex,
- &OutLocalFolderState,
- BaseTargetOffset = TargetOffset,
- TargetCount,
- &TargetsComplete](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_FinalizeChunkSequence");
-
- size_t TargetOffset = BaseTargetOffset;
- const IoHash& RawHash = Targets[TargetOffset].RawHash;
-
- if (RawHash == IoHash::Zero)
- {
- ZEN_TRACE_CPU("CreateEmptyFiles");
- while (TargetOffset < (BaseTargetOffset + TargetCount))
- {
- const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
- ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
- const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex];
- std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred();
- if (!RemotePathIndexToLocalPathIndex[RemotePathIndex])
- {
- if (IsFileWithRetry(TargetFilePath))
- {
- SetFileReadOnlyWithRetry(TargetFilePath, false);
- }
- else
- {
- CreateDirectories(TargetFilePath.parent_path());
- }
- BasicFile OutputFile;
- OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate);
- }
- OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
- OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex];
-
- OutLocalFolderState.Attributes[RemotePathIndex] =
- m_RemoteContent.Attributes.empty()
- ? GetNativeFileAttributes(TargetFilePath)
- : SetNativeFileAttributes(TargetFilePath,
- m_RemoteContent.Platform,
- m_RemoteContent.Attributes[RemotePathIndex]);
- OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
-
- TargetOffset++;
- TargetsComplete++;
- }
- }
- else
- {
- ZEN_TRACE_CPU("FinalizeFile");
- ZEN_ASSERT(m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
- const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex;
- const std::filesystem::path& FirstTargetPath = m_RemoteContent.Paths[FirstRemotePathIndex];
- std::filesystem::path FirstTargetFilePath = (m_Path / FirstTargetPath).make_preferred();
-
- if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex);
- InPlaceIt != RemotePathIndexToLocalPathIndex.end())
- {
- ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
- }
- else
- {
- if (IsFileWithRetry(FirstTargetFilePath))
- {
- SetFileReadOnlyWithRetry(FirstTargetFilePath, false);
- }
- else
- {
- CreateDirectories(FirstTargetFilePath.parent_path());
- }
-
- if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash);
- InplaceIt != SequenceHashToLocalPathIndex.end())
- {
- ZEN_TRACE_CPU("Copy");
- const uint32_t LocalPathIndex = InplaceIt->second;
- const std::filesystem::path& SourcePath = m_LocalContent.Paths[LocalPathIndex];
- std::filesystem::path SourceFilePath = (m_Path / SourcePath).make_preferred();
- ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath));
-
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Copying from '{}' -> '{}'",
- SourceFilePath,
- FirstTargetFilePath);
- const uint64_t RawSize = m_LocalContent.RawSizes[LocalPathIndex];
- FastCopyFile(m_Options.AllowFileClone,
- m_Options.UseSparseFiles,
- SourceFilePath,
- FirstTargetFilePath,
- RawSize,
- m_DiskStats.WriteCount,
- m_DiskStats.WriteByteCount,
- m_DiskStats.CloneCount,
- m_DiskStats.CloneByteCount);
-
- m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
- }
- else
- {
- ZEN_TRACE_CPU("Rename");
- const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
- ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath));
-
- std::error_code Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
- CacheFilePath,
- FirstTargetFilePath,
- Ec.value(),
- Ec.message());
- Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
- if (Ec)
- {
- throw std::system_error(
- std::error_code(Ec.value(), std::system_category()),
- fmt::format("Failed to move file from '{}' to '{}', reason: ({}) {}",
- CacheFilePath,
- FirstTargetFilePath,
- Ec.value(),
- Ec.message()));
- }
- }
-
- m_RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
- }
- }
-
- OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath;
- OutLocalFolderState.RawSizes[FirstRemotePathIndex] = m_RemoteContent.RawSizes[FirstRemotePathIndex];
-
- OutLocalFolderState.Attributes[FirstRemotePathIndex] =
- m_RemoteContent.Attributes.empty()
- ? GetNativeFileAttributes(FirstTargetFilePath)
- : SetNativeFileAttributes(FirstTargetFilePath,
- m_RemoteContent.Platform,
- m_RemoteContent.Attributes[FirstRemotePathIndex]);
- OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] =
- GetModificationTickFromPath(FirstTargetFilePath);
-
- TargetOffset++;
- TargetsComplete++;
-
- while (TargetOffset < (BaseTargetOffset + TargetCount))
- {
- const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
- ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
- const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex];
- std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred();
-
- if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex);
- InPlaceIt != RemotePathIndexToLocalPathIndex.end())
- {
- ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath));
- }
- else
- {
- ZEN_TRACE_CPU("Copy");
- if (IsFileWithRetry(TargetFilePath))
- {
- SetFileReadOnlyWithRetry(TargetFilePath, false);
- }
- else
- {
- CreateDirectories(TargetFilePath.parent_path());
- }
-
- ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Copying from '{}' -> '{}'",
- FirstTargetFilePath,
- TargetFilePath);
- const uint64_t RawSize = m_RemoteContent.RawSizes[RemotePathIndex];
- FastCopyFile(m_Options.AllowFileClone,
- m_Options.UseSparseFiles,
- FirstTargetFilePath,
- TargetFilePath,
- RawSize,
- m_DiskStats.WriteCount,
- m_DiskStats.WriteByteCount,
- m_DiskStats.CloneCount,
- m_DiskStats.CloneByteCount);
-
- m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
- }
-
- OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
- OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex];
-
- OutLocalFolderState.Attributes[RemotePathIndex] =
- m_RemoteContent.Attributes.empty()
- ? GetNativeFileAttributes(TargetFilePath)
- : SetNativeFileAttributes(TargetFilePath,
- m_RemoteContent.Platform,
- m_RemoteContent.Attributes[RemotePathIndex]);
- OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
-
- TargetOffset++;
- TargetsComplete++;
- }
- }
- }
- });
-
- TargetOffset += TargetCount;
- }
-
- {
- ZEN_TRACE_CPU("FinalizeTree_Wait");
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size();
- const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load();
- std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal);
- RebuildProgressBar.UpdateState({.Task = "Rebuilding state ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(WorkTotal),
- .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
- }
-
- m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs();
- RebuildProgressBar.Finish();
- }
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount);
- }
- catch (const std::exception&)
- {
- m_AbortFlag = true;
- throw;
- }
-}
-
-void
-BuildsOperationUpdateFolder::ScanCacheFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedChunkHashesFound,
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedSequenceHashesFound)
-{
- ZEN_TRACE_CPU("ScanCacheFolder");
-
- Stopwatch CacheTimer;
-
- DirectoryContent CacheDirContent;
- GetDirectoryContent(m_CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, CacheDirContent);
- for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++)
- {
- if (m_Options.EnableTargetFolderScavenging)
- {
- IoHash FileHash;
- if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash))
- {
- if (auto ChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
- ChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end())
- {
- const uint32_t ChunkIndex = ChunkIt->second;
- const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- if (ChunkSize == CacheDirContent.FileSizes[Index])
- {
- OutCachedChunkHashesFound.insert({FileHash, ChunkIndex});
- m_CacheMappingStats.CacheChunkCount++;
- m_CacheMappingStats.CacheChunkByteCount += ChunkSize;
- continue;
- }
- }
- else if (auto SequenceIt = m_RemoteLookup.RawHashToSequenceIndex.find(FileHash);
- SequenceIt != m_RemoteLookup.RawHashToSequenceIndex.end())
- {
- const uint32_t SequenceIndex = SequenceIt->second;
- const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
- const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex];
- if (SequenceSize == CacheDirContent.FileSizes[Index])
- {
- OutCachedSequenceHashesFound.insert({FileHash, SequenceIndex});
- m_CacheMappingStats.CacheSequenceHashesCount++;
- m_CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
-
- const std::filesystem::path CacheFilePath =
- GetFinalChunkedSequenceFileName(m_CacheFolderPath,
- m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
- ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
-
- continue;
- }
- }
- }
- }
- std::error_code Ec = TryRemoveFile(CacheDirContent.Files[Index]);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- CacheDirContent.Files[Index],
- Ec.value(),
- Ec.message());
- }
- }
- m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
-}
-
-void
-BuildsOperationUpdateFolder::ScanTempBlocksFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedBlocksFound)
-{
- ZEN_TRACE_CPU("ScanTempBlocksFolder");
-
- Stopwatch CacheTimer;
-
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> AllBlockSizes;
- AllBlockSizes.reserve(m_BlockDescriptions.size());
- for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++)
- {
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
- AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex});
- }
-
- DirectoryContent BlockDirContent;
- GetDirectoryContent(m_TempBlockFolderPath,
- DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
- BlockDirContent);
- OutCachedBlocksFound.reserve(BlockDirContent.Files.size());
- for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++)
- {
- if (m_Options.EnableTargetFolderScavenging)
- {
- IoHash FileHash;
- if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash))
- {
- if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end())
- {
- const uint32_t BlockIndex = BlockIt->second;
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
- uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize;
- for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths)
- {
- BlockSize += ChunkSize;
- }
-
- if (BlockSize == BlockDirContent.FileSizes[Index])
- {
- OutCachedBlocksFound.insert({FileHash, BlockIndex});
- m_CacheMappingStats.CacheBlockCount++;
- m_CacheMappingStats.CacheBlocksByteCount += BlockSize;
- continue;
- }
- }
- }
- }
- std::error_code Ec = TryRemoveFile(BlockDirContent.Files[Index]);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- BlockDirContent.Files[Index],
- Ec.value(),
- Ec.message());
- }
- }
-
- m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
-}
-
-std::vector<BuildsOperationUpdateFolder::ScavengeSource>
-BuildsOperationUpdateFolder::FindScavengeSources()
-{
- ZEN_TRACE_CPU("FindScavengeSources");
-
- const bool TargetPathExists = IsDir(m_Path);
-
- std::vector<std::filesystem::path> StatePaths = GetDownloadedStatePaths(m_Options.SystemRootDir);
-
- std::vector<ScavengeSource> Result;
- for (const std::filesystem::path& EntryPath : StatePaths)
- {
- if (IsFile(EntryPath))
- {
- bool DeleteEntry = false;
-
- try
- {
- BuildsDownloadInfo Info = ReadDownloadedInfoFile(EntryPath);
- const bool LocalPathExists = !Info.LocalPath.empty() && IsDir(Info.LocalPath);
- const bool LocalStateFileExists = IsFile(Info.StateFilePath);
- if (LocalPathExists && LocalStateFileExists)
- {
- if (TargetPathExists && std::filesystem::equivalent(Info.LocalPath, m_Path))
- {
- DeleteEntry = true;
- }
- else
- {
- Result.push_back({.StateFilePath = std::move(Info.StateFilePath), .Path = std::move(Info.LocalPath)});
- }
- }
- else
- {
- DeleteEntry = true;
- }
- }
- catch (const std::exception& Ex)
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", Ex.what());
- DeleteEntry = true;
- }
-
- if (DeleteEntry)
- {
- std::error_code DummyEc;
- std::filesystem::remove(EntryPath, DummyEc);
- }
- }
- }
- return Result;
-}
-
-std::vector<uint32_t>
-BuildsOperationUpdateFolder::ScanTargetFolder(const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedChunkHashesFound,
- const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedSequenceHashesFound)
-{
- ZEN_TRACE_CPU("ScanTargetFolder");
-
- Stopwatch LocalTimer;
-
- std::vector<uint32_t> MissingSequenceIndexes;
-
- for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size();
- RemoteSequenceIndex++)
- {
- const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(m_RemoteLookup, RemoteSequenceIndex);
- const uint64_t RemoteRawSize = m_RemoteContent.RawSizes[RemotePathIndex];
- if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash);
- CacheSequenceIt != CachedSequenceHashesFound.end())
- {
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
- ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found sequence {} at {} ({})",
- RemoteSequenceRawHash,
- CacheFilePath,
- NiceBytes(RemoteRawSize));
- }
- }
- else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end())
- {
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
- ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found chunk {} at {} ({})",
- RemoteSequenceRawHash,
- CacheFilePath,
- NiceBytes(RemoteRawSize));
- }
- }
- else if (auto It = m_LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash);
- It != m_LocalLookup.RawHashToSequenceIndex.end())
- {
- const uint32_t LocalSequenceIndex = It->second;
- const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(m_LocalLookup, LocalSequenceIndex);
- const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
- ZEN_ASSERT_SLOW(IsFile(LocalFilePath));
- m_CacheMappingStats.LocalPathsMatchingSequencesCount++;
- m_CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize;
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found sequence {} at {} ({})",
- RemoteSequenceRawHash,
- LocalFilePath,
- NiceBytes(RemoteRawSize));
- }
- }
- else
- {
- MissingSequenceIndexes.push_back(RemoteSequenceIndex);
- }
- }
-
- m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
- return MissingSequenceIndexes;
-}
-
-bool
-BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source,
- ChunkedFolderContent& OutScavengedLocalContent,
- ChunkedContentLookup& OutScavengedLookup)
-{
- ZEN_TRACE_CPU("FindScavengeContent");
-
- FolderContent LocalFolderState;
- try
- {
- BuildSaveState SavedState = ReadBuildSaveStateFile(Source.StateFilePath);
- OutScavengedLocalContent = std::move(SavedState.State.ChunkedContent);
- LocalFolderState = std::move(SavedState.FolderState);
- }
- catch (const std::exception& Ex)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput, "Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what());
- return false;
- }
-
- tsl::robin_set<uint32_t> PathIndexesToScavenge;
- PathIndexesToScavenge.reserve(OutScavengedLocalContent.Paths.size());
- std::vector<uint32_t> ChunkOrderOffsets = BuildChunkOrderOffset(OutScavengedLocalContent.ChunkedContent.ChunkCounts);
-
- {
- tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToPathIndex;
-
- RawHashToPathIndex.reserve(OutScavengedLocalContent.Paths.size());
- for (uint32_t ScavengedPathIndex = 0; ScavengedPathIndex < OutScavengedLocalContent.RawHashes.size(); ScavengedPathIndex++)
- {
- if (!RawHashToPathIndex.contains(OutScavengedLocalContent.RawHashes[ScavengedPathIndex]))
- {
- RawHashToPathIndex.insert_or_assign(OutScavengedLocalContent.RawHashes[ScavengedPathIndex], ScavengedPathIndex);
- }
- }
-
- for (uint32_t ScavengeSequenceIndex = 0; ScavengeSequenceIndex < OutScavengedLocalContent.ChunkedContent.SequenceRawHashes.size();
- ScavengeSequenceIndex++)
- {
- const IoHash& SequenceHash = OutScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengeSequenceIndex];
- if (auto It = RawHashToPathIndex.find(SequenceHash); It != RawHashToPathIndex.end())
- {
- uint32_t PathIndex = It->second;
- if (!PathIndexesToScavenge.contains(PathIndex))
- {
- if (m_RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash))
- {
- PathIndexesToScavenge.insert(PathIndex);
- }
- else
- {
- uint32_t ChunkOrderIndexStart = ChunkOrderOffsets[ScavengeSequenceIndex];
- const uint32_t ChunkCount = OutScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex];
- for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < ChunkCount; ChunkOrderIndex++)
- {
- const uint32_t ChunkIndex =
- OutScavengedLocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndexStart + ChunkOrderIndex];
- const IoHash& ChunkHash = OutScavengedLocalContent.ChunkedContent.ChunkHashes[ChunkIndex];
- if (m_RemoteLookup.ChunkHashToChunkIndex.contains(ChunkHash))
- {
- PathIndexesToScavenge.insert(PathIndex);
- break;
- }
- }
- }
- }
- }
- else
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "Scavenged state file at '{}' for '{}' is invalid, skipping scavenging for sequence {}",
- Source.StateFilePath,
- Source.Path,
- SequenceHash);
- }
- }
- }
-
- if (PathIndexesToScavenge.empty())
- {
- OutScavengedLocalContent = {};
- return false;
- }
-
- std::vector<std::filesystem::path> PathsToScavenge;
- PathsToScavenge.reserve(PathIndexesToScavenge.size());
- for (uint32_t ScavengedStatePathIndex : PathIndexesToScavenge)
- {
- PathsToScavenge.push_back(OutScavengedLocalContent.Paths[ScavengedStatePathIndex]);
- }
-
- FolderContent ValidFolderContent =
- GetValidFolderContent(m_IOWorkerPool, m_ScavengedFolderScanStats, Source.Path, PathsToScavenge, {}, 0, m_AbortFlag, m_PauseFlag);
-
- if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent))
- {
- std::vector<std::filesystem::path> DeletedPaths;
- FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths);
-
- // If the files are modified since the state was saved we ignore the files since we don't
- // want to incur the cost of scanning/hashing scavenged files
- DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end());
- if (!DeletedPaths.empty())
- {
- OutScavengedLocalContent =
- DeletePathsFromChunkedContent(OutScavengedLocalContent,
- BuildHashLookup(OutScavengedLocalContent.ChunkedContent.SequenceRawHashes),
- ChunkOrderOffsets,
- DeletedPaths);
- }
- }
-
- if (OutScavengedLocalContent.Paths.empty())
- {
- OutScavengedLocalContent = {};
- return false;
- }
-
- OutScavengedLookup = BuildChunkedContentLookup(OutScavengedLocalContent);
-
- return true;
-}
-
-void
-BuildsOperationUpdateFolder::ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount,
- std::vector<bool>& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags,
- tsl::robin_map<IoHash, size_t, IoHash::Hasher>& InOutRawHashToCopyChunkDataIndex,
- const std::vector<std::atomic<uint32_t>>& SequenceIndexChunksLeftToWriteCounters,
- const ChunkedFolderContent& ScavengedContent,
- const ChunkedContentLookup& ScavengedLookup,
- std::vector<CopyChunkData>& InOutCopyChunkDatas,
- uint32_t ScavengedContentIndex,
- uint64_t& InOutChunkMatchingRemoteCount,
- uint64_t& InOutChunkMatchingRemoteByteCount)
-{
- for (uint32_t RemoteChunkIndex = 0;
- RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size() && (InOutRemainingChunkCount > 0);
- RemoteChunkIndex++)
- {
- if (!InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
- {
- const IoHash& RemoteChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
- if (auto It = ScavengedLookup.ChunkHashToChunkIndex.find(RemoteChunkHash); It != ScavengedLookup.ChunkHashToChunkIndex.end())
- {
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
-
- if (!ChunkTargetPtrs.empty())
- {
- const uint32_t ScavengedChunkIndex = It->second;
- const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex];
- const size_t ChunkSequenceLocationOffset = ScavengedLookup.ChunkSequenceLocationOffset[ScavengedChunkIndex];
- const ChunkedContentLookup::ChunkSequenceLocation& ScavengeLocation =
- ScavengedLookup.ChunkSequenceLocations[ChunkSequenceLocationOffset];
- const IoHash& ScavengedSequenceRawHash =
- ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengeLocation.SequenceIndex];
-
- CopyChunkData::ChunkTarget Target = {.TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()),
- .RemoteChunkIndex = RemoteChunkIndex,
- .CacheFileOffset = ScavengeLocation.Offset};
- if (auto CopySourceIt = InOutRawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash);
- CopySourceIt != InOutRawHashToCopyChunkDataIndex.end())
- {
- CopyChunkData& Data = InOutCopyChunkDatas[CopySourceIt->second];
- if (Data.TargetChunkLocationPtrs.size() > 1024)
- {
- InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size());
- InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex,
- .SourceSequenceIndex = ScavengeLocation.SequenceIndex,
- .TargetChunkLocationPtrs = ChunkTargetPtrs,
- .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}});
- }
- else
- {
- Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(),
- ChunkTargetPtrs.begin(),
- ChunkTargetPtrs.end());
- Data.ChunkTargets.push_back(Target);
- }
- }
- else
- {
- InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size());
- InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex,
- .SourceSequenceIndex = ScavengeLocation.SequenceIndex,
- .TargetChunkLocationPtrs = ChunkTargetPtrs,
- .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}});
- }
- InOutChunkMatchingRemoteCount++;
- InOutChunkMatchingRemoteByteCount += ScavengedChunkRawSize;
- InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
- InOutRemainingChunkCount--;
- }
- }
- }
- }
-}
-
-std::filesystem::path
-BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash)
-{
- ZEN_TRACE_CPU("FindDownloadedChunk");
-
- std::filesystem::path CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString();
- if (IsFile(CompressedChunkPath))
- {
- IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
- if (ExistingCompressedPart)
- {
- IoHash RawHash;
- uint64_t RawSize;
- if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart,
- RawHash,
- RawSize,
- /*OutOptionalTotalCompressedSize*/ nullptr))
- {
- return CompressedChunkPath;
- }
- else
- {
- std::error_code DummyEc;
- RemoveFile(CompressedChunkPath, DummyEc);
- }
- }
- }
- return {};
-}
-
-std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>
-BuildsOperationUpdateFolder::GetRemainingChunkTargets(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- uint32_t ChunkIndex)
-{
- ZEN_TRACE_CPU("GetRemainingChunkTargets");
-
- std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex);
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs;
- if (!ChunkSources.empty())
- {
- ChunkTargetPtrs.reserve(ChunkSources.size());
- for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
- {
- if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
- {
- ChunkTargetPtrs.push_back(&Source);
- }
- }
- }
- return ChunkTargetPtrs;
-};
-
-uint64_t
-BuildsOperationUpdateFolder::GetChunkWriteCount(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- uint32_t ChunkIndex)
-{
- ZEN_TRACE_CPU("GetChunkWriteCount");
-
- uint64_t WriteCount = 0;
- std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex);
- for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
- {
- if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
- {
- WriteCount++;
- }
- }
- return WriteCount;
-};
-
-void
-BuildsOperationUpdateFolder::CheckRequiredDiskSpace(const tsl::robin_map<std::string, uint32_t>& RemotePathToRemoteIndex)
-{
- tsl::robin_set<uint32_t> ExistingRemotePaths;
-
- if (m_Options.EnableTargetFolderScavenging)
- {
- for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++)
- {
- const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
- const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
-
- if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end())
- {
- const uint32_t RemotePathIndex = RemotePathIt->second;
- if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash)
- {
- ExistingRemotePaths.insert(RemotePathIndex);
- }
- }
- }
- }
-
- uint64_t RequiredSpace = 0;
- for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
- {
- if (!ExistingRemotePaths.contains(RemotePathIndex))
- {
- RequiredSpace += m_RemoteContent.RawSizes[RemotePathIndex];
- }
- }
-
- std::error_code Ec;
- DiskSpace Space = DiskSpaceInfo(m_Path, Ec);
- if (Ec)
- {
- throw std::runtime_error(fmt::format("Get free disk space for target path '{}' FAILED, reason: {}", m_Path, Ec.message()));
- }
- if (Space.Free < (RequiredSpace + 16u * 1024u * 1024u))
- {
- throw std::runtime_error(
- fmt::format("Not enough free space for target path '{}', {} of free space is needed", m_Path, RequiredSpace));
- }
-}
-
-void
-BuildsOperationUpdateFolder::WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath,
- const ChunkedFolderContent& ScavengedContent,
- const ScavengedSequenceCopyOperation& ScavengeOp)
-{
- ZEN_TRACE_CPU("WriteScavengedSequenceToCache");
-
- const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex];
- const std::filesystem::path ScavengedFilePath = (ScavengeRootPath / ScavengedPath).make_preferred();
- ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize);
-
- const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex];
- const std::filesystem::path TempFilePath = GetTempChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
-
- const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedPathIndex];
- FastCopyFile(m_Options.AllowFileClone,
- m_Options.UseSparseFiles,
- ScavengedFilePath,
- TempFilePath,
- RawSize,
- m_DiskStats.WriteCount,
- m_DiskStats.WriteByteCount,
- m_DiskStats.CloneCount,
- m_DiskStats.CloneByteCount);
-
- const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
- RenameFile(TempFilePath, CacheFilePath);
-
- m_WrittenChunkByteCount += RawSize;
- if (m_Options.ValidateCompletedSequences)
- {
- m_ValidatedChunkByteCount += RawSize;
- }
-}
-
-void
-BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkIndex,
- const BlobsExistsResult& ExistsResult,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- std::atomic<uint64_t>& WritePartsComplete,
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
- BufferedWriteFileCache& WriteCache,
- ParallelWork& Work,
- uint64_t TotalRequestCount,
- uint64_t TotalPartWriteCount,
- FilteredRate& FilteredDownloadedBytesPerSecond,
- FilteredRate& FilteredWrittenBytesPerSecond)
-{
- std::filesystem::path ExistingCompressedChunkPath;
- if (!m_Options.PrimeCacheOnly)
- {
- const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
- ExistingCompressedChunkPath = FindDownloadedChunk(ChunkHash);
- if (!ExistingCompressedChunkPath.empty())
- {
- m_DownloadStats.RequestsCompleteCount++;
- if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- }
- }
- if (!m_AbortFlag)
- {
- if (!ExistingCompressedChunkPath.empty())
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- SequenceIndexChunksLeftToWriteCounters,
- &WriteCache,
- &Work,
- &WritePartsComplete,
- TotalPartWriteCount,
- &FilteredWrittenBytesPerSecond,
- RemoteChunkIndex,
- ChunkTargetPtrs = std::move(ChunkTargetPtrs),
- CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>& AbortFlag) mutable {
- if (!AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WritePreDownloadedChunk");
-
- FilteredWrittenBytesPerSecond.Start();
-
- const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
-
- IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
- if (!CompressedPart)
- {
- throw std::runtime_error(
- fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath));
- }
-
- bool NeedHashVerify =
- WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart));
- WritePartsComplete++;
-
- if (!AbortFlag)
- {
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
-
- std::error_code Ec = TryRemoveFile(CompressedChunkPath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- CompressedChunkPath,
- Ec.value(),
- Ec.message());
- }
-
- std::vector<uint32_t> CompletedSequences =
- CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
- WriteCache.Close(CompletedSequences);
- if (NeedHashVerify)
- {
- VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work);
- }
- else
- {
- FinalizeChunkSequences(CompletedSequences);
- }
- }
- }
- });
- }
- else
- {
- Work.ScheduleWork(m_NetworkPool,
- [this,
- &ExistsResult,
- SequenceIndexChunksLeftToWriteCounters,
- &WriteCache,
- &Work,
- &WritePartsComplete,
- TotalPartWriteCount,
- TotalRequestCount,
- &FilteredDownloadedBytesPerSecond,
- &FilteredWrittenBytesPerSecond,
- RemoteChunkIndex,
- ChunkTargetPtrs = std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>(
- std::move(ChunkTargetPtrs))](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_DownloadChunk");
-
- FilteredDownloadedBytesPerSecond.Start();
- DownloadBuildBlob(RemoteChunkIndex,
- ExistsResult,
- Work,
- [this,
- &ExistsResult,
- SequenceIndexChunksLeftToWriteCounters,
- &WriteCache,
- &Work,
- &WritePartsComplete,
- TotalPartWriteCount,
- TotalRequestCount,
- RemoteChunkIndex,
- &FilteredDownloadedBytesPerSecond,
- &FilteredWrittenBytesPerSecond,
- ChunkTargetPtrs = std::move(ChunkTargetPtrs)](IoBuffer&& Payload) mutable {
- if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- IoBufferFileReference FileRef;
- bool EnableBacklog = Payload.GetFileReference(FileRef);
- AsyncWriteDownloadedChunk(m_Options.ZenFolderPath,
- RemoteChunkIndex,
- std::move(ChunkTargetPtrs),
- WriteCache,
- Work,
- std::move(Payload),
- SequenceIndexChunksLeftToWriteCounters,
- WritePartsComplete,
- TotalPartWriteCount,
- FilteredWrittenBytesPerSecond,
- EnableBacklog);
- });
- }
- });
- }
- }
-}
-
-void
-BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkIndex,
- const BlobsExistsResult& ExistsResult,
- ParallelWork& Work,
- std::function<void(IoBuffer&& Payload)>&& OnDownloaded)
-{
- const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
- // FilteredDownloadedBytesPerSecond.Start();
- IoBuffer BuildBlob;
- const bool ExistsInCache = m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash);
- if (ExistsInCache)
- {
- BuildBlob = m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, ChunkHash);
- }
- if (BuildBlob)
- {
- uint64_t BlobSize = BuildBlob.GetSize();
- m_DownloadStats.DownloadedChunkCount++;
- m_DownloadStats.DownloadedChunkByteCount += BlobSize;
- m_DownloadStats.RequestsCompleteCount++;
- OnDownloaded(std::move(BuildBlob));
- }
- else
- {
- if (m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= m_Options.LargeAttachmentSize)
- {
- DownloadLargeBlob(
- *m_Storage.BuildStorage,
- m_TempDownloadFolderPath,
- m_BuildId,
- ChunkHash,
- m_Options.PreferredMultipartChunkSize,
- Work,
- m_NetworkPool,
- m_DownloadStats.DownloadedChunkByteCount,
- m_DownloadStats.MultipartAttachmentCount,
- [this, &Work, ChunkHash, RemoteChunkIndex, OnDownloaded = std::move(OnDownloaded)](IoBuffer&& Payload) mutable {
- m_DownloadStats.DownloadedChunkCount++;
- m_DownloadStats.RequestsCompleteCount++;
-
- if (Payload && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- ChunkHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(Payload)));
- }
-
- OnDownloaded(std::move(Payload));
- });
- }
- else
- {
- BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash);
- if (BuildBlob && m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- ChunkHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(BuildBlob)));
- }
- if (!BuildBlob)
- {
- throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
- }
- if (!m_Options.PrimeCacheOnly)
- {
- if (!m_AbortFlag)
- {
- uint64_t BlobSize = BuildBlob.GetSize();
- m_DownloadStats.DownloadedChunkCount++;
- m_DownloadStats.DownloadedChunkByteCount += BlobSize;
- m_DownloadStats.RequestsCompleteCount++;
-
- OnDownloaded(std::move(BuildBlob));
- }
- }
- }
- }
-}
-
-void
-BuildsOperationUpdateFolder::DownloadPartialBlock(
- const ChunkBlockAnalyser::BlockRangeDescriptor BlockRange,
- const BlobsExistsResult& ExistsResult,
- std::function<void(IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath)>&& OnDownloaded)
-{
- const uint32_t BlockIndex = BlockRange.BlockIndex;
-
- const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
-
- IoBuffer BlockBuffer;
- if (m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash))
- {
- BlockBuffer =
- m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
- }
- if (!BlockBuffer)
- {
- BlockBuffer =
- m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
- }
- if (!BlockBuffer)
- {
- throw std::runtime_error(fmt::format("Block {} is missing when fetching range {} -> {}",
- BlockDescription.BlockHash,
- BlockRange.RangeStart,
- BlockRange.RangeStart + BlockRange.RangeLength));
- }
- if (!m_AbortFlag)
- {
- uint64_t BlockSize = BlockBuffer.GetSize();
- m_DownloadStats.DownloadedBlockCount++;
- m_DownloadStats.DownloadedBlockByteCount += BlockSize;
- m_DownloadStats.RequestsCompleteCount++;
-
- std::filesystem::path BlockChunkPath;
-
- // Check if the dowloaded block is file based and we can move it directly without rewriting it
- {
- IoBufferFileReference FileRef;
- if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == BlockSize))
- {
- ZEN_TRACE_CPU("MoveTempPartialBlock");
-
- std::error_code Ec;
- std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
- if (!Ec)
- {
- BlockBuffer.SetDeleteOnClose(false);
- BlockBuffer = {};
- BlockChunkPath = m_TempBlockFolderPath /
- fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
- RenameFile(TempBlobPath, BlockChunkPath, Ec);
- if (Ec)
- {
- BlockChunkPath = std::filesystem::path{};
-
- // Re-open the temp file again
- BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
- BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
- BlockBuffer.SetDeleteOnClose(true);
- }
- }
- }
- }
-
- if (BlockChunkPath.empty() && (BlockSize > m_Options.MaximumInMemoryPayloadSize))
- {
- ZEN_TRACE_CPU("WriteTempPartialBlock");
- // Could not be moved and rather large, lets store it on disk
- BlockChunkPath = m_TempBlockFolderPath /
- fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength);
- TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer);
- BlockBuffer = {};
- }
- if (!m_AbortFlag)
- {
- OnDownloaded(std::move(BlockBuffer), std::move(BlockChunkPath));
- }
- }
-}
-
-std::vector<uint32_t>
-BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface* CloneQuery,
- const CopyChunkData& CopyData,
- const std::vector<ChunkedFolderContent>& ScavengedContents,
- const std::vector<ChunkedContentLookup>& ScavengedLookups,
- const std::vector<std::filesystem::path>& ScavengedPaths,
- BufferedWriteFileCache& WriteCache)
-{
- ZEN_TRACE_CPU("WriteLocalChunkToCache");
-
- std::filesystem::path SourceFilePath;
-
- if (CopyData.ScavengeSourceIndex == (uint32_t)-1)
- {
- const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
- SourceFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
- }
- else
- {
- const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex];
- const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex];
- const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex];
- const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
- SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred();
- }
- ZEN_ASSERT_SLOW(IsFile(SourceFilePath));
- ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty());
-
- uint64_t CacheLocalFileBytesRead = 0;
-
- size_t TargetStart = 0;
- const std::span<const ChunkedContentLookup::ChunkSequenceLocation* const> AllTargets(CopyData.TargetChunkLocationPtrs);
-
- struct WriteOp
- {
- const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr;
- uint64_t CacheFileOffset = (uint64_t)-1;
- uint32_t ChunkIndex = (uint32_t)-1;
- };
-
- std::vector<WriteOp> WriteOps;
-
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Sort");
- WriteOps.reserve(AllTargets.size());
- for (const CopyChunkData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets)
- {
- std::span<const ChunkedContentLookup::ChunkSequenceLocation* const> TargetRange =
- AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount);
- for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange)
- {
- WriteOps.push_back(
- WriteOp{.Target = Target, .CacheFileOffset = ChunkTarget.CacheFileOffset, .ChunkIndex = ChunkTarget.RemoteChunkIndex});
- }
- TargetStart += ChunkTarget.TargetChunkLocationCount;
- }
-
- std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) {
- if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
- {
- return true;
- }
- else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
- {
- return false;
- }
- if (Lhs.Target->Offset < Rhs.Target->Offset)
- {
- return true;
- }
- return false;
- });
- }
-
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Write");
-
- tsl::robin_set<uint32_t> ChunkIndexesWritten;
-
- BufferedOpenFile SourceFile(SourceFilePath,
- m_DiskStats.OpenReadCount,
- m_DiskStats.CurrentOpenFileCount,
- m_DiskStats.ReadCount,
- m_DiskStats.ReadByteCount);
-
- bool CanCloneSource = CloneQuery && CloneQuery->CanClone(SourceFile.Handle());
-
- BufferedWriteFileCache::Local LocalWriter(WriteCache);
-
- for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();)
- {
- if (m_AbortFlag)
- {
- break;
- }
- const WriteOp& Op = WriteOps[WriteOpIndex];
-
- const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
- const uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
- const uint64_t TargetSize = m_RemoteContent.RawSizes[RemotePathIndex];
- const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex];
-
- uint64_t ReadLength = ChunkSize;
- size_t WriteCount = 1;
- uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize;
- uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize;
- while ((WriteOpIndex + WriteCount) < WriteOps.size())
- {
- const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount];
- if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex)
- {
- break;
- }
- if (NextOp.Target->Offset != OpTargetEnd)
- {
- break;
- }
- if (NextOp.CacheFileOffset != OpSourceEnd)
- {
- break;
- }
- const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
- if (ReadLength + NextChunkLength > BufferedOpenFile::BlockSize)
- {
- break;
- }
- ReadLength += NextChunkLength;
- OpSourceEnd += NextChunkLength;
- OpTargetEnd += NextChunkLength;
- WriteCount++;
- }
-
- {
- bool DidClone = false;
-
- if (CanCloneSource)
- {
- uint64_t PreBytes = 0;
- uint64_t PostBytes = 0;
- uint64_t ClonableBytes =
- CloneQuery->GetClonableRange(Op.CacheFileOffset, Op.Target->Offset, ReadLength, PreBytes, PostBytes);
- if (ClonableBytes > 0)
- {
- // We need to open the file...
- BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(RemoteSequenceIndex);
- if (!Writer)
- {
- Writer = LocalWriter.PutWriter(RemoteSequenceIndex, std::make_unique<BufferedWriteFileCache::Local::Writer>());
-
- Writer->File = std::make_unique<BasicFile>();
-
- const std::filesystem::path FileName =
- GetTempChunkedSequenceFileName(m_CacheFolderPath,
- m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]);
- Writer->File->Open(FileName, BasicFile::Mode::kWrite);
- if (m_Options.UseSparseFiles)
- {
- PrepareFileForScatteredWrite(Writer->File->Handle(), TargetSize);
- }
- }
- DidClone = CloneQuery->TryClone(SourceFile.Handle(),
- Writer->File->Handle(),
- Op.CacheFileOffset + PreBytes,
- Op.Target->Offset + PreBytes,
- ClonableBytes,
- TargetSize);
- if (DidClone)
- {
- m_DiskStats.WriteCount++;
- m_DiskStats.WriteByteCount += ClonableBytes;
-
- m_DiskStats.CloneCount++;
- m_DiskStats.CloneByteCount += ClonableBytes;
-
- m_WrittenChunkByteCount += ClonableBytes;
-
- if (PreBytes > 0)
- {
- CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, PreBytes);
- const uint64_t FileOffset = Op.Target->Offset;
-
- WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
- }
- if (PostBytes > 0)
- {
- CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset + ReadLength - PostBytes, PostBytes);
- const uint64_t FileOffset = Op.Target->Offset + ReadLength - PostBytes;
-
- WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
- }
- }
- }
- }
-
- if (!DidClone)
- {
- CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength);
-
- const uint64_t FileOffset = Op.Target->Offset;
-
- WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
- }
- }
-
- CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes?
-
- WriteOpIndex += WriteCount;
- }
- }
-
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath);
- }
-
- std::vector<uint32_t> Result;
- Result.reserve(WriteOps.size());
-
- for (const WriteOp& Op : WriteOps)
- {
- Result.push_back(Op.Target->SequenceIndex);
- }
- return Result;
-}
-
-bool
-BuildsOperationUpdateFolder::WriteCompressedChunkToCache(
- const IoHash& ChunkHash,
- const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
- BufferedWriteFileCache& WriteCache,
- IoBuffer&& CompressedPart)
-{
- ZEN_TRACE_CPU("WriteCompressedChunkToCache");
-
- auto ChunkHashToChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
- ZEN_ASSERT(ChunkHashToChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
- if (IsSingleFileChunk(m_RemoteContent, ChunkTargetPtrs))
- {
- const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex;
- const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex];
- StreamDecompress(SequenceRawHash, CompositeBuffer(std::move(CompressedPart)));
- return false;
- }
- else
- {
- IoHash RawHash;
- uint64_t RawSize;
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize);
- if (!Compressed)
- {
- throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash));
- }
- if (RawHash != ChunkHash)
- {
- throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash));
- }
-
- BufferedWriteFileCache::Local LocalWriter(WriteCache);
-
- IoHashStream Hash;
- bool CouldDecompress = Compressed.DecompressToStream(
- 0,
- (uint64_t)-1,
- [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
- ZEN_UNUSED(SourceOffset);
- ZEN_TRACE_CPU("Async_StreamDecompress_Write");
- m_DiskStats.ReadByteCount += SourceSize;
- if (!m_AbortFlag)
- {
- for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs)
- {
- const auto& Target = *TargetPtr;
- const uint64_t FileOffset = Target.Offset + Offset;
- const uint32_t SequenceIndex = Target.SequenceIndex;
- const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
-
- WriteSequenceChunkToCache(LocalWriter, RangeBuffer, SequenceIndex, FileOffset, PathIndex);
- }
-
- return true;
- }
- return false;
- });
-
- if (m_AbortFlag)
- {
- return false;
- }
-
- if (!CouldDecompress)
- {
- throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash));
- }
-
- return true;
- }
-}
-
-void
-BuildsOperationUpdateFolder::StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart)
-{
- ZEN_TRACE_CPU("StreamDecompress");
- const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash);
- TemporaryFile DecompressedTemp;
- std::error_code Ec;
- DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec);
- if (Ec)
- {
- throw std::runtime_error(fmt::format("Failed creating temporary file for decompressing large blob {}, reason: ({}) {}",
- SequenceRawHash,
- Ec.value(),
- Ec.message()));
- }
- IoHash RawHash;
- uint64_t RawSize;
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedPart, RawHash, RawSize);
- if (!Compressed)
- {
- throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", SequenceRawHash));
- }
- if (RawHash != SequenceRawHash)
- {
- throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash));
- }
- PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize);
-
- IoHashStream Hash;
- bool CouldDecompress =
- Compressed.DecompressToStream(0,
- (uint64_t)-1,
- [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
- ZEN_UNUSED(SourceOffset);
- ZEN_TRACE_CPU("StreamDecompress_Write");
- m_DiskStats.ReadCount++;
- m_DiskStats.ReadByteCount += SourceSize;
- if (!m_AbortFlag)
- {
- for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
- {
- if (m_Options.ValidateCompletedSequences)
- {
- Hash.Append(Segment.GetView());
- m_ValidatedChunkByteCount += Segment.GetSize();
- }
- DecompressedTemp.Write(Segment, Offset);
- Offset += Segment.GetSize();
- m_DiskStats.WriteByteCount += Segment.GetSize();
- m_DiskStats.WriteCount++;
- m_WrittenChunkByteCount += Segment.GetSize();
- }
- return true;
- }
- return false;
- });
-
- if (m_AbortFlag)
- {
- return;
- }
-
- if (!CouldDecompress)
- {
- throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash));
- }
- if (m_Options.ValidateCompletedSequences)
- {
- const IoHash VerifyHash = Hash.GetHash();
- if (VerifyHash != SequenceRawHash)
- {
- throw std::runtime_error(
- fmt::format("Decompressed blob payload hash {} does not match expected hash {}", VerifyHash, SequenceRawHash));
- }
- }
- DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec);
- if (Ec)
- {
- throw std::runtime_error(fmt::format("Failed moving temporary file for decompressing large blob {}, reason: ({}) {}",
- SequenceRawHash,
- Ec.value(),
- Ec.message()));
- }
- // WriteChunkStats.ChunkCountWritten++;
-}
-
-void
-BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter,
- const CompositeBuffer& Chunk,
- const uint32_t SequenceIndex,
- const uint64_t FileOffset,
- const uint32_t PathIndex)
-{
- ZEN_TRACE_CPU("WriteSequenceChunkToCache");
-
- const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex];
-
- auto OpenFile = [&](BasicFile& File) {
- const std::filesystem::path FileName =
- GetTempChunkedSequenceFileName(m_CacheFolderPath, m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
- File.Open(FileName, BasicFile::Mode::kWrite);
- if (m_Options.UseSparseFiles)
- {
- PrepareFileForScatteredWrite(File.Handle(), SequenceSize);
- }
- };
-
- const uint64_t ChunkSize = Chunk.GetSize();
- ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize);
- if (ChunkSize == SequenceSize)
- {
- BasicFile SingleChunkFile;
- OpenFile(SingleChunkFile);
-
- m_DiskStats.CurrentOpenFileCount++;
- auto _ = MakeGuard([this]() { m_DiskStats.CurrentOpenFileCount--; });
- SingleChunkFile.Write(Chunk, FileOffset);
- }
- else
- {
- const uint64_t MaxWriterBufferSize = 256u * 1025u;
-
- BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex);
- if (Writer)
- {
- if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize))
- {
- Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
- }
- Writer->Write(Chunk, FileOffset);
- }
- else
- {
- Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique<BufferedWriteFileCache::Local::Writer>());
-
- Writer->File = std::make_unique<BasicFile>();
- OpenFile(*Writer->File);
- if (ChunkSize < MaxWriterBufferSize)
- {
- Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
- }
- Writer->Write(Chunk, FileOffset);
- }
- }
- m_DiskStats.WriteCount++;
- m_DiskStats.WriteByteCount += ChunkSize;
- m_WrittenChunkByteCount += ChunkSize;
-}
-
-bool
-BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash& BlockRawHash,
- std::span<const IoHash> ChunkRawHashes,
- std::span<const uint32_t> ChunkCompressedLengths,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
- const MemoryView BlockView,
- uint32_t FirstIncludedBlockChunkIndex,
- uint32_t LastIncludedBlockChunkIndex,
- BlockWriteOps& OutOps)
-{
- ZEN_TRACE_CPU("GetBlockWriteOps");
-
- uint32_t OffsetInBlock = 0;
- for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++)
- {
- const uint32_t ChunkCompressedSize = ChunkCompressedLengths[ChunkBlockIndex];
- const IoHash& ChunkHash = ChunkRawHashes[ChunkBlockIndex];
- if (auto It = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != m_RemoteLookup.ChunkHashToChunkIndex.end())
- {
- const uint32_t ChunkIndex = It->second;
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
- GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, ChunkIndex);
-
- if (!ChunkTargetPtrs.empty())
- {
- bool NeedsWrite = true;
- if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false))
- {
- MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize);
- IoHash VerifyChunkHash;
- uint64_t VerifyChunkSize;
- CompressedBuffer CompressedChunk =
- CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize);
- if (!CompressedChunk)
- {
- throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer",
- ChunkHash,
- OffsetInBlock,
- ChunkCompressedSize,
- BlockRawHash));
- }
- if (VerifyChunkHash != ChunkHash)
- {
- throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}",
- ChunkHash,
- OffsetInBlock,
- ChunkCompressedSize,
- BlockRawHash,
- VerifyChunkHash));
- }
- if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
- {
- throw std::runtime_error(
- fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}",
- ChunkHash,
- OffsetInBlock,
- ChunkCompressedSize,
- BlockRawHash,
- VerifyChunkSize,
- m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
- }
-
- OodleCompressor ChunkCompressor;
- OodleCompressionLevel ChunkCompressionLevel;
- uint64_t ChunkBlockSize;
-
- bool GetCompressParametersSuccess =
- CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize);
- ZEN_ASSERT(GetCompressParametersSuccess);
-
- IoBuffer Decompressed;
- if (ChunkCompressionLevel == OodleCompressionLevel::None)
- {
- MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder());
- Decompressed =
- IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize());
- }
- else
- {
- Decompressed = CompressedChunk.Decompress().AsIoBuffer();
- }
-
- if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
- {
- throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}",
- ChunkHash,
- OffsetInBlock,
- ChunkCompressedSize,
- BlockRawHash,
- Decompressed.GetSize(),
- m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
- }
-
- ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed));
- for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs)
- {
- OutOps.WriteOps.push_back(
- BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()});
- }
- OutOps.ChunkBuffers.emplace_back(std::move(Decompressed));
- }
- }
- }
-
- OffsetInBlock += ChunkCompressedSize;
- }
- {
- ZEN_TRACE_CPU("Sort");
- std::sort(OutOps.WriteOps.begin(),
- OutOps.WriteOps.end(),
- [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) {
- if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
- {
- return true;
- }
- if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
- {
- return false;
- }
- return Lhs.Target->Offset < Rhs.Target->Offset;
- });
- }
- return true;
-}
-
-void
-BuildsOperationUpdateFolder::WriteBlockChunkOpsToCache(std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- const BlockWriteOps& Ops,
- BufferedWriteFileCache& WriteCache,
- ParallelWork& Work)
-{
- ZEN_TRACE_CPU("WriteBlockChunkOpsToCache");
-
- {
- BufferedWriteFileCache::Local LocalWriter(WriteCache);
- for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
- {
- if (Work.IsAborted())
- {
- break;
- }
- const CompositeBuffer& Chunk = Ops.ChunkBuffers[WriteOp.ChunkBufferIndex];
- const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex;
- ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <=
- m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]);
- ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0);
- const uint64_t FileOffset = WriteOp.Target->Offset;
- const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
-
- WriteSequenceChunkToCache(LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex);
- }
- }
- if (!Work.IsAborted())
- {
- // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local)
- std::vector<uint32_t> CompletedChunkSequences;
- for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
- {
- const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex;
- if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
- {
- CompletedChunkSequences.push_back(RemoteSequenceIndex);
- }
- }
- WriteCache.Close(CompletedChunkSequences);
- VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Work);
- }
-}
-
-bool
-BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- ParallelWork& Work,
- CompositeBuffer&& BlockBuffer,
- std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
- BufferedWriteFileCache& WriteCache)
-{
- ZEN_TRACE_CPU("WriteChunksBlockToCache");
-
- IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer);
- const MemoryView BlockView = BlockMemoryBuffer.GetView();
-
- BlockWriteOps Ops;
- if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty())
- {
- ZEN_TRACE_CPU("WriteChunksBlockToCache_Legacy");
-
- uint64_t HeaderSize;
- const std::vector<uint32_t> ChunkCompressedLengths =
- ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize);
-
- if (GetBlockWriteOps(BlockDescription.BlockHash,
- BlockDescription.ChunkRawHashes,
- ChunkCompressedLengths,
- SequenceIndexChunksLeftToWriteCounters,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize),
- 0,
- gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
- Ops))
- {
- WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
- return true;
- }
- return false;
- }
-
- if (GetBlockWriteOps(BlockDescription.BlockHash,
- BlockDescription.ChunkRawHashes,
- BlockDescription.ChunkCompressedLengths,
- SequenceIndexChunksLeftToWriteCounters,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize),
- 0,
- gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
- Ops))
- {
- WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
- return true;
- }
- return false;
-}
-
-bool
-BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- ParallelWork& Work,
- CompositeBuffer&& PartialBlockBuffer,
- uint32_t FirstIncludedBlockChunkIndex,
- uint32_t LastIncludedBlockChunkIndex,
- std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
- BufferedWriteFileCache& WriteCache)
-{
- ZEN_TRACE_CPU("WritePartialBlockChunksToCache");
-
- IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer);
- const MemoryView BlockView = BlockMemoryBuffer.GetView();
-
- BlockWriteOps Ops;
- if (GetBlockWriteOps(BlockDescription.BlockHash,
- BlockDescription.ChunkRawHashes,
- BlockDescription.ChunkCompressedLengths,
- SequenceIndexChunksLeftToWriteCounters,
- RemoteChunkIndexNeedsCopyFromSourceFlags,
- BlockView,
- FirstIncludedBlockChunkIndex,
- LastIncludedBlockChunkIndex,
- Ops))
- {
- WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
- return true;
- }
- else
- {
- return false;
- }
-}
-
-void
-BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(const std::filesystem::path& ZenFolderPath,
- uint32_t RemoteChunkIndex,
- std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
- BufferedWriteFileCache& WriteCache,
- ParallelWork& Work,
- IoBuffer&& Payload,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
- std::atomic<uint64_t>& WritePartsComplete,
- const uint64_t TotalPartWriteCount,
- FilteredRate& FilteredWrittenBytesPerSecond,
- bool EnableBacklog)
-{
- ZEN_TRACE_CPU("AsyncWriteDownloadedChunk");
-
- const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
-
- const uint64_t Size = Payload.GetSize();
-
- std::filesystem::path CompressedChunkPath;
-
- // Check if the dowloaded chunk is file based and we can move it directly without rewriting it
- {
- IoBufferFileReference FileRef;
- if (Payload.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == Size))
- {
- ZEN_TRACE_CPU("MoveTempChunk");
- std::error_code Ec;
- std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
- if (!Ec)
- {
- Payload.SetDeleteOnClose(false);
- Payload = {};
- CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString();
- RenameFile(TempBlobPath, CompressedChunkPath, Ec);
- if (Ec)
- {
- CompressedChunkPath = std::filesystem::path{};
-
- // Re-open the temp file again
- BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
- Payload = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, Size, true);
- Payload.SetDeleteOnClose(true);
- }
- }
- }
- }
-
- if (CompressedChunkPath.empty() && (Size > m_Options.MaximumInMemoryPayloadSize))
- {
- ZEN_TRACE_CPU("WriteTempChunk");
- // Could not be moved and rather large, lets store it on disk
- CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString();
- TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload);
- Payload = {};
- }
-
- Work.ScheduleWork(
- m_IOWorkerPool,
- [&ZenFolderPath,
- this,
- SequenceIndexChunksLeftToWriteCounters,
- &Work,
- CompressedChunkPath,
- RemoteChunkIndex,
- TotalPartWriteCount,
- &WriteCache,
- &WritePartsComplete,
- &FilteredWrittenBytesPerSecond,
- ChunkTargetPtrs = std::move(ChunkTargetPtrs),
- CompressedPart = IoBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_WriteChunk");
-
- FilteredWrittenBytesPerSecond.Start();
-
- const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
- if (CompressedChunkPath.empty())
- {
- ZEN_ASSERT(CompressedPart);
- }
- else
- {
- ZEN_ASSERT(!CompressedPart);
- CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
- if (!CompressedPart)
- {
- throw std::runtime_error(
- fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath));
- }
- }
-
- bool NeedHashVerify = WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart));
- if (!m_AbortFlag)
- {
- WritePartsComplete++;
- if (WritePartsComplete == TotalPartWriteCount)
- {
- FilteredWrittenBytesPerSecond.Stop();
- }
-
- if (!CompressedChunkPath.empty())
- {
- std::error_code Ec = TryRemoveFile(CompressedChunkPath);
- if (Ec)
- {
- ZEN_OPERATION_LOG_DEBUG(m_LogOutput,
- "Failed removing file '{}', reason: ({}) {}",
- CompressedChunkPath,
- Ec.value(),
- Ec.message());
- }
- }
-
- std::vector<uint32_t> CompletedSequences =
- CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
- WriteCache.Close(CompletedSequences);
- if (NeedHashVerify)
- {
- VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work);
- }
- else
- {
- FinalizeChunkSequences(CompletedSequences);
- }
- }
- }
- },
- EnableBacklog ? WorkerThreadPool::EMode::EnableBacklog : WorkerThreadPool::EMode::DisableBacklog);
-}
-
-void
-BuildsOperationUpdateFolder::VerifyAndCompleteChunkSequencesAsync(std::span<const uint32_t> RemoteSequenceIndexes, ParallelWork& Work)
-{
- if (RemoteSequenceIndexes.empty())
- {
- return;
- }
- ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence");
- if (m_Options.ValidateCompletedSequences)
- {
- for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++)
- {
- const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset];
- Work.ScheduleWork(m_IOWorkerPool, [this, RemoteSequenceIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("Async_VerifyAndFinalizeSequence");
-
- VerifySequence(RemoteSequenceIndex);
- if (!m_AbortFlag)
- {
- const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- FinalizeChunkSequence(SequenceRawHash);
- }
- }
- });
- }
- const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0];
-
- VerifySequence(RemoteSequenceIndex);
- const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- FinalizeChunkSequence(SequenceRawHash);
- }
- else
- {
- for (uint32_t RemoteSequenceIndexOffset = 0; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++)
- {
- const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset];
- const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- FinalizeChunkSequence(SequenceRawHash);
- }
- }
-}
-
-bool
-BuildsOperationUpdateFolder::CompleteSequenceChunk(uint32_t RemoteSequenceIndex,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
-{
- uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1);
- ZEN_ASSERT(PreviousValue >= 1);
- ZEN_ASSERT(PreviousValue != (uint32_t)-1);
- return PreviousValue == 1;
-}
-
-std::vector<uint32_t>
-BuildsOperationUpdateFolder::CompleteChunkTargets(const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
- std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
-{
- ZEN_TRACE_CPU("CompleteChunkTargets");
-
- std::vector<uint32_t> CompletedSequenceIndexes;
- for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs)
- {
- const uint32_t RemoteSequenceIndex = Location->SequenceIndex;
- if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
- {
- CompletedSequenceIndexes.push_back(RemoteSequenceIndex);
- }
- }
- return CompletedSequenceIndexes;
-}
-
-void
-BuildsOperationUpdateFolder::FinalizeChunkSequence(const IoHash& SequenceRawHash)
-{
- ZEN_TRACE_CPU("FinalizeChunkSequence");
-
- ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash)));
- std::error_code Ec;
- RenameFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash),
- GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash),
- Ec);
- if (Ec)
- {
- throw std::system_error(Ec);
- }
-}
-
-void
-BuildsOperationUpdateFolder::FinalizeChunkSequences(std::span<const uint32_t> RemoteSequenceIndexes)
-{
- ZEN_TRACE_CPU("FinalizeChunkSequences");
-
- for (uint32_t SequenceIndex : RemoteSequenceIndexes)
- {
- FinalizeChunkSequence(m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
- }
-}
-
-void
-BuildsOperationUpdateFolder::VerifySequence(uint32_t RemoteSequenceIndex)
-{
- ZEN_TRACE_CPU("VerifySequence");
-
- ZEN_ASSERT(m_Options.ValidateCompletedSequences);
-
- const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
- {
- ZEN_TRACE_CPU("HashSequence");
- const std::uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
- const uint64_t ExpectedSize = m_RemoteContent.RawSizes[RemotePathIndex];
- IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash));
- const uint64_t VerifySize = VerifyBuffer.GetSize();
- if (VerifySize != ExpectedSize)
- {
- throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}",
- SequenceRawHash,
- VerifySize,
- ExpectedSize));
- }
-
- const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer), &m_ValidatedChunkByteCount);
- if (VerifyChunkHash != SequenceRawHash)
- {
- throw std::runtime_error(
- fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash));
- }
- }
-}
-
-////////////////////// BuildsOperationUploadFolder
-
-BuildsOperationUploadFolder::BuildsOperationUploadFolder(OperationLogOutput& OperationLogOutput,
- StorageInstance& Storage,
- std::atomic<bool>& AbortFlag,
- std::atomic<bool>& PauseFlag,
- WorkerThreadPool& IOWorkerPool,
- WorkerThreadPool& NetworkPool,
- const Oid& BuildId,
- const std::filesystem::path& Path,
- bool CreateBuild,
- const CbObject& MetaData,
- const Options& Options)
-: m_LogOutput(OperationLogOutput)
-, m_Storage(Storage)
-, m_AbortFlag(AbortFlag)
-, m_PauseFlag(PauseFlag)
-, m_IOWorkerPool(IOWorkerPool)
-, m_NetworkPool(NetworkPool)
-, m_BuildId(BuildId)
-, m_Path(Path)
-, m_CreateBuild(CreateBuild)
-, m_MetaData(MetaData)
-, m_Options(Options)
-{
- m_NonCompressableExtensionHashes.reserve(Options.NonCompressableExtensions.size());
- for (const std::string& Extension : Options.NonCompressableExtensions)
- {
- m_NonCompressableExtensionHashes.insert(HashStringAsLowerDjb2(Extension));
- }
-}
-
-BuildsOperationUploadFolder::PrepareBuildResult
-BuildsOperationUploadFolder::PrepareBuild()
-{
- ZEN_TRACE_CPU("PrepareBuild");
-
- PrepareBuildResult Result;
- Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize;
- Stopwatch Timer;
- if (m_CreateBuild)
- {
- ZEN_TRACE_CPU("CreateBuild");
-
- Stopwatch PutBuildTimer;
- CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
- Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs();
- if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0)
- {
- Result.PreferredMultipartChunkSize = ChunkSize;
- }
- Result.PayloadSize = m_MetaData.GetSize();
- }
- else
- {
- ZEN_TRACE_CPU("PutBuild");
- Stopwatch GetBuildTimer;
- CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId);
- Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
- Result.PayloadSize = Build.GetSize();
- if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
- {
- Result.PreferredMultipartChunkSize = ChunkSize;
- }
- else if (m_Options.AllowMultiparts)
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "PreferredMultipartChunkSize is unknown. Defaulting to '{}'",
- NiceBytes(Result.PreferredMultipartChunkSize));
- }
- }
-
- if (!m_Options.IgnoreExistingBlocks)
- {
- ZEN_TRACE_CPU("FindBlocks");
- Stopwatch KnownBlocksTimer;
- CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount);
- if (BlockDescriptionList)
- {
- Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList);
- }
- Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs();
- }
- Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
- return Result;
-}
-
-std::vector<BuildsOperationUploadFolder::UploadPart>
-BuildsOperationUploadFolder::ReadFolder()
-{
- std::vector<UploadPart> UploadParts;
- std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName;
- tsl::robin_set<std::string> ExcludeAssetPaths;
- if (IsFile(ExcludeManifestPath))
- {
- std::filesystem::path AbsoluteExcludeManifestPath =
- MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath);
- BuildManifest Manifest = ParseBuildManifest(AbsoluteExcludeManifestPath);
- const std::vector<std::filesystem::path>& AssetPaths = Manifest.Parts.front().Files;
- ExcludeAssetPaths.reserve(AssetPaths.size());
- for (const std::filesystem::path& AssetPath : AssetPaths)
- {
- ExcludeAssetPaths.insert(AssetPath.generic_string());
- }
- }
-
- UploadParts.resize(1);
-
- UploadPart& Part = UploadParts.front();
- GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
-
- Part.Content = GetFolderContent(
- Part.LocalFolderScanStats,
- m_Path,
- [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); },
- [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool {
- ZEN_UNUSED(Size, Attributes);
- if (!IsAcceptedFile(RelativePath))
- {
- return false;
- }
- if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string()))
- {
- return false;
- }
- return true;
- },
- m_IOWorkerPool,
- m_LogOutput.GetProgressUpdateDelayMS(),
- [&](bool, std::ptrdiff_t) {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path);
- },
- m_AbortFlag);
- Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
-
- return UploadParts;
-}
-
-std::vector<BuildsOperationUploadFolder::UploadPart>
-BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath)
-{
- std::vector<UploadPart> UploadParts;
- Stopwatch ManifestParseTimer;
- std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath);
- BuildManifest Manifest = ParseBuildManifest(AbsoluteManifestPath);
- if (Manifest.Parts.empty())
- {
- throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath));
- }
-
- UploadParts.resize(Manifest.Parts.size());
- for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++)
- {
- BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex];
- if (ManifestPath.is_relative())
- {
- PartManifest.Files.push_back(ManifestPath);
- }
-
- UploadPart& Part = UploadParts[PartIndex];
- FolderContent& Content = Part.Content;
-
- GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
-
- const std::vector<std::filesystem::path>& AssetPaths = PartManifest.Files;
- Content = GetValidFolderContent(
- m_IOWorkerPool,
- LocalFolderScanStats,
- m_Path,
- AssetPaths,
- [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); },
- 1000,
- m_AbortFlag,
- m_PauseFlag);
-
- if (Content.Paths.size() != AssetPaths.size())
- {
- const tsl::robin_set<std::filesystem::path> FoundPaths(Content.Paths.begin(), Content.Paths.end());
- ExtendableStringBuilder<1024> SB;
- for (const std::filesystem::path& AssetPath : AssetPaths)
- {
- if (!FoundPaths.contains(AssetPath))
- {
- SB << "\n " << AssetPath.generic_string();
- }
- }
- throw std::runtime_error(
- fmt::format("Manifest file at '{}' references files that does not exist{}", ManifestPath, SB.ToView()));
- }
-
- Part.PartId = PartManifest.PartId;
- Part.PartName = PartManifest.PartName;
- Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
- }
-
- return UploadParts;
-}
-
-std::vector<std::pair<Oid, std::string>>
-BuildsOperationUploadFolder::Execute(const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& ManifestPath,
- ChunkingController& ChunkController,
- ChunkingCache& ChunkCache)
-{
- ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute");
- try
- {
- Stopwatch ReadPartsTimer;
- std::vector<UploadPart> UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath);
-
- for (UploadPart& Part : UploadParts)
- {
- if (Part.PartId == Oid::Zero)
- {
- if (UploadParts.size() != 1)
- {
- throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath));
- }
-
- if (BuildPartId == Oid::Zero)
- {
- Part.PartId = Oid::NewOid();
- }
- else
- {
- Part.PartId = BuildPartId;
- }
- }
- if (Part.PartName.empty())
- {
- if (UploadParts.size() != 1)
- {
- throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath));
- }
- if (BuildPartName.empty())
- {
- throw std::runtime_error("Build part name must be set");
- }
- Part.PartName = std::string(BuildPartName);
- }
- }
-
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Reading {} parts took {}",
- UploadParts.size(),
- NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs()));
- }
-
- const uint32_t PartsUploadStepCount = gsl::narrow<uint32_t>(uint32_t(PartTaskSteps::StepCount) * UploadParts.size());
-
- const uint32_t PrepareBuildStep = 0;
- const uint32_t UploadPartsStep = 1;
- const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount;
- const uint32_t CleanupStep = FinalizeBuildStep + 1;
- const uint32_t StepCount = CleanupStep + 1;
-
- auto EndProgress = MakeGuard([&]() { m_LogOutput.SetLogOperationProgress(StepCount, StepCount); });
-
- Stopwatch ProcessTimer;
-
- CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir);
- CreateDirectories(m_Options.TempDir);
- auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); });
-
- m_LogOutput.SetLogOperationProgress(PrepareBuildStep, StepCount);
-
- m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task<PrepareBuildResult()>{[this] { return PrepareBuild(); }},
- WorkerThreadPool::EMode::EnableBacklog);
-
- for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++)
- {
- const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount));
-
- const UploadPart& Part = UploadParts[PartIndex];
- UploadBuildPart(ChunkController, ChunkCache, PartIndex, Part, PartStepOffset, StepCount);
- if (m_AbortFlag)
- {
- return {};
- }
- }
-
- m_LogOutput.SetLogOperationProgress(FinalizeBuildStep, StepCount);
-
- if (m_CreateBuild && !m_AbortFlag)
- {
- Stopwatch FinalizeBuildTimer;
- m_Storage.BuildStorage->FinalizeBuild(m_BuildId);
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
- }
- }
-
- m_LogOutput.SetLogOperationProgress(CleanupStep, StepCount);
-
- std::vector<std::pair<Oid, std::string>> Result;
- Result.reserve(UploadParts.size());
- for (UploadPart& Part : UploadParts)
- {
- Result.push_back(std::make_pair(Part.PartId, Part.PartName));
- }
- return Result;
- }
- catch (const std::exception&)
- {
- m_AbortFlag = true;
- throw;
- }
-}
-
-bool
-BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const
-{
- for (const std::string& ExcludeFolder : m_Options.ExcludeFolders)
- {
- if (RelativePath.starts_with(ExcludeFolder))
- {
- if (RelativePath.length() == ExcludeFolder.length())
- {
- return false;
- }
- else if (RelativePath[ExcludeFolder.length()] == '/')
- {
- return false;
- }
- }
- }
- return true;
-}
-
-bool
-BuildsOperationUploadFolder::IsAcceptedFile(const std::string_view& RelativePath) const
-{
- if (RelativePath == m_Options.ZenExcludeManifestName)
- {
- return false;
- }
- for (const std::string& ExcludeExtension : m_Options.ExcludeExtensions)
- {
- if (RelativePath.ends_with(ExcludeExtension))
- {
- return false;
- }
- }
- return true;
-}
-
-void
-BuildsOperationUploadFolder::ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- std::vector<uint32_t>& ChunkIndexes,
- std::vector<std::vector<uint32_t>>& OutBlocks)
-{
- ZEN_TRACE_CPU("ArrangeChunksIntoBlocks");
- std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) {
- const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0];
- const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0];
- if (LhsLocation.SequenceIndex < RhsLocation.SequenceIndex)
- {
- return true;
- }
- else if (LhsLocation.SequenceIndex > RhsLocation.SequenceIndex)
- {
- return false;
- }
- return LhsLocation.Offset < RhsLocation.Offset;
- });
-
- uint64_t MaxBlockSizeLowThreshold = m_Options.BlockParameters.MaxBlockSize - (m_Options.BlockParameters.MaxBlockSize / 16);
-
- uint64_t BlockSize = 0;
-
- uint32_t ChunkIndexStart = 0;
- for (uint32_t ChunkIndexOffset = 0; ChunkIndexOffset < ChunkIndexes.size();)
- {
- const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset];
- const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
-
- if (((BlockSize + ChunkSize) > m_Options.BlockParameters.MaxBlockSize) ||
- (ChunkIndexOffset - ChunkIndexStart) > m_Options.BlockParameters.MaxChunksPerBlock)
- {
- // Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break
- // between source paths for chunks. Break the block at the last such break if any.
- ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart);
-
- const uint32_t ChunkSequenceIndex = Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[ChunkIndex]].SequenceIndex;
-
- uint64_t ScanBlockSize = BlockSize;
-
- uint32_t ScanChunkIndexOffset = ChunkIndexOffset - 1;
- while (ScanChunkIndexOffset > (ChunkIndexStart + 2))
- {
- const uint32_t TestChunkIndex = ChunkIndexes[ScanChunkIndexOffset];
- const uint64_t TestChunkSize = Content.ChunkedContent.ChunkRawSizes[TestChunkIndex];
- if ((ScanBlockSize - TestChunkSize) < MaxBlockSizeLowThreshold)
- {
- break;
- }
-
- const uint32_t TestSequenceIndex =
- Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[TestChunkIndex]].SequenceIndex;
- if (ChunkSequenceIndex != TestSequenceIndex)
- {
- ChunkIndexOffset = ScanChunkIndexOffset + 1;
- break;
- }
-
- ScanBlockSize -= TestChunkSize;
- ScanChunkIndexOffset--;
- }
-
- std::vector<uint32_t> ChunksInBlock;
- ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart);
- for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++)
- {
- const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset];
- ChunksInBlock.push_back(AddChunkIndex);
- }
- OutBlocks.emplace_back(std::move(ChunksInBlock));
- BlockSize = 0;
- ChunkIndexStart = ChunkIndexOffset;
- }
- else
- {
- ChunkIndexOffset++;
- BlockSize += ChunkSize;
- }
- }
- if (ChunkIndexStart < ChunkIndexes.size())
- {
- std::vector<uint32_t> ChunksInBlock;
- ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart);
- for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++)
- {
- const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset];
- ChunksInBlock.push_back(AddChunkIndex);
- }
- OutBlocks.emplace_back(std::move(ChunksInBlock));
- }
-}
-
-void
-BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- const std::vector<std::vector<uint32_t>>& NewBlockChunks,
- GeneratedBlocks& OutBlocks,
- GenerateBlocksStatistics& GenerateBlocksStats,
- UploadStatistics& UploadStats)
-{
- ZEN_TRACE_CPU("GenerateBuildBlocks");
- const std::size_t NewBlockCount = NewBlockChunks.size();
- if (NewBlockCount > 0)
- {
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Generate Blocks"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- OutBlocks.BlockDescriptions.resize(NewBlockCount);
- OutBlocks.BlockSizes.resize(NewBlockCount);
- OutBlocks.BlockMetaDatas.resize(NewBlockCount);
- OutBlocks.BlockHeaders.resize(NewBlockCount);
- OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, 0);
- OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount);
-
- RwLock Lock;
-
- WorkerThreadPool& GenerateBlobsPool = m_IOWorkerPool;
- WorkerThreadPool& UploadBlocksPool = m_NetworkPool;
-
- FilteredRate FilteredGeneratedBytesPerSecond;
- FilteredRate FilteredUploadedBytesPerSecond;
-
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- std::atomic<uint64_t> QueuedPendingBlocksForUpload = 0;
-
- for (size_t BlockIndex = 0; BlockIndex < NewBlockCount; BlockIndex++)
- {
- if (Work.IsAborted())
- {
- break;
- }
- const std::vector<uint32_t>& ChunksInBlock = NewBlockChunks[BlockIndex];
- Work.ScheduleWork(
- GenerateBlobsPool,
- [this,
- &Content,
- &Lookup,
- &Work,
- &UploadBlocksPool,
- NewBlockCount,
- ChunksInBlock,
- &Lock,
- &OutBlocks,
- &GenerateBlocksStats,
- &UploadStats,
- &FilteredGeneratedBytesPerSecond,
- &QueuedPendingBlocksForUpload,
- &FilteredUploadedBytesPerSecond,
- BlockIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("GenerateBuildBlocks_Generate");
-
- FilteredGeneratedBytesPerSecond.Start();
-
- Stopwatch GenerateTimer;
- CompressedBuffer CompressedBlock =
- GenerateBlock(Content, Lookup, ChunksInBlock, OutBlocks.BlockDescriptions[BlockIndex]);
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Generated block {} ({}) containing {} chunks in {}",
- OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
- NiceBytes(CompressedBlock.GetCompressedSize()),
- OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
- NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
- }
-
- OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize();
- {
- CbObjectWriter Writer;
- Writer.AddString("createdBy", "zen");
- OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save();
- }
- GenerateBlocksStats.GeneratedBlockByteCount += OutBlocks.BlockSizes[BlockIndex];
- GenerateBlocksStats.GeneratedBlockCount++;
-
- Lock.WithExclusiveLock([&]() {
- OutBlocks.BlockHashToBlockIndex.insert_or_assign(OutBlocks.BlockDescriptions[BlockIndex].BlockHash, BlockIndex);
- });
-
- {
- std::span<const SharedBuffer> Segments = CompressedBlock.GetCompressed().GetSegments();
- ZEN_ASSERT(Segments.size() >= 2);
- OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
- }
-
- if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
- {
- FilteredGeneratedBytesPerSecond.Stop();
- }
-
- if (QueuedPendingBlocksForUpload.load() > 16)
- {
- std::span<const SharedBuffer> Segments = CompressedBlock.GetCompressed().GetSegments();
- ZEN_ASSERT(Segments.size() >= 2);
- OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
- }
- else
- {
- if (!m_AbortFlag)
- {
- QueuedPendingBlocksForUpload++;
-
- Work.ScheduleWork(
- UploadBlocksPool,
- [this,
- NewBlockCount,
- &GenerateBlocksStats,
- &UploadStats,
- &FilteredUploadedBytesPerSecond,
- &QueuedPendingBlocksForUpload,
- &OutBlocks,
- BlockIndex,
- Payload = std::move(CompressedBlock)](std::atomic<bool>&) mutable {
- auto _ = MakeGuard([&QueuedPendingBlocksForUpload] { QueuedPendingBlocksForUpload--; });
- if (!m_AbortFlag)
- {
- if (GenerateBlocksStats.GeneratedBlockCount == NewBlockCount)
- {
- ZEN_TRACE_CPU("GenerateBuildBlocks_Save");
-
- FilteredUploadedBytesPerSecond.Stop();
- std::span<const SharedBuffer> Segments = Payload.GetCompressed().GetSegments();
- ZEN_ASSERT(Segments.size() >= 2);
- OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
- }
- else
- {
- ZEN_TRACE_CPU("GenerateBuildBlocks_Upload");
-
- FilteredUploadedBytesPerSecond.Start();
-
- const CbObject BlockMetaData =
- BuildChunkBlockDescription(OutBlocks.BlockDescriptions[BlockIndex],
- OutBlocks.BlockMetaDatas[BlockIndex]);
-
- const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash;
- const uint64_t CompressedBlockSize = Payload.GetCompressedSize();
-
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- BlockHash,
- ZenContentType::kCompressedBinary,
- Payload.GetCompressed());
- }
-
- m_Storage.BuildStorage->PutBuildBlob(m_BuildId,
- BlockHash,
- ZenContentType::kCompressedBinary,
- std::move(Payload).GetCompressed());
- UploadStats.BlocksBytes += CompressedBlockSize;
-
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded block {} ({}) containing {} chunks",
- BlockHash,
- NiceBytes(CompressedBlockSize),
- OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
- }
-
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
- std::vector<IoHash>({BlockHash}),
- std::vector<CbObject>({BlockMetaData}));
- }
-
- bool MetadataSucceeded =
- m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
- if (MetadataSucceeded)
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded block {} metadata ({})",
- BlockHash,
- NiceBytes(BlockMetaData.GetSize()));
- }
-
- OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
- UploadStats.BlocksBytes += BlockMetaData.GetSize();
- }
-
- UploadStats.BlockCount++;
- if (UploadStats.BlockCount == NewBlockCount)
- {
- FilteredUploadedBytesPerSecond.Stop();
- }
- }
- }
- });
- }
- }
- }
- });
- }
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
-
- FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load());
- FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load());
-
- std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)",
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NewBlockCount,
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
- NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()),
- UploadStats.BlockCount.load(),
- NewBlockCount,
- NiceBytes(UploadStats.BlocksBytes.load()),
- NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8));
-
- Progress.UpdateState({.Task = "Generating blocks",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(NewBlockCount),
- .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
-
- ZEN_ASSERT(m_AbortFlag || QueuedPendingBlocksForUpload.load() == 0);
-
- Progress.Finish();
-
- GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS();
- UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
- }
-}
-
-std::vector<uint32_t>
-BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
- const std::span<const IoHash> LocalChunkHashes,
- const std::span<const uint32_t> LocalChunkOrder,
- const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex,
- const std::span<const uint32_t>& LooseChunkIndexes,
- const std::span<const ChunkBlockDescription>& BlockDescriptions)
-{
- ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders");
-
- std::vector<IoHash> TmpAbsoluteChunkHashes;
- if (m_Options.DoExtraContentValidation)
- {
- TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size());
- }
- std::vector<uint32_t> LocalChunkIndexToAbsoluteChunkIndex;
- LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1);
- std::uint32_t AbsoluteChunkCount = 0;
- for (uint32_t ChunkIndex : LooseChunkIndexes)
- {
- LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount;
- if (m_Options.DoExtraContentValidation)
- {
- TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]);
- }
- AbsoluteChunkCount++;
- }
- for (const ChunkBlockDescription& Block : BlockDescriptions)
- {
- for (const IoHash& ChunkHash : Block.ChunkRawHashes)
- {
- if (auto It = ChunkHashToLocalChunkIndex.find(ChunkHash); It != ChunkHashToLocalChunkIndex.end())
- {
- const uint32_t LocalChunkIndex = It->second;
- ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash);
- LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount;
- }
- if (m_Options.DoExtraContentValidation)
- {
- TmpAbsoluteChunkHashes.push_back(ChunkHash);
- }
- AbsoluteChunkCount++;
- }
- }
- std::vector<uint32_t> AbsoluteChunkOrder;
- AbsoluteChunkOrder.reserve(LocalChunkHashes.size());
- for (const uint32_t LocalChunkIndex : LocalChunkOrder)
- {
- const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex];
- if (m_Options.DoExtraContentValidation)
- {
- ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]);
- }
- AbsoluteChunkOrder.push_back(AbsoluteChunkIndex);
- }
- if (m_Options.DoExtraContentValidation)
- {
- uint32_t OrderIndex = 0;
- while (OrderIndex < LocalChunkOrder.size())
- {
- const uint32_t LocalChunkIndex = LocalChunkOrder[OrderIndex];
- const IoHash& LocalChunkHash = LocalChunkHashes[LocalChunkIndex];
- const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrder[OrderIndex];
- const IoHash& AbsoluteChunkHash = TmpAbsoluteChunkHashes[AbsoluteChunkIndex];
- ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
- OrderIndex++;
- }
- }
- return AbsoluteChunkOrder;
-}
-
-CompositeBuffer
-BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- const IoHash& ChunkHash,
- ReadFileCache& OpenFileCache)
-{
- ZEN_TRACE_CPU("FetchChunk");
- auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash);
- ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end());
- uint32_t ChunkIndex = It->second;
- std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex);
- ZEN_ASSERT(!ChunkLocations.empty());
- CompositeBuffer Chunk =
- OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
- if (!Chunk)
- {
- throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'",
- ChunkLocations[0].Offset,
- Content.ChunkedContent.ChunkRawSizes[ChunkIndex],
- Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]]));
- }
- ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash);
- return Chunk;
-};
-
-CompressedBuffer
-BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- const std::vector<uint32_t>& ChunksInBlock,
- ChunkBlockDescription& OutBlockDescription)
-{
- ZEN_TRACE_CPU("GenerateBlock");
- ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
- m_DiskStats.CurrentOpenFileCount,
- m_DiskStats.ReadCount,
- m_DiskStats.ReadByteCount,
- m_Path,
- Content,
- Lookup,
- 4);
-
- std::vector<std::pair<IoHash, FetchChunkFunc>> BlockContent;
- BlockContent.reserve(ChunksInBlock.size());
- for (uint32_t ChunkIndex : ChunksInBlock)
- {
- BlockContent.emplace_back(std::make_pair(
- Content.ChunkedContent.ChunkHashes[ChunkIndex],
- [this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair<uint64_t, CompressedBuffer> {
- CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache);
- ZEN_ASSERT(Chunk);
- uint64_t RawSize = Chunk.GetSize();
-
- const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock &&
- IsChunkCompressable(m_NonCompressableExtensionHashes, Content, Lookup, ChunkIndex);
-
- const OodleCompressionLevel CompressionLevel =
- ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
- return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel)};
- }));
- }
-
- return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription);
-};
-
-CompressedBuffer
-BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- CompositeBuffer&& HeaderBuffer,
- const std::vector<uint32_t>& ChunksInBlock)
-{
- ZEN_TRACE_CPU("RebuildBlock");
- ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
- m_DiskStats.CurrentOpenFileCount,
- m_DiskStats.ReadCount,
- m_DiskStats.ReadByteCount,
- m_Path,
- Content,
- Lookup,
- 4);
-
- std::vector<SharedBuffer> ResultBuffers;
- ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size());
- ResultBuffers.insert(ResultBuffers.end(), HeaderBuffer.GetSegments().begin(), HeaderBuffer.GetSegments().end());
- for (uint32_t ChunkIndex : ChunksInBlock)
- {
- std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex);
- ZEN_ASSERT(!ChunkLocations.empty());
- CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex,
- ChunkLocations[0].Offset,
- Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
- ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]);
-
- const uint64_t RawSize = Chunk.GetSize();
- const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock &&
- IsChunkCompressable(m_NonCompressableExtensionHashes, Content, Lookup, ChunkIndex);
-
- const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
-
- CompositeBuffer CompressedChunk =
- CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed();
- ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end());
- }
- return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers)));
-};
-
-void
-BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController,
- ChunkingCache& ChunkCache,
- uint32_t PartIndex,
- const UploadPart& Part,
- uint32_t PartStepOffset,
- uint32_t StepCount)
-{
- Stopwatch UploadTimer;
-
- ChunkingStatistics ChunkingStats;
- FindBlocksStatistics FindBlocksStats;
- ReuseBlocksStatistics ReuseBlocksStats;
- UploadStatistics UploadStats;
- GenerateBlocksStatistics GenerateBlocksStats;
-
- LooseChunksStatistics LooseChunksStats;
- ChunkedFolderContent LocalContent;
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount);
-
- Stopwatch ScanTimer;
- {
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Scan Folder"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- FilteredRate FilteredBytesHashed;
- FilteredBytesHashed.Start();
- LocalContent = ChunkFolderContent(
- ChunkingStats,
- m_IOWorkerPool,
- m_Path,
- Part.Content,
- ChunkController,
- ChunkCache,
- m_LogOutput.GetProgressUpdateDelayMS(),
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
- FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
- std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
- ChunkingStats.FilesProcessed.load(),
- Part.Content.Paths.size(),
- NiceBytes(ChunkingStats.BytesHashed.load()),
- NiceBytes(Part.TotalRawSize),
- NiceNum(FilteredBytesHashed.GetCurrent()),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()));
- Progress.UpdateState({.Task = "Scanning files ",
- .Details = Details,
- .TotalCount = Part.TotalRawSize,
- .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- },
- m_AbortFlag,
- m_PauseFlag);
- FilteredBytesHashed.Stop();
- Progress.Finish();
- if (m_AbortFlag)
- {
- return;
- }
- }
-
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
- Part.Content.Paths.size(),
- NiceBytes(Part.TotalRawSize),
- ChunkingStats.UniqueChunksFound.load(),
- NiceBytes(ChunkingStats.UniqueBytesFound.load()),
- m_Path,
- NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
- NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
- }
-
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
-
- std::vector<size_t> ReuseBlockIndexes;
- std::vector<uint32_t> NewBlockChunkIndexes;
-
- if (PartIndex == 0)
- {
- const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get();
-
- m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs;
- m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size();
-
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Build prepare took {}. {} took {}, payload size {}{}",
- NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
- m_CreateBuild ? "PutBuild" : "GetBuild",
- NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
- NiceBytes(PrepBuildResult.PayloadSize),
- m_Options.IgnoreExistingBlocks ? ""
- : fmt::format(". Found {} blocks in {}",
- PrepBuildResult.KnownBlocks.size(),
- NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
- }
-
- m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize;
-
- m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
-
- m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks);
- }
-
- ZEN_ASSERT(m_PreferredMultipartChunkSize != 0);
- ZEN_ASSERT(m_LargeAttachmentSize != 0);
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount);
-
- Stopwatch BlockArrangeTimer;
-
- std::vector<std::uint32_t> LooseChunkIndexes;
- {
- bool EnableBlocks = true;
- std::vector<std::uint32_t> BlockChunkIndexes;
- for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
- {
- const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize)
- {
- LooseChunkIndexes.push_back(ChunkIndex);
- LooseChunksStats.ChunkByteCount += ChunkRawSize;
- }
- else
- {
- BlockChunkIndexes.push_back(ChunkIndex);
- FindBlocksStats.PotentialChunkByteCount += ChunkRawSize;
- }
- }
- FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size();
- LooseChunksStats.ChunkCount = LooseChunkIndexes.size();
-
- if (m_Options.IgnoreExistingBlocks)
- {
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Ignoring any existing blocks in store");
- }
- NewBlockChunkIndexes = std::move(BlockChunkIndexes);
- }
- else
- {
- ReuseBlockIndexes = FindReuseBlocks(m_LogOutput,
- m_Options.BlockReuseMinPercentLimit,
- m_Options.IsVerbose,
- ReuseBlocksStats,
- m_KnownBlocks,
- LocalContent.ChunkedContent.ChunkHashes,
- BlockChunkIndexes,
- NewBlockChunkIndexes);
- FindBlocksStats.AcceptedBlockCount += ReuseBlockIndexes.size();
-
- for (const ChunkBlockDescription& Description : m_KnownBlocks)
- {
- for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
- {
- FindBlocksStats.FoundBlockByteCount += ChunkRawLength;
- }
- FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size();
- }
- }
- }
-
- std::vector<std::vector<uint32_t>> NewBlockChunks;
- ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks);
-
- FindBlocksStats.NewBlocksCount += NewBlockChunks.size();
- for (uint32_t ChunkIndex : NewBlockChunkIndexes)
- {
- FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size();
-
- const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0
- ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount)
- : 0.0;
-
- const double AcceptedReduntantByteCountPercent =
- ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) /
- (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount)
- : 0.0;
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
- " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
- " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
- " Rejected {} ({}) chunks in {} blocks\n"
- " Arranged {} ({}) chunks in {} new blocks\n"
- " Keeping {} ({}) chunks as loose chunks\n"
- " Discovery completed in {}",
- FindBlocksStats.FoundBlockChunkCount,
- FindBlocksStats.FoundBlockCount,
- NiceBytes(FindBlocksStats.FoundBlockByteCount),
- NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
-
- ReuseBlocksStats.AcceptedChunkCount,
- NiceBytes(ReuseBlocksStats.AcceptedRawByteCount),
- FindBlocksStats.AcceptedBlockCount,
- AcceptedByteCountPercent,
-
- ReuseBlocksStats.AcceptedReduntantChunkCount,
- NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount),
- AcceptedReduntantByteCountPercent,
-
- ReuseBlocksStats.RejectedChunkCount,
- NiceBytes(ReuseBlocksStats.RejectedByteCount),
- ReuseBlocksStats.RejectedBlockCount,
-
- FindBlocksStats.NewBlocksChunkCount,
- NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
- FindBlocksStats.NewBlocksCount,
-
- LooseChunksStats.ChunkCount,
- NiceBytes(LooseChunksStats.ChunkByteCount),
-
- NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
- }
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount);
- GeneratedBlocks NewBlocks;
-
- if (!NewBlockChunks.empty())
- {
- Stopwatch GenerateBuildBlocksTimer;
- auto __ = MakeGuard([&]() {
- uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(
- m_LogOutput,
- "Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
- GenerateBlocksStats.GeneratedBlockCount.load(),
- NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
- UploadStats.BlockCount.load(),
- NiceBytes(UploadStats.BlocksBytes.load()),
- NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
- NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
- GenerateBlocksStats.GeneratedBlockByteCount)),
- NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
- }
- });
- GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats);
- }
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount);
-
- CbObject PartManifest;
- {
- CbObjectWriter PartManifestWriter;
- Stopwatch ManifestGenerationTimer;
- auto __ = MakeGuard([&]() {
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Generated build part manifest in {} ({})",
- NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifestWriter.GetSaveSize()));
- }
- });
-
- PartManifestWriter.BeginObject("chunker"sv);
- {
- PartManifestWriter.AddString("name"sv, ChunkController.GetName());
- PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters());
- }
- PartManifestWriter.EndObject(); // chunker
-
- std::vector<IoHash> AllChunkBlockHashes;
- std::vector<ChunkBlockDescription> AllChunkBlockDescriptions;
- AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
- AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
- for (size_t ReuseBlockIndex : ReuseBlockIndexes)
- {
- AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]);
- AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash);
- }
- AllChunkBlockDescriptions.insert(AllChunkBlockDescriptions.end(),
- NewBlocks.BlockDescriptions.begin(),
- NewBlocks.BlockDescriptions.end());
- for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions)
- {
- AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
- }
-
- std::vector<IoHash> AbsoluteChunkHashes;
- if (m_Options.DoExtraContentValidation)
- {
- tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
- AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
- for (uint32_t ChunkIndex : LooseChunkIndexes)
- {
- ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
- AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
- }
- for (const ChunkBlockDescription& Block : AllChunkBlockDescriptions)
- {
- for (const IoHash& ChunkHash : Block.ChunkRawHashes)
- {
- ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
- AbsoluteChunkHashes.push_back(ChunkHash);
- }
- }
- for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
- {
- ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
- ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
- }
- for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
- {
- ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
- LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
- ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
- }
- }
- std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes,
- LocalContent.ChunkedContent.ChunkOrders,
- LocalLookup.ChunkHashToChunkIndex,
- LooseChunkIndexes,
- AllChunkBlockDescriptions);
-
- if (m_Options.DoExtraContentValidation)
- {
- for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
- {
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
- uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
- const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
- const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex];
- ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
- }
- }
-
- WriteBuildContentToCompactBinary(PartManifestWriter,
- LocalContent.Platform,
- LocalContent.Paths,
- LocalContent.RawHashes,
- LocalContent.RawSizes,
- LocalContent.Attributes,
- LocalContent.ChunkedContent.SequenceRawHashes,
- LocalContent.ChunkedContent.ChunkCounts,
- LocalContent.ChunkedContent.ChunkHashes,
- LocalContent.ChunkedContent.ChunkRawSizes,
- AbsoluteChunkOrders,
- LooseChunkIndexes,
- AllChunkBlockHashes);
-
- if (m_Options.DoExtraContentValidation)
- {
- ChunkedFolderContent VerifyFolderContent;
-
- std::vector<uint32_t> OutAbsoluteChunkOrders;
- std::vector<IoHash> OutLooseChunkHashes;
- std::vector<uint64_t> OutLooseChunkRawSizes;
- std::vector<IoHash> OutBlockRawHashes;
- ReadBuildContentFromCompactBinary(PartManifestWriter.Save(),
- VerifyFolderContent.Platform,
- VerifyFolderContent.Paths,
- VerifyFolderContent.RawHashes,
- VerifyFolderContent.RawSizes,
- VerifyFolderContent.Attributes,
- VerifyFolderContent.ChunkedContent.SequenceRawHashes,
- VerifyFolderContent.ChunkedContent.ChunkCounts,
- OutAbsoluteChunkOrders,
- OutLooseChunkHashes,
- OutLooseChunkRawSizes,
- OutBlockRawHashes);
- ZEN_ASSERT(OutBlockRawHashes == AllChunkBlockHashes);
-
- for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++)
- {
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
-
- uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex];
- const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex];
-
- ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
- }
-
- CalculateLocalChunkOrders(OutAbsoluteChunkOrders,
- OutLooseChunkHashes,
- OutLooseChunkRawSizes,
- AllChunkBlockDescriptions,
- VerifyFolderContent.ChunkedContent.ChunkHashes,
- VerifyFolderContent.ChunkedContent.ChunkRawSizes,
- VerifyFolderContent.ChunkedContent.ChunkOrders,
- m_Options.DoExtraContentValidation);
-
- ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths);
- ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes);
- ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes);
- ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes);
- ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes);
- ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts);
-
- for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++)
- {
- uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
- uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
-
- uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex];
- const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex];
- uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex];
-
- ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
- ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
- }
- }
- PartManifest = PartManifestWriter.Save();
- }
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount);
-
- Stopwatch PutBuildPartResultTimer;
- std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
- m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, PartManifest);
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart took {}, payload size {}. {} attachments are needed.",
- NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
- NiceBytes(PartManifest.GetSize()),
- PutBuildPartResult.second.size());
- }
- IoHash PartHash = PutBuildPartResult.first;
-
- auto UploadAttachments =
- [this, &LooseChunksStats, &UploadStats, &LocalContent, &LocalLookup, &NewBlockChunks, &NewBlocks, &LooseChunkIndexes](
- std::span<IoHash> RawHashes,
- std::vector<IoHash>& OutUnknownChunks) {
- if (!m_AbortFlag)
- {
- UploadStatistics TempUploadStats;
- LooseChunksStatistics TempLooseChunksStats;
-
- Stopwatch TempUploadTimer;
- auto __ = MakeGuard([&]() {
- if (!m_Options.IsQuiet)
- {
- uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
- ZEN_OPERATION_LOG_INFO(
- m_LogOutput,
- "Uploaded {} ({}) blocks. "
- "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
- "Transferred {} ({}bits/s) in {}",
- TempUploadStats.BlockCount.load(),
- NiceBytes(TempUploadStats.BlocksBytes),
-
- TempLooseChunksStats.CompressedChunkCount.load(),
- NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
- NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS,
- TempLooseChunksStats.ChunkByteCount)),
- TempUploadStats.ChunkCount.load(),
- NiceBytes(TempUploadStats.ChunksBytes),
-
- NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
- NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
- NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
- }
- });
- UploadPartBlobs(LocalContent,
- LocalLookup,
- RawHashes,
- NewBlockChunks,
- NewBlocks,
- LooseChunkIndexes,
- m_LargeAttachmentSize,
- TempUploadStats,
- TempLooseChunksStats,
- OutUnknownChunks);
- UploadStats += TempUploadStats;
- LooseChunksStats += TempLooseChunksStats;
- }
- };
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount);
-
- std::vector<IoHash> UnknownChunks;
- if (m_Options.IgnoreExistingBlocks)
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart uploading all attachments, needs are: {}",
- FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
- }
-
- std::vector<IoHash> ForceUploadChunkHashes;
- ForceUploadChunkHashes.reserve(LooseChunkIndexes.size());
-
- for (uint32_t ChunkIndex : LooseChunkIndexes)
- {
- ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
- }
-
- for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++)
- {
- if (NewBlocks.BlockHeaders[BlockIndex])
- {
- // Block was not uploaded during generation
- ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash);
- }
- }
- UploadAttachments(ForceUploadChunkHashes, UnknownChunks);
- }
- else if (!PutBuildPartResult.second.empty())
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "PutBuildPart needs attachments: {}",
- FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
- }
- UploadAttachments(PutBuildPartResult.second, UnknownChunks);
- }
-
- auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) {
- return fmt::format(
- "The following build blobs was reported as needed for upload but was reported as existing at the start of the "
- "operation.{}{}",
- WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv,
- FormatArray<IoHash>(UnknownChunks, "\n "sv));
- };
-
- if (!UnknownChunks.empty())
- {
- ZEN_OPERATION_LOG_WARN(m_LogOutput, "{}", BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ true));
- }
-
- uint32_t FinalizeBuildPartRetryCount = 5;
- while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
- {
- Stopwatch FinalizeBuildPartTimer;
- std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash);
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "FinalizeBuildPart took {}. {} attachments are missing.",
- NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
- Needs.size());
- }
- if (Needs.empty())
- {
- break;
- }
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv));
- }
-
- std::vector<IoHash> RetryUnknownChunks;
- UploadAttachments(Needs, RetryUnknownChunks);
- if (RetryUnknownChunks == UnknownChunks)
- {
- if (FinalizeBuildPartRetryCount > 0)
- {
- // Back off a bit
- Sleep(1000);
- }
- }
- else
- {
- UnknownChunks = RetryUnknownChunks;
- ZEN_OPERATION_LOG_WARN(m_LogOutput,
- "{}",
- BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
- }
- }
-
- if (!UnknownChunks.empty())
- {
- throw std::runtime_error(BuildUnkownChunksResponse(UnknownChunks, /*WillRetry*/ false));
- }
-
- if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag)
- {
- uint64_t UploadBlockMetadataCount = 0;
- Stopwatch UploadBlockMetadataTimer;
-
- uint32_t FailedMetadataUploadCount = 1;
- int32_t MetadataUploadRetryCount = 3;
- while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
- {
- FailedMetadataUploadCount = 0;
- for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
- {
- if (m_AbortFlag)
- {
- break;
- }
- const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
- if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
- {
- const CbObject BlockMetaData =
- BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
- std::vector<IoHash>({BlockHash}),
- std::vector<CbObject>({BlockMetaData}));
- }
- bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
- if (MetadataSucceeded)
- {
- UploadStats.BlocksBytes += BlockMetaData.GetSize();
- NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
- UploadBlockMetadataCount++;
- }
- else
- {
- FailedMetadataUploadCount++;
- }
- }
- }
- }
- if (UploadBlockMetadataCount > 0)
- {
- uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
- UploadStats.ElapsedWallTimeUS += ElapsedUS;
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded metadata for {} blocks in {}",
- UploadBlockMetadataCount,
- NiceTimeSpanMs(ElapsedUS / 1000));
- }
- }
-
- // The newly generated blocks are now known blocks so the next part upload can use those blocks as well
- m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end());
- }
-
- m_LogOutput.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount);
-
- m_Storage.BuildStorage->PutBuildPartStats(
- m_BuildId,
- Part.PartId,
- {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())},
- {"reusedRatio", AcceptedByteCountPercent / 100.0},
- {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)},
- {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)},
- {"newBlockCount", double(FindBlocksStats.NewBlocksCount)},
- {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)},
- {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())},
- {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())},
- {"uploadedBytesPerSec",
- double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))},
- {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}});
-
- m_LocalFolderScanStats += Part.LocalFolderScanStats;
- m_ChunkingStats += ChunkingStats;
- m_FindBlocksStats += FindBlocksStats;
- m_ReuseBlocksStats += ReuseBlocksStats;
- m_UploadStats += UploadStats;
- m_GenerateBlocksStats += GenerateBlocksStats;
- m_LooseChunksStats += LooseChunksStats;
-}
-
-void
-BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- std::span<IoHash> RawHashes,
- const std::vector<std::vector<uint32_t>>& NewBlockChunks,
- GeneratedBlocks& NewBlocks,
- std::span<const uint32_t> LooseChunkIndexes,
- const std::uint64_t LargeAttachmentSize,
- UploadStatistics& TempUploadStats,
- LooseChunksStatistics& TempLooseChunksStats,
- std::vector<IoHash>& OutUnknownChunks)
-{
- ZEN_TRACE_CPU("UploadPartBlobs");
- {
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Upload Blobs"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- WorkerThreadPool& ReadChunkPool = m_IOWorkerPool;
- WorkerThreadPool& UploadChunkPool = m_NetworkPool;
-
- FilteredRate FilteredGenerateBlockBytesPerSecond;
- FilteredRate FilteredCompressedBytesPerSecond;
- FilteredRate FilteredUploadedBytesPerSecond;
-
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- std::atomic<size_t> UploadedBlockSize = 0;
- std::atomic<size_t> UploadedBlockCount = 0;
- std::atomic<size_t> UploadedRawChunkSize = 0;
- std::atomic<size_t> UploadedCompressedChunkSize = 0;
- std::atomic<uint32_t> UploadedChunkCount = 0;
-
- tsl::robin_map<uint32_t, uint32_t> ChunkIndexToLooseChunkOrderIndex;
- ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size());
- for (uint32_t OrderIndex = 0; OrderIndex < LooseChunkIndexes.size(); OrderIndex++)
- {
- ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex);
- }
-
- std::vector<size_t> BlockIndexes;
- std::vector<uint32_t> LooseChunkOrderIndexes;
-
- uint64_t TotalLooseChunksSize = 0;
- uint64_t TotalBlocksSize = 0;
- for (const IoHash& RawHash : RawHashes)
- {
- if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end())
- {
- BlockIndexes.push_back(It->second);
- TotalBlocksSize += NewBlocks.BlockSizes[It->second];
- }
- else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
- {
- const uint32_t ChunkIndex = ChunkIndexIt->second;
- if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex);
- LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end())
- {
- LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second);
- TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
- }
- }
- else
- {
- OutUnknownChunks.push_back(RawHash);
- }
- }
- if (BlockIndexes.empty() && LooseChunkOrderIndexes.empty())
- {
- return;
- }
-
- uint64_t TotalRawSize = TotalLooseChunksSize + TotalBlocksSize;
-
- const size_t UploadBlockCount = BlockIndexes.size();
- const uint32_t UploadChunkCount = gsl::narrow<uint32_t>(LooseChunkOrderIndexes.size());
-
- auto AsyncUploadBlock = [this,
- &Work,
- &NewBlocks,
- UploadBlockCount,
- &UploadedBlockCount,
- UploadChunkCount,
- &UploadedChunkCount,
- &UploadedBlockSize,
- &TempUploadStats,
- &FilteredUploadedBytesPerSecond,
- &UploadChunkPool](const size_t BlockIndex,
- const IoHash BlockHash,
- CompositeBuffer&& Payload,
- std::atomic<uint64_t>& QueuedPendingInMemoryBlocksForUpload) {
- bool IsInMemoryBlock = true;
- if (QueuedPendingInMemoryBlocksForUpload.load() > 16)
- {
- ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock");
- std::filesystem::path TempFilePath = m_Options.TempDir / (BlockHash.ToHexString());
- Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), TempFilePath));
- IsInMemoryBlock = false;
- }
- else
- {
- QueuedPendingInMemoryBlocksForUpload++;
- }
-
- Work.ScheduleWork(
- UploadChunkPool,
- [this,
- &QueuedPendingInMemoryBlocksForUpload,
- &NewBlocks,
- UploadBlockCount,
- &UploadedBlockCount,
- UploadChunkCount,
- &UploadedChunkCount,
- &UploadedBlockSize,
- &TempUploadStats,
- &FilteredUploadedBytesPerSecond,
- IsInMemoryBlock,
- BlockIndex,
- BlockHash,
- Payload = CompositeBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
- auto _ = MakeGuard([IsInMemoryBlock, &QueuedPendingInMemoryBlocksForUpload] {
- if (IsInMemoryBlock)
- {
- QueuedPendingInMemoryBlocksForUpload--;
- }
- });
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("AsyncUploadBlock");
-
- const uint64_t PayloadSize = Payload.GetSize();
-
- FilteredUploadedBytesPerSecond.Start();
- const CbObject BlockMetaData =
- BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
-
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
- }
- m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded block {} ({}) containing {} chunks",
- BlockHash,
- NiceBytes(PayloadSize),
- NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
- }
- UploadedBlockSize += PayloadSize;
- TempUploadStats.BlocksBytes += PayloadSize;
-
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId,
- std::vector<IoHash>({BlockHash}),
- std::vector<CbObject>({BlockMetaData}));
- }
- bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
- if (MetadataSucceeded)
- {
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Uploaded block {} metadata ({})",
- BlockHash,
- NiceBytes(BlockMetaData.GetSize()));
- }
-
- NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
- TempUploadStats.BlocksBytes += BlockMetaData.GetSize();
- }
-
- TempUploadStats.BlockCount++;
-
- UploadedBlockCount++;
- if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
- {
- FilteredUploadedBytesPerSecond.Stop();
- }
- }
- });
- };
-
- auto AsyncUploadLooseChunk = [this,
- LargeAttachmentSize,
- &Work,
- &UploadChunkPool,
- &FilteredUploadedBytesPerSecond,
- &UploadedBlockCount,
- &UploadedChunkCount,
- UploadBlockCount,
- UploadChunkCount,
- &UploadedCompressedChunkSize,
- &UploadedRawChunkSize,
- &TempUploadStats](const IoHash& RawHash, uint64_t RawSize, CompositeBuffer&& Payload) {
- Work.ScheduleWork(
- UploadChunkPool,
- [this,
- &Work,
- LargeAttachmentSize,
- &FilteredUploadedBytesPerSecond,
- &UploadChunkPool,
- &UploadedBlockCount,
- &UploadedChunkCount,
- UploadBlockCount,
- UploadChunkCount,
- &UploadedCompressedChunkSize,
- &UploadedRawChunkSize,
- &TempUploadStats,
- RawHash,
- RawSize,
- Payload = CompositeBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("AsyncUploadLooseChunk");
-
- const uint64_t PayloadSize = Payload.GetSize();
-
- if (m_Storage.BuildCacheStorage && m_Options.PopulateCache)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- }
-
- if (PayloadSize >= LargeAttachmentSize)
- {
- ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart");
- TempUploadStats.MultipartAttachmentCount++;
- std::vector<std::function<void()>> MultipartWork = m_Storage.BuildStorage->PutLargeBuildBlob(
- m_BuildId,
- RawHash,
- ZenContentType::kCompressedBinary,
- PayloadSize,
- [Payload = std::move(Payload), &FilteredUploadedBytesPerSecond](uint64_t Offset,
- uint64_t Size) mutable -> IoBuffer {
- FilteredUploadedBytesPerSecond.Start();
-
- IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer();
- PartPayload.SetContentType(ZenContentType::kBinary);
- return PartPayload;
- },
- [RawSize,
- &TempUploadStats,
- &UploadedCompressedChunkSize,
- &UploadChunkPool,
- &UploadedBlockCount,
- UploadBlockCount,
- &UploadedChunkCount,
- UploadChunkCount,
- &FilteredUploadedBytesPerSecond,
- &UploadedRawChunkSize](uint64_t SentBytes, bool IsComplete) {
- TempUploadStats.ChunksBytes += SentBytes;
- UploadedCompressedChunkSize += SentBytes;
- if (IsComplete)
- {
- TempUploadStats.ChunkCount++;
- UploadedChunkCount++;
- if (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
- {
- FilteredUploadedBytesPerSecond.Stop();
- }
- UploadedRawChunkSize += RawSize;
- }
- });
- for (auto& WorkPart : MultipartWork)
- {
- Work.ScheduleWork(UploadChunkPool, [Work = std::move(WorkPart)](std::atomic<bool>& AbortFlag) {
- ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work");
- if (!AbortFlag)
- {
- Work();
- }
- });
- }
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize));
- }
- }
- else
- {
- ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart");
- m_Storage.BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize));
- }
- TempUploadStats.ChunksBytes += Payload.GetSize();
- TempUploadStats.ChunkCount++;
- UploadedCompressedChunkSize += Payload.GetSize();
- UploadedRawChunkSize += RawSize;
- UploadedChunkCount++;
- if (UploadedChunkCount == UploadChunkCount)
- {
- FilteredUploadedBytesPerSecond.Stop();
- }
- }
- }
- });
- };
-
- std::vector<size_t> GenerateBlockIndexes;
-
- std::atomic<uint64_t> GeneratedBlockCount = 0;
- std::atomic<uint64_t> GeneratedBlockByteCount = 0;
-
- std::atomic<uint64_t> QueuedPendingInMemoryBlocksForUpload = 0;
-
- // Start generation of any non-prebuilt blocks and schedule upload
- for (const size_t BlockIndex : BlockIndexes)
- {
- const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
- if (!m_AbortFlag)
- {
- Work.ScheduleWork(
- ReadChunkPool,
- [this,
- BlockHash = IoHash(BlockHash),
- BlockIndex,
- &FilteredGenerateBlockBytesPerSecond,
- &Content,
- &Lookup,
- &NewBlocks,
- &NewBlockChunks,
- &GenerateBlockIndexes,
- &GeneratedBlockCount,
- &GeneratedBlockByteCount,
- &AsyncUploadBlock,
- &QueuedPendingInMemoryBlocksForUpload](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock");
-
- FilteredGenerateBlockBytesPerSecond.Start();
-
- Stopwatch GenerateTimer;
- CompositeBuffer Payload;
- if (NewBlocks.BlockHeaders[BlockIndex])
- {
- Payload =
- RebuildBlock(Content, Lookup, std::move(NewBlocks.BlockHeaders[BlockIndex]), NewBlockChunks[BlockIndex])
- .GetCompressed();
- }
- else
- {
- ChunkBlockDescription BlockDescription;
- CompressedBuffer CompressedBlock =
- GenerateBlock(Content, Lookup, NewBlockChunks[BlockIndex], BlockDescription);
- if (!CompressedBlock)
- {
- throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash));
- }
- ZEN_ASSERT(BlockDescription.BlockHash == BlockHash);
- Payload = std::move(CompressedBlock).GetCompressed();
- }
-
- GeneratedBlockByteCount += NewBlocks.BlockSizes[BlockIndex];
- GeneratedBlockCount++;
- if (GeneratedBlockCount == GenerateBlockIndexes.size())
- {
- FilteredGenerateBlockBytesPerSecond.Stop();
- }
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "{} block {} ({}) containing {} chunks in {}",
- NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated",
- NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
- NiceBytes(NewBlocks.BlockSizes[BlockIndex]),
- NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
- NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
- }
- if (!m_AbortFlag)
- {
- AsyncUploadBlock(BlockIndex, BlockHash, std::move(Payload), QueuedPendingInMemoryBlocksForUpload);
- }
- }
- });
- }
- }
-
- // Start compression of any non-precompressed loose chunks and schedule upload
- for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes)
- {
- const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex];
- Work.ScheduleWork(
- ReadChunkPool,
- [this,
- &Content,
- &Lookup,
- &TempLooseChunksStats,
- &LooseChunkOrderIndexes,
- &FilteredCompressedBytesPerSecond,
- &TempUploadStats,
- &AsyncUploadLooseChunk,
- ChunkIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk");
-
- FilteredCompressedBytesPerSecond.Start();
- Stopwatch CompressTimer;
- CompositeBuffer Payload = CompressChunk(Content, Lookup, ChunkIndex, TempLooseChunksStats);
- if (m_Options.IsVerbose)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Compressed chunk {} ({} -> {}) in {}",
- Content.ChunkedContent.ChunkHashes[ChunkIndex],
- NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]),
- NiceBytes(Payload.GetSize()),
- NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs()));
- }
- const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
- TempUploadStats.ReadFromDiskBytes += ChunkRawSize;
- if (TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size())
- {
- FilteredCompressedBytesPerSecond.Stop();
- }
- if (!m_AbortFlag)
- {
- AsyncUploadLooseChunk(Content.ChunkedContent.ChunkHashes[ChunkIndex], ChunkRawSize, std::move(Payload));
- }
- }
- });
- }
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
- FilteredCompressedBytesPerSecond.Update(TempLooseChunksStats.CompressedChunkRawBytes.load());
- FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load());
- FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load());
- uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load();
- uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load();
-
- std::string Details = fmt::format(
- "Compressed {}/{} ({}/{}{}) chunks. "
- "Uploaded {}/{} ({}/{}) blobs "
- "({}{})",
- TempLooseChunksStats.CompressedChunkCount.load(),
- LooseChunkOrderIndexes.size(),
- NiceBytes(TempLooseChunksStats.CompressedChunkRawBytes),
- NiceBytes(TotalLooseChunksSize),
- (TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size())
- ? ""
- : fmt::format(" {}B/s", NiceNum(FilteredCompressedBytesPerSecond.GetCurrent())),
-
- UploadedBlockCount.load() + UploadedChunkCount.load(),
- UploadBlockCount + UploadChunkCount,
- NiceBytes(UploadedRawSize),
- NiceBytes(TotalRawSize),
-
- NiceBytes(UploadedCompressedSize),
- (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
- ? ""
- : fmt::format(" {}bits/s", NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())));
-
- Progress.UpdateState({.Task = "Uploading blobs ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(TotalRawSize),
- .RemainingCount = gsl::narrow<uint64_t>(TotalRawSize - UploadedRawSize),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
-
- ZEN_ASSERT(m_AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0);
-
- Progress.Finish();
-
- TempUploadStats.ElapsedWallTimeUS += FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
- TempLooseChunksStats.CompressChunksElapsedWallTimeUS += FilteredCompressedBytesPerSecond.GetElapsedTimeUS();
- }
-}
-
-CompositeBuffer
-BuildsOperationUploadFolder::CompressChunk(const ChunkedFolderContent& Content,
- const ChunkedContentLookup& Lookup,
- uint32_t ChunkIndex,
- LooseChunksStatistics& TempLooseChunksStats)
-{
- ZEN_TRACE_CPU("CompressChunk");
- ZEN_ASSERT(!m_Options.TempDir.empty());
- const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
- const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
-
- const ChunkedContentLookup::ChunkSequenceLocation& Source = GetChunkSequenceLocations(Lookup, ChunkIndex)[0];
- const std::uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[Source.SequenceIndex];
- IoBuffer RawSource = IoBufferBuilder::MakeFromFile((m_Path / Content.Paths[PathIndex]).make_preferred(), Source.Offset, ChunkSize);
- if (!RawSource)
- {
- throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash));
- }
- if (RawSource.GetSize() != ChunkSize)
- {
- throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash));
- }
-
- const bool ShouldCompressChunk = IsChunkCompressable(m_NonCompressableExtensionHashes, Content, Lookup, ChunkIndex);
- const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
-
- if (ShouldCompressChunk)
- {
- std::filesystem::path TempFilePath = m_Options.TempDir / ChunkHash.ToHexString();
-
- BasicFile CompressedFile;
- std::error_code Ec;
- CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec);
- if (Ec)
- {
- throw std::runtime_error(fmt::format("Failed creating temporary file for compressing blob {}, reason: ({}) {}",
- ChunkHash,
- Ec.value(),
- Ec.message()));
- }
-
- uint64_t StreamRawBytes = 0;
- uint64_t StreamCompressedBytes = 0;
-
- bool CouldCompress = CompressedBuffer::CompressToStream(
- CompositeBuffer(SharedBuffer(RawSource)),
- [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
- ZEN_UNUSED(SourceOffset);
- TempLooseChunksStats.CompressedChunkRawBytes += SourceSize;
- CompressedFile.Write(RangeBuffer, Offset);
- TempLooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize();
- StreamRawBytes += SourceSize;
- StreamCompressedBytes += RangeBuffer.GetSize();
- },
- OodleCompressor::Mermaid,
- CompressionLevel);
- if (CouldCompress)
- {
- uint64_t CompressedSize = CompressedFile.FileSize();
- void* FileHandle = CompressedFile.Detach();
- IoBuffer TempPayload = IoBuffer(IoBuffer::File,
- FileHandle,
- 0,
- CompressedSize,
- /*IsWholeFile*/ true);
- ZEN_ASSERT(TempPayload);
- TempPayload.SetDeleteOnClose(true);
- IoHash RawHash;
- uint64_t RawSize;
- CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(TempPayload), RawHash, RawSize);
- ZEN_ASSERT(Compressed);
- ZEN_ASSERT(RawHash == ChunkHash);
- ZEN_ASSERT(RawSize == ChunkSize);
-
- TempLooseChunksStats.CompressedChunkCount++;
-
- return Compressed.GetCompressed();
- }
- else
- {
- TempLooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes;
- TempLooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes;
- }
- CompressedFile.Close();
- RemoveFile(TempFilePath, Ec);
- ZEN_UNUSED(Ec);
- }
-
- CompressedBuffer CompressedBlob =
- CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel);
- if (!CompressedBlob)
- {
- throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash));
- }
- ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash);
- ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize);
-
- TempLooseChunksStats.CompressedChunkRawBytes += ChunkSize;
- TempLooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize();
-
- // If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk
- if (ShouldCompressChunk)
- {
- std::filesystem::path TempFilePath = m_Options.TempDir / (ChunkHash.ToHexString());
- IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFilePath);
- CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload));
- }
-
- TempLooseChunksStats.CompressedChunkCount++;
- return std::move(CompressedBlob).GetCompressed();
-}
-
-BuildsOperationValidateBuildPart::BuildsOperationValidateBuildPart(OperationLogOutput& OperationLogOutput,
- BuildStorageBase& Storage,
- std::atomic<bool>& AbortFlag,
- std::atomic<bool>& PauseFlag,
- WorkerThreadPool& IOWorkerPool,
- WorkerThreadPool& NetworkPool,
- const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const Options& Options)
-
-: m_LogOutput(OperationLogOutput)
-, m_Storage(Storage)
-, m_AbortFlag(AbortFlag)
-, m_PauseFlag(PauseFlag)
-, m_IOWorkerPool(IOWorkerPool)
-, m_NetworkPool(NetworkPool)
-, m_BuildId(BuildId)
-, m_BuildPartId(BuildPartId)
-, m_BuildPartName(BuildPartName)
-, m_Options(Options)
-{
-}
-
-void
-BuildsOperationValidateBuildPart::Execute()
-{
- ZEN_TRACE_CPU("ValidateBuildPart");
- try
- {
- enum class TaskSteps : uint32_t
- {
- FetchBuild,
- FetchBuildPart,
- ValidateBlobs,
- Cleanup,
- StepCount
- };
-
- auto EndProgress =
- MakeGuard([&]() { m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); });
-
- Stopwatch Timer;
- auto _ = MakeGuard([&]() {
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Validated build part {}/{} ('{}') in {}",
- m_BuildId,
- m_BuildPartId,
- m_BuildPartName,
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- }
- });
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuild, (uint32_t)TaskSteps::StepCount);
-
- CbObject Build = m_Storage.GetBuild(m_BuildId);
- if (!m_BuildPartName.empty())
- {
- m_BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId();
- if (m_BuildPartId == Oid::Zero)
- {
- throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName));
- }
- }
- m_ValidateStats.BuildBlobSize = Build.GetSize();
- uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
- {
- PreferredMultipartChunkSize = ChunkSize;
- }
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuildPart, (uint32_t)TaskSteps::StepCount);
-
- CbObject BuildPart = m_Storage.GetBuildPart(m_BuildId, m_BuildPartId);
- m_ValidateStats.BuildPartSize = BuildPart.GetSize();
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Validating build part {}/{} ({})",
- m_BuildId,
- m_BuildPartId,
- NiceBytes(BuildPart.GetSize()));
- }
- std::vector<IoHash> ChunkAttachments;
- if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView())
- {
- for (CbFieldView LooseFileView : ChunkAttachmentsView["rawHashes"sv])
- {
- ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment());
- }
- }
- m_ValidateStats.ChunkAttachmentCount = ChunkAttachments.size();
- std::vector<IoHash> BlockAttachments;
- if (const CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView())
- {
- {
- for (CbFieldView BlocksView : BlockAttachmentsView["rawHashes"sv])
- {
- BlockAttachments.push_back(BlocksView.AsBinaryAttachment());
- }
- }
- }
- m_ValidateStats.BlockAttachmentCount = BlockAttachments.size();
-
- std::vector<ChunkBlockDescription> VerifyBlockDescriptions =
- ParseChunkBlockDescriptionList(m_Storage.GetBlockMetadatas(m_BuildId, BlockAttachments));
- if (VerifyBlockDescriptions.size() != BlockAttachments.size())
- {
- throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing",
- BlockAttachments.size() - VerifyBlockDescriptions.size()));
- }
-
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- const std::filesystem::path TempFolder = ".zen-tmp";
-
- CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder);
- CreateDirectories(TempFolder);
- auto __ = MakeGuard([this, TempFolder]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); });
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::ValidateBlobs, (uint32_t)TaskSteps::StepCount);
-
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Validate Blobs"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- uint64_t AttachmentsToVerifyCount = ChunkAttachments.size() + BlockAttachments.size();
- FilteredRate FilteredDownloadedBytesPerSecond;
- FilteredRate FilteredVerifiedBytesPerSecond;
-
- std::atomic<uint64_t> MultipartAttachmentCount = 0;
-
- for (const IoHash& ChunkAttachment : ChunkAttachments)
- {
- Work.ScheduleWork(
- m_NetworkPool,
- [this,
- &Work,
- AttachmentsToVerifyCount,
- &TempFolder,
- PreferredMultipartChunkSize,
- &FilteredDownloadedBytesPerSecond,
- &FilteredVerifiedBytesPerSecond,
- &ChunkAttachments,
- ChunkAttachment = IoHash(ChunkAttachment)](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("ValidateBuildPart_GetChunk");
-
- FilteredDownloadedBytesPerSecond.Start();
- DownloadLargeBlob(
- m_Storage,
- TempFolder,
- m_BuildId,
- ChunkAttachment,
- PreferredMultipartChunkSize,
- Work,
- m_NetworkPool,
- m_DownloadStats.DownloadedChunkByteCount,
- m_DownloadStats.MultipartAttachmentCount,
- [this,
- &Work,
- AttachmentsToVerifyCount,
- &FilteredDownloadedBytesPerSecond,
- &FilteredVerifiedBytesPerSecond,
- ChunkHash = IoHash(ChunkAttachment)](IoBuffer&& Payload) {
- m_DownloadStats.DownloadedChunkCount++;
- Payload.SetContentType(ZenContentType::kCompressedBinary);
- if (!m_AbortFlag)
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- AttachmentsToVerifyCount,
- &FilteredDownloadedBytesPerSecond,
- &FilteredVerifiedBytesPerSecond,
- Payload = IoBuffer(std::move(Payload)),
- ChunkHash](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("ValidateBuildPart_Validate");
-
- if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount ==
- AttachmentsToVerifyCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
-
- FilteredVerifiedBytesPerSecond.Start();
-
- uint64_t CompressedSize;
- uint64_t DecompressedSize;
- ValidateBlob(m_AbortFlag, std::move(Payload), ChunkHash, CompressedSize, DecompressedSize);
- m_ValidateStats.VerifiedAttachmentCount++;
- m_ValidateStats.VerifiedByteCount += DecompressedSize;
- if (m_ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
- {
- FilteredVerifiedBytesPerSecond.Stop();
- }
- }
- });
- }
- });
- }
- });
- }
-
- for (const IoHash& BlockAttachment : BlockAttachments)
- {
- Work.ScheduleWork(
- m_NetworkPool,
- [this,
- &Work,
- AttachmentsToVerifyCount,
- &FilteredDownloadedBytesPerSecond,
- &FilteredVerifiedBytesPerSecond,
- BlockAttachment = IoHash(BlockAttachment)](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("ValidateBuildPart_GetBlock");
-
- FilteredDownloadedBytesPerSecond.Start();
- IoBuffer Payload = m_Storage.GetBuildBlob(m_BuildId, BlockAttachment);
- m_DownloadStats.DownloadedBlockCount++;
- m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize();
- if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == AttachmentsToVerifyCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- if (!Payload)
- {
- throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment));
- }
- if (!m_AbortFlag)
- {
- Work.ScheduleWork(
- m_IOWorkerPool,
- [this,
- &FilteredVerifiedBytesPerSecond,
- AttachmentsToVerifyCount,
- Payload = std::move(Payload),
- BlockAttachment](std::atomic<bool>&) mutable {
- if (!m_AbortFlag)
- {
- ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock");
-
- FilteredVerifiedBytesPerSecond.Start();
-
- uint64_t CompressedSize;
- uint64_t DecompressedSize;
- ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize);
- m_ValidateStats.VerifiedAttachmentCount++;
- m_ValidateStats.VerifiedByteCount += DecompressedSize;
- if (m_ValidateStats.VerifiedAttachmentCount.load() == AttachmentsToVerifyCount)
- {
- FilteredVerifiedBytesPerSecond.Stop();
- }
- }
- });
- }
- }
- });
- }
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
-
- const uint64_t DownloadedAttachmentCount = m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount;
- const uint64_t DownloadedByteCount = m_DownloadStats.DownloadedChunkByteCount + m_DownloadStats.DownloadedBlockByteCount;
-
- FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount);
- FilteredVerifiedBytesPerSecond.Update(m_ValidateStats.VerifiedByteCount);
-
- std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)",
- DownloadedAttachmentCount,
- AttachmentsToVerifyCount,
- NiceBytes(DownloadedByteCount),
- NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8),
- m_ValidateStats.VerifiedAttachmentCount.load(),
- AttachmentsToVerifyCount,
- NiceBytes(m_ValidateStats.VerifiedByteCount.load()),
- NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent()));
-
- Progress.UpdateState(
- {.Task = "Validating blobs ",
- .Details = Details,
- .TotalCount = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2),
- .RemainingCount = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2 -
- (DownloadedAttachmentCount + m_ValidateStats.VerifiedAttachmentCount.load())),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
-
- Progress.Finish();
- m_ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs();
-
- m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount);
- }
- catch (const std::exception&)
- {
- m_AbortFlag = true;
- throw;
- }
-}
-
-BuildsOperationPrimeCache::BuildsOperationPrimeCache(OperationLogOutput& OperationLogOutput,
- StorageInstance& Storage,
- std::atomic<bool>& AbortFlag,
- std::atomic<bool>& PauseFlag,
- WorkerThreadPool& NetworkPool,
- const Oid& BuildId,
- std::span<const Oid> BuildPartIds,
- const Options& Options,
- BuildStorageCache::Statistics& StorageCacheStats)
-: m_LogOutput(OperationLogOutput)
-, m_Storage(Storage)
-, m_AbortFlag(AbortFlag)
-, m_PauseFlag(PauseFlag)
-, m_NetworkPool(NetworkPool)
-, m_BuildId(BuildId)
-, m_BuildPartIds(BuildPartIds.begin(), BuildPartIds.end())
-, m_Options(Options)
-, m_StorageCacheStats(StorageCacheStats)
-{
- m_TempPath = m_Options.ZenFolderPath / "tmp";
- CreateDirectories(m_TempPath);
-}
-
-void
-BuildsOperationPrimeCache::Execute()
-{
- ZEN_TRACE_CPU("BuildsOperationPrimeCache::Execute");
-
- Stopwatch PrimeTimer;
-
- tsl::robin_map<IoHash, uint64_t, IoHash::Hasher> LooseChunkRawSizes;
-
- tsl::robin_set<IoHash, IoHash::Hasher> BuildBlobs;
-
- for (const Oid& BuildPartId : m_BuildPartIds)
- {
- CbObject BuildPart = m_Storage.BuildStorage->GetBuildPart(m_BuildId, BuildPartId);
-
- CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView();
- std::vector<IoHash> BlockAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, BlockAttachmentsView);
-
- CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView();
- std::vector<IoHash> ChunkAttachments = compactbinary_helpers::ReadBinaryAttachmentArray("rawHashes"sv, ChunkAttachmentsView);
- std::vector<uint64_t> ChunkRawSizes = compactbinary_helpers::ReadArray<uint64_t>("chunkRawSizes"sv, ChunkAttachmentsView);
- if (ChunkAttachments.size() != ChunkRawSizes.size())
- {
- throw std::runtime_error(fmt::format("Mismatch of loose chunk raw size array, expected {}, found {}",
- ChunkAttachments.size(),
- ChunkRawSizes.size()));
- }
-
- BuildBlobs.reserve(ChunkAttachments.size() + BlockAttachments.size());
- BuildBlobs.insert(BlockAttachments.begin(), BlockAttachments.end());
- BuildBlobs.insert(ChunkAttachments.begin(), ChunkAttachments.end());
-
- for (size_t ChunkAttachmentIndex = 0; ChunkAttachmentIndex < ChunkAttachments.size(); ChunkAttachmentIndex++)
- {
- LooseChunkRawSizes.insert_or_assign(ChunkAttachments[ChunkAttachmentIndex], ChunkRawSizes[ChunkAttachmentIndex]);
- }
- }
-
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Found {} referenced blobs", BuildBlobs.size());
- }
-
- if (BuildBlobs.empty())
- {
- return;
- }
-
- std::vector<IoHash> BlobsToDownload;
- BlobsToDownload.reserve(BuildBlobs.size());
-
- if (m_Storage.BuildCacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload)
- {
- ZEN_TRACE_CPU("BlobCacheExistCheck");
- Stopwatch Timer;
-
- const std::vector<IoHash> BlobHashes(BuildBlobs.begin(), BuildBlobs.end());
- const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult =
- m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes);
-
- if (CacheExistsResult.size() == BlobHashes.size())
- {
- for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++)
- {
- if (!CacheExistsResult[BlobIndex].HasBody)
- {
- BlobsToDownload.push_back(BlobHashes[BlobIndex]);
- }
- }
- size_t FoundCount = BuildBlobs.size() - BlobsToDownload.size();
-
- if (FoundCount > 0 && !m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Remote cache : Found {} out of {} needed blobs in {}",
- FoundCount,
- BuildBlobs.size(),
- NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
- }
- }
- }
- else
- {
- BlobsToDownload.insert(BlobsToDownload.end(), BuildBlobs.begin(), BuildBlobs.end());
- }
-
- if (BlobsToDownload.empty())
- {
- return;
- }
-
- std::atomic<uint64_t> MultipartAttachmentCount;
- std::atomic<size_t> CompletedDownloadCount;
- FilteredRate FilteredDownloadedBytesPerSecond;
-
- {
- std::unique_ptr<OperationLogOutput::ProgressBar> ProgressBarPtr(m_LogOutput.CreateProgressBar("Downloading"));
- OperationLogOutput::ProgressBar& Progress(*ProgressBarPtr);
-
- ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
-
- const size_t BlobCount = BlobsToDownload.size();
-
- for (size_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++)
- {
- Work.ScheduleWork(
- m_NetworkPool,
- [this,
- &Work,
- &BlobsToDownload,
- BlobCount,
- &LooseChunkRawSizes,
- &CompletedDownloadCount,
- &FilteredDownloadedBytesPerSecond,
- &MultipartAttachmentCount,
- BlobIndex](std::atomic<bool>&) {
- if (!m_AbortFlag)
- {
- const IoHash& BlobHash = BlobsToDownload[BlobIndex];
-
- bool IsLargeBlob = false;
-
- if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end())
- {
- IsLargeBlob = It->second >= m_Options.LargeAttachmentSize;
- }
-
- FilteredDownloadedBytesPerSecond.Start();
-
- if (IsLargeBlob)
- {
- DownloadLargeBlob(*m_Storage.BuildStorage,
- m_TempPath,
- m_BuildId,
- BlobHash,
- m_Options.PreferredMultipartChunkSize,
- Work,
- m_NetworkPool,
- m_DownloadStats.DownloadedChunkByteCount,
- MultipartAttachmentCount,
- [this, BlobCount, BlobHash, &FilteredDownloadedBytesPerSecond, &CompletedDownloadCount](
- IoBuffer&& Payload) {
- m_DownloadStats.DownloadedChunkCount++;
- m_DownloadStats.RequestsCompleteCount++;
-
- if (!m_AbortFlag)
- {
- if (Payload && m_Storage.BuildCacheStorage)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- BlobHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(Payload)));
- }
- }
- CompletedDownloadCount++;
- if (CompletedDownloadCount == BlobCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- });
- }
- else
- {
- IoBuffer Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash);
- m_DownloadStats.DownloadedBlockCount++;
- m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize();
- m_DownloadStats.RequestsCompleteCount++;
-
- if (!m_AbortFlag)
- {
- if (Payload && m_Storage.BuildCacheStorage)
- {
- m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId,
- BlobHash,
- ZenContentType::kCompressedBinary,
- CompositeBuffer(SharedBuffer(std::move(Payload))));
- }
- }
- CompletedDownloadCount++;
- if (CompletedDownloadCount == BlobCount)
- {
- FilteredDownloadedBytesPerSecond.Stop();
- }
- }
- }
- });
- }
-
- Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
- ZEN_UNUSED(PendingWork);
-
- uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load();
- FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
-
- std::string DownloadRateString = (CompletedDownloadCount == BlobCount)
- ? ""
- : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
- std::string UploadDetails = m_Storage.BuildCacheStorage ? fmt::format(" {} ({}) uploaded.",
- m_StorageCacheStats.PutBlobCount.load(),
- NiceBytes(m_StorageCacheStats.PutBlobByteCount.load()))
- : "";
-
- std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
- CompletedDownloadCount.load(),
- BlobCount,
- NiceBytes(DownloadedBytes),
- DownloadRateString,
- UploadDetails);
- Progress.UpdateState({.Task = "Downloading",
- .Details = Details,
- .TotalCount = BlobCount,
- .RemainingCount = BlobCount - CompletedDownloadCount.load(),
- .Status = OperationLogOutput::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
- false);
- });
-
- FilteredDownloadedBytesPerSecond.Stop();
-
- Progress.Finish();
- }
- if (m_AbortFlag)
- {
- return;
- }
-
- if (m_Storage.BuildCacheStorage)
- {
- m_Storage.BuildCacheStorage->Flush(m_LogOutput.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool {
- ZEN_UNUSED(Remaining);
- if (!m_Options.IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(m_LogOutput, "Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheName);
- }
- return !m_AbortFlag;
- });
- }
-
- if (!m_Options.IsQuiet)
- {
- uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() + m_DownloadStats.DownloadedBlockByteCount.load();
- ZEN_OPERATION_LOG_INFO(m_LogOutput,
- "Downloaded {} ({}bits/s) in {}. {} as multipart. Completed in {}",
- NiceBytes(DownloadedBytes),
- NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
- NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
- MultipartAttachmentCount.load(),
- NiceTimeSpanMs(PrimeTimer.GetElapsedTimeMs()));
- }
-}
-
-CompositeBuffer
-ValidateBlob(std::atomic<bool>& AbortFlag,
- BuildStorageBase& Storage,
- const Oid& BuildId,
- const IoHash& BlobHash,
- uint64_t& OutCompressedSize,
- uint64_t& OutDecompressedSize)
-{
- ZEN_TRACE_CPU("ValidateBlob");
- IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash);
- if (!Payload)
- {
- throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash));
- }
- return ValidateBlob(AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize);
-}
-
-ChunkBlockDescription
-BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload,
- const IoHash& BlobHash,
- uint64_t& OutCompressedSize,
- uint64_t& OutDecompressedSize)
-{
- CompositeBuffer BlockBuffer = ValidateBlob(m_AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize);
- if (!BlockBuffer)
- {
- throw std::runtime_error(fmt::format("Chunk block blob {} is not compressed using 'None' compression level", BlobHash));
- }
- return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash);
-}
-
-std::vector<std::pair<Oid, std::string>>
-ResolveBuildPartNames(CbObjectView BuildObject,
- const Oid& BuildId,
- const std::vector<Oid>& BuildPartIds,
- std::span<const std::string> BuildPartNames,
- std::uint64_t& OutPreferredMultipartChunkSize)
-{
- std::vector<std::pair<Oid, std::string>> Result;
- {
- CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
- if (!PartsObject)
- {
- throw std::runtime_error("Build object does not have a 'parts' object");
- }
-
- OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize);
-
- std::vector<std::pair<Oid, std::string>> AvailableParts;
-
- for (CbFieldView PartView : PartsObject)
- {
- const std::string BuildPartName = std::string(PartView.GetName());
- const Oid BuildPartId = PartView.AsObjectId();
- if (BuildPartId == Oid::Zero)
- {
- ExtendableStringBuilder<128> SB;
- for (CbFieldView ScanPartView : PartsObject)
- {
- SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId()));
- }
- throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView()));
- }
- AvailableParts.push_back({BuildPartId, BuildPartName});
- }
-
- if (BuildPartIds.empty() && BuildPartNames.empty())
- {
- Result = AvailableParts;
- }
- else
- {
- for (const std::string& BuildPartName : BuildPartNames)
- {
- if (auto It = std::find_if(AvailableParts.begin(),
- AvailableParts.end(),
- [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; });
- It != AvailableParts.end())
- {
- Result.push_back(*It);
- }
- else
- {
- throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName));
- }
- }
- for (const Oid& BuildPartId : BuildPartIds)
- {
- if (auto It = std::find_if(AvailableParts.begin(),
- AvailableParts.end(),
- [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; });
- It != AvailableParts.end())
- {
- Result.push_back(*It);
- }
- else
- {
- throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId));
- }
- }
- }
-
- if (Result.empty())
- {
- throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId));
- }
- }
- return Result;
-}
-
-ChunkedFolderContent
-GetRemoteContent(OperationLogOutput& Output,
- StorageInstance& Storage,
- const Oid& BuildId,
- const std::vector<std::pair<Oid, std::string>>& BuildParts,
- const BuildManifest& Manifest,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- std::unique_ptr<ChunkingController>& OutChunkController,
- std::vector<ChunkedFolderContent>& OutPartContents,
- std::vector<ChunkBlockDescription>& OutBlockDescriptions,
- std::vector<IoHash>& OutLooseChunkHashes,
- bool IsQuiet,
- bool IsVerbose,
- bool DoExtraContentVerify)
-{
- ZEN_TRACE_CPU("GetRemoteContent");
-
- Stopwatch GetBuildPartTimer;
- const Oid BuildPartId = BuildParts[0].first;
- const std::string_view BuildPartName = BuildParts[0].second;
- CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
- if (!IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(Output,
- "GetBuildPart {} ('{}') took {}. Payload size: {}",
- BuildPartId,
- BuildPartName,
- NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(BuildPartManifest.GetSize()));
- ZEN_OPERATION_LOG_INFO(Output, "{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
- }
-
- {
- CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
- std::string_view ChunkerName = Chunker["name"sv].AsString();
- CbObjectView Parameters = Chunker["parameters"sv].AsObjectView();
- OutChunkController = CreateChunkingController(ChunkerName, Parameters);
- }
-
- auto ParseBuildPartManifest = [&Output, IsQuiet, IsVerbose, DoExtraContentVerify](
- StorageInstance& Storage,
- const Oid& BuildId,
- const Oid& BuildPartId,
- CbObject BuildPartManifest,
- std::span<const std::string> IncludeWildcards,
- std::span<const std::string> ExcludeWildcards,
- const BuildManifest::Part* OptionalManifest,
- ChunkedFolderContent& OutRemoteContent,
- std::vector<ChunkBlockDescription>& OutBlockDescriptions,
- std::vector<IoHash>& OutLooseChunkHashes) {
- std::vector<uint32_t> AbsoluteChunkOrders;
- std::vector<uint64_t> LooseChunkRawSizes;
- std::vector<IoHash> BlockRawHashes;
-
- ReadBuildContentFromCompactBinary(BuildPartManifest,
- OutRemoteContent.Platform,
- OutRemoteContent.Paths,
- OutRemoteContent.RawHashes,
- OutRemoteContent.RawSizes,
- OutRemoteContent.Attributes,
- OutRemoteContent.ChunkedContent.SequenceRawHashes,
- OutRemoteContent.ChunkedContent.ChunkCounts,
- AbsoluteChunkOrders,
- OutLooseChunkHashes,
- LooseChunkRawSizes,
- BlockRawHashes);
-
- // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
-
- {
- bool AttemptFallback = false;
- OutBlockDescriptions = GetBlockDescriptions(Output,
- *Storage.BuildStorage,
- Storage.BuildCacheStorage.get(),
- BuildId,
- BuildPartId,
- BlockRawHashes,
- AttemptFallback,
- IsQuiet,
- IsVerbose);
- }
-
- CalculateLocalChunkOrders(AbsoluteChunkOrders,
- OutLooseChunkHashes,
- LooseChunkRawSizes,
- OutBlockDescriptions,
- OutRemoteContent.ChunkedContent.ChunkHashes,
- OutRemoteContent.ChunkedContent.ChunkRawSizes,
- OutRemoteContent.ChunkedContent.ChunkOrders,
- DoExtraContentVerify);
-
- std::vector<std::filesystem::path> DeletedPaths;
-
- if (OptionalManifest)
- {
- tsl::robin_set<std::string> PathsInManifest;
- PathsInManifest.reserve(OptionalManifest->Files.size());
- for (const std::filesystem::path& ManifestPath : OptionalManifest->Files)
- {
- PathsInManifest.insert(ToLower(ManifestPath.generic_string()));
- }
- for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
- {
- if (!PathsInManifest.contains(ToLower(RemotePath.generic_string())))
- {
- DeletedPaths.push_back(RemotePath);
- }
- }
- }
-
- if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
- {
- for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
- {
- if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true))
- {
- DeletedPaths.push_back(RemotePath);
- }
- }
- }
-
- if (!DeletedPaths.empty())
- {
- OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
- InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
- }
-
-#if ZEN_BUILD_DEBUG
- ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards);
-#endif // ZEN_BUILD_DEBUG
- };
-
- auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* {
- if (Manifest.Parts.empty())
- {
- return nullptr;
- }
- if (Manifest.Parts.size() == 1)
- {
- if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty())
- {
- return &Manifest.Parts[0];
- }
- }
-
- auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) {
- if (Part.PartId != Oid::Zero)
- {
- return Part.PartId == BuildPartId;
- }
- if (!Part.PartName.empty())
- {
- return Part.PartName == BuildPartName;
- }
- return false;
- });
- if (It != Manifest.Parts.end())
- {
- return &(*It);
- }
- return nullptr;
- };
-
- OutPartContents.resize(1);
- ParseBuildPartManifest(Storage,
- BuildId,
- BuildPartId,
- BuildPartManifest,
- IncludeWildcards,
- ExcludeWildcards,
- FindManifest(BuildPartId, BuildPartName),
- OutPartContents[0],
- OutBlockDescriptions,
- OutLooseChunkHashes);
- ChunkedFolderContent RemoteContent;
- if (BuildParts.size() > 1)
- {
- std::vector<ChunkBlockDescription> OverlayBlockDescriptions;
- std::vector<IoHash> OverlayLooseChunkHashes;
- for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++)
- {
- const Oid& OverlayBuildPartId = BuildParts[PartIndex].first;
- const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
- Stopwatch GetOverlayBuildPartTimer;
- CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
- if (!IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(Output,
- "GetBuildPart {} ('{}') took {}. Payload size: {}",
- OverlayBuildPartId,
- OverlayBuildPartName,
- NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
- NiceBytes(OverlayBuildPartManifest.GetSize()));
- }
-
- ChunkedFolderContent OverlayPartContent;
- std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
- std::vector<IoHash> OverlayPartLooseChunkHashes;
-
- ParseBuildPartManifest(Storage,
- BuildId,
- OverlayBuildPartId,
- OverlayBuildPartManifest,
- IncludeWildcards,
- ExcludeWildcards,
- FindManifest(OverlayBuildPartId, OverlayBuildPartName),
- OverlayPartContent,
- OverlayPartBlockDescriptions,
- OverlayPartLooseChunkHashes);
- OutPartContents.push_back(OverlayPartContent);
- OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(),
- OverlayPartBlockDescriptions.begin(),
- OverlayPartBlockDescriptions.end());
- OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(),
- OverlayPartLooseChunkHashes.begin(),
- OverlayPartLooseChunkHashes.end());
- }
-
- RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1));
- {
- tsl::robin_set<IoHash> AllBlockHashes;
- for (const ChunkBlockDescription& Description : OutBlockDescriptions)
- {
- AllBlockHashes.insert(Description.BlockHash);
- }
- for (const ChunkBlockDescription& Description : OverlayBlockDescriptions)
- {
- if (!AllBlockHashes.contains(Description.BlockHash))
- {
- AllBlockHashes.insert(Description.BlockHash);
- OutBlockDescriptions.push_back(Description);
- }
- }
- }
- {
- tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end());
- for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes)
- {
- if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash))
- {
- AllLooseChunkHashes.insert(OverlayLooseChunkHash);
- OutLooseChunkHashes.push_back(OverlayLooseChunkHash);
- }
- }
- }
- }
- else
- {
- RemoteContent = OutPartContents[0];
- }
- return RemoteContent;
-}
-
-std::string
-GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
-{
- ExtendableStringBuilder<512> SB;
- std::vector<std::pair<std::string, std::string>> NameStringValuePairs;
- for (CbFieldView Field : Object)
- {
- std::string_view Name = Field.GetName();
- switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
- {
- case CbFieldType::String:
- NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())});
- break;
- case CbFieldType::IntegerPositive:
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())});
- break;
- case CbFieldType::IntegerNegative:
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())});
- break;
- case CbFieldType::Float32:
- {
- const float Value = Accessor.AsFloat32();
- if (std::isfinite(Value))
- {
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)});
- }
- else
- {
- NameStringValuePairs.push_back({std::string(Name), "null"});
- }
- }
- break;
- case CbFieldType::Float64:
- {
- const double Value = Accessor.AsFloat64();
- if (std::isfinite(Value))
- {
- NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)});
- }
- else
- {
- NameStringValuePairs.push_back({std::string(Name), "null"});
- }
- }
- break;
- case CbFieldType::BoolFalse:
- NameStringValuePairs.push_back({std::string(Name), "false"});
- break;
- case CbFieldType::BoolTrue:
- NameStringValuePairs.push_back({std::string(Name), "true"});
- break;
- case CbFieldType::Hash:
- {
- NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()});
- }
- break;
- case CbFieldType::Uuid:
- {
- StringBuilder<Oid::StringLength + 1> Builder;
- Accessor.AsUuid().ToString(Builder);
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- }
- break;
- case CbFieldType::DateTime:
- {
- ExtendableStringBuilder<64> Builder;
- Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- }
- break;
- case CbFieldType::TimeSpan:
- {
- ExtendableStringBuilder<64> Builder;
- const TimeSpan Span(Accessor.AsTimeSpanTicks());
- if (Span.GetDays() == 0)
- {
- Builder << Span.ToString("%h:%m:%s.%n");
- }
- else
- {
- Builder << Span.ToString("%d.%h:%m:%s.%n");
- }
- NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
- break;
- }
- case CbFieldType::ObjectId:
- NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()});
- break;
- }
- }
- std::string::size_type LongestKey = 0;
- for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
- {
- LongestKey = Max(KeyValue.first.length(), LongestKey);
- }
- for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
- {
- SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix));
- }
- return SB.ToString();
-}
-
-#if ZEN_WITH_TESTS
-
-namespace buildstorageoperations_testutils {
- struct TestState
- {
- TestState(const std::filesystem::path& InRootPath)
- : RootPath(InRootPath)
- , LogOutput(CreateStandardLogOutput(Log))
- , ChunkController(CreateStandardChunkingController(StandardChunkingControllerSettings{}))
- , ChunkCache(CreateMemoryChunkingCache())
- , WorkerPool(2)
- , NetworkPool(2)
- {
- }
-
- void Initialize()
- {
- StoragePath = RootPath / "storage";
- TempPath = RootPath / "temp";
- SystemRootDir = RootPath / "sysroot";
- ZenFolderPath = RootPath / ".zen";
-
- CreateDirectories(TempPath);
- CreateDirectories(StoragePath);
-
- Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false);
- }
-
- void CreateSourceData(const std::filesystem::path& Source, std::span<const std::string> Paths, std::span<const uint64_t> Sizes)
- {
- const std::filesystem::path SourcePath = RootPath / Source;
- CreateDirectories(SourcePath);
- for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++)
- {
- const std::string& FilePath = Paths[FileIndex];
- const uint64_t FileSize = Sizes[FileIndex];
- IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{};
- WriteFile(SourcePath / FilePath, FileData);
- }
- }
-
- std::vector<std::pair<Oid, std::string>> Upload(const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& Source,
- const std::filesystem::path& ManifestPath)
- {
- const std::filesystem::path SourcePath = RootPath / Source;
- CbObject MetaData;
- BuildsOperationUploadFolder Upload(*LogOutput,
- Storage,
- AbortFlag,
- PauseFlag,
- WorkerPool,
- NetworkPool,
- BuildId,
- SourcePath,
- true,
- MetaData,
- BuildsOperationUploadFolder::Options{.TempDir = TempPath});
- return Upload.Execute(BuildPartId, BuildPartName, ManifestPath, *ChunkController, *ChunkCache);
- }
-
- void ValidateUpload(const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& Parts)
- {
- for (auto Part : Parts)
- {
- BuildsOperationValidateBuildPart Validate(*LogOutput,
- *Storage.BuildStorage,
- AbortFlag,
- PauseFlag,
- WorkerPool,
- NetworkPool,
- BuildId,
- Part.first,
- Part.second,
- BuildsOperationValidateBuildPart::Options{});
- Validate.Execute();
- }
- }
-
- FolderContent Download(const Oid& BuildId,
- const Oid& BuildPartId,
- const std::string_view BuildPartName,
- const std::filesystem::path& Target,
- bool Append)
- {
- const std::filesystem::path TargetPath = RootPath / Target;
-
- CreateDirectories(TargetPath);
-
- uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
- CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId);
- std::vector<Oid> PartIds;
- if (BuildPartId != Oid::Zero)
- {
- PartIds.push_back(BuildPartId);
- }
- std::vector<std::string> PartNames;
- if (!BuildPartName.empty())
- {
- PartNames.push_back(std::string(BuildPartName));
- }
- std::vector<std::pair<Oid, std::string>> AllBuildParts =
- ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize);
-
- std::vector<ChunkedFolderContent> PartContents;
-
- std::vector<ChunkBlockDescription> BlockDescriptions;
- std::vector<IoHash> LooseChunkHashes;
-
- ChunkedFolderContent RemoteContent = GetRemoteContent(*LogOutput,
- Storage,
- BuildId,
- AllBuildParts,
- {},
- {},
- {},
- ChunkController,
- PartContents,
- BlockDescriptions,
- LooseChunkHashes,
- /*IsQuiet*/ false,
- /*IsVerbose*/ false,
- /*DoExtraContentVerify*/ true);
-
- GetFolderContentStatistics LocalFolderScanStats;
-
- struct ContentVisitor : public GetDirectoryContentVisitor
- {
- virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content)
- {
- RwLock::ExclusiveLockScope _(ExistingPathsLock);
- for (const std::filesystem::path& FileName : Content.FileNames)
- {
- if (RelativeRoot.empty())
- {
- ExistingPaths.push_back(FileName);
- }
- else
- {
- ExistingPaths.push_back(RelativeRoot / FileName);
- }
- }
- }
-
- RwLock ExistingPathsLock;
- std::vector<std::filesystem::path> ExistingPaths;
- } Visitor;
-
- Latch PendingWorkCount(1);
-
- GetDirectoryContent(TargetPath,
- DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive,
- Visitor,
- WorkerPool,
- PendingWorkCount);
-
- PendingWorkCount.CountDown();
- PendingWorkCount.Wait();
-
- FolderContent CurrentLocalFolderState = GetValidFolderContent(
- WorkerPool,
- LocalFolderScanStats,
- TargetPath,
- Visitor.ExistingPaths,
- [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); },
- 1000,
- AbortFlag,
- PauseFlag);
-
- ChunkingStatistics LocalChunkingStats;
- ChunkedFolderContent LocalContent = ChunkFolderContent(
- LocalChunkingStats,
- WorkerPool,
- TargetPath,
- CurrentLocalFolderState,
- *ChunkController,
- *ChunkCache,
- 1000,
- [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); },
- AbortFlag,
- PauseFlag);
-
- if (Append)
- {
- RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {});
- }
-
- const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
- const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
-
- BuildsOperationUpdateFolder Download(*LogOutput,
- Storage,
- AbortFlag,
- PauseFlag,
- WorkerPool,
- NetworkPool,
- BuildId,
- TargetPath,
- LocalContent,
- LocalLookup,
- RemoteContent,
- RemoteLookup,
- BlockDescriptions,
- LooseChunkHashes,
- BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir,
- .ZenFolderPath = ZenFolderPath,
- .ValidateCompletedSequences = true});
- FolderContent ResultingState;
- Download.Execute(ResultingState);
-
- return ResultingState;
- }
-
- void ValidateDownload(std::span<const std::string> Paths,
- std::span<const uint64_t> Sizes,
- const std::filesystem::path& Source,
- const std::filesystem::path& Target,
- const FolderContent& DownloadContent)
- {
- const std::filesystem::path SourcePath = RootPath / Source;
- const std::filesystem::path TargetPath = RootPath / Target;
-
- CHECK_EQ(Paths.size(), DownloadContent.Paths.size());
- tsl::robin_map<std::string, uint64_t> ExpectedSizes;
- tsl::robin_map<std::string, IoHash> ExpectedHashes;
- for (size_t Index = 0; Index < Paths.size(); Index++)
- {
- const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string();
- ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]);
- std::filesystem::path FilePath = SourcePath / Paths[Index];
- const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
- ExpectedHashes.insert_or_assign(LookupString, SourceHash);
- }
- for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++)
- {
- const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string();
- auto SizeIt = ExpectedSizes.find(LookupString);
- CHECK_NE(SizeIt, ExpectedSizes.end());
- CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]);
- std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index];
- const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
- auto HashIt = ExpectedHashes.find(LookupString);
- CHECK_NE(HashIt, ExpectedHashes.end());
- CHECK_EQ(HashIt->second, DownloadedHash);
- }
- }
-
- const std::filesystem::path RootPath;
- std::filesystem::path StoragePath;
- std::filesystem::path TempPath;
- std::filesystem::path SystemRootDir;
- std::filesystem::path ZenFolderPath;
-
- LoggerRef Log = ConsoleLog();
- std::unique_ptr<OperationLogOutput> LogOutput;
-
- std::unique_ptr<ChunkingController> ChunkController;
- std::unique_ptr<ChunkingCache> ChunkCache;
-
- StorageInstance Storage;
- BuildStorageBase::Statistics StorageStats;
-
- WorkerThreadPool WorkerPool;
- WorkerThreadPool NetworkPool;
-
- std::atomic<bool> AbortFlag;
- std::atomic<bool> PauseFlag;
- };
-
-} // namespace buildstorageoperations_testutils
-
-TEST_CASE("buildstorageoperations.upload.folder")
-{
- using namespace buildstorageoperations_testutils;
-
- FastRandom BaseRandom;
-
- const size_t FileCount = 11;
-
- const std::string Paths[FileCount] = {{"file_1"},
- {"file_2.exe"},
- {"file_3.txt"},
- {"dir_1/dir1_file_1.exe"},
- {"dir_1/dir1_file_2.pdb"},
- {"dir_1/dir1_file_3.txt"},
- {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
- {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
- {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
- {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
- {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
- const uint64_t Sizes[FileCount] =
- {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
-
- ScopedTemporaryDirectory SourceFolder;
- TestState State(SourceFolder.Path());
- State.Initialize();
- State.CreateSourceData("source", Paths, Sizes);
-
- const Oid BuildId = Oid::NewOid();
- const Oid BuildPartId = Oid::NewOid();
- const std::string BuildPartName = "default";
-
- auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {});
-
- CHECK_EQ(Result.size(), 1u);
- CHECK_EQ(Result[0].first, BuildPartId);
- CHECK_EQ(Result[0].second, BuildPartName);
- State.ValidateUpload(BuildId, Result);
-
- FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
- CHECK_EQ(DownloadContent.Paths.size(), FileCount);
- State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
-}
-
-TEST_CASE("buildstorageoperations.upload.manifest")
-{
- using namespace buildstorageoperations_testutils;
-
- FastRandom BaseRandom;
-
- const size_t FileCount = 11;
-
- const std::string Paths[FileCount] = {{"file_1"},
- {"file_2.exe"},
- {"file_3.txt"},
- {"dir_1/dir1_file_1.exe"},
- {"dir_1/dir1_file_2.pdb"},
- {"dir_1/dir1_file_3.txt"},
- {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
- {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
- {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
- {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
- {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
- const uint64_t Sizes[FileCount] =
- {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
-
- ScopedTemporaryDirectory SourceFolder;
- TestState State(SourceFolder.Path());
- State.Initialize();
- State.CreateSourceData("source", Paths, Sizes);
-
- std::span<const std::string> ManifestFiles(Paths);
- ManifestFiles = ManifestFiles.subspan(0, FileCount / 2);
-
- std::span<const uint64_t> ManifestSizes(Sizes);
- ManifestSizes = ManifestSizes.subspan(0, FileCount / 2);
-
- ExtendableStringBuilder<1024> Manifest;
- for (const std::string& FilePath : ManifestFiles)
- {
- Manifest << FilePath << "\n";
- }
-
- WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
-
- const Oid BuildId = Oid::NewOid();
- const Oid BuildPartId = Oid::NewOid();
- const std::string BuildPartName = "default";
-
- auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt");
-
- CHECK_EQ(Result.size(), 1u);
- CHECK_EQ(Result[0].first, BuildPartId);
- CHECK_EQ(Result[0].second, BuildPartName);
- State.ValidateUpload(BuildId, Result);
-
- FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
- State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent);
-}
-
-TEST_CASE("buildstorageoperations.memorychunkingcache")
-{
- using namespace buildstorageoperations_testutils;
-
- FastRandom BaseRandom;
-
- const size_t FileCount = 11;
-
- const std::string Paths[FileCount] = {{"file_1"},
- {"file_2.exe"},
- {"file_3.txt"},
- {"dir_1/dir1_file_1.exe"},
- {"dir_1/dir1_file_2.pdb"},
- {"dir_1/dir1_file_3.txt"},
- {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
- {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
- {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
- {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
- {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
- const uint64_t Sizes[FileCount] =
- {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
-
- ScopedTemporaryDirectory SourceFolder;
- TestState State(SourceFolder.Path());
- State.Initialize();
- State.CreateSourceData("source", Paths, Sizes);
-
- const Oid BuildId = Oid::NewOid();
- const Oid BuildPartId = Oid::NewOid();
- const std::string BuildPartName = "default";
-
- {
- const std::filesystem::path SourcePath = SourceFolder.Path() / "source";
- CbObject MetaData;
- BuildsOperationUploadFolder Upload(*State.LogOutput,
- State.Storage,
- State.AbortFlag,
- State.PauseFlag,
- State.WorkerPool,
- State.NetworkPool,
- BuildId,
- SourcePath,
- true,
- MetaData,
- BuildsOperationUploadFolder::Options{.TempDir = State.TempPath});
- auto Result = Upload.Execute(BuildPartId, BuildPartName, {}, *State.ChunkController, *State.ChunkCache);
-
- CHECK_EQ(Upload.m_ChunkingStats.FilesStoredInCache.load(), FileCount - 1); // Zero size files are not stored in cache
- CHECK_EQ(Upload.m_ChunkingStats.BytesStoredInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0)));
- CHECK(Upload.m_ChunkingStats.ChunksStoredInCache.load() >= FileCount - 1); // Zero size files are not stored in cache
-
- CHECK_EQ(Result.size(), 1u);
- CHECK_EQ(Result[0].first, BuildPartId);
- CHECK_EQ(Result[0].second, BuildPartName);
- }
-
- auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {});
-
- const Oid BuildId2 = Oid::NewOid();
- const Oid BuildPartId2 = Oid::NewOid();
-
- {
- const std::filesystem::path SourcePath = SourceFolder.Path() / "source";
- CbObject MetaData;
- BuildsOperationUploadFolder Upload(*State.LogOutput,
- State.Storage,
- State.AbortFlag,
- State.PauseFlag,
- State.WorkerPool,
- State.NetworkPool,
- BuildId2,
- SourcePath,
- true,
- MetaData,
- BuildsOperationUploadFolder::Options{.TempDir = State.TempPath});
- Upload.Execute(BuildPartId2, BuildPartName, {}, *State.ChunkController, *State.ChunkCache);
-
- CHECK_EQ(Upload.m_ChunkingStats.FilesFoundInCache.load(), FileCount - 1); // Zero size files are not stored in cache
- CHECK_EQ(Upload.m_ChunkingStats.BytesFoundInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0)));
- CHECK(Upload.m_ChunkingStats.ChunksFoundInCache.load() >= FileCount - 1); // Zero size files are not stored in cache
- }
-
- FolderContent DownloadContent = State.Download(BuildId2, BuildPartId2, {}, "download", /* Append */ false);
- State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
-}
-
-TEST_CASE("buildstorageoperations.upload.multipart")
-{
- // Disabled since it relies on authentication and specific block being present in cloud storage
- if (false)
- {
- using namespace buildstorageoperations_testutils;
-
- FastRandom BaseRandom;
-
- const size_t FileCount = 11;
-
- const std::string Paths[FileCount] = {{"file_1"},
- {"file_2.exe"},
- {"file_3.txt"},
- {"dir_1/dir1_file_1.exe"},
- {"dir_1/dir1_file_2.pdb"},
- {"dir_1/dir1_file_3.txt"},
- {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
- {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
- {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
- {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
- {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
- const uint64_t Sizes[FileCount] =
- {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
-
- ScopedTemporaryDirectory SourceFolder;
- TestState State(SourceFolder.Path());
- State.Initialize();
- State.CreateSourceData("source", Paths, Sizes);
-
- std::span<const std::string> ManifestFiles1(Paths);
- ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2);
-
- std::span<const uint64_t> ManifestSizes1(Sizes);
- ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2);
-
- std::span<const std::string> ManifestFiles2(Paths);
- ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1);
-
- std::span<const uint64_t> ManifestSizes2(Sizes);
- ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1);
-
- const Oid BuildPart1Id = Oid::NewOid();
- const std::string BuildPart1Name = "part1";
- const Oid BuildPart2Id = Oid::NewOid();
- const std::string BuildPart2Name = "part2";
- {
- CbObjectWriter Writer;
- Writer.BeginObject("parts"sv);
- {
- Writer.BeginObject(BuildPart1Name);
- {
- Writer.AddObjectId("partId"sv, BuildPart1Id);
- Writer.BeginArray("files"sv);
- for (const std::string& ManifestFile : ManifestFiles1)
- {
- Writer.AddString(ManifestFile);
- }
- Writer.EndArray(); // files
- }
- Writer.EndObject(); // part1
-
- Writer.BeginObject(BuildPart2Name);
- {
- Writer.AddObjectId("partId"sv, BuildPart2Id);
- Writer.BeginArray("files"sv);
- for (const std::string& ManifestFile : ManifestFiles2)
- {
- Writer.AddString(ManifestFile);
- }
- Writer.EndArray(); // files
- }
- Writer.EndObject(); // part2
- }
- Writer.EndObject(); // parts
-
- ExtendableStringBuilder<1024> Manifest;
- CompactBinaryToJson(Writer.Save(), Manifest);
- WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
- }
-
- const Oid BuildId = Oid::NewOid();
-
- auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json");
-
- CHECK_EQ(Result.size(), 2u);
- CHECK_EQ(Result[0].first, BuildPart1Id);
- CHECK_EQ(Result[0].second, BuildPart1Name);
- CHECK_EQ(Result[1].first, BuildPart2Id);
- CHECK_EQ(Result[1].second, BuildPart2Name);
- State.ValidateUpload(BuildId, Result);
-
- FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
- State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
-
- FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false);
- State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent);
-
- FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false);
- State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent);
-
- (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false);
- FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true);
- State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent);
- }
-}
-
-void
-buildstorageoperations_forcelink()
-{
-}
-
-#endif // ZEN_WITH_TESTS
-
-} // namespace zen
diff --git a/src/zenremotestore/builds/buildstorageresolve.cpp b/src/zenremotestore/builds/buildstorageresolve.cpp
new file mode 100644
index 000000000..b33d7af29
--- /dev/null
+++ b/src/zenremotestore/builds/buildstorageresolve.cpp
@@ -0,0 +1,249 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildstorageresolve.h>
+
+#include <zencore/fmtutils.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/jupiter/jupiterhost.h>
+#include <zenutil/zenserverprocess.h>
+
+namespace zen {
+
+namespace {
+ std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings)
+ {
+ ExtendableStringBuilder<128> SB;
+ SB << "\n LogCategory: " << ClientSettings.LogCategory;
+ SB << "\n ConnectTimeout: " << ClientSettings.ConnectTimeout.count() << " ms";
+ SB << "\n Timeout: " << ClientSettings.Timeout.count() << " ms";
+ SB << "\n AccessTokenProvider: " << ClientSettings.AccessTokenProvider.has_value();
+ SB << "\n AssumeHttp2: " << ClientSettings.AssumeHttp2;
+ SB << "\n AllowResume: " << ClientSettings.AllowResume;
+ SB << "\n RetryCount: " << ClientSettings.RetryCount;
+ SB << "\n SessionId: " << ClientSettings.SessionId.ToString();
+ SB << "\n Verbose: " << ClientSettings.Verbose;
+ SB << "\n MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize;
+ return SB.ToString();
+ }
+} // namespace
+
+BuildStorageResolveResult
+ResolveBuildStorage(LoggerRef InLog,
+ const HttpClientSettings& ClientSettings,
+ std::string_view Host,
+ std::string_view OverrideHost,
+ std::string_view ZenCacheHost,
+ ZenCacheResolveMode ZenResolveMode,
+ bool Verbose)
+{
+ ZEN_SCOPED_LOG(InLog);
+
+ bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All;
+ bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All;
+
+ auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view {
+ std::string::size_type HostnameStart = 0;
+ std::string::size_type HostnameLength = std::string::npos;
+ if (auto StartPos = Url.find("//"); StartPos != std::string::npos)
+ {
+ HostnameStart = StartPos + 2;
+ }
+ if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos)
+ {
+ HostnameLength = EndPos - HostnameStart;
+ }
+ if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos)
+ {
+ HostnameLength = EndPos - HostnameStart;
+ }
+ return Url.substr(HostnameStart, HostnameLength);
+ };
+
+ std::string HostUrl;
+ std::string HostName;
+ double HostLatencySec = -1.0;
+ uint64_t HostMaxRangeCountPerRequest = 1;
+
+ std::string CacheUrl;
+ std::string CacheName;
+ bool HostAssumeHttp2 = ClientSettings.AssumeHttp2;
+ bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2;
+ double CacheLatencySec = -1.0;
+ uint64_t CacheMaxRangeCountPerRequest = 1;
+
+ JupiterServerDiscovery DiscoveryResponse;
+ const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host;
+
+ if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery))
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}",
+ DiscoveryHost,
+ ConnectionSettingsToString(ClientSettings));
+ }
+
+ DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings);
+ }
+
+ if (!OverrideHost.empty())
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2);
+ }
+ if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose);
+ TestResult.Success)
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost);
+ }
+ HostUrl = OverrideHost;
+ HostName = GetHostNameFromUrl(OverrideHost);
+ HostLatencySec = TestResult.LatencySeconds;
+ HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason));
+ }
+ }
+ else
+ {
+ if (DiscoveryResponse.ServerEndPoints.empty())
+ {
+ throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost));
+ }
+
+ for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints)
+ {
+ if (!ServerEndpoint.BaseUrl.empty())
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Testing server endpoint at '{}/health/live'. Assume http2: {}",
+ ServerEndpoint.BaseUrl,
+ ServerEndpoint.AssumeHttp2);
+ }
+
+ if (JupiterEndpointTestResult TestResult =
+ TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose);
+ TestResult.Success)
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
+ }
+
+ HostUrl = ServerEndpoint.BaseUrl;
+ HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
+ HostName = ServerEndpoint.Name;
+ HostLatencySec = TestResult.LatencySeconds;
+ HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+ break;
+ }
+ else
+ {
+ ZEN_DEBUG("Unable to reach host {}. Reason: {}", ServerEndpoint.BaseUrl, TestResult.FailureReason);
+ }
+ }
+ }
+ if (HostUrl.empty())
+ {
+ throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}",
+ DiscoveryResponse.ServerEndPoints.size(),
+ DiscoveryHost));
+ }
+ }
+ if (ZenCacheHost.empty())
+ {
+ if (AllowZenCacheDiscovery)
+ {
+ for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints)
+ {
+ if (!CacheEndpoint.BaseUrl.empty())
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}",
+ CacheEndpoint.BaseUrl,
+ CacheEndpoint.AssumeHttp2);
+ }
+
+ if (ZenCacheEndpointTestResult TestResult =
+ TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose);
+ TestResult.Success)
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
+ }
+
+ CacheUrl = CacheEndpoint.BaseUrl;
+ CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
+ CacheName = CacheEndpoint.Name;
+ CacheLatencySec = TestResult.LatencySeconds;
+ CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+ break;
+ }
+ }
+ }
+ }
+ if (CacheUrl.empty() && AllowLocalZenCache)
+ {
+ ZenServerState State;
+ if (State.InitializeReadOnly())
+ {
+ State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
+ if (CacheUrl.empty())
+ {
+ std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
+ if (ZenCacheEndpointTestResult TestResult =
+ TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose);
+ TestResult.Success)
+ {
+ CacheUrl = ZenServerLocalHostUrl;
+ CacheAssumeHttp2 = false;
+ CacheName = "localhost";
+ CacheLatencySec = TestResult.LatencySeconds;
+ }
+ }
+ });
+ }
+ }
+ }
+ else
+ {
+ if (Verbose)
+ {
+ ZEN_INFO("Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false);
+ }
+ if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose);
+ TestResult.Success)
+ {
+ CacheUrl = ZenCacheHost;
+ CacheName = GetHostNameFromUrl(ZenCacheHost);
+ CacheLatencySec = TestResult.LatencySeconds;
+ CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest;
+ }
+ else
+ {
+ ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason);
+ }
+ }
+
+ return BuildStorageResolveResult{
+ .Cloud = {.Address = HostUrl,
+ .Name = HostName,
+ .AssumeHttp2 = HostAssumeHttp2,
+ .LatencySec = HostLatencySec,
+ .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = HostMaxRangeCountPerRequest}},
+ .Cache = {.Address = CacheUrl,
+ .Name = CacheName,
+ .AssumeHttp2 = CacheAssumeHttp2,
+ .LatencySec = CacheLatencySec,
+ .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = CacheMaxRangeCountPerRequest}}};
+}
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp
index b249d7d52..dc8f79a47 100644
--- a/src/zenremotestore/builds/buildstorageutil.cpp
+++ b/src/zenremotestore/builds/buildstorageutil.cpp
@@ -2,293 +2,116 @@
#include <zenremotestore/builds/buildstorageutil.h>
+#include <zencore/basicfile.h>
+#include <zencore/compactbinary.h>
+#include <zencore/compactbinaryutil.h>
+#include <zencore/compactbinaryvalue.h>
+#include <zencore/except.h>
#include <zencore/fmtutils.h>
+#include <zencore/logging/broadcastsink.h>
+#include <zencore/parallelwork.h>
#include <zencore/timer.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/buildcontent.h>
+#include <zenremotestore/builds/buildmanifest.h>
+#include <zenremotestore/builds/buildprimecache.h>
#include <zenremotestore/builds/buildstorage.h>
#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/builds/buildupdatefolder.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/builds/buildvalidatebuildpart.h>
#include <zenremotestore/builds/jupiterbuildstorage.h>
#include <zenremotestore/chunking/chunkblock.h>
+#include <zenremotestore/chunking/chunkingcache.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
#include <zenremotestore/jupiter/jupiterhost.h>
-#include <zenremotestore/operationlogoutput.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/logging.h>
+#include <zenutil/progress.h>
+#include <zenutil/wildcard.h>
#include <zenutil/zenserverprocess.h>
-namespace zen {
-namespace {
- std::string ConnectionSettingsToString(const HttpClientSettings& ClientSettings)
- {
- ExtendableStringBuilder<128> SB;
- SB << "\n LogCategory: " << ClientSettings.LogCategory;
- SB << "\n ConnectTimeout: " << ClientSettings.ConnectTimeout.count() << " ms";
- SB << "\n Timeout: " << ClientSettings.Timeout.count() << " ms";
- SB << "\n AccessTokenProvider: " << ClientSettings.AccessTokenProvider.has_value();
- SB << "\n AssumeHttp2: " << ClientSettings.AssumeHttp2;
- SB << "\n AllowResume: " << ClientSettings.AllowResume;
- SB << "\n RetryCount: " << ClientSettings.RetryCount;
- SB << "\n SessionId: " << ClientSettings.SessionId.ToString();
- SB << "\n Verbose: " << ClientSettings.Verbose;
- SB << "\n MaximumInMemoryDownloadSize: " << ClientSettings.MaximumInMemoryDownloadSize;
- return SB.ToString();
- }
-} // namespace
-
-BuildStorageResolveResult
-ResolveBuildStorage(OperationLogOutput& Output,
- const HttpClientSettings& ClientSettings,
- std::string_view Host,
- std::string_view OverrideHost,
- std::string_view ZenCacheHost,
- ZenCacheResolveMode ZenResolveMode,
- bool Verbose)
-{
- bool AllowZenCacheDiscovery = ZenResolveMode == ZenCacheResolveMode::Discovery || ZenResolveMode == ZenCacheResolveMode::All;
- bool AllowLocalZenCache = ZenResolveMode == ZenCacheResolveMode::LocalHost || ZenResolveMode == ZenCacheResolveMode::All;
-
- auto GetHostNameFromUrl = [](std::string_view Url) -> std::string_view {
- std::string::size_type HostnameStart = 0;
- std::string::size_type HostnameLength = std::string::npos;
- if (auto StartPos = Url.find("//"); StartPos != std::string::npos)
- {
- HostnameStart = StartPos + 2;
- }
- if (auto EndPos = Url.find("/", HostnameStart); EndPos != std::string::npos)
- {
- HostnameLength = EndPos - HostnameStart;
- }
- if (auto EndPos = Url.find(":", HostnameStart); EndPos != std::string::npos)
- {
- HostnameLength = EndPos - HostnameStart;
- }
- return Url.substr(HostnameStart, HostnameLength);
- };
+#include <numeric>
- std::string HostUrl;
- std::string HostName;
- double HostLatencySec = -1.0;
+#if ZEN_WITH_TESTS
+# include <zencore/testing.h>
+# include <zencore/testutils.h>
+# include <zenhttp/httpclientauth.h>
+# include <zenremotestore/builds/filebuildstorage.h>
+#endif // ZEN_WITH_TESTS
- std::string CacheUrl;
- std::string CacheName;
- bool HostAssumeHttp2 = ClientSettings.AssumeHttp2;
- bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2;
- double CacheLatencySec = -1.0;
-
- JupiterServerDiscovery DiscoveryResponse;
- const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host;
+namespace zen {
- if (OverrideHost.empty() || (ZenCacheHost.empty() && AllowZenCacheDiscovery))
+StorageInstance::~StorageInstance()
+{
+ if (CacheLogSink)
{
- if (Verbose)
+ if (Ref<logging::BroadcastSink> Broadcast = GetDefaultBroadcastSink())
{
- ZEN_OPERATION_LOG_INFO(Output,
- "Querying servers at '{}/api/v1/status/servers'\n Connection settings:{}",
- DiscoveryHost,
- ConnectionSettingsToString(ClientSettings));
+ Broadcast->RemoveSink(CacheLogSink);
}
-
- DiscoveryResponse = DiscoverJupiterEndpoints(DiscoveryHost, ClientSettings);
}
+}
- if (!OverrideHost.empty())
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Testing server endpoint at '{}/health/live'. Assume http2: {}", OverrideHost, HostAssumeHttp2);
- }
- if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(OverrideHost, HostAssumeHttp2, ClientSettings.Verbose);
- TestResult.Success)
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost);
- }
- HostUrl = OverrideHost;
- HostName = GetHostNameFromUrl(OverrideHost);
- HostLatencySec = TestResult.LatencySeconds;
- }
- else
- {
- throw std::runtime_error(fmt::format("Host {} could not be reached. Reason: {}", OverrideHost, TestResult.FailureReason));
- }
- }
- else
- {
- if (DiscoveryResponse.ServerEndPoints.empty())
- {
- throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", DiscoveryHost));
- }
-
- for (const JupiterServerDiscovery::EndPoint& ServerEndpoint : DiscoveryResponse.ServerEndPoints)
- {
- if (!ServerEndpoint.BaseUrl.empty())
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output,
- "Testing server endpoint at '{}/health/live'. Assume http2: {}",
- ServerEndpoint.BaseUrl,
- ServerEndpoint.AssumeHttp2);
- }
-
- if (JupiterEndpointTestResult TestResult =
- TestJupiterEndpoint(ServerEndpoint.BaseUrl, ServerEndpoint.AssumeHttp2, ClientSettings.Verbose);
- TestResult.Success)
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl);
- }
-
- HostUrl = ServerEndpoint.BaseUrl;
- HostAssumeHttp2 = ServerEndpoint.AssumeHttp2;
- HostName = ServerEndpoint.Name;
- HostLatencySec = TestResult.LatencySeconds;
- break;
- }
- else
- {
- ZEN_OPERATION_LOG_DEBUG(Output,
- "Unable to reach host {}. Reason: {}",
- ServerEndpoint.BaseUrl,
- TestResult.FailureReason);
- }
- }
- }
- if (HostUrl.empty())
- {
- throw std::runtime_error(fmt::format("Failed to find any usable builds hosts out of {} using {}",
- DiscoveryResponse.ServerEndPoints.size(),
- DiscoveryHost));
- }
- }
- if (ZenCacheHost.empty())
- {
- if (AllowZenCacheDiscovery)
- {
- for (const JupiterServerDiscovery::EndPoint& CacheEndpoint : DiscoveryResponse.CacheEndPoints)
- {
- if (!CacheEndpoint.BaseUrl.empty())
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output,
- "Testing cache endpoint at '{}/status/builds'. Assume http2: {}",
- CacheEndpoint.BaseUrl,
- CacheEndpoint.AssumeHttp2);
- }
+void
+StorageInstance::SetupCacheSession(std::string_view TargetUrl, std::string_view Mode, const Oid& SessionId)
+{
+ CacheSession = std::make_unique<SessionsServiceClient>(SessionsServiceClient::Options{
+ .TargetUrl = std::string(TargetUrl),
+ .AppName = "zen",
+ .Mode = std::string(Mode),
+ .SessionId = SessionId,
+ });
+ CacheSession->Announce();
+ CacheLogSink = CacheSession->CreateLogSink();
+ GetDefaultBroadcastSink()->AddSink(CacheLogSink);
+}
- if (ZenCacheEndpointTestResult TestResult =
- TestZenCacheEndpoint(CacheEndpoint.BaseUrl, CacheEndpoint.AssumeHttp2, ClientSettings.Verbose);
- TestResult.Success)
- {
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl);
- }
+using namespace std::literals;
- CacheUrl = CacheEndpoint.BaseUrl;
- CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2;
- CacheName = CacheEndpoint.Name;
- CacheLatencySec = TestResult.LatencySeconds;
- break;
- }
- }
- }
- }
- if (CacheUrl.empty() && AllowLocalZenCache)
- {
- ZenServerState State;
- if (State.InitializeReadOnly())
- {
- State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) {
- if (CacheUrl.empty())
- {
- std::string ZenServerLocalHostUrl = fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load());
- if (ZenCacheEndpointTestResult TestResult =
- TestZenCacheEndpoint(ZenServerLocalHostUrl, /*AssumeHttp2*/ false, ClientSettings.Verbose);
- TestResult.Success)
- {
- CacheUrl = ZenServerLocalHostUrl;
- CacheAssumeHttp2 = false;
- CacheName = "localhost";
- CacheLatencySec = TestResult.LatencySeconds;
- }
- }
- });
- }
- }
- }
- else
+std::vector<ChunkBlockDescription>
+ParseBlockMetadatas(std::span<const CbObject> BlockMetadatas)
+{
+ std::vector<ChunkBlockDescription> UnorderedList;
+ UnorderedList.reserve(BlockMetadatas.size());
+ for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < BlockMetadatas.size(); CacheBlockMetadataIndex++)
{
- if (Verbose)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Testing cache endpoint at '{}/status/builds'. Assume http2: {}", ZenCacheHost, false);
- }
- if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose);
- TestResult.Success)
+ const CbObject& CacheBlockMetadata = BlockMetadatas[CacheBlockMetadataIndex];
+ ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata);
+ if (Description.BlockHash != IoHash::Zero)
{
- CacheUrl = ZenCacheHost;
- CacheName = GetHostNameFromUrl(ZenCacheHost);
- CacheLatencySec = TestResult.LatencySeconds;
- }
- else
- {
- ZEN_WARN("Unable to reach cache host {}. Reason: {}", ZenCacheHost, TestResult.FailureReason);
+ UnorderedList.emplace_back(std::move(Description));
}
}
-
- return BuildStorageResolveResult{.HostUrl = HostUrl,
- .HostName = HostName,
- .HostAssumeHttp2 = HostAssumeHttp2,
- .HostLatencySec = HostLatencySec,
-
- .CacheUrl = CacheUrl,
- .CacheName = CacheName,
- .CacheAssumeHttp2 = CacheAssumeHttp2,
- .CacheLatencySec = CacheLatencySec};
+ return UnorderedList;
}
std::vector<ChunkBlockDescription>
-GetBlockDescriptions(OperationLogOutput& Output,
+GetBlockDescriptions(LoggerRef InLog,
BuildStorageBase& Storage,
BuildStorageCache* OptionalCacheStorage,
const Oid& BuildId,
- const Oid& BuildPartId,
std::span<const IoHash> BlockRawHashes,
bool AttemptFallback,
bool IsQuiet,
bool IsVerbose)
{
using namespace std::literals;
-
- if (!IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size());
- }
-
- Stopwatch GetBlockMetadataTimer;
+ ZEN_SCOPED_LOG(InLog);
std::vector<ChunkBlockDescription> UnorderedList;
tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup;
if (OptionalCacheStorage && !BlockRawHashes.empty())
{
std::vector<CbObject> CacheBlockMetadatas = OptionalCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes);
- UnorderedList.reserve(CacheBlockMetadatas.size());
- for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size(); CacheBlockMetadataIndex++)
+ if (!CacheBlockMetadatas.empty())
{
- const CbObject& CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex];
- ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata);
- if (Description.BlockHash == IoHash::Zero)
+ UnorderedList = ParseBlockMetadatas(CacheBlockMetadatas);
+ for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
{
- ZEN_OPERATION_LOG_WARN(Output, "Unexpected/invalid block metadata received from remote cache, skipping block");
+ const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
+ BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
}
- else
- {
- UnorderedList.emplace_back(std::move(Description));
- }
- }
- for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++)
- {
- const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex];
- BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex);
}
}
@@ -315,7 +138,7 @@ GetBlockDescriptions(OperationLogOutput& Output,
if (Description.BlockHash == IoHash::Zero)
{
- ZEN_OPERATION_LOG_WARN(Output, "Unexpected/invalid block metadata received from remote store, skipping block");
+ ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block");
}
else
{
@@ -355,15 +178,6 @@ GetBlockDescriptions(OperationLogOutput& Output,
}
}
- if (!IsQuiet)
- {
- ZEN_OPERATION_LOG_INFO(Output,
- "GetBlockMetadata for {} took {}. Found {} blocks",
- BuildPartId,
- NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
- Result.size());
- }
-
if (Result.size() != BlockRawHashes.size())
{
std::string ErrorDescription =
@@ -385,7 +199,7 @@ GetBlockDescriptions(OperationLogOutput& Output,
}
if (AttemptFallback)
{
- ZEN_OPERATION_LOG_WARN(Output, "{} Attemping fallback options.", ErrorDescription);
+ ZEN_WARN("{} Attemping fallback options.", ErrorDescription);
std::vector<ChunkBlockDescription> AugmentedBlockDescriptions;
AugmentedBlockDescriptions.reserve(BlockRawHashes.size());
std::vector<ChunkBlockDescription> FoundBlocks = ParseChunkBlockDescriptionList(Storage.FindBlocks(BuildId, (uint64_t)-1));
@@ -408,7 +222,7 @@ GetBlockDescriptions(OperationLogOutput& Output,
{
if (!IsQuiet)
{
- ZEN_OPERATION_LOG_INFO(Output, "Found block {} via context find successfully", BlockHash);
+ ZEN_INFO("Found block {} via context find successfully", BlockHash);
}
AugmentedBlockDescriptions.emplace_back(std::move(*ListBlocksIt));
}
@@ -453,4 +267,1472 @@ GetBlockDescriptions(OperationLogOutput& Output,
return Result;
}
+////////////////////// Shared helpers
+
+std::filesystem::path
+ZenStateFilePath(const std::filesystem::path& ZenFolderPath)
+{
+ return ZenFolderPath / "current_state.cbo";
+}
+std::filesystem::path
+ZenTempFolderPath(const std::filesystem::path& ZenFolderPath)
+{
+ return ZenFolderPath / "tmp";
+}
+
+CbObject
+GetBuild(BuildStorageBase& Storage, const Oid& BuildId, bool IsQuiet)
+{
+ Stopwatch GetBuildTimer;
+ CbObject BuildObject = Storage.GetBuild(BuildId);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}",
+ NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()),
+ BuildObject["name"sv].AsString(),
+ NiceBytes(BuildObject.GetSize()));
+
+ ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv));
+ }
+ return BuildObject;
+}
+
+uint64_t
+GetMaxMemoryBufferSize(size_t MaxBlockSize, bool BoostWorkerMemory)
+{
+ return BoostWorkerMemory ? (MaxBlockSize + 16u * 1024u) : 1024u * 1024u;
+}
+
+void
+DownloadLargeBlob(BuildStorageBase& Storage,
+ const std::filesystem::path& DownloadFolder,
+ const Oid& BuildId,
+ const IoHash& ChunkHash,
+ const std::uint64_t PreferredMultipartChunkSize,
+ ParallelWork& Work,
+ WorkerThreadPool& NetworkPool,
+ std::atomic<uint64_t>& DownloadedChunkByteCount,
+ std::atomic<uint64_t>& MultipartAttachmentCount,
+ std::function<void(IoBuffer&& Payload)>&& OnDownloadComplete)
+{
+ ZEN_TRACE_CPU("DownloadLargeBlob");
+
+ struct WorkloadData
+ {
+ TemporaryFile TempFile;
+ };
+ std::shared_ptr<WorkloadData> Workload(std::make_shared<WorkloadData>());
+
+ std::error_code Ec;
+ Workload->TempFile.CreateTemporary(DownloadFolder, Ec);
+ if (Ec)
+ {
+ throw std::runtime_error(
+ fmt::format("Failed opening temporary file '{}', reason: ({}) {}", Workload->TempFile.GetPath(), Ec.message(), Ec.value()));
+ }
+ std::vector<std::function<void()>> WorkItems = Storage.GetLargeBuildBlob(
+ BuildId,
+ ChunkHash,
+ PreferredMultipartChunkSize,
+ [&Work, Workload, &DownloadedChunkByteCount](uint64_t Offset, const IoBuffer& Chunk) {
+ DownloadedChunkByteCount += Chunk.GetSize();
+
+ if (!Work.IsAborted())
+ {
+ ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnReceive");
+ Workload->TempFile.Write(Chunk.GetView(), Offset);
+ }
+ },
+ [&Work, Workload, OnDownloadComplete = std::move(OnDownloadComplete)]() {
+ if (!Work.IsAborted())
+ {
+ ZEN_TRACE_CPU("Async_DownloadLargeBlob_OnComplete");
+
+ uint64_t PayloadSize = Workload->TempFile.FileSize();
+ void* FileHandle = Workload->TempFile.Detach();
+ ZEN_ASSERT(FileHandle != nullptr);
+ IoBuffer Payload(IoBuffer::File, FileHandle, 0, PayloadSize, true);
+ Payload.SetDeleteOnClose(true);
+ OnDownloadComplete(std::move(Payload));
+ }
+ });
+ if (!WorkItems.empty())
+ {
+ MultipartAttachmentCount++;
+ }
+ for (auto& WorkItem : WorkItems)
+ {
+ Work.ScheduleWork(NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic<bool>& AbortFlag) {
+ if (!AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_DownloadLargeBlob_Work");
+
+ WorkItem();
+ }
+ });
+ }
+}
+
+CompositeBuffer
+ValidateBlob(std::atomic<bool>& AbortFlag,
+ IoBuffer&& Payload,
+ const IoHash& BlobHash,
+ uint64_t& OutCompressedSize,
+ uint64_t& OutDecompressedSize)
+{
+ ZEN_TRACE_CPU("ValidateBlob");
+
+ if (Payload.GetContentType() != ZenContentType::kCompressedBinary)
+ {
+ throw std::runtime_error(fmt::format("Blob {} ({} bytes) has unexpected content type '{}'",
+ BlobHash,
+ Payload.GetSize(),
+ ToString(Payload.GetContentType())));
+ }
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Payload), RawHash, RawSize);
+ if (!Compressed)
+ {
+ throw std::runtime_error(fmt::format("Blob {} ({} bytes) compressed header is invalid", BlobHash, Payload.GetSize()));
+ }
+ if (RawHash != BlobHash)
+ {
+ throw std::runtime_error(
+ fmt::format("Blob {} ({} bytes) compressed header has a mismatching raw hash {}", BlobHash, Payload.GetSize(), RawHash));
+ }
+
+ IoHashStream Hash;
+ bool CouldDecompress = Compressed.DecompressToStream(
+ 0,
+ RawSize,
+ [&AbortFlag, &Hash](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+ ZEN_UNUSED(SourceOffset, SourceSize, Offset);
+ if (!AbortFlag)
+ {
+ for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+ {
+ Hash.Append(Segment.GetView());
+ }
+ return true;
+ }
+ return false;
+ });
+
+ if (AbortFlag)
+ {
+ return CompositeBuffer{};
+ }
+
+ if (!CouldDecompress)
+ {
+ throw std::runtime_error(
+ fmt::format("Blob {} ({} bytes) failed to decompress - header information mismatch", BlobHash, Payload.GetSize()));
+ }
+ IoHash ValidateRawHash = Hash.GetHash();
+ if (ValidateRawHash != BlobHash)
+ {
+ throw std::runtime_error(fmt::format("Blob {} ({} bytes) decompressed hash {} does not match header information",
+ BlobHash,
+ Payload.GetSize(),
+ ValidateRawHash));
+ }
+ OodleCompressor Compressor;
+ OodleCompressionLevel CompressionLevel;
+ uint64_t BlockSize;
+ if (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize))
+ {
+ throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to get compression details", BlobHash, Payload.GetSize()));
+ }
+ OutCompressedSize = Payload.GetSize();
+ OutDecompressedSize = RawSize;
+ if (CompressionLevel == OodleCompressionLevel::None)
+ {
+ // Only decompress to composite if we need it for block verification
+ CompositeBuffer DecompressedComposite = Compressed.DecompressToComposite();
+ if (!DecompressedComposite)
+ {
+ throw std::runtime_error(fmt::format("Blob {} ({} bytes) failed to decompress to composite", BlobHash, Payload.GetSize()));
+ }
+ return DecompressedComposite;
+ }
+ return CompositeBuffer{};
+}
+
+CompositeBuffer
+ValidateBlob(std::atomic<bool>& AbortFlag,
+ BuildStorageBase& Storage,
+ const Oid& BuildId,
+ const IoHash& BlobHash,
+ uint64_t& OutCompressedSize,
+ uint64_t& OutDecompressedSize)
+{
+ ZEN_TRACE_CPU("ValidateBlob");
+ IoBuffer Payload = Storage.GetBuildBlob(BuildId, BlobHash);
+ if (!Payload)
+ {
+ throw std::runtime_error(fmt::format("Blob {} could not be found", BlobHash));
+ }
+ return ValidateBlob(AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize);
+}
+
+std::vector<std::pair<Oid, std::string>>
+ResolveBuildPartNames(CbObjectView BuildObject,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ std::uint64_t& OutPreferredMultipartChunkSize)
+{
+ std::vector<std::pair<Oid, std::string>> Result;
+ {
+ CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView();
+ if (!PartsObject)
+ {
+ throw std::runtime_error("Build object does not have a 'parts' object");
+ }
+
+ OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize);
+
+ std::vector<std::pair<Oid, std::string>> AvailableParts;
+
+ for (CbFieldView PartView : PartsObject)
+ {
+ const std::string BuildPartName = std::string(PartView.GetName());
+ const Oid BuildPartId = PartView.AsObjectId();
+ if (BuildPartId == Oid::Zero)
+ {
+ ExtendableStringBuilder<128> SB;
+ for (CbFieldView ScanPartView : PartsObject)
+ {
+ SB.Append(fmt::format("\n {}: {}", ScanPartView.GetName(), ScanPartView.AsObjectId()));
+ }
+ throw std::runtime_error(fmt::format("Build object parts does not have a '{}' object id{}", BuildPartName, SB.ToView()));
+ }
+ AvailableParts.push_back({BuildPartId, BuildPartName});
+ }
+
+ if (BuildPartIds.empty() && BuildPartNames.empty())
+ {
+ Result = AvailableParts;
+ }
+ else
+ {
+ for (const std::string& BuildPartName : BuildPartNames)
+ {
+ if (auto It = std::find_if(AvailableParts.begin(),
+ AvailableParts.end(),
+ [&BuildPartName](const auto& Part) { return Part.second == BuildPartName; });
+ It != AvailableParts.end())
+ {
+ Result.push_back(*It);
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Build {} object does not have a part named '{}'", BuildId, BuildPartName));
+ }
+ }
+ for (const Oid& BuildPartId : BuildPartIds)
+ {
+ if (auto It = std::find_if(AvailableParts.begin(),
+ AvailableParts.end(),
+ [&BuildPartId](const auto& Part) { return Part.first == BuildPartId; });
+ It != AvailableParts.end())
+ {
+ Result.push_back(*It);
+ }
+ else
+ {
+ throw std::runtime_error(fmt::format("Build {} object does not have a part with id '{}'", BuildId, BuildPartId));
+ }
+ }
+ }
+
+ if (Result.empty())
+ {
+ throw std::runtime_error(fmt::format("Build object does not have any parts", BuildId));
+ }
+ }
+ return Result;
+}
+
+void
+NormalizePartSelection(std::vector<Oid>& BuildPartIds, std::vector<std::string>& BuildPartNames, std::string_view HelpText)
+{
+ const bool HasWildcard = std::find(BuildPartNames.begin(), BuildPartNames.end(), "*") != BuildPartNames.end();
+ if (HasWildcard)
+ {
+ if (BuildPartNames.size() != 1 || !BuildPartIds.empty())
+ {
+ throw OptionParseException("'*' cannot be combined with other part names or ids", std::string(HelpText));
+ }
+ BuildPartNames.clear();
+ return;
+ }
+
+ if (BuildPartIds.empty() && BuildPartNames.empty())
+ {
+ BuildPartNames.push_back("default");
+ }
+}
+
+ChunkedFolderContent
+GetRemoteContent(LoggerRef InLog,
+ StorageInstance& Storage,
+ const Oid& BuildId,
+ const std::vector<std::pair<Oid, std::string>>& BuildParts,
+ const BuildManifest& Manifest,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ std::unique_ptr<ChunkingController>& OutChunkController,
+ std::vector<ChunkedFolderContent>& OutPartContents,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes,
+ bool IsQuiet,
+ bool IsVerbose,
+ bool DoExtraContentVerify)
+{
+ ZEN_TRACE_CPU("GetRemoteContent");
+ ZEN_SCOPED_LOG(InLog);
+
+ Stopwatch GetBuildPartTimer;
+ const Oid BuildPartId = BuildParts[0].first;
+ const std::string_view BuildPartName = BuildParts[0].second;
+ CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId);
+ if (!IsQuiet)
+ {
+ ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ BuildPartId,
+ BuildPartName,
+ NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(BuildPartManifest.GetSize()));
+ ZEN_INFO("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv));
+ }
+
+ {
+ CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView();
+ std::string_view ChunkerName = Chunker["name"sv].AsString();
+ CbObjectView Parameters = Chunker["parameters"sv].AsObjectView();
+ OutChunkController = CreateChunkingController(ChunkerName, Parameters);
+ }
+
+ auto ParseBuildPartManifest = [&Log, IsQuiet, IsVerbose, DoExtraContentVerify](StorageInstance& Storage,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ CbObject BuildPartManifest,
+ std::span<const std::string> IncludeWildcards,
+ std::span<const std::string> ExcludeWildcards,
+ const BuildManifest::Part* OptionalManifest,
+ ChunkedFolderContent& OutRemoteContent,
+ std::vector<ChunkBlockDescription>& OutBlockDescriptions,
+ std::vector<IoHash>& OutLooseChunkHashes) {
+ std::vector<uint32_t> AbsoluteChunkOrders;
+ std::vector<uint64_t> LooseChunkRawSizes;
+ std::vector<IoHash> BlockRawHashes;
+
+ ReadBuildContentFromCompactBinary(BuildPartManifest,
+ OutRemoteContent.Platform,
+ OutRemoteContent.Paths,
+ OutRemoteContent.RawHashes,
+ OutRemoteContent.RawSizes,
+ OutRemoteContent.Attributes,
+ OutRemoteContent.ChunkedContent.SequenceRawHashes,
+ OutRemoteContent.ChunkedContent.ChunkCounts,
+ AbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ LooseChunkRawSizes,
+ BlockRawHashes);
+
+ // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them
+
+ {
+ if (!IsQuiet)
+ {
+ ZEN_INFO("Fetching metadata for {} blocks", BlockRawHashes.size());
+ }
+
+ Stopwatch GetBlockMetadataTimer;
+
+ bool AttemptFallback = false;
+ OutBlockDescriptions = GetBlockDescriptions(Log(),
+ *Storage.BuildStorage,
+ Storage.CacheStorage.get(),
+ BuildId,
+ BlockRawHashes,
+ AttemptFallback,
+ IsQuiet,
+ IsVerbose);
+
+ if (!IsQuiet)
+ {
+ ZEN_INFO("GetBlockMetadata for {} took {}. Found {} blocks",
+ BuildPartId,
+ NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()),
+ OutBlockDescriptions.size());
+ }
+ }
+
+ CalculateLocalChunkOrders(AbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ LooseChunkRawSizes,
+ OutBlockDescriptions,
+ OutRemoteContent.ChunkedContent.ChunkHashes,
+ OutRemoteContent.ChunkedContent.ChunkRawSizes,
+ OutRemoteContent.ChunkedContent.ChunkOrders,
+ DoExtraContentVerify);
+
+ std::vector<std::filesystem::path> DeletedPaths;
+
+ if (OptionalManifest)
+ {
+ tsl::robin_set<std::string> PathsInManifest;
+ PathsInManifest.reserve(OptionalManifest->Files.size());
+ for (const std::filesystem::path& ManifestPath : OptionalManifest->Files)
+ {
+ PathsInManifest.insert(ToLower(ManifestPath.generic_string()));
+ }
+ for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
+ {
+ if (!PathsInManifest.contains(ToLower(RemotePath.generic_string())))
+ {
+ DeletedPaths.push_back(RemotePath);
+ }
+ }
+ }
+
+ if (!IncludeWildcards.empty() || !ExcludeWildcards.empty())
+ {
+ for (const std::filesystem::path& RemotePath : OutRemoteContent.Paths)
+ {
+ if (!IncludePath(IncludeWildcards, ExcludeWildcards, ToLower(RemotePath.generic_string()), /*CaseSensitive*/ true))
+ {
+ DeletedPaths.push_back(RemotePath);
+ }
+ }
+ }
+
+ if (!DeletedPaths.empty())
+ {
+ OutRemoteContent = DeletePathsFromChunkedContent(OutRemoteContent, DeletedPaths);
+ InlineRemoveUnusedHashes(OutLooseChunkHashes, OutRemoteContent.ChunkedContent.ChunkHashes);
+ }
+
+#if ZEN_BUILD_DEBUG
+ ValidateChunkedFolderContent(OutRemoteContent, OutBlockDescriptions, OutLooseChunkHashes, IncludeWildcards, ExcludeWildcards);
+#endif // ZEN_BUILD_DEBUG
+ };
+
+ auto FindManifest = [&Manifest](const Oid& BuildPartId, std::string_view BuildPartName) -> const BuildManifest::Part* {
+ if (Manifest.Parts.empty())
+ {
+ return nullptr;
+ }
+ if (Manifest.Parts.size() == 1)
+ {
+ if (Manifest.Parts[0].PartId == Oid::Zero && Manifest.Parts[0].PartName.empty())
+ {
+ return &Manifest.Parts[0];
+ }
+ }
+
+ auto It = std::find_if(Manifest.Parts.begin(), Manifest.Parts.end(), [BuildPartId, BuildPartName](const BuildManifest::Part& Part) {
+ if (Part.PartId != Oid::Zero)
+ {
+ return Part.PartId == BuildPartId;
+ }
+ if (!Part.PartName.empty())
+ {
+ return Part.PartName == BuildPartName;
+ }
+ return false;
+ });
+ if (It != Manifest.Parts.end())
+ {
+ return &(*It);
+ }
+ return nullptr;
+ };
+
+ OutPartContents.resize(1);
+ ParseBuildPartManifest(Storage,
+ BuildId,
+ BuildPartId,
+ BuildPartManifest,
+ IncludeWildcards,
+ ExcludeWildcards,
+ FindManifest(BuildPartId, BuildPartName),
+ OutPartContents[0],
+ OutBlockDescriptions,
+ OutLooseChunkHashes);
+ ChunkedFolderContent RemoteContent;
+ if (BuildParts.size() > 1)
+ {
+ std::vector<ChunkBlockDescription> OverlayBlockDescriptions;
+ std::vector<IoHash> OverlayLooseChunkHashes;
+ for (size_t PartIndex = 1; PartIndex < BuildParts.size(); PartIndex++)
+ {
+ const Oid& OverlayBuildPartId = BuildParts[PartIndex].first;
+ const std::string& OverlayBuildPartName = BuildParts[PartIndex].second;
+ Stopwatch GetOverlayBuildPartTimer;
+ CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId);
+ if (!IsQuiet)
+ {
+ ZEN_INFO("GetBuildPart {} ('{}') took {}. Payload size: {}",
+ OverlayBuildPartId,
+ OverlayBuildPartName,
+ NiceTimeSpanMs(GetOverlayBuildPartTimer.GetElapsedTimeMs()),
+ NiceBytes(OverlayBuildPartManifest.GetSize()));
+ }
+
+ ChunkedFolderContent OverlayPartContent;
+ std::vector<ChunkBlockDescription> OverlayPartBlockDescriptions;
+ std::vector<IoHash> OverlayPartLooseChunkHashes;
+
+ ParseBuildPartManifest(Storage,
+ BuildId,
+ OverlayBuildPartId,
+ OverlayBuildPartManifest,
+ IncludeWildcards,
+ ExcludeWildcards,
+ FindManifest(OverlayBuildPartId, OverlayBuildPartName),
+ OverlayPartContent,
+ OverlayPartBlockDescriptions,
+ OverlayPartLooseChunkHashes);
+ OutPartContents.push_back(OverlayPartContent);
+ OverlayBlockDescriptions.insert(OverlayBlockDescriptions.end(),
+ OverlayPartBlockDescriptions.begin(),
+ OverlayPartBlockDescriptions.end());
+ OverlayLooseChunkHashes.insert(OverlayLooseChunkHashes.end(),
+ OverlayPartLooseChunkHashes.begin(),
+ OverlayPartLooseChunkHashes.end());
+ }
+
+ RemoteContent = MergeChunkedFolderContents(OutPartContents[0], std::span<const ChunkedFolderContent>(OutPartContents).subspan(1));
+ {
+ tsl::robin_set<IoHash> AllBlockHashes;
+ for (const ChunkBlockDescription& Description : OutBlockDescriptions)
+ {
+ AllBlockHashes.insert(Description.BlockHash);
+ }
+ for (const ChunkBlockDescription& Description : OverlayBlockDescriptions)
+ {
+ if (!AllBlockHashes.contains(Description.BlockHash))
+ {
+ AllBlockHashes.insert(Description.BlockHash);
+ OutBlockDescriptions.push_back(Description);
+ }
+ }
+ }
+ {
+ tsl::robin_set<IoHash> AllLooseChunkHashes(OutLooseChunkHashes.begin(), OutLooseChunkHashes.end());
+ for (const IoHash& OverlayLooseChunkHash : OverlayLooseChunkHashes)
+ {
+ if (!AllLooseChunkHashes.contains(OverlayLooseChunkHash))
+ {
+ AllLooseChunkHashes.insert(OverlayLooseChunkHash);
+ OutLooseChunkHashes.push_back(OverlayLooseChunkHash);
+ }
+ }
+ }
+ }
+ else
+ {
+ RemoteContent = OutPartContents[0];
+ }
+ return RemoteContent;
+}
+std::string
+GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix)
+{
+ ExtendableStringBuilder<512> SB;
+ std::vector<std::pair<std::string, std::string>> NameStringValuePairs;
+ for (CbFieldView Field : Object)
+ {
+ std::string_view Name = Field.GetName();
+ switch (CbValue Accessor = Field.GetValue(); Accessor.GetType())
+ {
+ case CbFieldType::String:
+ NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())});
+ break;
+ case CbFieldType::IntegerPositive:
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())});
+ break;
+ case CbFieldType::IntegerNegative:
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())});
+ break;
+ case CbFieldType::Float32:
+ {
+ const float Value = Accessor.AsFloat32();
+ if (std::isfinite(Value))
+ {
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)});
+ }
+ else
+ {
+ NameStringValuePairs.push_back({std::string(Name), "null"});
+ }
+ }
+ break;
+ case CbFieldType::Float64:
+ {
+ const double Value = Accessor.AsFloat64();
+ if (std::isfinite(Value))
+ {
+ NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)});
+ }
+ else
+ {
+ NameStringValuePairs.push_back({std::string(Name), "null"});
+ }
+ }
+ break;
+ case CbFieldType::BoolFalse:
+ NameStringValuePairs.push_back({std::string(Name), "false"});
+ break;
+ case CbFieldType::BoolTrue:
+ NameStringValuePairs.push_back({std::string(Name), "true"});
+ break;
+ case CbFieldType::Hash:
+ {
+ NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()});
+ }
+ break;
+ case CbFieldType::Uuid:
+ {
+ StringBuilder<Oid::StringLength + 1> Builder;
+ Accessor.AsUuid().ToString(Builder);
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ }
+ break;
+ case CbFieldType::DateTime:
+ {
+ ExtendableStringBuilder<64> Builder;
+ Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601();
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ }
+ break;
+ case CbFieldType::TimeSpan:
+ {
+ ExtendableStringBuilder<64> Builder;
+ const TimeSpan Span(Accessor.AsTimeSpanTicks());
+ if (Span.GetDays() == 0)
+ {
+ Builder << Span.ToString("%h:%m:%s.%n");
+ }
+ else
+ {
+ Builder << Span.ToString("%d.%h:%m:%s.%n");
+ }
+ NameStringValuePairs.push_back({std::string(Name), Builder.ToString()});
+ break;
+ }
+ case CbFieldType::ObjectId:
+ NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()});
+ break;
+ }
+ }
+ std::string::size_type LongestKey = 0;
+ for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+ {
+ LongestKey = Max(KeyValue.first.length(), LongestKey);
+ }
+ for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs)
+ {
+ SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix));
+ }
+ return SB.ToString();
+}
+
+#if ZEN_WITH_TESTS
+
+namespace buildstorageoperations_testutils {
+ struct TestState
+ {
+ TestState(const std::filesystem::path& InRootPath)
+ : RootPath(InRootPath)
+ , LogOutput(CreateStandardProgress(Log))
+ , ChunkController(CreateStandardChunkingController(StandardChunkingControllerSettings{}))
+ , ChunkCache(CreateMemoryChunkingCache())
+ , WorkerPool(2)
+ , NetworkPool(2)
+ {
+ }
+
+ void Initialize()
+ {
+ StoragePath = RootPath / "storage";
+ TempPath = RootPath / "temp";
+ SystemRootDir = RootPath / "sysroot";
+ ZenFolderPath = RootPath / ".zen";
+
+ CreateDirectories(TempPath);
+ CreateDirectories(StoragePath);
+
+ Storage.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false);
+ }
+
+ void CreateSourceData(const std::filesystem::path& Source, std::span<const std::string> Paths, std::span<const uint64_t> Sizes)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ CreateDirectories(SourcePath);
+ for (size_t FileIndex = 0; FileIndex < Paths.size(); FileIndex++)
+ {
+ const std::string& FilePath = Paths[FileIndex];
+ const uint64_t FileSize = Sizes[FileIndex];
+ IoBuffer FileData = FileSize > 0 ? CreateSemiRandomBlob(FileSize) : IoBuffer{};
+ WriteFile(SourcePath / FilePath, FileData);
+ }
+ }
+
+ std::vector<std::pair<Oid, std::string>> Upload(const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& Source,
+ const std::filesystem::path& ManifestPath)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ CbObject MetaData;
+ BuildsOperationUploadFolder Upload(Log,
+ *LogOutput,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ SourcePath,
+ true,
+ MetaData,
+ BuildsOperationUploadFolder::Options{.TempDir = TempPath});
+ return Upload.Execute(BuildPartId, BuildPartName, ManifestPath, *ChunkController, *ChunkCache);
+ }
+
+ void ValidateUpload(const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& Parts)
+ {
+ for (auto Part : Parts)
+ {
+ BuildsOperationValidateBuildPart Validate(Log,
+ *LogOutput,
+ *Storage.BuildStorage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ Part.first,
+ Part.second,
+ BuildsOperationValidateBuildPart::Options{.TempFolder = TempPath / "validate"});
+ Validate.Execute();
+ }
+ }
+
+ FolderContent Download(const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& Target,
+ bool Append)
+ {
+ const std::filesystem::path TargetPath = RootPath / Target;
+
+ CreateDirectories(TargetPath);
+
+ uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+ CbObject BuildObject = Storage.BuildStorage->GetBuild(BuildId);
+ std::vector<Oid> PartIds;
+ if (BuildPartId != Oid::Zero)
+ {
+ PartIds.push_back(BuildPartId);
+ }
+ std::vector<std::string> PartNames;
+ if (!BuildPartName.empty())
+ {
+ PartNames.push_back(std::string(BuildPartName));
+ }
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(BuildObject, BuildId, PartIds, PartNames, PreferredMultipartChunkSize);
+
+ std::vector<ChunkedFolderContent> PartContents;
+
+ std::vector<ChunkBlockDescription> BlockDescriptions;
+ std::vector<IoHash> LooseChunkHashes;
+
+ ChunkedFolderContent RemoteContent = GetRemoteContent(Log,
+ Storage,
+ BuildId,
+ AllBuildParts,
+ {},
+ {},
+ {},
+ ChunkController,
+ PartContents,
+ BlockDescriptions,
+ LooseChunkHashes,
+ /*IsQuiet*/ false,
+ /*IsVerbose*/ false,
+ /*DoExtraContentVerify*/ true);
+
+ GetFolderContentStatistics LocalFolderScanStats;
+
+ struct ContentVisitor : public GetDirectoryContentVisitor
+ {
+ virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content)
+ {
+ RwLock::ExclusiveLockScope _(ExistingPathsLock);
+ for (const std::filesystem::path& FileName : Content.FileNames)
+ {
+ if (RelativeRoot.empty())
+ {
+ ExistingPaths.push_back(FileName);
+ }
+ else
+ {
+ ExistingPaths.push_back(RelativeRoot / FileName);
+ }
+ }
+ }
+
+ RwLock ExistingPathsLock;
+ std::vector<std::filesystem::path> ExistingPaths;
+ } Visitor;
+
+ Latch PendingWorkCount(1);
+
+ GetDirectoryContent(TargetPath,
+ DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive,
+ Visitor,
+ WorkerPool,
+ PendingWorkCount);
+
+ PendingWorkCount.CountDown();
+ PendingWorkCount.Wait();
+
+ FolderContent CurrentLocalFolderState = GetValidFolderContent(
+ WorkerPool,
+ LocalFolderScanStats,
+ TargetPath,
+ Visitor.ExistingPaths,
+ [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); },
+ 1000,
+ AbortFlag,
+ PauseFlag);
+
+ ChunkingStatistics LocalChunkingStats;
+ ChunkedFolderContent LocalContent = ChunkFolderContent(
+ LocalChunkingStats,
+ WorkerPool,
+ TargetPath,
+ CurrentLocalFolderState,
+ *ChunkController,
+ *ChunkCache,
+ 1000,
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) { ZEN_UNUSED(IsAborted, IsPaused); },
+ AbortFlag,
+ PauseFlag);
+
+ if (Append)
+ {
+ RemoteContent = ApplyChunkedContentOverlay(LocalContent, RemoteContent, {}, {});
+ }
+
+ const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
+ const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
+
+ BuildsOperationUpdateFolder Download(Log,
+ *LogOutput,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ WorkerPool,
+ NetworkPool,
+ BuildId,
+ TargetPath,
+ LocalContent,
+ LocalLookup,
+ RemoteContent,
+ RemoteLookup,
+ BlockDescriptions,
+ LooseChunkHashes,
+ BuildsOperationUpdateFolder::Options{.SystemRootDir = SystemRootDir,
+ .ZenFolderPath = ZenFolderPath,
+ .ValidateCompletedSequences = true});
+ FolderContent ResultingState;
+ Download.Execute(ResultingState);
+
+ return ResultingState;
+ }
+
+ void ValidateDownload(std::span<const std::string> Paths,
+ std::span<const uint64_t> Sizes,
+ const std::filesystem::path& Source,
+ const std::filesystem::path& Target,
+ const FolderContent& DownloadContent)
+ {
+ const std::filesystem::path SourcePath = RootPath / Source;
+ const std::filesystem::path TargetPath = RootPath / Target;
+
+ CHECK_EQ(Paths.size(), DownloadContent.Paths.size());
+ tsl::robin_map<std::string, uint64_t> ExpectedSizes;
+ tsl::robin_map<std::string, IoHash> ExpectedHashes;
+ for (size_t Index = 0; Index < Paths.size(); Index++)
+ {
+ const std::string LookupString = std::filesystem::path(Paths[Index]).generic_string();
+ ExpectedSizes.insert_or_assign(LookupString, Sizes[Index]);
+ std::filesystem::path FilePath = SourcePath / Paths[Index];
+ const IoHash SourceHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
+ ExpectedHashes.insert_or_assign(LookupString, SourceHash);
+ }
+ for (size_t Index = 0; Index < DownloadContent.Paths.size(); Index++)
+ {
+ const std::string LookupString = std::filesystem::path(DownloadContent.Paths[Index]).generic_string();
+ auto SizeIt = ExpectedSizes.find(LookupString);
+ CHECK_NE(SizeIt, ExpectedSizes.end());
+ CHECK_EQ(SizeIt->second, DownloadContent.RawSizes[Index]);
+ std::filesystem::path FilePath = TargetPath / DownloadContent.Paths[Index];
+ const IoHash DownloadedHash = IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(FilePath.make_preferred()));
+ auto HashIt = ExpectedHashes.find(LookupString);
+ CHECK_NE(HashIt, ExpectedHashes.end());
+ CHECK_EQ(HashIt->second, DownloadedHash);
+ }
+ }
+
+ const std::filesystem::path RootPath;
+ std::filesystem::path StoragePath;
+ std::filesystem::path TempPath;
+ std::filesystem::path SystemRootDir;
+ std::filesystem::path ZenFolderPath;
+
+ LoggerRef Log = ConsoleLog();
+ std::unique_ptr<ProgressBase> LogOutput;
+
+ std::unique_ptr<ChunkingController> ChunkController;
+ std::unique_ptr<ChunkingCache> ChunkCache;
+
+ StorageInstance Storage;
+ BuildStorageBase::Statistics StorageStats;
+
+ WorkerThreadPool WorkerPool;
+ WorkerThreadPool NetworkPool;
+
+ std::atomic<bool> AbortFlag;
+ std::atomic<bool> PauseFlag;
+ };
+
+} // namespace buildstorageoperations_testutils
+
+TEST_SUITE_BEGIN("remotestore.buildstorageutil");
+
+TEST_CASE("normalizepartselection.empty_defaults_to_default")
+{
+ std::vector<Oid> Ids;
+ std::vector<std::string> Names;
+ NormalizePartSelection(Ids, Names, {});
+ CHECK(Ids.empty());
+ REQUIRE_EQ(Names.size(), 1u);
+ CHECK_EQ(Names[0], "default");
+}
+
+TEST_CASE("normalizepartselection.wildcard_alone_clears_names")
+{
+ std::vector<Oid> Ids;
+ std::vector<std::string> Names = {"*"};
+ NormalizePartSelection(Ids, Names, {});
+ CHECK(Ids.empty());
+ CHECK(Names.empty());
+}
+
+TEST_CASE("normalizepartselection.wildcard_with_other_name_throws")
+{
+ std::vector<Oid> Ids;
+ std::vector<std::string> Names = {"*", "foo"};
+ CHECK_THROWS_AS(NormalizePartSelection(Ids, Names, {}), OptionParseException);
+}
+
+TEST_CASE("normalizepartselection.wildcard_with_ids_throws")
+{
+ std::vector<Oid> Ids = {Oid::NewOid()};
+ std::vector<std::string> Names = {"*"};
+ CHECK_THROWS_AS(NormalizePartSelection(Ids, Names, {}), OptionParseException);
+}
+
+TEST_CASE("normalizepartselection.explicit_name_unchanged")
+{
+ std::vector<Oid> Ids;
+ std::vector<std::string> Names = {"foo"};
+ NormalizePartSelection(Ids, Names, {});
+ CHECK(Ids.empty());
+ REQUIRE_EQ(Names.size(), 1u);
+ CHECK_EQ(Names[0], "foo");
+}
+
+TEST_CASE("normalizepartselection.ids_only_unchanged")
+{
+ const Oid Id = Oid::NewOid();
+ std::vector<Oid> Ids = {Id};
+ std::vector<std::string> Names;
+ NormalizePartSelection(Ids, Names, {});
+ REQUIRE_EQ(Ids.size(), 1u);
+ CHECK_EQ(Ids[0], Id);
+ CHECK(Names.empty());
+}
+
+TEST_CASE("buildstorageoperations.upload.folder")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ const std::string BuildPartName = "default";
+
+ auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {});
+
+ CHECK_EQ(Result.size(), 1u);
+ CHECK_EQ(Result[0].first, BuildPartId);
+ CHECK_EQ(Result[0].second, BuildPartName);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ CHECK_EQ(DownloadContent.Paths.size(), FileCount);
+ State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
+}
+
+TEST_CASE("buildstorageoperations.upload.manifest")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ std::span<const std::string> ManifestFiles(Paths);
+ ManifestFiles = ManifestFiles.subspan(0, FileCount / 2);
+
+ std::span<const uint64_t> ManifestSizes(Sizes);
+ ManifestSizes = ManifestSizes.subspan(0, FileCount / 2);
+
+ ExtendableStringBuilder<1024> Manifest;
+ for (const std::string& FilePath : ManifestFiles)
+ {
+ Manifest << FilePath << "\n";
+ }
+
+ WriteFile(State.RootPath / "manifest.txt", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ const std::string BuildPartName = "default";
+
+ auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", State.RootPath / "manifest.txt");
+
+ CHECK_EQ(Result.size(), 1u);
+ CHECK_EQ(Result[0].first, BuildPartId);
+ CHECK_EQ(Result[0].second, BuildPartName);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ State.ValidateDownload(ManifestFiles, ManifestSizes, "source", "download", DownloadContent);
+}
+
+TEST_CASE("buildstorageoperations.memorychunkingcache")
+{
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ const Oid BuildId = Oid::NewOid();
+ const Oid BuildPartId = Oid::NewOid();
+ const std::string BuildPartName = "default";
+
+ {
+ const std::filesystem::path SourcePath = SourceFolder.Path() / "source";
+ CbObject MetaData;
+ BuildsOperationUploadFolder Upload(State.Log,
+ *State.LogOutput,
+ State.Storage,
+ State.AbortFlag,
+ State.PauseFlag,
+ State.WorkerPool,
+ State.NetworkPool,
+ BuildId,
+ SourcePath,
+ true,
+ MetaData,
+ BuildsOperationUploadFolder::Options{.TempDir = State.TempPath});
+ auto Result = Upload.Execute(BuildPartId, BuildPartName, {}, *State.ChunkController, *State.ChunkCache);
+
+ CHECK_EQ(Upload.m_ChunkingStats.FilesStoredInCache.load(), FileCount - 1); // Zero size files are not stored in cache
+ CHECK_EQ(Upload.m_ChunkingStats.BytesStoredInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0)));
+ CHECK(Upload.m_ChunkingStats.ChunksStoredInCache.load() >= FileCount - 1); // Zero size files are not stored in cache
+
+ CHECK_EQ(Result.size(), 1u);
+ CHECK_EQ(Result[0].first, BuildPartId);
+ CHECK_EQ(Result[0].second, BuildPartName);
+ }
+
+ auto Result = State.Upload(BuildId, BuildPartId, BuildPartName, "source", {});
+
+ const Oid BuildId2 = Oid::NewOid();
+ const Oid BuildPartId2 = Oid::NewOid();
+
+ {
+ const std::filesystem::path SourcePath = SourceFolder.Path() / "source";
+ CbObject MetaData;
+ BuildsOperationUploadFolder Upload(State.Log,
+ *State.LogOutput,
+ State.Storage,
+ State.AbortFlag,
+ State.PauseFlag,
+ State.WorkerPool,
+ State.NetworkPool,
+ BuildId2,
+ SourcePath,
+ true,
+ MetaData,
+ BuildsOperationUploadFolder::Options{.TempDir = State.TempPath});
+ Upload.Execute(BuildPartId2, BuildPartName, {}, *State.ChunkController, *State.ChunkCache);
+
+ CHECK_EQ(Upload.m_ChunkingStats.FilesFoundInCache.load(), FileCount - 1); // Zero size files are not stored in cache
+ CHECK_EQ(Upload.m_ChunkingStats.BytesFoundInCache.load(), std::accumulate(&Sizes[0], &Sizes[FileCount], uint64_t(0)));
+ CHECK(Upload.m_ChunkingStats.ChunksFoundInCache.load() >= FileCount - 1); // Zero size files are not stored in cache
+ }
+
+ FolderContent DownloadContent = State.Download(BuildId2, BuildPartId2, {}, "download", /* Append */ false);
+ State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
+}
+
+TEST_CASE("buildstorageoperations.upload.multipart")
+{
+ // Disabled since it relies on authentication and specific block being present in cloud storage
+ if (false)
+ {
+ using namespace buildstorageoperations_testutils;
+
+ FastRandom BaseRandom;
+
+ const size_t FileCount = 11;
+
+ const std::string Paths[FileCount] = {{"file_1"},
+ {"file_2.exe"},
+ {"file_3.txt"},
+ {"dir_1/dir1_file_1.exe"},
+ {"dir_1/dir1_file_2.pdb"},
+ {"dir_1/dir1_file_3.txt"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"},
+ {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"},
+ {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}};
+ const uint64_t Sizes[FileCount] =
+ {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u};
+
+ ScopedTemporaryDirectory SourceFolder;
+ TestState State(SourceFolder.Path());
+ State.Initialize();
+ State.CreateSourceData("source", Paths, Sizes);
+
+ std::span<const std::string> ManifestFiles1(Paths);
+ ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2);
+
+ std::span<const uint64_t> ManifestSizes1(Sizes);
+ ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2);
+
+ std::span<const std::string> ManifestFiles2(Paths);
+ ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1);
+
+ std::span<const uint64_t> ManifestSizes2(Sizes);
+ ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1);
+
+ const Oid BuildPart1Id = Oid::NewOid();
+ const std::string BuildPart1Name = "part1";
+ const Oid BuildPart2Id = Oid::NewOid();
+ const std::string BuildPart2Name = "part2";
+ {
+ CbObjectWriter Writer;
+ Writer.BeginObject("parts"sv);
+ {
+ Writer.BeginObject(BuildPart1Name);
+ {
+ Writer.AddObjectId("partId"sv, BuildPart1Id);
+ Writer.BeginArray("files"sv);
+ for (const std::string& ManifestFile : ManifestFiles1)
+ {
+ Writer.AddString(ManifestFile);
+ }
+ Writer.EndArray(); // files
+ }
+ Writer.EndObject(); // part1
+
+ Writer.BeginObject(BuildPart2Name);
+ {
+ Writer.AddObjectId("partId"sv, BuildPart2Id);
+ Writer.BeginArray("files"sv);
+ for (const std::string& ManifestFile : ManifestFiles2)
+ {
+ Writer.AddString(ManifestFile);
+ }
+ Writer.EndArray(); // files
+ }
+ Writer.EndObject(); // part2
+ }
+ Writer.EndObject(); // parts
+
+ ExtendableStringBuilder<1024> Manifest;
+ CompactBinaryToJson(Writer.Save(), Manifest);
+ WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size()));
+ }
+
+ const Oid BuildId = Oid::NewOid();
+
+ auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json");
+
+ CHECK_EQ(Result.size(), 2u);
+ CHECK_EQ(Result[0].first, BuildPart1Id);
+ CHECK_EQ(Result[0].second, BuildPart1Name);
+ CHECK_EQ(Result[1].first, BuildPart2Id);
+ CHECK_EQ(Result[1].second, BuildPart2Name);
+ State.ValidateUpload(BuildId, Result);
+
+ FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false);
+ State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent);
+
+ FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false);
+ State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent);
+
+ FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false);
+ State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent);
+
+ (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false);
+ FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true);
+ State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent);
+ }
+}
+
+TEST_CASE("buildstorageoperations.partial.block.download" * doctest::skip(true))
+{
+ const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL;
+ std::filesystem::path OidcTokenExePath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred();
+
+ HttpClientSettings ClientSettings{
+ .LogCategory = "httpbuildsclient",
+ .AccessTokenProvider =
+ httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, "https://jupiter.devtools.epicgames.com", true, false, false),
+ .AssumeHttp2 = false,
+ .AllowResume = true,
+ .RetryCount = 0,
+ .Verbose = false};
+
+ HttpClient HttpClient("https://euc.jupiter.devtools.epicgames.com", ClientSettings);
+
+ const std::string_view Namespace = "fortnite.oplog";
+ const std::string_view Bucket = "fortnitegame.staged-build.fortnite-main.ps4-client";
+ const Oid BuildId = Oid::FromHexString("09a76ea92ad301d4724fafad");
+
+ {
+ HttpClient::Response Response = HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, Bucket, BuildId),
+ HttpClient::Accept(ZenContentType::kCbObject));
+ CbValidateError ValidateResult = CbValidateError::None;
+ CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Response.ResponsePayload), ValidateResult);
+ REQUIRE(ValidateResult == CbValidateError::None);
+ }
+
+ std::vector<ChunkBlockDescription> BlockDescriptions;
+ {
+ CbObjectWriter Request;
+
+ Request.BeginArray("blocks"sv);
+ {
+ Request.AddHash(IoHash::FromHexString("7c353ed782675a5e8f968e61e51fc797ecdc2882"));
+ }
+ Request.EndArray();
+
+ IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer();
+ Payload.SetContentType(ZenContentType::kCbObject);
+
+ HttpClient::Response BlockDescriptionsResponse =
+ HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, Bucket, BuildId),
+ Payload,
+ HttpClient::Accept(ZenContentType::kCbObject));
+ REQUIRE(BlockDescriptionsResponse.IsSuccess());
+
+ CbValidateError ValidateResult = CbValidateError::None;
+ CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(BlockDescriptionsResponse.ResponsePayload), ValidateResult);
+ REQUIRE(ValidateResult == CbValidateError::None);
+
+ {
+ CbArrayView BlocksArray = Object["blocks"sv].AsArrayView();
+ for (CbFieldView Block : BlocksArray)
+ {
+ ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView());
+ BlockDescriptions.emplace_back(std::move(Description));
+ }
+ }
+ }
+
+ REQUIRE(!BlockDescriptions.empty());
+
+ const IoHash BlockHash = BlockDescriptions.back().BlockHash;
+
+ const ChunkBlockDescription& BlockDescription = BlockDescriptions.front();
+ REQUIRE(!BlockDescription.ChunkRawHashes.empty());
+ REQUIRE(!BlockDescription.ChunkCompressedLengths.empty());
+
+ std::vector<std::pair<uint64_t, uint64_t>> ChunkOffsetAndSizes;
+ uint64_t Offset = gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize);
+
+ for (uint32_t ChunkCompressedSize : BlockDescription.ChunkCompressedLengths)
+ {
+ ChunkOffsetAndSizes.push_back(std::make_pair(Offset, ChunkCompressedSize));
+ Offset += ChunkCompressedSize;
+ }
+
+ ScopedTemporaryDirectory SourceFolder;
+
+ auto Validate = [&](std::span<const uint32_t> ChunkIndexesToFetch) {
+ std::vector<std::pair<uint64_t, uint64_t>> Ranges;
+ for (uint32_t ChunkIndex : ChunkIndexesToFetch)
+ {
+ Ranges.push_back(ChunkOffsetAndSizes[ChunkIndex]);
+ }
+
+ HttpClient::KeyValueMap Headers;
+ if (!Ranges.empty())
+ {
+ ExtendableStringBuilder<512> SB;
+ for (const std::pair<uint64_t, uint64_t>& R : Ranges)
+ {
+ if (SB.Size() > 0)
+ {
+ SB << ", ";
+ }
+ SB << R.first << "-" << R.first + R.second - 1;
+ }
+ Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())});
+ }
+
+ HttpClient::Response GetBlobRangesResponse = HttpClient.Download(
+ fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect=false", Namespace, Bucket, BuildId, BlockHash),
+ SourceFolder.Path(),
+ Headers);
+
+ REQUIRE(GetBlobRangesResponse.IsSuccess());
+ [[maybe_unused]] MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView();
+
+ std::vector<std::pair<uint64_t, uint64_t>> PayloadRanges = GetBlobRangesResponse.GetRanges(Ranges);
+ if (PayloadRanges.empty())
+ {
+ // We got the whole blob, use the ranges as is
+ PayloadRanges = Ranges;
+ }
+
+ REQUIRE(PayloadRanges.size() == Ranges.size());
+
+ for (uint32_t RangeIndex = 0; RangeIndex < PayloadRanges.size(); RangeIndex++)
+ {
+ const std::pair<uint64_t, uint64_t>& PayloadRange = PayloadRanges[RangeIndex];
+
+ CHECK_EQ(PayloadRange.second, Ranges[RangeIndex].second);
+
+ IoBuffer ChunkPayload(GetBlobRangesResponse.ResponsePayload, PayloadRange.first, PayloadRange.second);
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(ChunkPayload), RawHash, RawSize);
+ CHECK(CompressedChunk);
+ CHECK_EQ(RawHash, BlockDescription.ChunkRawHashes[ChunkIndexesToFetch[RangeIndex]]);
+ CHECK_EQ(RawSize, BlockDescription.ChunkRawLengths[ChunkIndexesToFetch[RangeIndex]]);
+ }
+ };
+
+ {
+ // Single
+ std::vector<uint32_t> ChunkIndexesToFetch{uint32_t(BlockDescription.ChunkCompressedLengths.size() / 2)};
+ Validate(ChunkIndexesToFetch);
+ }
+ {
+ // Many
+ std::vector<uint32_t> ChunkIndexesToFetch;
+ for (uint32_t Index = 0; Index < BlockDescription.ChunkCompressedLengths.size() / 16; Index++)
+ {
+ ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7));
+ ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 1));
+ ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 3));
+ }
+ Validate(ChunkIndexesToFetch);
+ }
+
+ {
+ // First and last
+ std::vector<uint32_t> ChunkIndexesToFetch{0, uint32_t(BlockDescription.ChunkCompressedLengths.size() - 1)};
+ Validate(ChunkIndexesToFetch);
+ }
+}
+TEST_SUITE_END();
+
+void
+buildstorageutil_forcelink()
+{
+}
+
+#endif // ZEN_WITH_TESTS
+
} // namespace zen
diff --git a/src/zenremotestore/builds/buildupdatefolder.cpp b/src/zenremotestore/builds/buildupdatefolder.cpp
new file mode 100644
index 000000000..ebef12900
--- /dev/null
+++ b/src/zenremotestore/builds/buildupdatefolder.cpp
@@ -0,0 +1,4948 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildupdatefolder.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/fmtutils.h>
+#include <zencore/parallelwork.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/buildcontent.h>
+#include <zenremotestore/builds/buildmanifest.h>
+#include <zenremotestore/chunking/chunkingcache.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
+#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/filteredrate.h>
+#include <zenutil/progress.h>
+
+#include <numeric>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace {
+ std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath)
+ {
+ return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences
+ }
+ std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath)
+ {
+ return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks
+ }
+ std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath)
+ {
+ return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks
+ }
+ std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
+ {
+ return CacheFolderPath / (RawHash.ToHexString() + ".tmp");
+ }
+
+ std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash)
+ {
+ return CacheFolderPath / RawHash.ToHexString();
+ }
+ bool CleanDirectory(LoggerRef InLog,
+ ProgressBase& Progress,
+ WorkerThreadPool& IOWorkerPool,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ bool IsQuiet,
+ const std::filesystem::path& Path,
+ std::span<const std::string> ExcludeDirectories)
+ {
+ ZEN_TRACE_CPU("CleanDirectory");
+ ZEN_SCOPED_LOG(InLog);
+ Stopwatch Timer;
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = Progress.CreateProgressBar("Clean Folder");
+
+ CleanDirectoryResult Result = CleanDirectory(
+ IOWorkerPool,
+ AbortFlag,
+ PauseFlag,
+ Path,
+ ExcludeDirectories,
+ [&](const std::string_view Details, uint64_t TotalCount, uint64_t RemainingCount, bool IsPaused, bool IsAborted) {
+ ProgressBar->UpdateState({.Task = "Cleaning folder ",
+ .Details = std::string(Details),
+ .TotalCount = TotalCount,
+ .RemainingCount = RemainingCount,
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ },
+ Progress.GetProgressUpdateDelayMS());
+
+ ProgressBar->Finish();
+
+ if (AbortFlag)
+ {
+ return false;
+ }
+
+ uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs();
+
+ if (!Result.FailedRemovePaths.empty())
+ {
+ ExtendableStringBuilder<512> SB;
+ for (size_t FailedPathIndex = 0; FailedPathIndex < Result.FailedRemovePaths.size(); FailedPathIndex++)
+ {
+ SB << fmt::format("\n '{}': ({}) {}",
+ Result.FailedRemovePaths[FailedPathIndex].first,
+ Result.FailedRemovePaths[FailedPathIndex].second.value(),
+ Result.FailedRemovePaths[FailedPathIndex].second.message());
+ }
+ ZEN_WARN("Clean failed to remove files from '{}': {}", Path, SB.ToView());
+ }
+
+ if (ElapsedTimeMs >= 200 && !IsQuiet)
+ {
+ ZEN_INFO("Wiped folder '{}' {} ({}) in {}",
+ Path,
+ Result.FoundCount,
+ NiceBytes(Result.DeletedByteCount),
+ NiceTimeSpanMs(ElapsedTimeMs));
+ }
+
+ return Result.FailedRemovePaths.empty();
+ }
+ uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ if (SourcePlatform == SourcePlatform::Windows)
+ {
+ SetFileAttributesToPath(FilePath, Attributes);
+ return Attributes;
+ }
+ else
+ {
+ uint32_t CurrentAttributes = GetFileAttributesFromPath(FilePath);
+ uint32_t NewAttributes = zen::MakeFileAttributeReadOnly(CurrentAttributes, zen::IsFileModeReadOnly(Attributes));
+ if (CurrentAttributes != NewAttributes)
+ {
+ SetFileAttributesToPath(FilePath, NewAttributes);
+ }
+ return NewAttributes;
+ }
+#endif // ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ if (SourcePlatform != SourcePlatform::Windows)
+ {
+ zen::SetFileMode(FilePath, Attributes);
+ return Attributes;
+ }
+ else
+ {
+ uint32_t CurrentMode = zen::GetFileMode(FilePath);
+ uint32_t NewMode = zen::MakeFileModeReadOnly(CurrentMode, zen::IsFileAttributeReadOnly(Attributes));
+ if (CurrentMode != NewMode)
+ {
+ zen::SetFileMode(FilePath, NewMode);
+ }
+ return NewMode;
+ }
+#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ };
+
+ uint32_t GetNativeFileAttributes(const std::filesystem::path FilePath)
+ {
+#if ZEN_PLATFORM_WINDOWS
+ return GetFileAttributesFromPath(FilePath);
+#endif // ZEN_PLATFORM_WINDOWS
+#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ return GetFileMode(FilePath);
+#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
+ }
+ std::filesystem::path TryMoveDownloadedChunk(IoBuffer& BlockBuffer, const std::filesystem::path& Path, bool ForceDiskBased)
+ {
+ uint64_t BlockSize = BlockBuffer.GetSize();
+ IoBufferFileReference FileRef;
+ if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == BlockSize))
+ {
+ ZEN_TRACE_CPU("MoveTempFullBlock");
+ std::error_code Ec;
+ std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec);
+ if (!Ec)
+ {
+ BlockBuffer.SetDeleteOnClose(false);
+ BlockBuffer = {};
+ RenameFile(TempBlobPath, Path, Ec);
+ if (Ec)
+ {
+ // Re-open the temp file again
+ BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete);
+ BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true);
+ BlockBuffer.SetDeleteOnClose(true);
+ }
+ else
+ {
+ return Path;
+ }
+ }
+ }
+
+ if (ForceDiskBased)
+ {
+ // Could not be moved and rather large, lets store it on disk
+ ZEN_TRACE_CPU("WriteTempFullBlock");
+ TemporaryFile::SafeWriteFile(Path, BlockBuffer);
+ BlockBuffer = {};
+ return Path;
+ }
+
+ return {};
+ }
+ bool IsSingleFileChunk(const ChunkedFolderContent& RemoteContent,
+ const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> Locations)
+ {
+ if (Locations.size() == 1)
+ {
+ const uint32_t FirstSequenceIndex = Locations[0]->SequenceIndex;
+ if (RemoteContent.ChunkedContent.ChunkCounts[FirstSequenceIndex] == 1)
+ {
+ ZEN_ASSERT_SLOW(Locations[0]->Offset == 0);
+ return true;
+ }
+ }
+ return false;
+ }
+ IoBuffer MakeBufferMemoryBased(const CompositeBuffer& PartialBlockBuffer)
+ {
+ ZEN_TRACE_CPU("MakeBufferMemoryBased");
+ IoBuffer BlockMemoryBuffer;
+ std::span<const SharedBuffer> Segments = PartialBlockBuffer.GetSegments();
+ if (Segments.size() == 1)
+ {
+ IoBufferFileReference FileRef = {};
+ if (PartialBlockBuffer.GetSegments().front().AsIoBuffer().GetFileReference(FileRef))
+ {
+ BlockMemoryBuffer = UniqueBuffer::Alloc(FileRef.FileChunkSize).MoveToShared().AsIoBuffer();
+ BasicFile Reader;
+ Reader.Attach(FileRef.FileHandle);
+ auto _ = MakeGuard([&Reader]() { Reader.Detach(); });
+ MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
+ Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
+ return BlockMemoryBuffer;
+ }
+ else
+ {
+ return PartialBlockBuffer.GetSegments().front().AsIoBuffer();
+ }
+ }
+ else
+ {
+ // Not a homogenous memory buffer, read all to memory
+
+ BlockMemoryBuffer = UniqueBuffer::Alloc(PartialBlockBuffer.GetSize()).MoveToShared().AsIoBuffer();
+ MutableMemoryView ReadMem = BlockMemoryBuffer.GetMutableView();
+ for (const SharedBuffer& Segment : Segments)
+ {
+ IoBufferFileReference FileRef = {};
+ if (Segment.AsIoBuffer().GetFileReference(FileRef))
+ {
+ BasicFile Reader;
+ Reader.Attach(FileRef.FileHandle);
+ auto _ = MakeGuard([&Reader]() { Reader.Detach(); });
+ Reader.Read(ReadMem.GetData(), FileRef.FileChunkSize, FileRef.FileChunkOffset);
+ ReadMem = ReadMem.Mid(FileRef.FileChunkSize);
+ }
+ else
+ {
+ ReadMem = ReadMem.CopyFrom(Segment.AsIoBuffer().GetView());
+ }
+ }
+ return BlockMemoryBuffer;
+ }
+ }
+
+ FolderContent CheckFolderFiles(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ std::string_view ProgressLabel,
+ TransferThreadWorkers& Workers,
+ GetFolderContentStatistics& LocalFolderScanStats,
+ const std::filesystem::path& Path,
+ std::span<const std::filesystem::path> PathsToCheck)
+ {
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = Progress.CreateProgressBar(ProgressLabel);
+ FolderContent Result = GetValidFolderContent(
+ Workers.GetIOWorkerPool(),
+ LocalFolderScanStats,
+ Path,
+ PathsToCheck,
+ [&ProgressBar, &LocalFolderScanStats, &AbortFlag, &PauseFlag](uint64_t PathCount, uint64_t CompletedPathCount) {
+ std::string Details =
+ fmt::format("{}/{} checked, {} found", CompletedPathCount, PathCount, LocalFolderScanStats.FoundFileCount.load());
+ ProgressBar->UpdateState({.Task = "Checking files ",
+ .Details = Details,
+ .TotalCount = PathCount,
+ .RemainingCount = PathCount - CompletedPathCount,
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(AbortFlag, PauseFlag)},
+ false);
+ },
+ Progress.GetProgressUpdateDelayMS(),
+ AbortFlag,
+ PauseFlag);
+ ProgressBar->Finish();
+ return Result;
+ }
+
+ ChunkedFolderContent ScanFolderFiles(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ std::string_view ProgressLabel,
+ TransferThreadWorkers& Workers,
+ const std::filesystem::path& Path,
+ const FolderContent& FolderSource,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ ChunkingStatistics& OutChunkingStats)
+ {
+ uint64_t ByteCountToScan = 0;
+ for (const uint64_t RawSize : FolderSource.RawSizes)
+ {
+ ByteCountToScan += RawSize;
+ }
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = Progress.CreateProgressBar(ProgressLabel);
+ FilteredRate FilteredBytesHashed;
+ FilteredBytesHashed.Start();
+ ChunkingStatistics LocalChunkingStats;
+ ChunkedFolderContent Result = ChunkFolderContent(
+ LocalChunkingStats,
+ Workers.GetIOWorkerPool(),
+ Path,
+ FolderSource,
+ ChunkController,
+ ChunkCache,
+ Progress.GetProgressUpdateDelayMS(),
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
+ FilteredBytesHashed.Update(LocalChunkingStats.BytesHashed.load());
+ std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+ LocalChunkingStats.FilesProcessed.load(),
+ FolderSource.Paths.size(),
+ NiceBytes(LocalChunkingStats.BytesHashed.load()),
+ NiceBytes(ByteCountToScan),
+ NiceNum(FilteredBytesHashed.GetCurrent()),
+ LocalChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(LocalChunkingStats.UniqueBytesFound.load()));
+ ProgressBar->UpdateState({.Task = "Scanning files ",
+ .Details = Details,
+ .TotalCount = ByteCountToScan,
+ .RemainingCount = ByteCountToScan - LocalChunkingStats.BytesHashed.load(),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ },
+ AbortFlag,
+ PauseFlag);
+ OutChunkingStats += LocalChunkingStats;
+ FilteredBytesHashed.Stop();
+ ProgressBar->Finish();
+ return Result;
+ }
+} // namespace
+
+BuildsOperationUpdateFolder::BuildsOperationUpdateFolder(LoggerRef Log,
+ ProgressBase& Progress,
+ StorageInstance& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ WorkerThreadPool& IOWorkerPool,
+ WorkerThreadPool& NetworkPool,
+ const Oid& BuildId,
+ const std::filesystem::path& Path,
+ const ChunkedFolderContent& LocalContent,
+ const ChunkedContentLookup& LocalLookup,
+ const ChunkedFolderContent& RemoteContent,
+ const ChunkedContentLookup& RemoteLookup,
+ const std::vector<ChunkBlockDescription>& BlockDescriptions,
+ const std::vector<IoHash>& LooseChunkHashes,
+ const Options& Options)
+: m_Log(Log)
+, m_Progress(Progress)
+, m_Storage(Storage)
+, m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_IOWorkerPool(IOWorkerPool)
+, m_NetworkPool(NetworkPool)
+, m_BuildId(BuildId)
+, m_Path(Path)
+, m_LocalContent(LocalContent)
+, m_LocalLookup(LocalLookup)
+, m_RemoteContent(RemoteContent)
+, m_RemoteLookup(RemoteLookup)
+, m_BlockDescriptions(BlockDescriptions)
+, m_LooseChunkHashes(LooseChunkHashes)
+, m_Options(Options)
+, m_CacheFolderPath(ZenTempCacheFolderPath(m_Options.ZenFolderPath))
+, m_TempDownloadFolderPath(ZenTempDownloadFolderPath(m_Options.ZenFolderPath))
+, m_TempBlockFolderPath(ZenTempBlockFolderPath(m_Options.ZenFolderPath))
+{
+}
+
+void
+BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState)
+{
+ ZEN_TRACE_CPU("BuildsOperationUpdateFolder::Execute");
+ try
+ {
+ enum class TaskSteps : uint32_t
+ {
+ ScanExistingData,
+ WriteChunks,
+ PrepareTarget,
+ FinalizeTarget,
+ Cleanup,
+ StepCount
+ };
+
+ auto EndProgress =
+ MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); });
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ScanExistingData, (uint32_t)TaskSteps::StepCount);
+
+ CreateDirectories(m_CacheFolderPath);
+ CreateDirectories(m_TempDownloadFolderPath);
+ CreateDirectories(m_TempBlockFolderPath);
+
+ std::vector<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters(m_RemoteContent.ChunkedContent.SequenceRawHashes.size());
+ std::vector<bool> RemoteChunkIndexNeedsCopyFromLocalFileFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size());
+ std::vector<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags(m_RemoteContent.ChunkedContent.ChunkHashes.size());
+
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound;
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound;
+ ScanCacheFolder(CachedChunkHashesFound, CachedSequenceHashesFound);
+
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound;
+ ScanTempBlocksFolder(CachedBlocksFound);
+
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceIndexesLeftToFindToRemoteIndex;
+ InitializeSequenceCounters(SequenceIndexChunksLeftToWriteCounters,
+ SequenceIndexesLeftToFindToRemoteIndex,
+ CachedChunkHashesFound,
+ CachedSequenceHashesFound);
+
+ std::vector<ChunkedFolderContent> ScavengedContents;
+ std::vector<ChunkedContentLookup> ScavengedLookups;
+ std::vector<std::filesystem::path> ScavengedPaths;
+
+ std::vector<ScavengedSequenceCopyOperation> ScavengedSequenceCopyOperations;
+ uint64_t ScavengedPathsCount = 0;
+
+ if (m_Options.EnableOtherDownloadsScavenging)
+ {
+ ZEN_TRACE_CPU("GetScavengedSequences");
+
+ Stopwatch ScavengeTimer;
+
+ if (!SequenceIndexesLeftToFindToRemoteIndex.empty())
+ {
+ std::vector<ScavengeSource> ScavengeSources = FindScavengeSources();
+ ScanScavengeSources(ScavengeSources, ScavengedContents, ScavengedLookups, ScavengedPaths);
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ MatchScavengedSequencesToRemote(ScavengedContents,
+ ScavengedLookups,
+ ScavengedPaths,
+ SequenceIndexesLeftToFindToRemoteIndex,
+ SequenceIndexChunksLeftToWriteCounters,
+ ScavengedSequenceCopyOperations,
+ ScavengedPathsCount);
+ }
+ m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
+ }
+
+ uint32_t RemainingChunkCount = 0;
+ for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
+ {
+ uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
+ if (ChunkWriteCount > 0)
+ {
+ RemainingChunkCount++;
+ }
+ }
+
+ // Pick up all chunks in current local state
+ tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCopyChunkDataIndex;
+ std::vector<CopyChunkData> CopyChunkDatas;
+
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ ZEN_TRACE_CPU("GetLocalChunks");
+
+ Stopwatch LocalTimer;
+
+ ScavengeSourceForChunks(RemainingChunkCount,
+ RemoteChunkIndexNeedsCopyFromLocalFileFlags,
+ RawHashToCopyChunkDataIndex,
+ SequenceIndexChunksLeftToWriteCounters,
+ m_LocalContent,
+ m_LocalLookup,
+ CopyChunkDatas,
+ uint32_t(-1),
+ m_CacheMappingStats.LocalChunkMatchingRemoteCount,
+ m_CacheMappingStats.LocalChunkMatchingRemoteByteCount);
+
+ m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
+ }
+
+ if (m_Options.EnableOtherDownloadsScavenging)
+ {
+ ZEN_TRACE_CPU("GetScavengeChunks");
+
+ Stopwatch ScavengeTimer;
+
+ for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < ScavengedContents.size() && (RemainingChunkCount > 0);
+ ScavengedContentIndex++)
+ {
+ const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengedContentIndex];
+ const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[ScavengedContentIndex];
+
+ ScavengeSourceForChunks(RemainingChunkCount,
+ RemoteChunkIndexNeedsCopyFromLocalFileFlags,
+ RawHashToCopyChunkDataIndex,
+ SequenceIndexChunksLeftToWriteCounters,
+ ScavengedContent,
+ ScavengedLookup,
+ CopyChunkDatas,
+ ScavengedContentIndex,
+ m_CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount);
+ }
+ m_CacheMappingStats.ScavengeElapsedWallTimeUs += ScavengeTimer.GetElapsedTimeUs();
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ if (m_CacheMappingStats.CacheSequenceHashesCount > 0 || m_CacheMappingStats.CacheChunkCount > 0 ||
+ m_CacheMappingStats.CacheBlockCount > 0)
+ {
+ ZEN_INFO("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks in {}",
+ m_CacheMappingStats.CacheSequenceHashesCount,
+ NiceBytes(m_CacheMappingStats.CacheSequenceHashesByteCount),
+ m_CacheMappingStats.CacheChunkCount,
+ NiceBytes(m_CacheMappingStats.CacheChunkByteCount),
+ m_CacheMappingStats.CacheBlockCount,
+ NiceBytes(m_CacheMappingStats.CacheBlocksByteCount),
+ NiceTimeSpanMs(m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000));
+ }
+
+ if (m_CacheMappingStats.LocalPathsMatchingSequencesCount > 0 || m_CacheMappingStats.LocalChunkMatchingRemoteCount > 0)
+ {
+ ZEN_INFO("Local state : Found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ m_CacheMappingStats.LocalPathsMatchingSequencesCount,
+ NiceBytes(m_CacheMappingStats.LocalPathsMatchingSequencesByteCount),
+ m_CacheMappingStats.LocalChunkMatchingRemoteCount,
+ NiceBytes(m_CacheMappingStats.LocalChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000));
+ }
+ if (m_CacheMappingStats.ScavengedPathsMatchingSequencesCount > 0 || m_CacheMappingStats.ScavengedChunkMatchingRemoteCount > 0)
+ {
+ ZEN_INFO("Scavenge of {} paths, found {} ({}) chunk sequences, {} ({}) chunks in {}",
+ ScavengedPathsCount,
+ m_CacheMappingStats.ScavengedPathsMatchingSequencesCount,
+ NiceBytes(m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount),
+ m_CacheMappingStats.ScavengedChunkMatchingRemoteCount,
+ NiceBytes(m_CacheMappingStats.ScavengedChunkMatchingRemoteByteCount),
+ NiceTimeSpanMs(m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000));
+ }
+ }
+
+ uint64_t BytesToWrite = CalculateBytesToWriteAndFlagNeededChunks(SequenceIndexChunksLeftToWriteCounters,
+ RemoteChunkIndexNeedsCopyFromLocalFileFlags,
+ RemoteChunkIndexNeedsCopyFromSourceFlags);
+
+ for (const ScavengedSequenceCopyOperation& ScavengeCopyOp : ScavengedSequenceCopyOperations)
+ {
+ BytesToWrite += ScavengeCopyOp.RawSize;
+ }
+
+ uint64_t BytesToValidate = m_Options.ValidateCompletedSequences ? BytesToWrite : 0;
+
+ uint64_t TotalRequestCount = 0;
+ uint64_t TotalPartWriteCount = 0;
+ std::atomic<uint64_t> WritePartsComplete = 0;
+
+ tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex;
+ RemotePathToRemoteIndex.reserve(m_RemoteContent.Paths.size());
+ for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
+ {
+ RemotePathToRemoteIndex.insert({m_RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex});
+ }
+
+ CheckRequiredDiskSpace(RemotePathToRemoteIndex);
+
+ BlobsExistsResult ExistsResult;
+ {
+ ChunkBlockAnalyser BlockAnalyser(
+ Log(),
+ m_BlockDescriptions,
+ ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet,
+ .IsVerbose = m_Options.IsVerbose,
+ .HostLatencySec = m_Storage.BuildStorageHost.LatencySec,
+ .HostHighSpeedLatencySec = m_Storage.CacheHost.LatencySec,
+ .HostMaxRangeCountPerRequest = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest,
+ .HostHighSpeedMaxRangeCountPerRequest = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest});
+
+ std::vector<ChunkBlockAnalyser::NeededBlock> NeededBlocks = BlockAnalyser.GetNeeded(
+ m_RemoteLookup.ChunkHashToChunkIndex,
+ [&](uint32_t RemoteChunkIndex) -> bool { return RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]; });
+
+ std::vector<uint32_t> FetchBlockIndexes;
+ std::vector<uint32_t> CachedChunkBlockIndexes;
+ ClassifyCachedAndFetchBlocks(NeededBlocks, CachedBlocksFound, TotalPartWriteCount, CachedChunkBlockIndexes, FetchBlockIndexes);
+
+ std::vector<uint32_t> NeededLooseChunkIndexes = DetermineNeededLooseChunkIndexes(SequenceIndexChunksLeftToWriteCounters,
+ RemoteChunkIndexNeedsCopyFromLocalFileFlags,
+ RemoteChunkIndexNeedsCopyFromSourceFlags);
+
+ ExistsResult = QueryBlobCacheExists(NeededLooseChunkIndexes, FetchBlockIndexes);
+
+ std::vector<ChunkBlockAnalyser::EPartialBlockDownloadMode> BlockPartialDownloadModes =
+ DeterminePartialDownloadModes(ExistsResult);
+ ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size());
+
+ ChunkBlockAnalyser::BlockResult PartialBlocks =
+ BlockAnalyser.CalculatePartialBlockDownloads(NeededBlocks, BlockPartialDownloadModes);
+
+ TotalRequestCount += NeededLooseChunkIndexes.size();
+ TotalPartWriteCount += NeededLooseChunkIndexes.size();
+ TotalRequestCount += PartialBlocks.BlockRanges.size();
+ TotalPartWriteCount += PartialBlocks.BlockRanges.size();
+ TotalRequestCount += PartialBlocks.FullBlockIndexes.size();
+ TotalPartWriteCount += PartialBlocks.FullBlockIndexes.size();
+
+ std::vector<LooseChunkHashWorkData> LooseChunkHashWorks =
+ BuildLooseChunkHashWorks(NeededLooseChunkIndexes, SequenceIndexChunksLeftToWriteCounters);
+
+ ZEN_TRACE_CPU("WriteChunks");
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount);
+
+ Stopwatch WriteTimer;
+
+ FilteredRate FilteredDownloadedBytesPerSecond;
+ FilteredRate FilteredWrittenBytesPerSecond;
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Writing");
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ TotalPartWriteCount += CopyChunkDatas.size();
+ TotalPartWriteCount += ScavengedSequenceCopyOperations.size();
+
+ BufferedWriteFileCache WriteCache;
+
+ WriteChunksContext Context{.Work = Work,
+ .WriteCache = WriteCache,
+ .SequenceIndexChunksLeftToWriteCounters = SequenceIndexChunksLeftToWriteCounters,
+ .RemoteChunkIndexNeedsCopyFromSourceFlags = RemoteChunkIndexNeedsCopyFromSourceFlags,
+ .WritePartsComplete = WritePartsComplete,
+ .TotalPartWriteCount = TotalPartWriteCount,
+ .TotalRequestCount = TotalRequestCount,
+ .ExistsResult = ExistsResult,
+ .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond,
+ .FilteredWrittenBytesPerSecond = FilteredWrittenBytesPerSecond};
+
+ ScheduleScavengedSequenceWrites(Context, ScavengedSequenceCopyOperations, ScavengedContents, ScavengedPaths);
+ ScheduleLooseChunkWrites(Context, LooseChunkHashWorks);
+
+ std::unique_ptr<CloneQueryInterface> CloneQuery =
+ m_Options.AllowFileClone ? GetCloneQueryInterface(m_CacheFolderPath) : nullptr;
+
+ ScheduleLocalChunkCopies(Context, CopyChunkDatas, CloneQuery.get(), ScavengedContents, ScavengedLookups, ScavengedPaths);
+ ScheduleCachedBlockWrites(Context, CachedChunkBlockIndexes);
+ SchedulePartialBlockDownloads(Context, PartialBlocks);
+ ScheduleFullBlockDownloads(Context, PartialBlocks.FullBlockIndexes);
+
+ {
+ ZEN_TRACE_CPU("WriteChunks_Wait");
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() +
+ m_DownloadStats.DownloadedBlockByteCount.load() +
+ +m_DownloadStats.DownloadedPartialBlockByteCount.load();
+ FilteredWrittenBytesPerSecond.Update(m_DiskStats.WriteByteCount.load());
+ FilteredDownloadedBytesPerSecond.Update(DownloadedBytes);
+ std::string DownloadRateString =
+ (m_DownloadStats.RequestsCompleteCount == TotalRequestCount)
+ ? ""
+ : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8));
+ std::string CloneDetails;
+ if (m_DiskStats.CloneCount.load() > 0)
+ {
+ CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load()));
+ }
+ std::string WriteDetails = fmt::format(" {}/{} ({}B/s) written{}",
+ NiceBytes(m_WrittenChunkByteCount.load()),
+ NiceBytes(BytesToWrite),
+ NiceNum(FilteredWrittenBytesPerSecond.GetCurrent()),
+ CloneDetails);
+
+ std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}",
+ m_DownloadStats.RequestsCompleteCount.load(),
+ TotalRequestCount,
+ NiceBytes(DownloadedBytes),
+ DownloadRateString,
+ WriteDetails);
+
+ std::string Task;
+ if ((m_WrittenChunkByteCount < BytesToWrite) || (BytesToValidate == 0))
+ {
+ Task = "Writing chunks ";
+ }
+ else
+ {
+ Task = "Verifying chunks ";
+ }
+
+ ProgressBar->UpdateState({.Task = Task,
+ .Details = Details,
+ .TotalCount = (BytesToWrite + BytesToValidate),
+ .RemainingCount = ((BytesToWrite + BytesToValidate) -
+ (m_WrittenChunkByteCount.load() + m_ValidatedChunkByteCount.load())),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+ }
+
+ CloneQuery.reset();
+
+ FilteredWrittenBytesPerSecond.Stop();
+ FilteredDownloadedBytesPerSecond.Stop();
+
+ ProgressBar->Finish();
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ VerifyWriteChunksComplete(SequenceIndexChunksLeftToWriteCounters, BytesToWrite, BytesToValidate);
+
+ const uint64_t DownloadedBytes = m_DownloadStats.DownloadedChunkByteCount.load() +
+ m_DownloadStats.DownloadedBlockByteCount.load() +
+ m_DownloadStats.DownloadedPartialBlockByteCount.load();
+ if (!m_Options.IsQuiet)
+ {
+ std::string CloneDetails;
+ if (m_DiskStats.CloneCount.load() > 0)
+ {
+ CloneDetails = fmt::format(" ({} cloned)", NiceBytes(m_DiskStats.CloneByteCount.load()));
+ }
+ ZEN_INFO("Downloaded {} ({}bits/s) in {}. Wrote {} ({}B/s){} in {}. Completed in {}",
+ NiceBytes(DownloadedBytes),
+ NiceNum(GetBytesPerSecond(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS(), DownloadedBytes * 8)),
+ NiceTimeSpanMs(FilteredDownloadedBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceBytes(m_WrittenChunkByteCount.load()),
+ NiceNum(GetBytesPerSecond(FilteredWrittenBytesPerSecond.GetElapsedTimeUS(), m_DiskStats.WriteByteCount.load())),
+ CloneDetails,
+ NiceTimeSpanMs(FilteredWrittenBytesPerSecond.GetElapsedTimeUS() / 1000),
+ NiceTimeSpanMs(WriteTimer.GetElapsedTimeMs()));
+ }
+
+ m_WriteChunkStats.WriteChunksElapsedWallTimeUs = WriteTimer.GetElapsedTimeUs();
+ m_WriteChunkStats.DownloadTimeUs = FilteredDownloadedBytesPerSecond.GetElapsedTimeUS();
+ m_WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS();
+ }
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::PrepareTarget, (uint32_t)TaskSteps::StepCount);
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ LocalPathCategorization Categorization = CategorizeLocalPaths(RemotePathToRemoteIndex);
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ std::atomic<uint64_t> CachedCount = 0;
+ std::atomic<uint64_t> CachedByteCount = 0;
+ ScheduleLocalFileCaching(Categorization.FilesToCache, CachedCount, CachedByteCount);
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ ZEN_DEBUG(
+ "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, "
+ "Delete: {}",
+ Categorization.MatchCount,
+ Categorization.PathMismatchCount,
+ Categorization.HashMismatchCount,
+ CachedCount.load(),
+ NiceBytes(CachedByteCount.load()),
+ Categorization.SkippedCount,
+ Categorization.DeleteCount);
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FinalizeTarget, (uint32_t)TaskSteps::StepCount);
+
+ if (m_Options.WipeTargetFolder)
+ {
+ ZEN_TRACE_CPU("WipeTarget");
+ Stopwatch Timer;
+
+ // Clean target folder
+ if (!CleanDirectory(Log(),
+ m_Progress,
+ m_IOWorkerPool,
+ m_AbortFlag,
+ m_PauseFlag,
+ m_Options.IsQuiet,
+ m_Path,
+ m_Options.ExcludeFolders))
+ {
+ ZEN_WARN("Some files in {} could not be removed", m_Path);
+ }
+ m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+ }
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ {
+ ZEN_TRACE_CPU("FinalizeTree");
+
+ Stopwatch Timer;
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Rebuild State");
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ OutLocalFolderState.Paths.resize(m_RemoteContent.Paths.size());
+ OutLocalFolderState.RawSizes.resize(m_RemoteContent.Paths.size());
+ OutLocalFolderState.Attributes.resize(m_RemoteContent.Paths.size());
+ OutLocalFolderState.ModificationTicks.resize(m_RemoteContent.Paths.size());
+
+ std::atomic<uint64_t> DeletedCount = 0;
+ std::atomic<uint64_t> TargetsComplete = 0;
+
+ ScheduleLocalFileRemovals(Work, Categorization.RemoveLocalPathIndexes, DeletedCount);
+
+ std::vector<FinalizeTarget> Targets = BuildSortedFinalizeTargets();
+
+ ScheduleTargetFinalization(Work,
+ Targets,
+ Categorization.SequenceHashToLocalPathIndex,
+ Categorization.RemotePathIndexToLocalPathIndex,
+ OutLocalFolderState,
+ TargetsComplete);
+
+ {
+ ZEN_TRACE_CPU("FinalizeTree_Wait");
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ const uint64_t WorkTotal = Targets.size() + Categorization.RemoveLocalPathIndexes.size();
+ const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load();
+ std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal);
+ ProgressBar->UpdateState({.Task = "Rebuilding state ",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(WorkTotal),
+ .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+ }
+
+ m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+ ProgressBar->Finish();
+ }
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount);
+ }
+ catch (const std::exception&)
+ {
+ m_AbortFlag = true;
+ throw;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScanCacheFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedChunkHashesFound,
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedSequenceHashesFound)
+{
+ ZEN_TRACE_CPU("ScanCacheFolder");
+
+ Stopwatch CacheTimer;
+
+ DirectoryContent CacheDirContent;
+ GetDirectoryContent(m_CacheFolderPath, DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, CacheDirContent);
+ for (size_t Index = 0; Index < CacheDirContent.Files.size(); Index++)
+ {
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ IoHash FileHash;
+ if (IoHash::TryParse(CacheDirContent.Files[Index].filename().string(), FileHash))
+ {
+ if (auto ChunkIt = m_RemoteLookup.ChunkHashToChunkIndex.find(FileHash);
+ ChunkIt != m_RemoteLookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t ChunkIndex = ChunkIt->second;
+ const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ if (ChunkSize == CacheDirContent.FileSizes[Index])
+ {
+ OutCachedChunkHashesFound.insert({FileHash, ChunkIndex});
+ m_CacheMappingStats.CacheChunkCount++;
+ m_CacheMappingStats.CacheChunkByteCount += ChunkSize;
+ continue;
+ }
+ }
+ else if (auto SequenceIt = m_RemoteLookup.RawHashToSequenceIndex.find(FileHash);
+ SequenceIt != m_RemoteLookup.RawHashToSequenceIndex.end())
+ {
+ const uint32_t SequenceIndex = SequenceIt->second;
+ const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex];
+ if (SequenceSize == CacheDirContent.FileSizes[Index])
+ {
+ OutCachedSequenceHashesFound.insert({FileHash, SequenceIndex});
+ m_CacheMappingStats.CacheSequenceHashesCount++;
+ m_CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize;
+
+ const std::filesystem::path CacheFilePath =
+ GetFinalChunkedSequenceFileName(m_CacheFolderPath,
+ m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+
+ continue;
+ }
+ }
+ }
+ }
+ std::error_code Ec = TryRemoveFile(CacheDirContent.Files[Index]);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CacheDirContent.Files[Index], Ec.value(), Ec.message());
+ }
+ }
+ m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
+}
+
+void
+BuildsOperationUpdateFolder::ScanTempBlocksFolder(tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutCachedBlocksFound)
+{
+ ZEN_TRACE_CPU("ScanTempBlocksFolder");
+
+ Stopwatch CacheTimer;
+
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> AllBlockSizes;
+ AllBlockSizes.reserve(m_BlockDescriptions.size());
+ for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++)
+ {
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+ AllBlockSizes.insert({BlockDescription.BlockHash, BlockIndex});
+ }
+
+ DirectoryContent BlockDirContent;
+ GetDirectoryContent(m_TempBlockFolderPath,
+ DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes,
+ BlockDirContent);
+ OutCachedBlocksFound.reserve(BlockDirContent.Files.size());
+ for (size_t Index = 0; Index < BlockDirContent.Files.size(); Index++)
+ {
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ IoHash FileHash;
+ if (IoHash::TryParse(BlockDirContent.Files[Index].filename().string(), FileHash))
+ {
+ if (auto BlockIt = AllBlockSizes.find(FileHash); BlockIt != AllBlockSizes.end())
+ {
+ const uint32_t BlockIndex = BlockIt->second;
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+ uint64_t BlockSize = CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize;
+ for (uint64_t ChunkSize : BlockDescription.ChunkCompressedLengths)
+ {
+ BlockSize += ChunkSize;
+ }
+
+ if (BlockSize == BlockDirContent.FileSizes[Index])
+ {
+ OutCachedBlocksFound.insert({FileHash, BlockIndex});
+ m_CacheMappingStats.CacheBlockCount++;
+ m_CacheMappingStats.CacheBlocksByteCount += BlockSize;
+ continue;
+ }
+ }
+ }
+ }
+ std::error_code Ec = TryRemoveFile(BlockDirContent.Files[Index]);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockDirContent.Files[Index], Ec.value(), Ec.message());
+ }
+ }
+
+ m_CacheMappingStats.CacheScanElapsedWallTimeUs += CacheTimer.GetElapsedTimeUs();
+}
+
+void
+BuildsOperationUpdateFolder::InitializeSequenceCounters(std::vector<std::atomic<uint32_t>>& OutSequenceCounters,
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& OutSequencesLeftToFind,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedChunkHashesFound,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedSequenceHashesFound)
+{
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ // Pick up all whole files we can use from current local state
+ ZEN_TRACE_CPU("GetLocalSequences");
+
+ std::vector<uint32_t> MissingSequenceIndexes = ScanTargetFolder(CachedChunkHashesFound, CachedSequenceHashesFound);
+
+ for (uint32_t RemoteSequenceIndex : MissingSequenceIndexes)
+ {
+ // We must write the sequence
+ const uint32_t ChunkCount = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+ const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ OutSequenceCounters[RemoteSequenceIndex] = ChunkCount;
+ OutSequencesLeftToFind.insert({RemoteSequenceRawHash, RemoteSequenceIndex});
+ }
+ }
+ else
+ {
+ for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size();
+ RemoteSequenceIndex++)
+ {
+ OutSequenceCounters[RemoteSequenceIndex] = m_RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex];
+ }
+ }
+}
+
+void
+BuildsOperationUpdateFolder::MatchScavengedSequencesToRemote(std::span<const ChunkedFolderContent> Contents,
+ std::span<const ChunkedContentLookup> Lookups,
+ std::span<const std::filesystem::path> Paths,
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& InOutSequencesLeftToFind,
+ std::vector<std::atomic<uint32_t>>& InOutSequenceCounters,
+ std::vector<ScavengedSequenceCopyOperation>& OutCopyOperations,
+ uint64_t& OutScavengedPathsCount)
+{
+ for (uint32_t ScavengedContentIndex = 0; ScavengedContentIndex < Contents.size() && !InOutSequencesLeftToFind.empty();
+ ScavengedContentIndex++)
+ {
+ const std::filesystem::path& ScavengePath = Paths[ScavengedContentIndex];
+ if (ScavengePath.empty())
+ {
+ continue;
+ }
+ const ChunkedFolderContent& ScavengedLocalContent = Contents[ScavengedContentIndex];
+ const ChunkedContentLookup& ScavengedLookup = Lookups[ScavengedContentIndex];
+
+ for (uint32_t ScavengedSequenceIndex = 0; ScavengedSequenceIndex < ScavengedLocalContent.ChunkedContent.SequenceRawHashes.size();
+ ScavengedSequenceIndex++)
+ {
+ const IoHash& SequenceRawHash = ScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengedSequenceIndex];
+ auto It = InOutSequencesLeftToFind.find(SequenceRawHash);
+ if (It == InOutSequencesLeftToFind.end())
+ {
+ continue;
+ }
+ const uint32_t RemoteSequenceIndex = It->second;
+ const uint64_t RawSize = m_RemoteContent.RawSizes[m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]];
+ ZEN_ASSERT(RawSize > 0);
+
+ const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[ScavengedSequenceIndex];
+ ZEN_ASSERT_SLOW(IsFile((ScavengePath / ScavengedLocalContent.Paths[ScavengedPathIndex]).make_preferred()));
+
+ OutCopyOperations.push_back({.ScavengedContentIndex = ScavengedContentIndex,
+ .ScavengedPathIndex = ScavengedPathIndex,
+ .RemoteSequenceIndex = RemoteSequenceIndex,
+ .RawSize = RawSize});
+
+ InOutSequencesLeftToFind.erase(SequenceRawHash);
+ InOutSequenceCounters[RemoteSequenceIndex] = 0;
+
+ m_CacheMappingStats.ScavengedPathsMatchingSequencesCount++;
+ m_CacheMappingStats.ScavengedPathsMatchingSequencesByteCount += RawSize;
+ }
+ OutScavengedPathsCount++;
+ }
+}
+
+uint64_t
+BuildsOperationUpdateFolder::CalculateBytesToWriteAndFlagNeededChunks(std::span<const std::atomic<uint32_t>> SequenceCounters,
+ const std::vector<bool>& NeedsCopyFromLocalFileFlags,
+ std::span<std::atomic<bool>> OutNeedsCopyFromSourceFlags)
+{
+ uint64_t BytesToWrite = 0;
+ for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++)
+ {
+ const uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex);
+ if (ChunkWriteCount > 0)
+ {
+ BytesToWrite += m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount;
+ if (!NeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ {
+ OutNeedsCopyFromSourceFlags[RemoteChunkIndex] = true;
+ }
+ }
+ }
+ return BytesToWrite;
+}
+
+void
+BuildsOperationUpdateFolder::ClassifyCachedAndFetchBlocks(std::span<const ChunkBlockAnalyser::NeededBlock> NeededBlocks,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedBlocksFound,
+ uint64_t& TotalPartWriteCount,
+ std::vector<uint32_t>& OutCachedChunkBlockIndexes,
+ std::vector<uint32_t>& OutFetchBlockIndexes)
+{
+ ZEN_TRACE_CPU("BlockCacheFileExists");
+ for (const ChunkBlockAnalyser::NeededBlock& NeededBlock : NeededBlocks)
+ {
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex];
+ bool UsingCachedBlock = false;
+ if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end())
+ {
+ TotalPartWriteCount++;
+
+ std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
+ if (IsFile(BlockPath))
+ {
+ OutCachedChunkBlockIndexes.push_back(NeededBlock.BlockIndex);
+ UsingCachedBlock = true;
+ }
+ }
+ if (!UsingCachedBlock)
+ {
+ OutFetchBlockIndexes.push_back(NeededBlock.BlockIndex);
+ }
+ }
+}
+
+std::vector<uint32_t>
+BuildsOperationUpdateFolder::DetermineNeededLooseChunkIndexes(std::span<const std::atomic<uint32_t>> SequenceCounters,
+ const std::vector<bool>& NeedsCopyFromLocalFileFlags,
+ std::span<std::atomic<bool>> NeedsCopyFromSourceFlags)
+{
+ std::vector<uint32_t> NeededLooseChunkIndexes;
+ NeededLooseChunkIndexes.reserve(m_LooseChunkHashes.size());
+ for (uint32_t LooseChunkIndex = 0; LooseChunkIndex < m_LooseChunkHashes.size(); LooseChunkIndex++)
+ {
+ const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex];
+ auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
+ ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
+ const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
+
+ if (NeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]);
+ }
+ continue;
+ }
+
+ bool NeedsCopy = true;
+ if (NeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false))
+ {
+ const uint64_t WriteCount = GetChunkWriteCount(SequenceCounters, RemoteChunkIndex);
+ if (WriteCount == 0)
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Skipping chunk {} due to cache reuse", m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]);
+ }
+ }
+ else
+ {
+ NeededLooseChunkIndexes.push_back(LooseChunkIndex);
+ }
+ }
+ }
+ return NeededLooseChunkIndexes;
+}
+
+BuildsOperationUpdateFolder::BlobsExistsResult
+BuildsOperationUpdateFolder::QueryBlobCacheExists(std::span<const uint32_t> NeededLooseChunkIndexes,
+ std::span<const uint32_t> FetchBlockIndexes)
+{
+ BlobsExistsResult Result;
+ if (!m_Storage.CacheStorage)
+ {
+ return Result;
+ }
+
+ ZEN_TRACE_CPU("BlobCacheExistCheck");
+ Stopwatch Timer;
+
+ std::vector<IoHash> BlobHashes;
+ BlobHashes.reserve(NeededLooseChunkIndexes.size() + FetchBlockIndexes.size());
+
+ for (const uint32_t LooseChunkIndex : NeededLooseChunkIndexes)
+ {
+ BlobHashes.push_back(m_LooseChunkHashes[LooseChunkIndex]);
+ }
+
+ for (uint32_t BlockIndex : FetchBlockIndexes)
+ {
+ BlobHashes.push_back(m_BlockDescriptions[BlockIndex].BlockHash);
+ }
+
+ const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult = m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes);
+
+ if (CacheExistsResult.size() == BlobHashes.size())
+ {
+ Result.ExistingBlobs.reserve(CacheExistsResult.size());
+ for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++)
+ {
+ if (CacheExistsResult[BlobIndex].HasBody)
+ {
+ Result.ExistingBlobs.insert(BlobHashes[BlobIndex]);
+ }
+ }
+ }
+ Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+ if (!Result.ExistingBlobs.empty() && !m_Options.IsQuiet)
+ {
+ ZEN_INFO("Remote cache : Found {} out of {} needed blobs in {}",
+ Result.ExistingBlobs.size(),
+ BlobHashes.size(),
+ NiceTimeSpanMs(Result.ElapsedTimeMs));
+ }
+ return Result;
+}
+
+std::vector<ChunkBlockAnalyser::EPartialBlockDownloadMode>
+BuildsOperationUpdateFolder::DeterminePartialDownloadModes(const BlobsExistsResult& ExistsResult)
+{
+ std::vector<ChunkBlockAnalyser::EPartialBlockDownloadMode> Modes;
+
+ if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off)
+ {
+ Modes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off);
+ return Modes;
+ }
+
+ const bool MultiRangeCache = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1;
+ const bool MultiRangeBuild = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest > 1;
+ ChunkBlockAnalyser::EPartialBlockDownloadMode CachePartialDownloadMode =
+ MultiRangeCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed
+ : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange;
+ ChunkBlockAnalyser::EPartialBlockDownloadMode CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off;
+
+ switch (m_Options.PartialBlockRequestMode)
+ {
+ case EPartialBlockRequestMode::Off:
+ break;
+ case EPartialBlockRequestMode::ZenCacheOnly:
+ CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off;
+ break;
+ case EPartialBlockRequestMode::Mixed:
+ CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange;
+ break;
+ case EPartialBlockRequestMode::All:
+ CloudPartialDownloadMode = MultiRangeBuild ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange
+ : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange;
+ break;
+ default:
+ ZEN_ASSERT(false);
+ break;
+ }
+
+ Modes.reserve(m_BlockDescriptions.size());
+ for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++)
+ {
+ const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash);
+ Modes.push_back(BlockExistInCache ? CachePartialDownloadMode : CloudPartialDownloadMode);
+ }
+ return Modes;
+}
+
+std::vector<BuildsOperationUpdateFolder::LooseChunkHashWorkData>
+BuildsOperationUpdateFolder::BuildLooseChunkHashWorks(std::span<const uint32_t> NeededLooseChunkIndexes,
+ std::span<const std::atomic<uint32_t>> SequenceCounters)
+{
+ std::vector<LooseChunkHashWorkData> LooseChunkHashWorks;
+ LooseChunkHashWorks.reserve(NeededLooseChunkIndexes.size());
+ for (uint32_t LooseChunkIndex : NeededLooseChunkIndexes)
+ {
+ const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex];
+ auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
+ ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
+ const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second;
+
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
+ GetRemainingChunkTargets(SequenceCounters, RemoteChunkIndex);
+
+ ZEN_ASSERT(!ChunkTargetPtrs.empty());
+ LooseChunkHashWorks.push_back(LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex});
+ }
+ return LooseChunkHashWorks;
+}
+
+void
+BuildsOperationUpdateFolder::VerifyWriteChunksComplete(std::span<const std::atomic<uint32_t>> SequenceCounters,
+ uint64_t BytesToWrite,
+ uint64_t BytesToValidate)
+{
+ uint32_t RawSequencesMissingWriteCount = 0;
+ for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceCounters.size(); SequenceIndex++)
+ {
+ const auto& Counter = SequenceCounters[SequenceIndex];
+ if (Counter.load() != 0)
+ {
+ RawSequencesMissingWriteCount++;
+ const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const std::filesystem::path& IncompletePath = m_RemoteContent.Paths[PathIndex];
+ ZEN_ASSERT(!IncompletePath.empty());
+ const uint32_t ExpectedSequenceCount = m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex];
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("{}: Max count {}, Current count {}", IncompletePath, ExpectedSequenceCount, Counter.load());
+ }
+ ZEN_ASSERT(Counter.load() <= ExpectedSequenceCount);
+ }
+ }
+ ZEN_ASSERT(RawSequencesMissingWriteCount == 0);
+ ZEN_ASSERT(m_WrittenChunkByteCount == BytesToWrite);
+ ZEN_ASSERT(m_ValidatedChunkByteCount == BytesToValidate);
+}
+
+std::vector<BuildsOperationUpdateFolder::FinalizeTarget>
+BuildsOperationUpdateFolder::BuildSortedFinalizeTargets()
+{
+ std::vector<FinalizeTarget> Targets;
+ Targets.reserve(m_RemoteContent.Paths.size());
+ for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
+ {
+ Targets.push_back(FinalizeTarget{.RawHash = m_RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex});
+ }
+ std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) {
+ return std::tie(Lhs.RawHash, Lhs.RemotePathIndex) < std::tie(Rhs.RawHash, Rhs.RemotePathIndex);
+ });
+ return Targets;
+}
+
+void
+BuildsOperationUpdateFolder::ScanScavengeSources(std::span<const ScavengeSource> Sources,
+ std::vector<ChunkedFolderContent>& OutContents,
+ std::vector<ChunkedContentLookup>& OutLookups,
+ std::vector<std::filesystem::path>& OutPaths)
+{
+ ZEN_TRACE_CPU("ScanScavengeSources");
+
+ const size_t ScavengePathCount = Sources.size();
+ OutContents.resize(ScavengePathCount);
+ OutLookups.resize(ScavengePathCount);
+ OutPaths.resize(ScavengePathCount);
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Scavenging");
+
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ std::atomic<uint64_t> PathsFound(0);
+ std::atomic<uint64_t> ChunksFound(0);
+ std::atomic<uint64_t> PathsScavenged(0);
+
+ for (size_t ScavengeIndex = 0; ScavengeIndex < ScavengePathCount; ScavengeIndex++)
+ {
+ Work.ScheduleWork(m_IOWorkerPool,
+ [this, &Sources, &OutContents, &OutPaths, &OutLookups, &PathsFound, &ChunksFound, &PathsScavenged, ScavengeIndex](
+ std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_FindScavengeContent");
+
+ const ScavengeSource& Source = Sources[ScavengeIndex];
+ ChunkedFolderContent& ScavengedLocalContent = OutContents[ScavengeIndex];
+ ChunkedContentLookup& ScavengedLookup = OutLookups[ScavengeIndex];
+
+ if (FindScavengeContent(Source, ScavengedLocalContent, ScavengedLookup))
+ {
+ OutPaths[ScavengeIndex] = Source.Path;
+ PathsFound += ScavengedLocalContent.Paths.size();
+ ChunksFound += ScavengedLocalContent.ChunkedContent.ChunkHashes.size();
+ }
+ else
+ {
+ OutPaths[ScavengeIndex].clear();
+ }
+ PathsScavenged++;
+ }
+ });
+ }
+ {
+ ZEN_TRACE_CPU("ScavengeScan_Wait");
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ std::string Details = fmt::format("{}/{} scanned. {} paths and {} chunks found for scavenging",
+ PathsScavenged.load(),
+ ScavengePathCount,
+ PathsFound.load(),
+ ChunksFound.load());
+ ProgressBar->UpdateState({.Task = "Scavenging ",
+ .Details = Details,
+ .TotalCount = ScavengePathCount,
+ .RemainingCount = ScavengePathCount - PathsScavenged.load(),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+ }
+
+ ProgressBar->Finish();
+}
+
+BuildsOperationUpdateFolder::LocalPathCategorization
+BuildsOperationUpdateFolder::CategorizeLocalPaths(const tsl::robin_map<std::string, uint32_t>& RemotePathToRemoteIndex)
+{
+ ZEN_TRACE_CPU("PrepareTarget");
+
+ LocalPathCategorization Result;
+ tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences;
+
+ Result.RemotePathIndexToLocalPathIndex.reserve(m_RemoteContent.Paths.size());
+
+ for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
+ const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
+
+ ZEN_ASSERT_SLOW(IsFile((m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred()));
+
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ if (!m_Options.WipeTargetFolder)
+ {
+ // Check if it is already in the correct place
+ if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string());
+ RemotePathIt != RemotePathToRemoteIndex.end())
+ {
+ const uint32_t RemotePathIndex = RemotePathIt->second;
+ if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash)
+ {
+ // It is already in it's correct place
+ Result.RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex;
+ Result.SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex});
+ Result.MatchCount++;
+ continue;
+ }
+ else
+ {
+ Result.HashMismatchCount++;
+ }
+ }
+ else
+ {
+ Result.PathMismatchCount++;
+ }
+ }
+
+ // Do we need it?
+ if (m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash))
+ {
+ if (!CachedRemoteSequences.contains(RawHash))
+ {
+ // We need it, make sure we move it to the cache
+ Result.FilesToCache.push_back(LocalPathIndex);
+ CachedRemoteSequences.insert(RawHash);
+ continue;
+ }
+ else
+ {
+ Result.SkippedCount++;
+ }
+ }
+ }
+
+ if (!m_Options.WipeTargetFolder)
+ {
+ // Explicitly delete the unneeded local file
+ Result.RemoveLocalPathIndexes.push_back(LocalPathIndex);
+ Result.DeleteCount++;
+ }
+ }
+
+ return Result;
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleLocalFileCaching(std::span<const uint32_t> FilesToCache,
+ std::atomic<uint64_t>& OutCachedCount,
+ std::atomic<uint64_t>& OutCachedByteCount)
+{
+ ZEN_TRACE_CPU("CopyToCache");
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Cache Local Data");
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ for (uint32_t LocalPathIndex : FilesToCache)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ Work.ScheduleWork(m_IOWorkerPool, [this, &OutCachedCount, &OutCachedByteCount, LocalPathIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_CopyToCache");
+
+ const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
+ const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
+ const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
+ ZEN_ASSERT_SLOW(!IsFileWithRetry(CacheFilePath));
+ const std::filesystem::path LocalFilePath = (m_Path / LocalPath).make_preferred();
+
+ std::error_code Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
+ if (Ec)
+ {
+ ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
+ LocalFilePath,
+ CacheFilePath,
+ Ec.value(),
+ Ec.message());
+ Ec = RenameFileWithRetry(LocalFilePath, CacheFilePath);
+ if (Ec)
+ {
+ throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+ fmt::format("Failed to file from '{}' to '{}', reason: ({}) {}",
+ LocalFilePath,
+ CacheFilePath,
+ Ec.value(),
+ Ec.message()));
+ }
+ }
+
+ OutCachedCount++;
+ OutCachedByteCount += m_LocalContent.RawSizes[LocalPathIndex];
+ }
+ });
+ }
+
+ {
+ ZEN_TRACE_CPU("CopyToCache_Wait");
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ const uint64_t WorkTotal = FilesToCache.size();
+ const uint64_t WorkComplete = OutCachedCount.load();
+ std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(OutCachedByteCount));
+ ProgressBar->UpdateState({.Task = "Caching local ",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(WorkTotal),
+ .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+ }
+
+ ProgressBar->Finish();
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleScavengedSequenceWrites(WriteChunksContext& Context,
+ std::span<const ScavengedSequenceCopyOperation> CopyOperations,
+ const std::vector<ChunkedFolderContent>& ScavengedContents,
+ const std::vector<std::filesystem::path>& ScavengedPaths)
+{
+ for (uint32_t ScavengeOpIndex = 0; ScavengeOpIndex < CopyOperations.size(); ScavengeOpIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this, &Context, CopyOperations, &ScavengedContents, &ScavengedPaths, ScavengeOpIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_WriteScavenged");
+
+ Context.FilteredWrittenBytesPerSecond.Start();
+
+ const ScavengedSequenceCopyOperation& ScavengeOp = CopyOperations[ScavengeOpIndex];
+ const ChunkedFolderContent& ScavengedContent = ScavengedContents[ScavengeOp.ScavengedContentIndex];
+ const std::filesystem::path& ScavengeRootPath = ScavengedPaths[ScavengeOp.ScavengedContentIndex];
+
+ WriteScavengedSequenceToCache(ScavengeRootPath, ScavengedContent, ScavengeOp);
+
+ if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount)
+ {
+ Context.FilteredWrittenBytesPerSecond.Stop();
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleLooseChunkWrites(WriteChunksContext& Context, std::vector<LooseChunkHashWorkData>& LooseChunkHashWorks)
+{
+ for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this, &Context, &LooseChunkHashWorks, LooseChunkHashWorkIndex](std::atomic<bool>&) {
+ ZEN_TRACE_CPU("Async_ReadPreDownloadedChunk");
+ if (!m_AbortFlag)
+ {
+ LooseChunkHashWorkData& LooseChunkHashWork = LooseChunkHashWorks[LooseChunkHashWorkIndex];
+ const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex;
+ WriteLooseChunk(RemoteChunkIndex,
+ Context.ExistsResult,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.WritePartsComplete,
+ std::move(LooseChunkHashWork.ChunkTargetPtrs),
+ Context.WriteCache,
+ Context.Work,
+ Context.TotalRequestCount,
+ Context.TotalPartWriteCount,
+ Context.FilteredDownloadedBytesPerSecond,
+ Context.FilteredWrittenBytesPerSecond);
+ }
+ },
+ WorkerThreadPool::EMode::EnableBacklog);
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleLocalChunkCopies(WriteChunksContext& Context,
+ std::span<const CopyChunkData> CopyChunkDatas,
+ CloneQueryInterface* CloneQuery,
+ const std::vector<ChunkedFolderContent>& ScavengedContents,
+ const std::vector<ChunkedContentLookup>& ScavengedLookups,
+ const std::vector<std::filesystem::path>& ScavengedPaths)
+{
+ for (size_t CopyDataIndex = 0; CopyDataIndex < CopyChunkDatas.size(); CopyDataIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this, &Context, CloneQuery, CopyChunkDatas, &ScavengedContents, &ScavengedLookups, &ScavengedPaths, CopyDataIndex](
+ std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_CopyLocal");
+
+ Context.FilteredWrittenBytesPerSecond.Start();
+ const CopyChunkData& CopyData = CopyChunkDatas[CopyDataIndex];
+
+ std::vector<uint32_t> WrittenSequenceIndexes = WriteLocalChunkToCache(CloneQuery,
+ CopyData,
+ ScavengedContents,
+ ScavengedLookups,
+ ScavengedPaths,
+ Context.WriteCache);
+ bool WritePartsDone = Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount;
+ if (!m_AbortFlag)
+ {
+ if (WritePartsDone)
+ {
+ Context.FilteredWrittenBytesPerSecond.Stop();
+ }
+
+ // Write tracking, updating this must be done without any files open
+ std::vector<uint32_t> CompletedChunkSequences;
+ for (uint32_t RemoteSequenceIndex : WrittenSequenceIndexes)
+ {
+ if (CompleteSequenceChunk(RemoteSequenceIndex, Context.SequenceIndexChunksLeftToWriteCounters))
+ {
+ CompletedChunkSequences.push_back(RemoteSequenceIndex);
+ }
+ }
+ Context.WriteCache.Close(CompletedChunkSequences);
+ VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Context.Work);
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleCachedBlockWrites(WriteChunksContext& Context, std::span<const uint32_t> CachedBlockIndexes)
+{
+ for (uint32_t BlockIndex : CachedBlockIndexes)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ Context.Work.ScheduleWork(m_IOWorkerPool, [this, &Context, BlockIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_WriteCachedBlock");
+
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+ Context.FilteredWrittenBytesPerSecond.Start();
+
+ std::filesystem::path BlockChunkPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString();
+ IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+ if (!BlockBuffer)
+ {
+ throw std::runtime_error(fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath));
+ }
+
+ if (!m_AbortFlag)
+ {
+ if (!WriteChunksBlockToCache(BlockDescription,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.Work,
+ CompositeBuffer(std::move(BlockBuffer)),
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Context.WriteCache))
+ {
+ std::error_code DummyEc;
+ RemoveFile(BlockChunkPath, DummyEc);
+ throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
+ }
+
+ std::error_code Ec = TryRemoveFile(BlockChunkPath);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message());
+ }
+
+ if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount)
+ {
+ Context.FilteredWrittenBytesPerSecond.Stop();
+ }
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUpdateFolder::SchedulePartialBlockDownloads(WriteChunksContext& Context,
+ const ChunkBlockAnalyser::BlockResult& PartialBlocks)
+{
+ for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocks.BlockRanges.size();)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ size_t RangeCount = 1;
+ size_t RangesLeft = PartialBlocks.BlockRanges.size() - BlockRangeIndex;
+ const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocks.BlockRanges[BlockRangeIndex];
+ while (RangeCount < RangesLeft &&
+ CurrentBlockRange.BlockIndex == PartialBlocks.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex)
+ {
+ RangeCount++;
+ }
+
+ Context.Work.ScheduleWork(
+ m_NetworkPool,
+ [this, &Context, &PartialBlocks, BlockRangeStartIndex = BlockRangeIndex, RangeCount = RangeCount](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_GetPartialBlockRanges");
+
+ Context.FilteredDownloadedBytesPerSecond.Start();
+
+ DownloadPartialBlock(
+ PartialBlocks.BlockRanges,
+ BlockRangeStartIndex,
+ RangeCount,
+ Context.ExistsResult,
+ Context.TotalRequestCount,
+ Context.FilteredDownloadedBytesPerSecond,
+ [this, &Context, &PartialBlocks](IoBuffer&& InMemoryBuffer,
+ const std::filesystem::path& OnDiskPath,
+ size_t BlockRangeStartIndex,
+ std::span<const std::pair<uint64_t, uint64_t>> OffsetAndLengths) {
+ if (!m_AbortFlag)
+ {
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this,
+ &Context,
+ &PartialBlocks,
+ BlockRangeStartIndex,
+ BlockChunkPath = std::filesystem::path(OnDiskPath),
+ BlockPartialBuffer = std::move(InMemoryBuffer),
+ OffsetAndLengths =
+ std::vector<std::pair<uint64_t, uint64_t>>(OffsetAndLengths.begin(), OffsetAndLengths.end())](
+ std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ WritePartialBlockToCache(Context,
+ BlockRangeStartIndex,
+ std::move(BlockPartialBuffer),
+ BlockChunkPath,
+ OffsetAndLengths,
+ PartialBlocks);
+ }
+ },
+ OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog);
+ }
+ });
+ }
+ });
+ BlockRangeIndex += RangeCount;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::WritePartialBlockToCache(WriteChunksContext& Context,
+ size_t BlockRangeStartIndex,
+ IoBuffer BlockPartialBuffer,
+ const std::filesystem::path& BlockChunkPath,
+ std::span<const std::pair<uint64_t, uint64_t>> OffsetAndLengths,
+ const ChunkBlockAnalyser::BlockResult& PartialBlocks)
+{
+ ZEN_TRACE_CPU("Async_WritePartialBlock");
+
+ const uint32_t BlockIndex = PartialBlocks.BlockRanges[BlockRangeStartIndex].BlockIndex;
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+
+ if (BlockChunkPath.empty())
+ {
+ ZEN_ASSERT(BlockPartialBuffer);
+ }
+ else
+ {
+ ZEN_ASSERT(!BlockPartialBuffer);
+ BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+ if (!BlockPartialBuffer)
+ {
+ throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath));
+ }
+ }
+
+ Context.FilteredWrittenBytesPerSecond.Start();
+
+ const size_t RangeCount = OffsetAndLengths.size();
+
+ for (size_t PartialRangeIndex = 0; PartialRangeIndex < RangeCount; PartialRangeIndex++)
+ {
+ const std::pair<uint64_t, uint64_t>& OffsetAndLength = OffsetAndLengths[PartialRangeIndex];
+ IoBuffer BlockRangeBuffer(BlockPartialBuffer, OffsetAndLength.first, OffsetAndLength.second);
+
+ const ChunkBlockAnalyser::BlockRangeDescriptor& RangeDescriptor =
+ PartialBlocks.BlockRanges[BlockRangeStartIndex + PartialRangeIndex];
+
+ if (!WritePartialBlockChunksToCache(BlockDescription,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.Work,
+ CompositeBuffer(std::move(BlockRangeBuffer)),
+ RangeDescriptor.ChunkBlockIndexStart,
+ RangeDescriptor.ChunkBlockIndexStart + RangeDescriptor.ChunkBlockIndexCount - 1,
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Context.WriteCache))
+ {
+ std::error_code DummyEc;
+ RemoveFile(BlockChunkPath, DummyEc);
+ throw std::runtime_error(fmt::format("Partial block {} is malformed", BlockDescription.BlockHash));
+ }
+
+ if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount)
+ {
+ Context.FilteredWrittenBytesPerSecond.Stop();
+ }
+ }
+ std::error_code Ec = TryRemoveFile(BlockChunkPath);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message());
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleFullBlockDownloads(WriteChunksContext& Context, std::span<const uint32_t> FullBlockIndexes)
+{
+ for (uint32_t BlockIndex : FullBlockIndexes)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_GetFullBlock");
+
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+
+ Context.FilteredDownloadedBytesPerSecond.Start();
+
+ IoBuffer BlockBuffer;
+ const bool ExistsInCache =
+ m_Storage.CacheStorage && Context.ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash);
+ if (ExistsInCache)
+ {
+ BlockBuffer = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash);
+ }
+ if (!BlockBuffer)
+ {
+ try
+ {
+ BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+ }
+ if (!m_AbortFlag)
+ {
+ if (!BlockBuffer)
+ {
+ throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash));
+ }
+
+ uint64_t BlockSize = BlockBuffer.GetSize();
+ m_DownloadStats.DownloadedBlockCount++;
+ m_DownloadStats.DownloadedBlockByteCount += BlockSize;
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == Context.TotalRequestCount)
+ {
+ Context.FilteredDownloadedBytesPerSecond.Stop();
+ }
+
+ const bool PutInCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache;
+
+ std::filesystem::path BlockChunkPath =
+ TryMoveDownloadedChunk(BlockBuffer,
+ m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(),
+ /* ForceDiskBased */ PutInCache || (BlockSize > m_Options.MaximumInMemoryPayloadSize));
+
+ if (PutInCache)
+ {
+ ZEN_ASSERT(!BlockChunkPath.empty());
+ IoBuffer CacheBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+ if (CacheBuffer)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId,
+ BlockDescription.BlockHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(CacheBuffer)));
+ }
+ }
+
+ if (!m_AbortFlag)
+ {
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this, &Context, BlockIndex, BlockChunkPath, BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ WriteFullBlockToCache(Context, BlockIndex, std::move(BlockBuffer), BlockChunkPath);
+ }
+ },
+ BlockChunkPath.empty() ? WorkerThreadPool::EMode::DisableBacklog : WorkerThreadPool::EMode::EnableBacklog);
+ }
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUpdateFolder::WriteFullBlockToCache(WriteChunksContext& Context,
+ uint32_t BlockIndex,
+ IoBuffer BlockBuffer,
+ const std::filesystem::path& BlockChunkPath)
+{
+ ZEN_TRACE_CPU("Async_WriteFullBlock");
+
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+
+ if (BlockChunkPath.empty())
+ {
+ ZEN_ASSERT(BlockBuffer);
+ }
+ else
+ {
+ ZEN_ASSERT(!BlockBuffer);
+ BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath);
+ if (!BlockBuffer)
+ {
+ throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", BlockDescription.BlockHash, BlockChunkPath));
+ }
+ }
+
+ Context.FilteredWrittenBytesPerSecond.Start();
+ if (!WriteChunksBlockToCache(BlockDescription,
+ Context.SequenceIndexChunksLeftToWriteCounters,
+ Context.Work,
+ CompositeBuffer(std::move(BlockBuffer)),
+ Context.RemoteChunkIndexNeedsCopyFromSourceFlags,
+ Context.WriteCache))
+ {
+ std::error_code DummyEc;
+ RemoveFile(BlockChunkPath, DummyEc);
+ throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash));
+ }
+
+ if (!BlockChunkPath.empty())
+ {
+ std::error_code Ec = TryRemoveFile(BlockChunkPath);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", BlockChunkPath, Ec.value(), Ec.message());
+ }
+ }
+
+ if (Context.WritePartsComplete.fetch_add(1) + 1 == Context.TotalPartWriteCount)
+ {
+ Context.FilteredWrittenBytesPerSecond.Stop();
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleLocalFileRemovals(ParallelWork& Work,
+ std::span<const uint32_t> RemoveLocalPathIndexes,
+ std::atomic<uint64_t>& DeletedCount)
+{
+ for (uint32_t LocalPathIndex : RemoveLocalPathIndexes)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ Work.ScheduleWork(m_IOWorkerPool, [this, &DeletedCount, LocalPathIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_RemoveFile");
+
+ const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
+ SetFileReadOnlyWithRetry(LocalFilePath, false);
+ RemoveFileWithRetry(LocalFilePath);
+ DeletedCount++;
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUpdateFolder::ScheduleTargetFinalization(
+ ParallelWork& Work,
+ std::span<const FinalizeTarget> Targets,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& SequenceHashToLocalPathIndex,
+ const tsl::robin_map<uint32_t, uint32_t>& RemotePathIndexToLocalPathIndex,
+ FolderContent& OutLocalFolderState,
+ std::atomic<uint64_t>& TargetsComplete)
+{
+ size_t TargetOffset = 0;
+ while (TargetOffset < Targets.size())
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ size_t TargetCount = 1;
+ while ((TargetOffset + TargetCount) < Targets.size() &&
+ (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash))
+ {
+ TargetCount++;
+ }
+
+ Work.ScheduleWork(m_IOWorkerPool,
+ [this,
+ &SequenceHashToLocalPathIndex,
+ Targets,
+ &RemotePathIndexToLocalPathIndex,
+ &OutLocalFolderState,
+ BaseTargetOffset = TargetOffset,
+ TargetCount,
+ &TargetsComplete](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ FinalizeTargetGroup(BaseTargetOffset,
+ TargetCount,
+ Targets,
+ SequenceHashToLocalPathIndex,
+ RemotePathIndexToLocalPathIndex,
+ OutLocalFolderState,
+ TargetsComplete);
+ }
+ });
+
+ TargetOffset += TargetCount;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::FinalizeTargetGroup(size_t BaseOffset,
+ size_t Count,
+ std::span<const FinalizeTarget> Targets,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& SequenceHashToLocalPathIndex,
+ const tsl::robin_map<uint32_t, uint32_t>& RemotePathIndexToLocalPathIndex,
+ FolderContent& OutLocalFolderState,
+ std::atomic<uint64_t>& TargetsComplete)
+{
+ ZEN_TRACE_CPU("Async_FinalizeChunkSequence");
+
+ size_t TargetOffset = BaseOffset;
+ const IoHash& RawHash = Targets[TargetOffset].RawHash;
+
+ if (RawHash == IoHash::Zero)
+ {
+ ZEN_TRACE_CPU("CreateEmptyFiles");
+ while (TargetOffset < (BaseOffset + Count))
+ {
+ const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+ ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
+ const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex];
+ std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred();
+ auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex);
+ if (InPlaceIt == RemotePathIndexToLocalPathIndex.end() || InPlaceIt->second == 0)
+ {
+ if (IsFileWithRetry(TargetFilePath))
+ {
+ SetFileReadOnlyWithRetry(TargetFilePath, false);
+ }
+ else
+ {
+ CreateDirectories(TargetFilePath.parent_path());
+ }
+ BasicFile OutputFile;
+ OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate);
+ }
+ OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
+ OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex];
+
+ OutLocalFolderState.Attributes[RemotePathIndex] =
+ m_RemoteContent.Attributes.empty()
+ ? GetNativeFileAttributes(TargetFilePath)
+ : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]);
+ OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
+
+ TargetOffset++;
+ TargetsComplete++;
+ }
+ }
+ else
+ {
+ ZEN_TRACE_CPU("FinalizeFile");
+ ZEN_ASSERT(m_RemoteLookup.RawHashToSequenceIndex.contains(RawHash));
+ const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+ const std::filesystem::path& FirstTargetPath = m_RemoteContent.Paths[FirstRemotePathIndex];
+ std::filesystem::path FirstTargetFilePath = (m_Path / FirstTargetPath).make_preferred();
+
+ if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end())
+ {
+ ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
+ }
+ else
+ {
+ if (IsFileWithRetry(FirstTargetFilePath))
+ {
+ SetFileReadOnlyWithRetry(FirstTargetFilePath, false);
+ }
+ else
+ {
+ CreateDirectories(FirstTargetFilePath.parent_path());
+ }
+
+ if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); InplaceIt != SequenceHashToLocalPathIndex.end())
+ {
+ ZEN_TRACE_CPU("Copy");
+ const uint32_t LocalPathIndex = InplaceIt->second;
+ const std::filesystem::path& SourcePath = m_LocalContent.Paths[LocalPathIndex];
+ std::filesystem::path SourceFilePath = (m_Path / SourcePath).make_preferred();
+ ZEN_ASSERT_SLOW(IsFileWithRetry(SourceFilePath));
+
+ ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath);
+ const uint64_t RawSize = m_LocalContent.RawSizes[LocalPathIndex];
+ FastCopyFile(m_Options.AllowFileClone,
+ m_Options.UseSparseFiles,
+ SourceFilePath,
+ FirstTargetFilePath,
+ RawSize,
+ m_DiskStats.WriteCount,
+ m_DiskStats.WriteByteCount,
+ m_DiskStats.CloneCount,
+ m_DiskStats.CloneByteCount);
+
+ m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+ }
+ else
+ {
+ ZEN_TRACE_CPU("Rename");
+ const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RawHash);
+ ZEN_ASSERT_SLOW(IsFileWithRetry(CacheFilePath));
+
+ std::error_code Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
+ if (Ec)
+ {
+ ZEN_WARN("Failed to move file from '{}' to '{}', reason: ({}) {}, retrying...",
+ CacheFilePath,
+ FirstTargetFilePath,
+ Ec.value(),
+ Ec.message());
+ Ec = RenameFileWithRetry(CacheFilePath, FirstTargetFilePath);
+ if (Ec)
+ {
+ throw std::system_error(std::error_code(Ec.value(), std::system_category()),
+ fmt::format("Failed to move file from '{}' to '{}', reason: ({}) {}",
+ CacheFilePath,
+ FirstTargetFilePath,
+ Ec.value(),
+ Ec.message()));
+ }
+ }
+
+ m_RebuildFolderStateStats.FinalizeTreeFilesMovedCount++;
+ }
+ }
+
+ OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath;
+ OutLocalFolderState.RawSizes[FirstRemotePathIndex] = m_RemoteContent.RawSizes[FirstRemotePathIndex];
+
+ OutLocalFolderState.Attributes[FirstRemotePathIndex] =
+ m_RemoteContent.Attributes.empty()
+ ? GetNativeFileAttributes(FirstTargetFilePath)
+ : SetNativeFileAttributes(FirstTargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[FirstRemotePathIndex]);
+ OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = GetModificationTickFromPath(FirstTargetFilePath);
+
+ TargetOffset++;
+ TargetsComplete++;
+
+ while (TargetOffset < (BaseOffset + Count))
+ {
+ const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex;
+ ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash);
+ const std::filesystem::path& TargetPath = m_RemoteContent.Paths[RemotePathIndex];
+ std::filesystem::path TargetFilePath = (m_Path / TargetPath).make_preferred();
+
+ if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end())
+ {
+ ZEN_ASSERT_SLOW(IsFileWithRetry(TargetFilePath));
+ }
+ else
+ {
+ ZEN_TRACE_CPU("Copy");
+ if (IsFileWithRetry(TargetFilePath))
+ {
+ SetFileReadOnlyWithRetry(TargetFilePath, false);
+ }
+ else
+ {
+ CreateDirectories(TargetFilePath.parent_path());
+ }
+
+ ZEN_ASSERT_SLOW(IsFileWithRetry(FirstTargetFilePath));
+ ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath);
+ const uint64_t RawSize = m_RemoteContent.RawSizes[RemotePathIndex];
+ FastCopyFile(m_Options.AllowFileClone,
+ m_Options.UseSparseFiles,
+ FirstTargetFilePath,
+ TargetFilePath,
+ RawSize,
+ m_DiskStats.WriteCount,
+ m_DiskStats.WriteByteCount,
+ m_DiskStats.CloneCount,
+ m_DiskStats.CloneByteCount);
+
+ m_RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++;
+ }
+
+ OutLocalFolderState.Paths[RemotePathIndex] = TargetPath;
+ OutLocalFolderState.RawSizes[RemotePathIndex] = m_RemoteContent.RawSizes[RemotePathIndex];
+
+ OutLocalFolderState.Attributes[RemotePathIndex] =
+ m_RemoteContent.Attributes.empty()
+ ? GetNativeFileAttributes(TargetFilePath)
+ : SetNativeFileAttributes(TargetFilePath, m_RemoteContent.Platform, m_RemoteContent.Attributes[RemotePathIndex]);
+ OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath);
+
+ TargetOffset++;
+ TargetsComplete++;
+ }
+ }
+}
+
+std::vector<BuildsOperationUpdateFolder::ScavengeSource>
+BuildsOperationUpdateFolder::FindScavengeSources()
+{
+ ZEN_TRACE_CPU("FindScavengeSources");
+
+ const bool TargetPathExists = IsDir(m_Path);
+
+ std::vector<std::filesystem::path> StatePaths = GetDownloadedStatePaths(m_Options.SystemRootDir);
+
+ std::vector<ScavengeSource> Result;
+ for (const std::filesystem::path& EntryPath : StatePaths)
+ {
+ if (IsFile(EntryPath))
+ {
+ bool DeleteEntry = false;
+
+ try
+ {
+ BuildsDownloadInfo Info = ReadDownloadedInfoFile(EntryPath);
+ const bool LocalPathExists = !Info.LocalPath.empty() && IsDir(Info.LocalPath);
+ const bool LocalStateFileExists = IsFile(Info.StateFilePath);
+ if (LocalPathExists && LocalStateFileExists)
+ {
+ if (TargetPathExists && std::filesystem::equivalent(Info.LocalPath, m_Path))
+ {
+ DeleteEntry = true;
+ }
+ else
+ {
+ Result.push_back({.StateFilePath = std::move(Info.StateFilePath), .Path = std::move(Info.LocalPath)});
+ }
+ }
+ else
+ {
+ DeleteEntry = true;
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_WARN("{}", Ex.what());
+ DeleteEntry = true;
+ }
+
+ if (DeleteEntry)
+ {
+ std::error_code DummyEc;
+ std::filesystem::remove(EntryPath, DummyEc);
+ }
+ }
+ }
+ return Result;
+}
+
+std::vector<uint32_t>
+BuildsOperationUpdateFolder::ScanTargetFolder(const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedChunkHashesFound,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& CachedSequenceHashesFound)
+{
+ ZEN_TRACE_CPU("ScanTargetFolder");
+
+ Stopwatch LocalTimer;
+
+ std::vector<uint32_t> MissingSequenceIndexes;
+
+ for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < m_RemoteContent.ChunkedContent.SequenceRawHashes.size();
+ RemoteSequenceIndex++)
+ {
+ const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(m_RemoteLookup, RemoteSequenceIndex);
+ const uint64_t RemoteRawSize = m_RemoteContent.RawSizes[RemotePathIndex];
+ if (auto CacheSequenceIt = CachedSequenceHashesFound.find(RemoteSequenceRawHash);
+ CacheSequenceIt != CachedSequenceHashesFound.end())
+ {
+ const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
+ ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
+ }
+ }
+ else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end())
+ {
+ const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
+ ZEN_ASSERT_SLOW(IsFile(CacheFilePath));
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Found chunk {} at {} ({})", RemoteSequenceRawHash, CacheFilePath, NiceBytes(RemoteRawSize));
+ }
+ }
+ else if (auto It = m_LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash);
+ It != m_LocalLookup.RawHashToSequenceIndex.end())
+ {
+ const uint32_t LocalSequenceIndex = It->second;
+ const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(m_LocalLookup, LocalSequenceIndex);
+ const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
+ ZEN_ASSERT_SLOW(IsFile(LocalFilePath));
+ m_CacheMappingStats.LocalPathsMatchingSequencesCount++;
+ m_CacheMappingStats.LocalPathsMatchingSequencesByteCount += RemoteRawSize;
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Found sequence {} at {} ({})", RemoteSequenceRawHash, LocalFilePath, NiceBytes(RemoteRawSize));
+ }
+ }
+ else
+ {
+ MissingSequenceIndexes.push_back(RemoteSequenceIndex);
+ }
+ }
+
+ m_CacheMappingStats.LocalScanElapsedWallTimeUs += LocalTimer.GetElapsedTimeUs();
+ return MissingSequenceIndexes;
+}
+
+bool
+BuildsOperationUpdateFolder::FindScavengeContent(const ScavengeSource& Source,
+ ChunkedFolderContent& OutScavengedLocalContent,
+ ChunkedContentLookup& OutScavengedLookup)
+{
+ ZEN_TRACE_CPU("FindScavengeContent");
+
+ FolderContent LocalFolderState;
+ try
+ {
+ BuildSaveState SavedState = ReadBuildSaveStateFile(Source.StateFilePath);
+ if (SavedState.Version == BuildSaveState::NoVersion)
+ {
+ ZEN_DEBUG("Skipping old build state at '{}', state files before version {} can not be trusted during scavenge",
+ Source.StateFilePath,
+ BuildSaveState::kVersion1);
+ return false;
+ }
+ OutScavengedLocalContent = std::move(SavedState.State.ChunkedContent);
+ LocalFolderState = std::move(SavedState.FolderState);
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_DEBUG("Skipping invalid build state at '{}', reason: {}", Source.StateFilePath, Ex.what());
+ return false;
+ }
+
+ tsl::robin_set<uint32_t> PathIndexesToScavenge;
+ PathIndexesToScavenge.reserve(OutScavengedLocalContent.Paths.size());
+ std::vector<uint32_t> ChunkOrderOffsets = BuildChunkOrderOffset(OutScavengedLocalContent.ChunkedContent.ChunkCounts);
+
+ {
+ tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToPathIndex;
+
+ RawHashToPathIndex.reserve(OutScavengedLocalContent.Paths.size());
+ for (uint32_t ScavengedPathIndex = 0; ScavengedPathIndex < OutScavengedLocalContent.RawHashes.size(); ScavengedPathIndex++)
+ {
+ if (!RawHashToPathIndex.contains(OutScavengedLocalContent.RawHashes[ScavengedPathIndex]))
+ {
+ RawHashToPathIndex.insert_or_assign(OutScavengedLocalContent.RawHashes[ScavengedPathIndex], ScavengedPathIndex);
+ }
+ }
+
+ for (uint32_t ScavengeSequenceIndex = 0; ScavengeSequenceIndex < OutScavengedLocalContent.ChunkedContent.SequenceRawHashes.size();
+ ScavengeSequenceIndex++)
+ {
+ const IoHash& SequenceHash = OutScavengedLocalContent.ChunkedContent.SequenceRawHashes[ScavengeSequenceIndex];
+ if (auto It = RawHashToPathIndex.find(SequenceHash); It != RawHashToPathIndex.end())
+ {
+ uint32_t PathIndex = It->second;
+ if (!PathIndexesToScavenge.contains(PathIndex))
+ {
+ if (m_RemoteLookup.RawHashToSequenceIndex.contains(SequenceHash))
+ {
+ PathIndexesToScavenge.insert(PathIndex);
+ }
+ else
+ {
+ uint32_t ChunkOrderIndexStart = ChunkOrderOffsets[ScavengeSequenceIndex];
+ const uint32_t ChunkCount = OutScavengedLocalContent.ChunkedContent.ChunkCounts[ScavengeSequenceIndex];
+ for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < ChunkCount; ChunkOrderIndex++)
+ {
+ const uint32_t ChunkIndex =
+ OutScavengedLocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndexStart + ChunkOrderIndex];
+ const IoHash& ChunkHash = OutScavengedLocalContent.ChunkedContent.ChunkHashes[ChunkIndex];
+ if (m_RemoteLookup.ChunkHashToChunkIndex.contains(ChunkHash))
+ {
+ PathIndexesToScavenge.insert(PathIndex);
+ break;
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ ZEN_WARN("Scavenged state file at '{}' for '{}' is invalid, skipping scavenging for sequence {}",
+ Source.StateFilePath,
+ Source.Path,
+ SequenceHash);
+ }
+ }
+ }
+
+ if (PathIndexesToScavenge.empty())
+ {
+ OutScavengedLocalContent = {};
+ return false;
+ }
+
+ std::vector<std::filesystem::path> PathsToScavenge;
+ PathsToScavenge.reserve(PathIndexesToScavenge.size());
+ for (uint32_t ScavengedStatePathIndex : PathIndexesToScavenge)
+ {
+ PathsToScavenge.push_back(OutScavengedLocalContent.Paths[ScavengedStatePathIndex]);
+ }
+
+ FolderContent ValidFolderContent =
+ GetValidFolderContent(m_IOWorkerPool, m_ScavengedFolderScanStats, Source.Path, PathsToScavenge, {}, 0, m_AbortFlag, m_PauseFlag);
+
+ if (!LocalFolderState.AreKnownFilesEqual(ValidFolderContent))
+ {
+ std::vector<std::filesystem::path> DeletedPaths;
+ FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, ValidFolderContent, DeletedPaths);
+
+ // If the files are modified since the state was saved we ignore the files since we don't
+ // want to incur the cost of scanning/hashing scavenged files
+ DeletedPaths.insert(DeletedPaths.end(), UpdatedContent.Paths.begin(), UpdatedContent.Paths.end());
+ if (!DeletedPaths.empty())
+ {
+ OutScavengedLocalContent =
+ DeletePathsFromChunkedContent(OutScavengedLocalContent,
+ BuildHashLookup(OutScavengedLocalContent.ChunkedContent.SequenceRawHashes),
+ ChunkOrderOffsets,
+ DeletedPaths);
+ }
+ }
+
+ if (OutScavengedLocalContent.Paths.empty())
+ {
+ OutScavengedLocalContent = {};
+ return false;
+ }
+
+ OutScavengedLookup = BuildChunkedContentLookup(OutScavengedLocalContent);
+
+ return true;
+}
+
+void
+BuildsOperationUpdateFolder::ScavengeSourceForChunks(uint32_t& InOutRemainingChunkCount,
+ std::vector<bool>& InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags,
+ tsl::robin_map<IoHash, size_t, IoHash::Hasher>& InOutRawHashToCopyChunkDataIndex,
+ const std::vector<std::atomic<uint32_t>>& SequenceIndexChunksLeftToWriteCounters,
+ const ChunkedFolderContent& ScavengedContent,
+ const ChunkedContentLookup& ScavengedLookup,
+ std::vector<CopyChunkData>& InOutCopyChunkDatas,
+ uint32_t ScavengedContentIndex,
+ uint64_t& InOutChunkMatchingRemoteCount,
+ uint64_t& InOutChunkMatchingRemoteByteCount)
+{
+ for (uint32_t RemoteChunkIndex = 0;
+ RemoteChunkIndex < m_RemoteContent.ChunkedContent.ChunkHashes.size() && (InOutRemainingChunkCount > 0);
+ RemoteChunkIndex++)
+ {
+ if (!InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex])
+ {
+ const IoHash& RemoteChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ if (auto It = ScavengedLookup.ChunkHashToChunkIndex.find(RemoteChunkHash); It != ScavengedLookup.ChunkHashToChunkIndex.end())
+ {
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
+ GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex);
+
+ if (!ChunkTargetPtrs.empty())
+ {
+ const uint32_t ScavengedChunkIndex = It->second;
+ const uint64_t ScavengedChunkRawSize = ScavengedContent.ChunkedContent.ChunkRawSizes[ScavengedChunkIndex];
+ const size_t ChunkSequenceLocationOffset = ScavengedLookup.ChunkSequenceLocationOffset[ScavengedChunkIndex];
+ const ChunkedContentLookup::ChunkSequenceLocation& ScavengeLocation =
+ ScavengedLookup.ChunkSequenceLocations[ChunkSequenceLocationOffset];
+ const IoHash& ScavengedSequenceRawHash =
+ ScavengedContent.ChunkedContent.SequenceRawHashes[ScavengeLocation.SequenceIndex];
+
+ CopyChunkData::ChunkTarget Target = {.TargetChunkLocationCount = gsl::narrow<uint32_t>(ChunkTargetPtrs.size()),
+ .RemoteChunkIndex = RemoteChunkIndex,
+ .CacheFileOffset = ScavengeLocation.Offset};
+ if (auto CopySourceIt = InOutRawHashToCopyChunkDataIndex.find(ScavengedSequenceRawHash);
+ CopySourceIt != InOutRawHashToCopyChunkDataIndex.end())
+ {
+ CopyChunkData& Data = InOutCopyChunkDatas[CopySourceIt->second];
+ if (Data.TargetChunkLocationPtrs.size() > 1024)
+ {
+ InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size());
+ InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex,
+ .SourceSequenceIndex = ScavengeLocation.SequenceIndex,
+ .TargetChunkLocationPtrs = ChunkTargetPtrs,
+ .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}});
+ }
+ else
+ {
+ Data.TargetChunkLocationPtrs.insert(Data.TargetChunkLocationPtrs.end(),
+ ChunkTargetPtrs.begin(),
+ ChunkTargetPtrs.end());
+ Data.ChunkTargets.push_back(Target);
+ }
+ }
+ else
+ {
+ InOutRawHashToCopyChunkDataIndex.insert_or_assign(ScavengedSequenceRawHash, InOutCopyChunkDatas.size());
+ InOutCopyChunkDatas.push_back(CopyChunkData{.ScavengeSourceIndex = ScavengedContentIndex,
+ .SourceSequenceIndex = ScavengeLocation.SequenceIndex,
+ .TargetChunkLocationPtrs = ChunkTargetPtrs,
+ .ChunkTargets = std::vector<CopyChunkData::ChunkTarget>{Target}});
+ }
+ InOutChunkMatchingRemoteCount++;
+ InOutChunkMatchingRemoteByteCount += ScavengedChunkRawSize;
+ InOutRemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex] = true;
+ InOutRemainingChunkCount--;
+ }
+ }
+ }
+ }
+}
+
+std::filesystem::path
+BuildsOperationUpdateFolder::FindDownloadedChunk(const IoHash& ChunkHash)
+{
+ ZEN_TRACE_CPU("FindDownloadedChunk");
+
+ std::filesystem::path CompressedChunkPath = m_TempDownloadFolderPath / ChunkHash.ToHexString();
+ if (IsFile(CompressedChunkPath))
+ {
+ IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
+ if (ExistingCompressedPart)
+ {
+ IoHash RawHash;
+ uint64_t RawSize;
+ if (CompressedBuffer::ValidateCompressedHeader(ExistingCompressedPart,
+ RawHash,
+ RawSize,
+ /*OutOptionalTotalCompressedSize*/ nullptr))
+ {
+ return CompressedChunkPath;
+ }
+ else
+ {
+ std::error_code DummyEc;
+ RemoveFile(CompressedChunkPath, DummyEc);
+ }
+ }
+ }
+ return {};
+}
+
+std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>
+BuildsOperationUpdateFolder::GetRemainingChunkTargets(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ uint32_t ChunkIndex)
+{
+ ZEN_TRACE_CPU("GetRemainingChunkTargets");
+
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex);
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs;
+ if (!ChunkSources.empty())
+ {
+ ChunkTargetPtrs.reserve(ChunkSources.size());
+ for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
+ {
+ if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
+ {
+ ChunkTargetPtrs.push_back(&Source);
+ }
+ }
+ }
+ return ChunkTargetPtrs;
+};
+
+uint64_t
+BuildsOperationUpdateFolder::GetChunkWriteCount(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ uint32_t ChunkIndex)
+{
+ ZEN_TRACE_CPU("GetChunkWriteCount");
+
+ uint64_t WriteCount = 0;
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(m_RemoteLookup, ChunkIndex);
+ for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources)
+ {
+ if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0)
+ {
+ WriteCount++;
+ }
+ }
+ return WriteCount;
+};
+
+void
+BuildsOperationUpdateFolder::CheckRequiredDiskSpace(const tsl::robin_map<std::string, uint32_t>& RemotePathToRemoteIndex)
+{
+ tsl::robin_set<uint32_t> ExistingRemotePaths;
+
+ if (m_Options.EnableTargetFolderScavenging)
+ {
+ for (uint32_t LocalPathIndex = 0; LocalPathIndex < m_LocalContent.Paths.size(); LocalPathIndex++)
+ {
+ const IoHash& RawHash = m_LocalContent.RawHashes[LocalPathIndex];
+ const std::filesystem::path& LocalPath = m_LocalContent.Paths[LocalPathIndex];
+
+ if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); RemotePathIt != RemotePathToRemoteIndex.end())
+ {
+ const uint32_t RemotePathIndex = RemotePathIt->second;
+ if (m_RemoteContent.RawHashes[RemotePathIndex] == RawHash)
+ {
+ ExistingRemotePaths.insert(RemotePathIndex);
+ }
+ }
+ }
+ }
+
+ uint64_t RequiredSpace = 0;
+ for (uint32_t RemotePathIndex = 0; RemotePathIndex < m_RemoteContent.Paths.size(); RemotePathIndex++)
+ {
+ if (!ExistingRemotePaths.contains(RemotePathIndex))
+ {
+ RequiredSpace += m_RemoteContent.RawSizes[RemotePathIndex];
+ }
+ }
+
+ std::error_code Ec;
+ DiskSpace Space = DiskSpaceInfo(m_Path, Ec);
+ if (Ec)
+ {
+ throw std::runtime_error(fmt::format("Get free disk space for target path '{}' FAILED, reason: {}", m_Path, Ec.message()));
+ }
+ if (Space.Free < (RequiredSpace + 16u * 1024u * 1024u))
+ {
+ throw std::runtime_error(
+ fmt::format("Not enough free space for target path '{}', {} of free space is needed but only {} is available",
+ m_Path,
+ NiceBytes(RequiredSpace),
+ NiceBytes(Space.Free)));
+ }
+}
+
+void
+BuildsOperationUpdateFolder::WriteScavengedSequenceToCache(const std::filesystem::path& ScavengeRootPath,
+ const ChunkedFolderContent& ScavengedContent,
+ const ScavengedSequenceCopyOperation& ScavengeOp)
+{
+ ZEN_TRACE_CPU("WriteScavengedSequenceToCache");
+
+ const std::filesystem::path ScavengedPath = ScavengedContent.Paths[ScavengeOp.ScavengedPathIndex];
+ const std::filesystem::path ScavengedFilePath = (ScavengeRootPath / ScavengedPath).make_preferred();
+ ZEN_ASSERT_SLOW(FileSizeFromPath(ScavengedFilePath) == ScavengeOp.RawSize);
+
+ const IoHash& RemoteSequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[ScavengeOp.RemoteSequenceIndex];
+ const std::filesystem::path TempFilePath = GetTempChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
+
+ const uint64_t RawSize = ScavengedContent.RawSizes[ScavengeOp.ScavengedPathIndex];
+ FastCopyFile(m_Options.AllowFileClone,
+ m_Options.UseSparseFiles,
+ ScavengedFilePath,
+ TempFilePath,
+ RawSize,
+ m_DiskStats.WriteCount,
+ m_DiskStats.WriteByteCount,
+ m_DiskStats.CloneCount,
+ m_DiskStats.CloneByteCount);
+
+ const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(m_CacheFolderPath, RemoteSequenceRawHash);
+ RenameFile(TempFilePath, CacheFilePath);
+
+ m_WrittenChunkByteCount += RawSize;
+ if (m_Options.ValidateCompletedSequences)
+ {
+ m_ValidatedChunkByteCount += RawSize;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::WriteLooseChunk(const uint32_t RemoteChunkIndex,
+ const BlobsExistsResult& ExistsResult,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ std::atomic<uint64_t>& WritePartsComplete,
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
+ BufferedWriteFileCache& WriteCache,
+ ParallelWork& Work,
+ uint64_t TotalRequestCount,
+ uint64_t TotalPartWriteCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond,
+ FilteredRate& FilteredWrittenBytesPerSecond)
+{
+ const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ std::filesystem::path ExistingCompressedChunkPath = FindDownloadedChunk(ChunkHash);
+ if (!ExistingCompressedChunkPath.empty())
+ {
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+ }
+ if (!m_AbortFlag)
+ {
+ if (!ExistingCompressedChunkPath.empty())
+ {
+ Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this,
+ SequenceIndexChunksLeftToWriteCounters,
+ &WriteCache,
+ &Work,
+ &WritePartsComplete,
+ TotalPartWriteCount,
+ &FilteredWrittenBytesPerSecond,
+ RemoteChunkIndex,
+ ChunkTargetPtrs = std::move(ChunkTargetPtrs),
+ CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>& AbortFlag) {
+ if (!AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_WritePreDownloadedChunk");
+
+ FilteredWrittenBytesPerSecond.Start();
+
+ const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+
+ IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
+ if (!CompressedPart)
+ {
+ throw std::runtime_error(
+ fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath));
+ }
+
+ bool NeedHashVerify =
+ WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart));
+ bool WritePartsDone = WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount;
+
+ if (!AbortFlag)
+ {
+ if (WritePartsDone)
+ {
+ FilteredWrittenBytesPerSecond.Stop();
+ }
+
+ std::error_code Ec = TryRemoveFile(CompressedChunkPath);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message());
+ }
+
+ std::vector<uint32_t> CompletedSequences =
+ CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
+ WriteCache.Close(CompletedSequences);
+ if (NeedHashVerify)
+ {
+ VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work);
+ }
+ else
+ {
+ FinalizeChunkSequences(CompletedSequences);
+ }
+ }
+ }
+ });
+ }
+ else
+ {
+ Work.ScheduleWork(m_NetworkPool,
+ [this,
+ &ExistsResult,
+ SequenceIndexChunksLeftToWriteCounters,
+ &WriteCache,
+ &Work,
+ &WritePartsComplete,
+ TotalPartWriteCount,
+ TotalRequestCount,
+ &FilteredDownloadedBytesPerSecond,
+ &FilteredWrittenBytesPerSecond,
+ RemoteChunkIndex,
+ ChunkTargetPtrs = std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>(
+ std::move(ChunkTargetPtrs))](std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_DownloadChunk");
+
+ FilteredDownloadedBytesPerSecond.Start();
+ DownloadBuildBlob(RemoteChunkIndex,
+ ExistsResult,
+ Work,
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ [this,
+ &ExistsResult,
+ SequenceIndexChunksLeftToWriteCounters,
+ &WriteCache,
+ &Work,
+ &WritePartsComplete,
+ TotalPartWriteCount,
+ RemoteChunkIndex,
+ &FilteredWrittenBytesPerSecond,
+ ChunkTargetPtrs = std::move(ChunkTargetPtrs)](IoBuffer&& Payload) mutable {
+ AsyncWriteDownloadedChunk(RemoteChunkIndex,
+ ExistsResult,
+ std::move(ChunkTargetPtrs),
+ WriteCache,
+ Work,
+ std::move(Payload),
+ SequenceIndexChunksLeftToWriteCounters,
+ WritePartsComplete,
+ TotalPartWriteCount,
+ FilteredWrittenBytesPerSecond);
+ });
+ }
+ });
+ }
+ }
+}
+
+void
+BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkIndex,
+ const BlobsExistsResult& ExistsResult,
+ ParallelWork& Work,
+ uint64_t TotalRequestCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond,
+ std::function<void(IoBuffer&& Payload)>&& OnDownloaded)
+{
+ const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ // FilteredDownloadedBytesPerSecond.Start();
+ IoBuffer BuildBlob;
+ const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash);
+ if (ExistsInCache)
+ {
+ BuildBlob = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, ChunkHash);
+ }
+ if (BuildBlob)
+ {
+ uint64_t BlobSize = BuildBlob.GetSize();
+ m_DownloadStats.DownloadedChunkCount++;
+ m_DownloadStats.DownloadedChunkByteCount += BlobSize;
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+ OnDownloaded(std::move(BuildBlob));
+ }
+ else
+ {
+ if (m_RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= m_Options.LargeAttachmentSize)
+ {
+ DownloadLargeBlob(
+ *m_Storage.BuildStorage,
+ m_TempDownloadFolderPath,
+ m_BuildId,
+ ChunkHash,
+ m_Options.PreferredMultipartChunkSize,
+ Work,
+ m_NetworkPool,
+ m_DownloadStats.DownloadedChunkByteCount,
+ m_DownloadStats.MultipartAttachmentCount,
+ [this, &FilteredDownloadedBytesPerSecond, TotalRequestCount, OnDownloaded = std::move(OnDownloaded)](IoBuffer&& Payload) {
+ m_DownloadStats.DownloadedChunkCount++;
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+
+ OnDownloaded(std::move(Payload));
+ });
+ }
+ else
+ {
+ try
+ {
+ BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+ if (!m_AbortFlag)
+ {
+ if (!BuildBlob)
+ {
+ throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash));
+ }
+
+ if (!m_AbortFlag)
+ {
+ uint64_t BlobSize = BuildBlob.GetSize();
+ m_DownloadStats.DownloadedChunkCount++;
+ m_DownloadStats.DownloadedChunkByteCount += BlobSize;
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(1) + 1 == TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+
+ OnDownloaded(std::move(BuildBlob));
+ }
+ }
+ }
+ }
+}
+
+void
+BuildsOperationUpdateFolder::DownloadPartialBlock(
+ std::span<const ChunkBlockAnalyser::BlockRangeDescriptor> BlockRanges,
+ size_t BlockRangeStartIndex,
+ size_t BlockRangeCount,
+ const BlobsExistsResult& ExistsResult,
+ uint64_t TotalRequestCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond,
+ std::function<void(IoBuffer&& InMemoryBuffer,
+ const std::filesystem::path& OnDiskPath,
+ size_t BlockRangeStartIndex,
+ std::span<const std::pair<uint64_t, uint64_t>> OffsetAndLengths)>&& OnDownloaded)
+{
+ const uint32_t BlockIndex = BlockRanges[BlockRangeStartIndex].BlockIndex;
+
+ const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex];
+
+ auto ProcessDownload = [this](
+ const ChunkBlockDescription& BlockDescription,
+ IoBuffer&& BlockRangeBuffer,
+ size_t BlockRangeStartIndex,
+ std::span<const std::pair<uint64_t, uint64_t>> BlockOffsetAndLengths,
+ uint64_t TotalRequestCount,
+ FilteredRate& FilteredDownloadedBytesPerSecond,
+ const std::function<void(IoBuffer && InMemoryBuffer,
+ const std::filesystem::path& OnDiskPath,
+ size_t BlockRangeStartIndex,
+ std::span<const std::pair<uint64_t, uint64_t>> OffsetAndLengths)>& OnDownloaded) {
+ uint64_t BlockRangeBufferSize = BlockRangeBuffer.GetSize();
+ m_DownloadStats.DownloadedBlockCount++;
+ m_DownloadStats.DownloadedBlockByteCount += BlockRangeBufferSize;
+ if (m_DownloadStats.RequestsCompleteCount.fetch_add(BlockOffsetAndLengths.size()) + BlockOffsetAndLengths.size() ==
+ TotalRequestCount)
+ {
+ FilteredDownloadedBytesPerSecond.Stop();
+ }
+
+ IoHashStream RangeId;
+ for (const std::pair<uint64_t, uint64_t>& Range : BlockOffsetAndLengths)
+ {
+ RangeId.Append(&Range.first, sizeof(uint64_t));
+ RangeId.Append(&Range.second, sizeof(uint64_t));
+ }
+ std::filesystem::path BlockChunkPath =
+ TryMoveDownloadedChunk(BlockRangeBuffer,
+ m_TempBlockFolderPath / fmt::format("{}_{}", BlockDescription.BlockHash, RangeId.GetHash()),
+ /* ForceDiskBased */ BlockRangeBufferSize > m_Options.MaximumInMemoryPayloadSize);
+
+ if (!m_AbortFlag)
+ {
+ OnDownloaded(std::move(BlockRangeBuffer), std::move(BlockChunkPath), BlockRangeStartIndex, BlockOffsetAndLengths);
+ }
+ };
+
+ std::vector<std::pair<uint64_t, uint64_t>> Ranges;
+ Ranges.reserve(BlockRangeCount);
+ for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + BlockRangeCount; BlockRangeIndex++)
+ {
+ const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRanges[BlockRangeIndex];
+ Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength));
+ }
+
+ const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash);
+
+ size_t SubBlockRangeCount = BlockRangeCount;
+ size_t SubRangeCountComplete = 0;
+ std::span<const std::pair<uint64_t, uint64_t>> RangesSpan(Ranges);
+ while (SubRangeCountComplete < SubBlockRangeCount)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ // First try to get subrange from cache.
+ // If not successful, try to get the ranges from the build store and adapt SubRangeCount...
+
+ size_t SubRangeStartIndex = BlockRangeStartIndex + SubRangeCountComplete;
+ if (ExistsInCache)
+ {
+ size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.CacheHost.Caps.MaxRangeCountPerRequest);
+
+ if (SubRangeCount == 1)
+ {
+ // Legacy single-range path, prefer that for max compatibility
+
+ const std::pair<uint64_t, uint64_t> SubRange = RangesSpan[SubRangeCountComplete];
+ IoBuffer PayloadBuffer =
+ m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, SubRange.first, SubRange.second);
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ if (PayloadBuffer)
+ {
+ ProcessDownload(BlockDescription,
+ std::move(PayloadBuffer),
+ SubRangeStartIndex,
+ std::vector<std::pair<uint64_t, uint64_t>>{std::make_pair(0u, SubRange.second)},
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ OnDownloaded);
+ SubRangeCountComplete += SubRangeCount;
+ continue;
+ }
+ }
+ else
+ {
+ auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount);
+
+ BuildStorageCache::BuildBlobRanges RangeBuffers =
+ m_Storage.CacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges);
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ if (RangeBuffers.PayloadBuffer)
+ {
+ if (RangeBuffers.Ranges.empty())
+ {
+ SubRangeCount = Ranges.size() - SubRangeCountComplete;
+ ProcessDownload(BlockDescription,
+ std::move(RangeBuffers.PayloadBuffer),
+ SubRangeStartIndex,
+ RangesSpan.subspan(SubRangeCountComplete, SubRangeCount),
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ OnDownloaded);
+ SubRangeCountComplete += SubRangeCount;
+ continue;
+ }
+ else if (RangeBuffers.Ranges.size() == SubRangeCount)
+ {
+ ProcessDownload(BlockDescription,
+ std::move(RangeBuffers.PayloadBuffer),
+ SubRangeStartIndex,
+ RangeBuffers.Ranges,
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ OnDownloaded);
+ SubRangeCountComplete += SubRangeCount;
+ continue;
+ }
+ }
+ }
+ }
+
+ size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest);
+
+ auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount);
+
+ BuildStorageBase::BuildBlobRanges RangeBuffers;
+
+ try
+ {
+ RangeBuffers = m_Storage.BuildStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+
+ if (!m_AbortFlag)
+ {
+ if (RangeBuffers.PayloadBuffer)
+ {
+ if (RangeBuffers.Ranges.empty())
+ {
+ // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3
+ // Upload to cache (if enabled) and use the whole payload for the remaining ranges
+
+ const uint64_t Size = RangeBuffers.PayloadBuffer.GetSize();
+
+ const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache;
+
+ std::filesystem::path BlockPath =
+ TryMoveDownloadedChunk(RangeBuffers.PayloadBuffer,
+ m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(),
+ /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize);
+ if (!BlockPath.empty())
+ {
+ RangeBuffers.PayloadBuffer = IoBufferBuilder::MakeFromFile(BlockPath);
+ if (!RangeBuffers.PayloadBuffer)
+ {
+ throw std::runtime_error(
+ fmt::format("Failed to read block {} from temporary path '{}'", BlockDescription.BlockHash, BlockPath));
+ }
+ RangeBuffers.PayloadBuffer.SetDeleteOnClose(true);
+ }
+
+ if (PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId,
+ BlockDescription.BlockHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(RangeBuffers.PayloadBuffer)));
+ }
+
+ if (m_AbortFlag)
+ {
+ break;
+ }
+
+ SubRangeCount = Ranges.size() - SubRangeCountComplete;
+ ProcessDownload(BlockDescription,
+ std::move(RangeBuffers.PayloadBuffer),
+ SubRangeStartIndex,
+ RangesSpan.subspan(SubRangeCountComplete, SubRangeCount),
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ OnDownloaded);
+ }
+ else
+ {
+ if (RangeBuffers.Ranges.size() != SubRanges.size())
+ {
+ throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges",
+ SubRanges.size(),
+ BlockDescription.BlockHash,
+ RangeBuffers.Ranges.size()));
+ }
+ ProcessDownload(BlockDescription,
+ std::move(RangeBuffers.PayloadBuffer),
+ SubRangeStartIndex,
+ RangeBuffers.Ranges,
+ TotalRequestCount,
+ FilteredDownloadedBytesPerSecond,
+ OnDownloaded);
+ }
+ }
+ else
+ {
+ throw std::runtime_error(
+ fmt::format("Block {} is missing when fetching {} ranges", BlockDescription.BlockHash, SubRangeCount));
+ }
+
+ SubRangeCountComplete += SubRangeCount;
+ }
+ }
+}
+
+std::vector<uint32_t>
+BuildsOperationUpdateFolder::WriteLocalChunkToCache(CloneQueryInterface* CloneQuery,
+ const CopyChunkData& CopyData,
+ const std::vector<ChunkedFolderContent>& ScavengedContents,
+ const std::vector<ChunkedContentLookup>& ScavengedLookups,
+ const std::vector<std::filesystem::path>& ScavengedPaths,
+ BufferedWriteFileCache& WriteCache)
+{
+ ZEN_TRACE_CPU("WriteLocalChunkToCache");
+
+ std::filesystem::path SourceFilePath;
+
+ if (CopyData.ScavengeSourceIndex == (uint32_t)-1)
+ {
+ const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
+ SourceFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
+ }
+ else
+ {
+ const ChunkedFolderContent& ScavengedContent = ScavengedContents[CopyData.ScavengeSourceIndex];
+ const ChunkedContentLookup& ScavengedLookup = ScavengedLookups[CopyData.ScavengeSourceIndex];
+ const std::filesystem::path ScavengedPath = ScavengedPaths[CopyData.ScavengeSourceIndex];
+ const uint32_t ScavengedPathIndex = ScavengedLookup.SequenceIndexFirstPathIndex[CopyData.SourceSequenceIndex];
+ SourceFilePath = (ScavengedPath / ScavengedContent.Paths[ScavengedPathIndex]).make_preferred();
+ }
+ ZEN_ASSERT_SLOW(IsFile(SourceFilePath));
+ ZEN_ASSERT(!CopyData.TargetChunkLocationPtrs.empty());
+
+ uint64_t CacheLocalFileBytesRead = 0;
+
+ size_t TargetStart = 0;
+ const std::span<const ChunkedContentLookup::ChunkSequenceLocation* const> AllTargets(CopyData.TargetChunkLocationPtrs);
+
+ struct WriteOp
+ {
+ const ChunkedContentLookup::ChunkSequenceLocation* Target = nullptr;
+ uint64_t CacheFileOffset = (uint64_t)-1;
+ uint32_t ChunkIndex = (uint32_t)-1;
+ };
+
+ std::vector<WriteOp> WriteOps;
+
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Sort");
+ WriteOps.reserve(AllTargets.size());
+ for (const CopyChunkData::ChunkTarget& ChunkTarget : CopyData.ChunkTargets)
+ {
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation* const> TargetRange =
+ AllTargets.subspan(TargetStart, ChunkTarget.TargetChunkLocationCount);
+ for (const ChunkedContentLookup::ChunkSequenceLocation* Target : TargetRange)
+ {
+ WriteOps.push_back(
+ WriteOp{.Target = Target, .CacheFileOffset = ChunkTarget.CacheFileOffset, .ChunkIndex = ChunkTarget.RemoteChunkIndex});
+ }
+ TargetStart += ChunkTarget.TargetChunkLocationCount;
+ }
+
+ std::sort(WriteOps.begin(), WriteOps.end(), [](const WriteOp& Lhs, const WriteOp& Rhs) {
+ if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
+ {
+ return true;
+ }
+ else if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
+ {
+ return false;
+ }
+ if (Lhs.Target->Offset < Rhs.Target->Offset)
+ {
+ return true;
+ }
+ return false;
+ });
+ }
+
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Write");
+
+ tsl::robin_set<uint32_t> ChunkIndexesWritten;
+
+ BufferedOpenFile SourceFile(SourceFilePath,
+ m_DiskStats.OpenReadCount,
+ m_DiskStats.CurrentOpenFileCount,
+ m_DiskStats.ReadCount,
+ m_DiskStats.ReadByteCount);
+
+ bool CanCloneSource = CloneQuery && CloneQuery->CanClone(SourceFile.Handle());
+
+ BufferedWriteFileCache::Local LocalWriter(WriteCache);
+
+ for (size_t WriteOpIndex = 0; WriteOpIndex < WriteOps.size();)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ const WriteOp& Op = WriteOps[WriteOpIndex];
+
+ const uint32_t RemoteSequenceIndex = Op.Target->SequenceIndex;
+ const uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
+ const uint64_t TargetSize = m_RemoteContent.RawSizes[RemotePathIndex];
+ const uint64_t ChunkSize = m_RemoteContent.ChunkedContent.ChunkRawSizes[Op.ChunkIndex];
+
+ uint64_t ReadLength = ChunkSize;
+ size_t WriteCount = 1;
+ uint64_t OpSourceEnd = Op.CacheFileOffset + ChunkSize;
+ uint64_t OpTargetEnd = Op.Target->Offset + ChunkSize;
+ while ((WriteOpIndex + WriteCount) < WriteOps.size())
+ {
+ const WriteOp& NextOp = WriteOps[WriteOpIndex + WriteCount];
+ if (NextOp.Target->SequenceIndex != Op.Target->SequenceIndex)
+ {
+ break;
+ }
+ if (NextOp.Target->Offset != OpTargetEnd)
+ {
+ break;
+ }
+ if (NextOp.CacheFileOffset != OpSourceEnd)
+ {
+ break;
+ }
+ const uint64_t NextChunkLength = m_RemoteContent.ChunkedContent.ChunkRawSizes[NextOp.ChunkIndex];
+ if (ReadLength + NextChunkLength > BufferedOpenFile::BlockSize)
+ {
+ break;
+ }
+ ReadLength += NextChunkLength;
+ OpSourceEnd += NextChunkLength;
+ OpTargetEnd += NextChunkLength;
+ WriteCount++;
+ }
+
+ {
+ bool DidClone = false;
+
+ if (CanCloneSource)
+ {
+ uint64_t PreBytes = 0;
+ uint64_t PostBytes = 0;
+ uint64_t ClonableBytes =
+ CloneQuery->GetClonableRange(Op.CacheFileOffset, Op.Target->Offset, ReadLength, PreBytes, PostBytes);
+ if (ClonableBytes > 0)
+ {
+ // We need to open the file...
+ BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(RemoteSequenceIndex);
+ if (!Writer)
+ {
+ Writer = LocalWriter.PutWriter(RemoteSequenceIndex, std::make_unique<BufferedWriteFileCache::Local::Writer>());
+
+ Writer->File = std::make_unique<BasicFile>();
+
+ const std::filesystem::path FileName =
+ GetTempChunkedSequenceFileName(m_CacheFolderPath,
+ m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]);
+ Writer->File->Open(FileName, BasicFile::Mode::kWrite);
+ if (m_Options.UseSparseFiles)
+ {
+ PrepareFileForScatteredWrite(Writer->File->Handle(), TargetSize);
+ }
+ }
+ DidClone = CloneQuery->TryClone(SourceFile.Handle(),
+ Writer->File->Handle(),
+ Op.CacheFileOffset + PreBytes,
+ Op.Target->Offset + PreBytes,
+ ClonableBytes,
+ TargetSize);
+ if (DidClone)
+ {
+ m_DiskStats.WriteCount++;
+ m_DiskStats.WriteByteCount += ClonableBytes;
+
+ m_DiskStats.CloneCount++;
+ m_DiskStats.CloneByteCount += ClonableBytes;
+
+ m_WrittenChunkByteCount += ClonableBytes;
+
+ if (PreBytes > 0)
+ {
+ CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, PreBytes);
+ const uint64_t FileOffset = Op.Target->Offset;
+
+ WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
+ }
+ if (PostBytes > 0)
+ {
+ CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset + ReadLength - PostBytes, PostBytes);
+ const uint64_t FileOffset = Op.Target->Offset + ReadLength - PostBytes;
+
+ WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
+ }
+ }
+ }
+ }
+
+ if (!DidClone)
+ {
+ CompositeBuffer ChunkSource = SourceFile.GetRange(Op.CacheFileOffset, ReadLength);
+
+ const uint64_t FileOffset = Op.Target->Offset;
+
+ WriteSequenceChunkToCache(LocalWriter, ChunkSource, RemoteSequenceIndex, FileOffset, RemotePathIndex);
+ }
+ }
+
+ CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes?
+
+ WriteOpIndex += WriteCount;
+ }
+ }
+
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), SourceFilePath);
+ }
+
+ std::vector<uint32_t> Result;
+ Result.reserve(WriteOps.size());
+
+ for (const WriteOp& Op : WriteOps)
+ {
+ Result.push_back(Op.Target->SequenceIndex);
+ }
+ return Result;
+}
+
+bool
+BuildsOperationUpdateFolder::WriteCompressedChunkToCache(
+ const IoHash& ChunkHash,
+ const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
+ BufferedWriteFileCache& WriteCache,
+ IoBuffer&& CompressedPart)
+{
+ ZEN_TRACE_CPU("WriteCompressedChunkToCache");
+
+ auto ChunkHashToChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash);
+ ZEN_ASSERT(ChunkHashToChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end());
+ if (IsSingleFileChunk(m_RemoteContent, ChunkTargetPtrs))
+ {
+ const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex;
+ const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex];
+ StreamDecompress(SequenceRawHash, CompositeBuffer(std::move(CompressedPart)));
+ return false;
+ }
+ else
+ {
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompositeBuffer(std::move(CompressedPart)), RawHash, RawSize);
+ if (!Compressed)
+ {
+ throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", ChunkHash));
+ }
+ if (RawHash != ChunkHash)
+ {
+ throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, ChunkHash));
+ }
+
+ BufferedWriteFileCache::Local LocalWriter(WriteCache);
+
+ IoHashStream Hash;
+ bool CouldDecompress = Compressed.DecompressToStream(
+ 0,
+ (uint64_t)-1,
+ [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+ ZEN_UNUSED(SourceOffset);
+ ZEN_TRACE_CPU("Async_StreamDecompress_Write");
+ m_DiskStats.ReadByteCount += SourceSize;
+ if (!m_AbortFlag)
+ {
+ for (const ChunkedContentLookup::ChunkSequenceLocation* TargetPtr : ChunkTargetPtrs)
+ {
+ const auto& Target = *TargetPtr;
+ const uint64_t FileOffset = Target.Offset + Offset;
+ const uint32_t SequenceIndex = Target.SequenceIndex;
+ const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+
+ WriteSequenceChunkToCache(LocalWriter, RangeBuffer, SequenceIndex, FileOffset, PathIndex);
+ }
+
+ return true;
+ }
+ return false;
+ });
+
+ if (m_AbortFlag)
+ {
+ return false;
+ }
+
+ if (!CouldDecompress)
+ {
+ throw std::runtime_error(fmt::format("Failed to decompress large chunk {}", ChunkHash));
+ }
+
+ return true;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::StreamDecompress(const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart)
+{
+ ZEN_TRACE_CPU("StreamDecompress");
+ const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash);
+ TemporaryFile DecompressedTemp;
+ std::error_code Ec;
+ DecompressedTemp.CreateTemporary(TempChunkSequenceFileName.parent_path(), Ec);
+ if (Ec)
+ {
+ throw std::runtime_error(fmt::format("Failed creating temporary file for decompressing large blob {}, reason: ({}) {}",
+ SequenceRawHash,
+ Ec.value(),
+ Ec.message()));
+ }
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(CompressedPart, RawHash, RawSize);
+ if (!Compressed)
+ {
+ throw std::runtime_error(fmt::format("Failed to parse header of compressed large blob {}", SequenceRawHash));
+ }
+ if (RawHash != SequenceRawHash)
+ {
+ throw std::runtime_error(fmt::format("RawHash in header {} in large blob {} does match.", RawHash, SequenceRawHash));
+ }
+ PrepareFileForScatteredWrite(DecompressedTemp.Handle(), RawSize);
+
+ IoHashStream Hash;
+ bool CouldDecompress =
+ Compressed.DecompressToStream(0,
+ (uint64_t)-1,
+ [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+ ZEN_UNUSED(SourceOffset);
+ ZEN_TRACE_CPU("StreamDecompress_Write");
+ m_DiskStats.ReadCount++;
+ m_DiskStats.ReadByteCount += SourceSize;
+ if (!m_AbortFlag)
+ {
+ for (const SharedBuffer& Segment : RangeBuffer.GetSegments())
+ {
+ if (m_Options.ValidateCompletedSequences)
+ {
+ Hash.Append(Segment.GetView());
+ m_ValidatedChunkByteCount += Segment.GetSize();
+ }
+ DecompressedTemp.Write(Segment, Offset);
+ Offset += Segment.GetSize();
+ m_DiskStats.WriteByteCount += Segment.GetSize();
+ m_DiskStats.WriteCount++;
+ m_WrittenChunkByteCount += Segment.GetSize();
+ }
+ return true;
+ }
+ return false;
+ });
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ if (!CouldDecompress)
+ {
+ throw std::runtime_error(fmt::format("Failed to decompress large blob {}", SequenceRawHash));
+ }
+ if (m_Options.ValidateCompletedSequences)
+ {
+ const IoHash VerifyHash = Hash.GetHash();
+ if (VerifyHash != SequenceRawHash)
+ {
+ throw std::runtime_error(
+ fmt::format("Decompressed blob payload hash {} does not match expected hash {}", VerifyHash, SequenceRawHash));
+ }
+ }
+ DecompressedTemp.MoveTemporaryIntoPlace(TempChunkSequenceFileName, Ec);
+ if (Ec)
+ {
+ throw std::runtime_error(fmt::format("Failed moving temporary file for decompressing large blob {}, reason: ({}) {}",
+ SequenceRawHash,
+ Ec.value(),
+ Ec.message()));
+ }
+ // WriteChunkStats.ChunkCountWritten++;
+}
+
+void
+BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::Local& LocalWriter,
+ const CompositeBuffer& Chunk,
+ const uint32_t SequenceIndex,
+ const uint64_t FileOffset,
+ const uint32_t PathIndex)
+{
+ ZEN_TRACE_CPU("WriteSequenceChunkToCache");
+
+ const uint64_t SequenceSize = m_RemoteContent.RawSizes[PathIndex];
+
+ auto OpenFile = [&](BasicFile& File) {
+ const std::filesystem::path FileName =
+ GetTempChunkedSequenceFileName(m_CacheFolderPath, m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ File.Open(FileName, BasicFile::Mode::kWrite);
+ if (m_Options.UseSparseFiles)
+ {
+ PrepareFileForScatteredWrite(File.Handle(), SequenceSize);
+ }
+ };
+
+ const uint64_t ChunkSize = Chunk.GetSize();
+ ZEN_ASSERT(FileOffset + ChunkSize <= SequenceSize);
+ if (ChunkSize == SequenceSize)
+ {
+ BasicFile SingleChunkFile;
+ OpenFile(SingleChunkFile);
+
+ m_DiskStats.CurrentOpenFileCount++;
+ auto _ = MakeGuard([this]() { m_DiskStats.CurrentOpenFileCount--; });
+ SingleChunkFile.Write(Chunk, FileOffset);
+ }
+ else
+ {
+ const uint64_t MaxWriterBufferSize = 256u * 1025u;
+
+ BufferedWriteFileCache::Local::Writer* Writer = LocalWriter.GetWriter(SequenceIndex);
+ if (Writer)
+ {
+ if ((!Writer->Writer) && (ChunkSize < MaxWriterBufferSize))
+ {
+ Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
+ }
+ Writer->Write(Chunk, FileOffset);
+ }
+ else
+ {
+ Writer = LocalWriter.PutWriter(SequenceIndex, std::make_unique<BufferedWriteFileCache::Local::Writer>());
+
+ Writer->File = std::make_unique<BasicFile>();
+ OpenFile(*Writer->File);
+ if (ChunkSize < MaxWriterBufferSize)
+ {
+ Writer->Writer = std::make_unique<BasicFileWriter>(*Writer->File, Min(SequenceSize, MaxWriterBufferSize));
+ }
+ Writer->Write(Chunk, FileOffset);
+ }
+ }
+ m_DiskStats.WriteCount++;
+ m_DiskStats.WriteByteCount += ChunkSize;
+ m_WrittenChunkByteCount += ChunkSize;
+}
+
+bool
+BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash& BlockRawHash,
+ std::span<const IoHash> ChunkRawHashes,
+ std::span<const uint32_t> ChunkCompressedLengths,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
+ const MemoryView BlockView,
+ uint32_t FirstIncludedBlockChunkIndex,
+ uint32_t LastIncludedBlockChunkIndex,
+ BlockWriteOps& OutOps)
+{
+ ZEN_TRACE_CPU("GetBlockWriteOps");
+
+ uint32_t OffsetInBlock = 0;
+ for (uint32_t ChunkBlockIndex = FirstIncludedBlockChunkIndex; ChunkBlockIndex <= LastIncludedBlockChunkIndex; ChunkBlockIndex++)
+ {
+ const uint32_t ChunkCompressedSize = ChunkCompressedLengths[ChunkBlockIndex];
+ const IoHash& ChunkHash = ChunkRawHashes[ChunkBlockIndex];
+ if (auto It = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != m_RemoteLookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t ChunkIndex = It->second;
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs =
+ GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, ChunkIndex);
+
+ if (!ChunkTargetPtrs.empty())
+ {
+ bool NeedsWrite = true;
+ if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false))
+ {
+ MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize);
+ IoHash VerifyChunkHash;
+ uint64_t VerifyChunkSize;
+ CompressedBuffer CompressedChunk =
+ CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize);
+ if (!CompressedChunk)
+ {
+ throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer",
+ ChunkHash,
+ OffsetInBlock,
+ ChunkCompressedSize,
+ BlockRawHash));
+ }
+ if (VerifyChunkHash != ChunkHash)
+ {
+ throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}",
+ ChunkHash,
+ OffsetInBlock,
+ ChunkCompressedSize,
+ BlockRawHash,
+ VerifyChunkHash));
+ }
+ if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
+ {
+ throw std::runtime_error(
+ fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}",
+ ChunkHash,
+ OffsetInBlock,
+ ChunkCompressedSize,
+ BlockRawHash,
+ VerifyChunkSize,
+ m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
+ }
+
+ OodleCompressor ChunkCompressor;
+ OodleCompressionLevel ChunkCompressionLevel;
+ uint64_t ChunkBlockSize;
+
+ bool GetCompressParametersSuccess =
+ CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize);
+ ZEN_ASSERT(GetCompressParametersSuccess);
+
+ IoBuffer Decompressed;
+ if (ChunkCompressionLevel == OodleCompressionLevel::None)
+ {
+ MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder());
+ Decompressed =
+ IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize());
+ }
+ else
+ {
+ Decompressed = CompressedChunk.Decompress().AsIoBuffer();
+ }
+
+ if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])
+ {
+ throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}",
+ ChunkHash,
+ OffsetInBlock,
+ ChunkCompressedSize,
+ BlockRawHash,
+ Decompressed.GetSize(),
+ m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]));
+ }
+
+ ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed));
+ for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs)
+ {
+ OutOps.WriteOps.push_back(
+ BlockWriteOps::WriteOpData{.Target = Target, .ChunkBufferIndex = OutOps.ChunkBuffers.size()});
+ }
+ OutOps.ChunkBuffers.emplace_back(std::move(Decompressed));
+ }
+ }
+ }
+
+ OffsetInBlock += ChunkCompressedSize;
+ }
+ {
+ ZEN_TRACE_CPU("Sort");
+ std::sort(OutOps.WriteOps.begin(),
+ OutOps.WriteOps.end(),
+ [](const BlockWriteOps::WriteOpData& Lhs, const BlockWriteOps::WriteOpData& Rhs) {
+ if (Lhs.Target->SequenceIndex < Rhs.Target->SequenceIndex)
+ {
+ return true;
+ }
+ if (Lhs.Target->SequenceIndex > Rhs.Target->SequenceIndex)
+ {
+ return false;
+ }
+ return Lhs.Target->Offset < Rhs.Target->Offset;
+ });
+ }
+ return true;
+}
+
+void
+BuildsOperationUpdateFolder::WriteBlockChunkOpsToCache(std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ const BlockWriteOps& Ops,
+ BufferedWriteFileCache& WriteCache,
+ ParallelWork& Work)
+{
+ ZEN_TRACE_CPU("WriteBlockChunkOpsToCache");
+
+ {
+ BufferedWriteFileCache::Local LocalWriter(WriteCache);
+ for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
+ {
+ if (Work.IsAborted())
+ {
+ break;
+ }
+ const CompositeBuffer& Chunk = Ops.ChunkBuffers[WriteOp.ChunkBufferIndex];
+ const uint32_t SequenceIndex = WriteOp.Target->SequenceIndex;
+ ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() <=
+ m_RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]);
+ ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounters[SequenceIndex].load() > 0);
+ const uint64_t FileOffset = WriteOp.Target->Offset;
+ const uint32_t PathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+
+ WriteSequenceChunkToCache(LocalWriter, Chunk, SequenceIndex, FileOffset, PathIndex);
+ }
+ }
+ if (!Work.IsAborted())
+ {
+ // Write tracking, updating this must be done without any files open (BufferedWriteFileCache::Local)
+ std::vector<uint32_t> CompletedChunkSequences;
+ for (const BlockWriteOps::WriteOpData& WriteOp : Ops.WriteOps)
+ {
+ const uint32_t RemoteSequenceIndex = WriteOp.Target->SequenceIndex;
+ if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
+ {
+ CompletedChunkSequences.push_back(RemoteSequenceIndex);
+ }
+ }
+ WriteCache.Close(CompletedChunkSequences);
+ VerifyAndCompleteChunkSequencesAsync(CompletedChunkSequences, Work);
+ }
+}
+
+bool
+BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription& BlockDescription,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ ParallelWork& Work,
+ CompositeBuffer&& BlockBuffer,
+ std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
+ BufferedWriteFileCache& WriteCache)
+{
+ ZEN_TRACE_CPU("WriteChunksBlockToCache");
+
+ IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(BlockBuffer);
+ const MemoryView BlockView = BlockMemoryBuffer.GetView();
+
+ BlockWriteOps Ops;
+ if ((BlockDescription.HeaderSize == 0) || BlockDescription.ChunkCompressedLengths.empty())
+ {
+ ZEN_TRACE_CPU("WriteChunksBlockToCache_Legacy");
+
+ uint64_t HeaderSize;
+ const std::vector<uint32_t> ChunkCompressedLengths =
+ ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize);
+
+ if (GetBlockWriteOps(BlockDescription.BlockHash,
+ BlockDescription.ChunkRawHashes,
+ ChunkCompressedLengths,
+ SequenceIndexChunksLeftToWriteCounters,
+ RemoteChunkIndexNeedsCopyFromSourceFlags,
+ BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + HeaderSize),
+ 0,
+ gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
+ Ops))
+ {
+ WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
+ return true;
+ }
+ return false;
+ }
+
+ if (GetBlockWriteOps(BlockDescription.BlockHash,
+ BlockDescription.ChunkRawHashes,
+ BlockDescription.ChunkCompressedLengths,
+ SequenceIndexChunksLeftToWriteCounters,
+ RemoteChunkIndexNeedsCopyFromSourceFlags,
+ BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize),
+ 0,
+ gsl::narrow<uint32_t>(BlockDescription.ChunkRawHashes.size() - 1),
+ Ops))
+ {
+ WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
+ return true;
+ }
+ return false;
+}
+
+bool
+BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDescription& BlockDescription,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ ParallelWork& Work,
+ CompositeBuffer&& PartialBlockBuffer,
+ uint32_t FirstIncludedBlockChunkIndex,
+ uint32_t LastIncludedBlockChunkIndex,
+ std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags,
+ BufferedWriteFileCache& WriteCache)
+{
+ ZEN_TRACE_CPU("WritePartialBlockChunksToCache");
+
+ IoBuffer BlockMemoryBuffer = MakeBufferMemoryBased(PartialBlockBuffer);
+ const MemoryView BlockView = BlockMemoryBuffer.GetView();
+
+ BlockWriteOps Ops;
+ if (GetBlockWriteOps(BlockDescription.BlockHash,
+ BlockDescription.ChunkRawHashes,
+ BlockDescription.ChunkCompressedLengths,
+ SequenceIndexChunksLeftToWriteCounters,
+ RemoteChunkIndexNeedsCopyFromSourceFlags,
+ BlockView,
+ FirstIncludedBlockChunkIndex,
+ LastIncludedBlockChunkIndex,
+ Ops))
+ {
+ WriteBlockChunkOpsToCache(SequenceIndexChunksLeftToWriteCounters, Ops, WriteCache, Work);
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+}
+
+void
+BuildsOperationUpdateFolder::AsyncWriteDownloadedChunk(uint32_t RemoteChunkIndex,
+ const BlobsExistsResult& ExistsResult,
+ std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>&& ChunkTargetPtrs,
+ BufferedWriteFileCache& WriteCache,
+ ParallelWork& Work,
+ IoBuffer&& Payload,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters,
+ std::atomic<uint64_t>& WritePartsComplete,
+ const uint64_t TotalPartWriteCount,
+ FilteredRate& FilteredWrittenBytesPerSecond)
+{
+ ZEN_TRACE_CPU("AsyncWriteDownloadedChunk");
+
+ const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+
+ const uint64_t Size = Payload.GetSize();
+
+ const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash);
+
+ const bool PopulateCache = !ExistsInCache && m_Storage.CacheStorage && m_Options.PopulateCache;
+
+ std::filesystem::path CompressedChunkPath =
+ TryMoveDownloadedChunk(Payload,
+ m_TempDownloadFolderPath / ChunkHash.ToHexString(),
+ /* ForceDiskBased */ PopulateCache || Size > m_Options.MaximumInMemoryPayloadSize);
+ if (PopulateCache)
+ {
+ IoBuffer CacheBlob = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
+ if (CacheBlob)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId,
+ ChunkHash,
+ ZenContentType::kCompressedBinary,
+ CompositeBuffer(SharedBuffer(CacheBlob)));
+ }
+ }
+
+ IoBufferFileReference FileRef;
+ bool EnableBacklog = !CompressedChunkPath.empty() || Payload.GetFileReference(FileRef);
+
+ Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this,
+ SequenceIndexChunksLeftToWriteCounters,
+ &Work,
+ CompressedChunkPath,
+ RemoteChunkIndex,
+ TotalPartWriteCount,
+ &WriteCache,
+ &WritePartsComplete,
+ &FilteredWrittenBytesPerSecond,
+ ChunkTargetPtrs = std::move(ChunkTargetPtrs),
+ CompressedPart = IoBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_WriteChunk");
+
+ FilteredWrittenBytesPerSecond.Start();
+
+ const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex];
+ if (CompressedChunkPath.empty())
+ {
+ ZEN_ASSERT(CompressedPart);
+ }
+ else
+ {
+ ZEN_ASSERT(!CompressedPart);
+ CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath);
+ if (!CompressedPart)
+ {
+ throw std::runtime_error(
+ fmt::format("Could not open dowloaded compressed chunk {} from {}", ChunkHash, CompressedChunkPath));
+ }
+ }
+
+ bool NeedHashVerify = WriteCompressedChunkToCache(ChunkHash, ChunkTargetPtrs, WriteCache, std::move(CompressedPart));
+ if (!m_AbortFlag)
+ {
+ if (WritePartsComplete.fetch_add(1) + 1 == TotalPartWriteCount)
+ {
+ FilteredWrittenBytesPerSecond.Stop();
+ }
+
+ if (!CompressedChunkPath.empty())
+ {
+ std::error_code Ec = TryRemoveFile(CompressedChunkPath);
+ if (Ec)
+ {
+ ZEN_DEBUG("Failed removing file '{}', reason: ({}) {}", CompressedChunkPath, Ec.value(), Ec.message());
+ }
+ }
+
+ std::vector<uint32_t> CompletedSequences =
+ CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters);
+ WriteCache.Close(CompletedSequences);
+ if (NeedHashVerify)
+ {
+ VerifyAndCompleteChunkSequencesAsync(CompletedSequences, Work);
+ }
+ else
+ {
+ FinalizeChunkSequences(CompletedSequences);
+ }
+ }
+ }
+ },
+ EnableBacklog ? WorkerThreadPool::EMode::EnableBacklog : WorkerThreadPool::EMode::DisableBacklog);
+}
+
+void
+BuildsOperationUpdateFolder::VerifyAndCompleteChunkSequencesAsync(std::span<const uint32_t> RemoteSequenceIndexes, ParallelWork& Work)
+{
+ if (RemoteSequenceIndexes.empty())
+ {
+ return;
+ }
+ ZEN_TRACE_CPU("VerifyAndCompleteChunkSequence");
+ if (m_Options.ValidateCompletedSequences)
+ {
+ for (uint32_t RemoteSequenceIndexOffset = 1; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++)
+ {
+ const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset];
+ Work.ScheduleWork(m_IOWorkerPool, [this, RemoteSequenceIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("Async_VerifyAndFinalizeSequence");
+
+ VerifySequence(RemoteSequenceIndex);
+ if (!m_AbortFlag)
+ {
+ const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ FinalizeChunkSequence(SequenceRawHash);
+ }
+ }
+ });
+ }
+ const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0];
+
+ VerifySequence(RemoteSequenceIndex);
+ const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ FinalizeChunkSequence(SequenceRawHash);
+ }
+ else
+ {
+ for (uint32_t RemoteSequenceIndexOffset = 0; RemoteSequenceIndexOffset < RemoteSequenceIndexes.size(); RemoteSequenceIndexOffset++)
+ {
+ const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset];
+ const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ FinalizeChunkSequence(SequenceRawHash);
+ }
+ }
+}
+
+bool
+BuildsOperationUpdateFolder::CompleteSequenceChunk(uint32_t RemoteSequenceIndex,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
+{
+ uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1);
+ ZEN_ASSERT(PreviousValue >= 1);
+ ZEN_ASSERT(PreviousValue != (uint32_t)-1);
+ return PreviousValue == 1;
+}
+
+std::vector<uint32_t>
+BuildsOperationUpdateFolder::CompleteChunkTargets(const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs,
+ std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters)
+{
+ ZEN_TRACE_CPU("CompleteChunkTargets");
+
+ std::vector<uint32_t> CompletedSequenceIndexes;
+ for (const ChunkedContentLookup::ChunkSequenceLocation* Location : ChunkTargetPtrs)
+ {
+ const uint32_t RemoteSequenceIndex = Location->SequenceIndex;
+ if (CompleteSequenceChunk(RemoteSequenceIndex, SequenceIndexChunksLeftToWriteCounters))
+ {
+ CompletedSequenceIndexes.push_back(RemoteSequenceIndex);
+ }
+ }
+ return CompletedSequenceIndexes;
+}
+
+void
+BuildsOperationUpdateFolder::FinalizeChunkSequence(const IoHash& SequenceRawHash)
+{
+ ZEN_TRACE_CPU("FinalizeChunkSequence");
+
+ ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash)));
+ std::error_code Ec;
+ RenameFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash),
+ GetFinalChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash),
+ Ec);
+ if (Ec)
+ {
+ throw std::system_error(Ec);
+ }
+}
+
+void
+BuildsOperationUpdateFolder::FinalizeChunkSequences(std::span<const uint32_t> RemoteSequenceIndexes)
+{
+ ZEN_TRACE_CPU("FinalizeChunkSequences");
+
+ for (uint32_t SequenceIndex : RemoteSequenceIndexes)
+ {
+ FinalizeChunkSequence(m_RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]);
+ }
+}
+
+void
+BuildsOperationUpdateFolder::VerifySequence(uint32_t RemoteSequenceIndex)
+{
+ ZEN_TRACE_CPU("VerifySequence");
+
+ ZEN_ASSERT(m_Options.ValidateCompletedSequences);
+
+ const IoHash& SequenceRawHash = m_RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex];
+ {
+ ZEN_TRACE_CPU("HashSequence");
+ const std::uint32_t RemotePathIndex = m_RemoteLookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex];
+ const uint64_t ExpectedSize = m_RemoteContent.RawSizes[RemotePathIndex];
+ IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(m_CacheFolderPath, SequenceRawHash));
+ const uint64_t VerifySize = VerifyBuffer.GetSize();
+ if (VerifySize != ExpectedSize)
+ {
+ throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}",
+ SequenceRawHash,
+ VerifySize,
+ ExpectedSize));
+ }
+
+ const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer), &m_ValidatedChunkByteCount);
+ if (VerifyChunkHash != SequenceRawHash)
+ {
+ throw std::runtime_error(
+ fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash));
+ }
+ }
+}
+
+void
+VerifyFolder(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ TransferThreadWorkers& Workers,
+ const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const std::filesystem::path& Path,
+ const std::vector<std::string>& ExcludeFolders,
+ bool VerifyFileHash,
+ VerifyFolderStatistics& VerifyFolderStats)
+{
+ ZEN_TRACE_CPU("VerifyFolder");
+
+ Stopwatch Timer;
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = Progress.CreateProgressBar("Verify Files");
+
+ WorkerThreadPool& VerifyPool = Workers.GetIOWorkerPool();
+
+ ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ const uint32_t PathCount = gsl::narrow<uint32_t>(Content.Paths.size());
+
+ RwLock ErrorLock;
+ std::vector<std::string> Errors;
+
+ auto IsAcceptedFolder = [ExcludeFolders = ExcludeFolders](const std::string_view& RelativePath) -> bool {
+ for (const std::string& ExcludeFolder : ExcludeFolders)
+ {
+ if (StrCaseStartsWith(RelativePath, ExcludeFolder))
+ {
+ if (RelativePath.length() == ExcludeFolder.length())
+ {
+ return false;
+ }
+ else if (RelativePath[ExcludeFolder.length()] == '/')
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+ };
+
+ for (uint32_t PathIndex = 0; PathIndex < PathCount; PathIndex++)
+ {
+ if (Work.IsAborted())
+ {
+ break;
+ }
+
+ Work.ScheduleWork(
+ VerifyPool,
+ [&Path, &Content, &Lookup, &ErrorLock, &Errors, &VerifyFolderStats, VerifyFileHash, &IsAcceptedFolder, PathIndex, &AbortFlag](
+ std::atomic<bool>&) {
+ if (!AbortFlag)
+ {
+ ZEN_TRACE_CPU("VerifyFile_work");
+
+ // TODO: Convert ScheduleWork body to function
+
+ const std::filesystem::path TargetPath = (Path / Content.Paths[PathIndex]).make_preferred();
+ if (IsAcceptedFolder(TargetPath.parent_path().generic_string()))
+ {
+ const uint64_t ExpectedSize = Content.RawSizes[PathIndex];
+ if (!IsFile(TargetPath))
+ {
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize));
+ });
+ VerifyFolderStats.FilesFailed++;
+ }
+ else
+ {
+ std::error_code Ec;
+ uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec));
+ if (Ec)
+ {
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(
+ fmt::format("Failed to get size of file {}: {} ({})", TargetPath, Ec.message(), Ec.value()));
+ });
+ VerifyFolderStats.FilesFailed++;
+ }
+ else if (SizeOnDisk < ExpectedSize)
+ {
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(fmt::format("Size of file {} is smaller than expected. Expected: {}, Found: {}",
+ TargetPath,
+ ExpectedSize,
+ SizeOnDisk));
+ });
+ VerifyFolderStats.FilesFailed++;
+ }
+ else if (SizeOnDisk > ExpectedSize)
+ {
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(fmt::format("Size of file {} is bigger than expected. Expected: {}, Found: {}",
+ TargetPath,
+ ExpectedSize,
+ SizeOnDisk));
+ });
+ VerifyFolderStats.FilesFailed++;
+ }
+ else if (SizeOnDisk > 0 && VerifyFileHash)
+ {
+ const IoHash& ExpectedRawHash = Content.RawHashes[PathIndex];
+ IoBuffer Buffer = IoBufferBuilder::MakeFromFile(TargetPath);
+ IoHash RawHash = IoHash::HashBuffer(Buffer);
+ if (RawHash != ExpectedRawHash)
+ {
+ uint64_t FileOffset = 0;
+ const uint32_t SequenceIndex = Lookup.RawHashToSequenceIndex.at(ExpectedRawHash);
+ const uint32_t OrderOffset = Lookup.SequenceIndexChunkOrderOffset[SequenceIndex];
+ for (uint32_t OrderIndex = OrderOffset;
+ OrderIndex < OrderOffset + Content.ChunkedContent.ChunkCounts[SequenceIndex];
+ OrderIndex++)
+ {
+ uint32_t ChunkIndex = Content.ChunkedContent.ChunkOrders[OrderIndex];
+ uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ IoHash ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
+ IoBuffer FileChunk = IoBuffer(Buffer, FileOffset, ChunkSize);
+ if (IoHash::HashBuffer(FileChunk) != ChunkHash)
+ {
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(fmt::format(
+ "WARNING: Hash of file {} does not match expected hash. Expected: {}, Found: {}. "
+ "Mismatch at chunk {}",
+ TargetPath,
+ ExpectedRawHash,
+ RawHash,
+ OrderIndex - OrderOffset));
+ });
+ break;
+ }
+ FileOffset += ChunkSize;
+ }
+ VerifyFolderStats.FilesFailed++;
+ }
+ VerifyFolderStats.ReadBytes += SizeOnDisk;
+ }
+ }
+ }
+ VerifyFolderStats.FilesVerified++;
+ }
+ },
+ [&, PathIndex](std::exception_ptr Ex, std::atomic<bool>&) {
+ std::string Description;
+ try
+ {
+ std::rethrow_exception(Ex);
+ }
+ catch (const std::exception& Ex)
+ {
+ Description = Ex.what();
+ }
+ ErrorLock.WithExclusiveLock([&]() {
+ Errors.push_back(fmt::format("Failed verifying file '{}'. Reason: {}",
+ (Path / Content.Paths[PathIndex]).make_preferred(),
+ Description));
+ });
+ VerifyFolderStats.FilesFailed++;
+ });
+ }
+
+ Work.Wait(Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ std::string Details = fmt::format("Verified {}/{} ({}). Failed files: {}",
+ VerifyFolderStats.FilesVerified.load(),
+ PathCount,
+ NiceBytes(VerifyFolderStats.ReadBytes.load()),
+ VerifyFolderStats.FilesFailed.load());
+ ProgressBar->UpdateState({.Task = "Verifying files ",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(PathCount),
+ .RemainingCount = gsl::narrow<uint64_t>(PathCount - VerifyFolderStats.FilesVerified.load()),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+ VerifyFolderStats.VerifyElapsedWallTimeUs = Timer.GetElapsedTimeUs();
+
+ ProgressBar->Finish();
+ if (AbortFlag)
+ {
+ return;
+ }
+
+ for (const std::string& Error : Errors)
+ {
+ ZEN_CONSOLE_ERROR("{}", Error);
+ }
+ if (!Errors.empty())
+ {
+ throw std::runtime_error(fmt::format("Verify failed with {} errors", Errors.size()));
+ }
+}
+
+std::vector<std::filesystem::path>
+GetNewPaths(const std::span<const std::filesystem::path> KnownPaths, const std::span<const std::filesystem::path> Paths)
+{
+ tsl::robin_set<std::string> KnownPathsSet;
+ KnownPathsSet.reserve(KnownPaths.size());
+ for (const std::filesystem::path& LocalPath : KnownPaths)
+ {
+ KnownPathsSet.insert(LocalPath.generic_string());
+ }
+
+ std::vector<std::filesystem::path> NewPaths;
+ for (const std::filesystem::path& UntrackedPath : Paths)
+ {
+ if (!KnownPathsSet.contains(UntrackedPath.generic_string()))
+ {
+ NewPaths.push_back(UntrackedPath);
+ }
+ }
+ return NewPaths;
+}
+
+BuildSaveState
+GetLocalStateFromPaths(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ TransferThreadWorkers& Workers,
+ GetFolderContentStatistics& LocalFolderScanStats,
+ ChunkingStatistics& ChunkingStats,
+ const std::filesystem::path& Path,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ std::span<const std::filesystem::path> PathsToCheck)
+{
+ FolderContent FolderState =
+ CheckFolderFiles(Progress, AbortFlag, PauseFlag, "Check Files", Workers, LocalFolderScanStats, Path, PathsToCheck);
+
+ ChunkedFolderContent ChunkedContent;
+ if (FolderState.Paths.size() > 0)
+ {
+ ChunkedContent = ScanFolderFiles(Progress,
+ AbortFlag,
+ PauseFlag,
+ "Scan Files",
+ Workers,
+ Path,
+ FolderState,
+ ChunkController,
+ ChunkCache,
+ ChunkingStats);
+ }
+
+ return BuildSaveState{.State = BuildState{.ChunkedContent = std::move(ChunkedContent)}, .FolderState = FolderState, .LocalPath = Path};
+}
+
+BuildSaveState
+GetLocalContent(ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ bool IsQuiet,
+ TransferThreadWorkers& Workers,
+ GetFolderContentStatistics& LocalFolderScanStats,
+ ChunkingStatistics& ChunkingStats,
+ const std::filesystem::path& Path,
+ const std::filesystem::path& StateFilePath,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache)
+{
+ Stopwatch ReadStateTimer;
+ bool FileExists = IsFile(StateFilePath);
+ if (!FileExists)
+ {
+ ZEN_CONSOLE("No known local state file in {}, falling back to scanning", Path);
+ return {};
+ }
+
+ BuildSaveState SavedLocalState;
+ try
+ {
+ SavedLocalState = ReadBuildSaveStateFile(StateFilePath);
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Read local state file {} in {}", StateFilePath, NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs()));
+ }
+ }
+ catch (const std::exception& Ex)
+ {
+ ZEN_CONSOLE_WARN("Failed reading state file {}, falling back to scannning. Reason: {}", StateFilePath, Ex.what());
+ return {};
+ }
+
+ FolderContent CurrentLocalFolderState = CheckFolderFiles(Progress,
+ AbortFlag,
+ PauseFlag,
+ "Check Known Files",
+ Workers,
+ LocalFolderScanStats,
+ Path,
+ SavedLocalState.FolderState.Paths);
+ if (AbortFlag)
+ {
+ return {};
+ }
+
+ if (!SavedLocalState.FolderState.AreKnownFilesEqual(CurrentLocalFolderState))
+ {
+ const size_t LocalStatePathCount = SavedLocalState.FolderState.Paths.size();
+ std::vector<std::filesystem::path> DeletedPaths;
+ FolderContent UpdatedContent = GetUpdatedContent(SavedLocalState.FolderState, CurrentLocalFolderState, DeletedPaths);
+ if (!DeletedPaths.empty())
+ {
+ SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
+ }
+
+ if (!IsQuiet)
+ {
+ ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}",
+ DeletedPaths.size(),
+ UpdatedContent.Paths.size(),
+ LocalStatePathCount);
+ }
+ if (UpdatedContent.Paths.size() > 0)
+ {
+ ChunkedFolderContent UpdatedLocalContent = ScanFolderFiles(Progress,
+ AbortFlag,
+ PauseFlag,
+ "Scan Known Files",
+ Workers,
+ Path,
+ UpdatedContent,
+ ChunkController,
+ ChunkCache,
+ ChunkingStats);
+ if (AbortFlag)
+ {
+ return {};
+ }
+ SavedLocalState.State.ChunkedContent =
+ MergeChunkedFolderContents(SavedLocalState.State.ChunkedContent, {{UpdatedLocalContent}});
+ }
+ }
+ else
+ {
+ // Remove files from LocalContent no longer in LocalFolderState
+ tsl::robin_set<std::string> LocalFolderPaths;
+ LocalFolderPaths.reserve(SavedLocalState.FolderState.Paths.size());
+ for (const std::filesystem::path& LocalFolderPath : SavedLocalState.FolderState.Paths)
+ {
+ LocalFolderPaths.insert(LocalFolderPath.generic_string());
+ }
+ std::vector<std::filesystem::path> DeletedPaths;
+ for (const std::filesystem::path& LocalContentPath : SavedLocalState.State.ChunkedContent.Paths)
+ {
+ if (!LocalFolderPaths.contains(LocalContentPath.generic_string()))
+ {
+ DeletedPaths.push_back(LocalContentPath);
+ }
+ }
+ if (!DeletedPaths.empty())
+ {
+ SavedLocalState.State.ChunkedContent = DeletePathsFromChunkedContent(SavedLocalState.State.ChunkedContent, DeletedPaths);
+ }
+ }
+
+ SavedLocalState.FolderState = CurrentLocalFolderState;
+
+ return SavedLocalState;
+}
+
+void
+DownloadFolder(LoggerRef InLog,
+ ProgressBase& Progress,
+ TransferThreadWorkers& Workers,
+ StorageInstance& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ const BuildStorageCache::Statistics& StorageCacheStats,
+ const Oid& BuildId,
+ const std::vector<Oid>& BuildPartIds,
+ std::span<const std::string> BuildPartNames,
+ const std::filesystem::path& DownloadSpecPath,
+ const std::filesystem::path& Path,
+ const DownloadOptions& Options)
+{
+ ZEN_TRACE_CPU("DownloadFolder");
+ ZEN_SCOPED_LOG(InLog);
+
+ Progress.SetLogOperationName("Download Folder");
+
+ enum TaskSteps : uint32_t
+ {
+ CheckState,
+ CompareState,
+ Download,
+ Verify,
+ Cleanup,
+ StepCount
+ };
+
+ auto EndProgress = MakeGuard([&]() { Progress.SetLogOperationProgress(TaskSteps::StepCount, TaskSteps::StepCount); });
+
+ Stopwatch DownloadTimer;
+
+ Progress.SetLogOperationProgress(TaskSteps::CheckState, TaskSteps::StepCount);
+
+ const std::filesystem::path ZenTempFolder = ZenTempFolderPath(Options.ZenFolderPath);
+ CreateDirectories(ZenTempFolder);
+
+ std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ CbObject BuildObject = GetBuild(*Storage.BuildStorage, BuildId, Options.IsQuiet);
+
+ std::vector<std::pair<Oid, std::string>> AllBuildParts =
+ ResolveBuildPartNames(BuildObject, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize);
+
+ BuildManifest Manifest;
+ if (!DownloadSpecPath.empty())
+ {
+ const std::filesystem::path AbsoluteDownloadSpecPath =
+ DownloadSpecPath.is_relative() ? MakeSafeAbsolutePath(Path / DownloadSpecPath) : MakeSafeAbsolutePath(DownloadSpecPath);
+ Manifest = ParseBuildManifest(AbsoluteDownloadSpecPath);
+ }
+
+ std::vector<ChunkedFolderContent> PartContents;
+
+ std::unique_ptr<ChunkingController> ChunkController;
+
+ std::vector<ChunkBlockDescription> BlockDescriptions;
+ std::vector<IoHash> LooseChunkHashes;
+
+ Progress.SetLogOperationProgress(TaskSteps::CompareState, TaskSteps::StepCount);
+
+ ChunkedFolderContent RemoteContent = GetRemoteContent(InLog,
+ Storage,
+ BuildId,
+ AllBuildParts,
+ Manifest,
+ Options.IncludeWildcards,
+ Options.ExcludeWildcards,
+ ChunkController,
+ PartContents,
+ BlockDescriptions,
+ LooseChunkHashes,
+ Options.IsQuiet,
+ Options.IsVerbose,
+ Options.DoExtraContentVerify);
+
+ const std::uint64_t LargeAttachmentSize = Options.AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+ GetFolderContentStatistics LocalFolderScanStats;
+ ChunkingStatistics ChunkingStats;
+
+ BuildSaveState LocalState;
+
+ if (IsDir(Path))
+ {
+ if (!ChunkController && !Options.IsQuiet)
+ {
+ ZEN_CONSOLE_INFO("Unspecified chunking algorithm, using default");
+ ChunkController = CreateStandardChunkingController(StandardChunkingControllerSettings{});
+ }
+ std::unique_ptr<ChunkingCache> ChunkCache(CreateNullChunkingCache());
+
+ LocalState = GetLocalContent(Progress,
+ AbortFlag,
+ PauseFlag,
+ Options.IsQuiet,
+ Workers,
+ LocalFolderScanStats,
+ ChunkingStats,
+ Path,
+ ZenStateFilePath(Path / ZenFolderName),
+ *ChunkController,
+ *ChunkCache);
+
+ std::vector<std::filesystem::path> UntrackedPaths = GetNewPaths(LocalState.State.ChunkedContent.Paths, RemoteContent.Paths);
+
+ BuildSaveState UntrackedLocalContent = GetLocalStateFromPaths(Progress,
+ AbortFlag,
+ PauseFlag,
+ Workers,
+ LocalFolderScanStats,
+ ChunkingStats,
+ Path,
+ *ChunkController,
+ *ChunkCache,
+ UntrackedPaths);
+
+ if (!UntrackedLocalContent.State.ChunkedContent.Paths.empty())
+ {
+ LocalState.State.ChunkedContent =
+ MergeChunkedFolderContents(LocalState.State.ChunkedContent,
+ std::vector<ChunkedFolderContent>{UntrackedLocalContent.State.ChunkedContent});
+
+ // TODO: Helper
+ LocalState.FolderState.Paths.insert(LocalState.FolderState.Paths.begin(),
+ UntrackedLocalContent.FolderState.Paths.begin(),
+ UntrackedLocalContent.FolderState.Paths.end());
+ LocalState.FolderState.RawSizes.insert(LocalState.FolderState.RawSizes.begin(),
+ UntrackedLocalContent.FolderState.RawSizes.begin(),
+ UntrackedLocalContent.FolderState.RawSizes.end());
+ LocalState.FolderState.Attributes.insert(LocalState.FolderState.Attributes.begin(),
+ UntrackedLocalContent.FolderState.Attributes.begin(),
+ UntrackedLocalContent.FolderState.Attributes.end());
+ LocalState.FolderState.ModificationTicks.insert(LocalState.FolderState.ModificationTicks.begin(),
+ UntrackedLocalContent.FolderState.ModificationTicks.begin(),
+ UntrackedLocalContent.FolderState.ModificationTicks.end());
+ }
+
+ if (Options.AppendNewContent)
+ {
+ RemoteContent = ApplyChunkedContentOverlay(LocalState.State.ChunkedContent,
+ RemoteContent,
+ Options.IncludeWildcards,
+ Options.ExcludeWildcards);
+ }
+#if ZEN_BUILD_DEBUG
+ ValidateChunkedFolderContent(RemoteContent,
+ BlockDescriptions,
+ LooseChunkHashes,
+ Options.IncludeWildcards,
+ Options.ExcludeWildcards);
+#endif // ZEN_BUILD_DEBUG
+ }
+ else
+ {
+ CreateDirectories(Path);
+ }
+ if (AbortFlag)
+ {
+ return;
+ }
+
+ LocalState.LocalPath = Path;
+
+ {
+ BuildsSelection::Build RemoteBuildState = {.Id = BuildId,
+ .IncludeWildcards = Options.IncludeWildcards,
+ .ExcludeWildcards = Options.ExcludeWildcards};
+ RemoteBuildState.Parts.reserve(BuildPartIds.size());
+ for (size_t PartIndex = 0; PartIndex < BuildPartIds.size(); PartIndex++)
+ {
+ RemoteBuildState.Parts.push_back(
+ {BuildsSelection::BuildPart{.Id = BuildPartIds[PartIndex],
+ .Name = PartIndex < BuildPartNames.size() ? BuildPartNames[PartIndex] : ""}});
+ }
+
+ if (Options.AppendNewContent)
+ {
+ LocalState.State.Selection.Builds.emplace_back(std::move(RemoteBuildState));
+ }
+ else
+ {
+ LocalState.State.Selection.Builds = std::vector<BuildsSelection::Build>{std::move(RemoteBuildState)};
+ }
+ }
+
+ if ((Options.EnableTargetFolderScavenging || Options.AppendNewContent) && !Options.CleanTargetFolder &&
+ CompareChunkedContent(RemoteContent, LocalState.State.ChunkedContent))
+ {
+ if (!Options.IsQuiet)
+ {
+ ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.",
+ NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs()));
+ }
+
+ Stopwatch WriteStateTimer;
+
+ CbObject StateObject = CreateBuildSaveStateObject(LocalState);
+ CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
+ if (!Options.IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
+
+ AddDownloadedPath(Options.SystemRootDir,
+ BuildsDownloadInfo{.Selection = LocalState.State.Selection,
+ .LocalPath = Path,
+ .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
+ .Iso8601Date = DateTime::Now().ToIso8601()});
+ }
+ else
+ {
+ ExtendableStringBuilder<128> BuildPartString;
+ for (const std::pair<Oid, std::string>& BuildPart : AllBuildParts)
+ {
+ BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first));
+ }
+
+ uint64_t RawSize = std::accumulate(RemoteContent.RawSizes.begin(), RemoteContent.RawSizes.end(), std::uint64_t(0));
+
+ if (!Options.IsQuiet)
+ {
+ ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize));
+ }
+
+ Stopwatch IndexTimer;
+
+ const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent);
+ const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent);
+
+ if (!Options.IsQuiet)
+ {
+ ZEN_INFO("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs()));
+ }
+
+ Progress.SetLogOperationProgress(TaskSteps::Download, TaskSteps::StepCount);
+
+ BuildsOperationUpdateFolder Updater(
+ InLog,
+ Progress,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ Workers.GetIOWorkerPool(),
+ Workers.GetNetworkPool(),
+ BuildId,
+ Path,
+ LocalState.State.ChunkedContent,
+ LocalLookup,
+ RemoteContent,
+ RemoteLookup,
+ BlockDescriptions,
+ LooseChunkHashes,
+ BuildsOperationUpdateFolder::Options{
+ .IsQuiet = Options.IsQuiet,
+ .IsVerbose = Options.IsVerbose,
+ .AllowFileClone = Options.AllowFileClone,
+ .UseSparseFiles = Options.UseSparseFiles,
+ .SystemRootDir = Options.SystemRootDir,
+ .ZenFolderPath = Options.ZenFolderPath,
+ .LargeAttachmentSize = LargeAttachmentSize,
+ .PreferredMultipartChunkSize = PreferredMultipartChunkSize,
+ .PartialBlockRequestMode = Options.PartialBlockRequestMode,
+ .WipeTargetFolder = Options.CleanTargetFolder,
+ .EnableOtherDownloadsScavenging = Options.EnableOtherDownloadsScavenging,
+ .EnableTargetFolderScavenging = Options.EnableTargetFolderScavenging || Options.AppendNewContent,
+ .ValidateCompletedSequences = Options.PostDownloadVerify,
+ .ExcludeFolders = Options.ExcludeFolders,
+ .MaximumInMemoryPayloadSize = Options.MaximumInMemoryPayloadSize,
+ .PopulateCache = Options.PopulateCache});
+ {
+ Progress.PushLogOperation("Download");
+ auto _ = MakeGuard([&Progress]() { Progress.PopLogOperation(); });
+ FolderContent UpdatedLocalFolderState;
+ Updater.Execute(UpdatedLocalFolderState);
+
+ LocalState.State.ChunkedContent = RemoteContent;
+ LocalState.FolderState = std::move(UpdatedLocalFolderState);
+ }
+
+ VerifyFolderStatistics VerifyFolderStats;
+ if (!AbortFlag)
+ {
+ AddDownloadedPath(Options.SystemRootDir,
+ BuildsDownloadInfo{.Selection = LocalState.State.Selection,
+ .LocalPath = Path,
+ .StateFilePath = ZenStateFilePath(Options.ZenFolderPath),
+ .Iso8601Date = DateTime::Now().ToIso8601()});
+
+ Progress.SetLogOperationProgress(TaskSteps::Verify, TaskSteps::StepCount);
+
+ VerifyFolder(Progress,
+ AbortFlag,
+ PauseFlag,
+ Workers,
+ RemoteContent,
+ RemoteLookup,
+ Path,
+ Options.ExcludeFolders,
+ Options.PostDownloadVerify,
+ VerifyFolderStats);
+
+ Stopwatch WriteStateTimer;
+ CbObject StateObject = CreateBuildSaveStateObject(LocalState);
+
+ CreateDirectories(ZenStateFilePath(Options.ZenFolderPath).parent_path());
+ TemporaryFile::SafeWriteFile(ZenStateFilePath(Options.ZenFolderPath), StateObject.GetView());
+ if (!Options.IsQuiet)
+ {
+ ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs()));
+ }
+
+#if 0
+ ExtendableStringBuilder<1024> SB;
+ CompactBinaryToJson(StateObject, SB);
+ WriteFile(ZenStateFileJsonPath(Options.ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size()));
+#endif // 0
+ const uint64_t DownloadCount = Updater.m_DownloadStats.DownloadedChunkCount.load() +
+ Updater.m_DownloadStats.DownloadedBlockCount.load() +
+ Updater.m_DownloadStats.DownloadedPartialBlockCount.load();
+ const uint64_t DownloadByteCount = Updater.m_DownloadStats.DownloadedChunkByteCount.load() +
+ Updater.m_DownloadStats.DownloadedBlockByteCount.load() +
+ Updater.m_DownloadStats.DownloadedPartialBlockByteCount.load();
+ const uint64_t DownloadTimeMs = DownloadTimer.GetElapsedTimeMs();
+
+ if (!Options.IsQuiet)
+ {
+ std::string CloneInfo;
+ if (Updater.m_DiskStats.CloneByteCount > 0)
+ {
+ CloneInfo = fmt::format(" ({} cloned)", NiceBytes(Updater.m_DiskStats.CloneByteCount.load()));
+ }
+
+ std::string DownloadDetails;
+ {
+ ExtendableStringBuilder<128> SB;
+ BuildStorageBase::ExtendedStatistics ExtendedDownloadStats;
+ if (Storage.BuildStorage->GetExtendedStatistics(ExtendedDownloadStats))
+ {
+ if (!ExtendedDownloadStats.ReceivedBytesPerSource.empty())
+ {
+ for (auto& It : ExtendedDownloadStats.ReceivedBytesPerSource)
+ {
+ if (SB.Size() > 0)
+ {
+ SB.Append(", "sv);
+ }
+ SB.Append(It.first);
+ SB.Append(": "sv);
+ SB.Append(NiceBytes(It.second));
+ }
+ }
+ }
+ if (Storage.CacheStorage)
+ {
+ if (SB.Size() > 0)
+ {
+ SB.Append(", "sv);
+ }
+ SB.Append("Cache: ");
+ SB.Append(NiceBytes(StorageCacheStats.TotalBytesRead.load()));
+ }
+ if (SB.Size() > 0)
+ {
+ DownloadDetails = fmt::format(" ({})", SB.ToView());
+ }
+ }
+
+ ZEN_CONSOLE(
+ "Downloaded build {}, parts:{} in {}\n"
+ " Scavenge: {} (Target: {}, Cache: {}, Others: {})\n"
+ " Download: {} ({}) {}bits/s{}\n"
+ " Write: {} ({}) {}B/s{}\n"
+ " Clean: {}\n"
+ " Finalize: {}\n"
+ " Verify: {}",
+ BuildId,
+ BuildPartString.ToView(),
+ NiceTimeSpanMs(DownloadTimeMs),
+
+ NiceTimeSpanMs((Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs +
+ Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs +
+ Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs) /
+ 1000),
+ NiceTimeSpanMs(Updater.m_CacheMappingStats.LocalScanElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(Updater.m_CacheMappingStats.CacheScanElapsedWallTimeUs / 1000),
+ NiceTimeSpanMs(Updater.m_CacheMappingStats.ScavengeElapsedWallTimeUs / 1000),
+
+ DownloadCount,
+ NiceBytes(DownloadByteCount),
+ NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.DownloadTimeUs, DownloadByteCount * 8)),
+ DownloadDetails,
+
+ Updater.m_DiskStats.WriteCount.load(),
+ NiceBytes(Updater.m_WrittenChunkByteCount.load()),
+ NiceNum(GetBytesPerSecond(Updater.m_WriteChunkStats.WriteTimeUs, Updater.m_DiskStats.WriteByteCount.load())),
+ CloneInfo,
+
+ NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.CleanFolderElapsedWallTimeUs / 1000),
+
+ NiceTimeSpanMs(Updater.m_RebuildFolderStateStats.FinalizeTreeElapsedWallTimeUs / 1000),
+
+ NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000));
+ }
+ }
+ }
+
+ Progress.SetLogOperationProgress(TaskSteps::Cleanup, TaskSteps::StepCount);
+
+ CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), AbortFlag, PauseFlag, ZenTempFolder);
+}
+} // namespace zen
diff --git a/src/zenremotestore/builds/builduploadfolder.cpp b/src/zenremotestore/builds/builduploadfolder.cpp
new file mode 100644
index 000000000..433d24619
--- /dev/null
+++ b/src/zenremotestore/builds/builduploadfolder.cpp
@@ -0,0 +1,2635 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/builduploadfolder.h>
+
+#include <zencore/basicfile.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/fmtutils.h>
+#include <zencore/parallelwork.h>
+#include <zencore/scopeguard.h>
+#include <zencore/string.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/buildcontent.h>
+#include <zenremotestore/builds/buildmanifest.h>
+#include <zenremotestore/builds/buildstoragecache.h>
+#include <zenremotestore/chunking/chunkingcache.h>
+#include <zenremotestore/chunking/chunkingcontroller.h>
+#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/filteredrate.h>
+#include <zenutil/progress.h>
+
+#include <numeric>
+
+ZEN_THIRD_PARTY_INCLUDES_START
+#include <tsl/robin_set.h>
+ZEN_THIRD_PARTY_INCLUDES_END
+
+namespace zen {
+
+using namespace std::literals;
+
+namespace {
+ bool IsExtensionHashCompressable(const tsl::robin_set<uint32_t>& NonCompressableExtensionHashes, const uint32_t PathHash)
+ {
+ return !NonCompressableExtensionHashes.contains(PathHash);
+ }
+
+ bool IsChunkCompressable(const tsl::robin_set<uint32_t>& NonCompressableExtensionHashes,
+ const ChunkedContentLookup& Lookup,
+ uint32_t ChunkIndex)
+ {
+ const uint32_t ChunkLocationCount = Lookup.ChunkSequenceLocationCounts[ChunkIndex];
+ if (ChunkLocationCount == 0)
+ {
+ return false;
+ }
+ const size_t ChunkLocationOffset = Lookup.ChunkSequenceLocationOffset[ChunkIndex];
+ const uint32_t SequenceIndex = Lookup.ChunkSequenceLocations[ChunkLocationOffset].SequenceIndex;
+ const uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const uint32_t ExtensionHash = Lookup.PathExtensionHash[PathIndex];
+
+ const bool IsCompressable = IsExtensionHashCompressable(NonCompressableExtensionHashes, ExtensionHash);
+ return IsCompressable;
+ }
+ template<typename T>
+ std::string FormatArray(std::span<const T> Items, std::string_view Prefix)
+ {
+ ExtendableStringBuilder<512> SB;
+ for (const T& Item : Items)
+ {
+ SB.Append(fmt::format("{}{}", Prefix, Item));
+ }
+ return SB.ToString();
+ }
+} // namespace
+
+class ReadFileCache
+{
+public:
+ // A buffered file reader that provides CompositeBuffer where the buffers are owned and the memory never overwritten
+ ReadFileCache(std::atomic<uint64_t>& OpenReadCount,
+ std::atomic<uint64_t>& CurrentOpenFileCount,
+ std::atomic<uint64_t>& ReadCount,
+ std::atomic<uint64_t>& ReadByteCount,
+ const std::filesystem::path& Path,
+ const ChunkedFolderContent& LocalContent,
+ const ChunkedContentLookup& LocalLookup,
+ size_t MaxOpenFileCount)
+ : m_Path(Path)
+ , m_LocalContent(LocalContent)
+ , m_LocalLookup(LocalLookup)
+ , m_OpenReadCount(OpenReadCount)
+ , m_CurrentOpenFileCount(CurrentOpenFileCount)
+ , m_ReadCount(ReadCount)
+ , m_ReadByteCount(ReadByteCount)
+ {
+ m_OpenFiles.reserve(MaxOpenFileCount);
+ }
+ ~ReadFileCache() { m_OpenFiles.clear(); }
+
+ CompositeBuffer GetRange(uint32_t SequenceIndex, uint64_t Offset, uint64_t Size)
+ {
+ ZEN_TRACE_CPU("ReadFileCache::GetRange");
+
+ auto CacheIt =
+ std::find_if(m_OpenFiles.begin(), m_OpenFiles.end(), [SequenceIndex](const auto& Lhs) { return Lhs.first == SequenceIndex; });
+ if (CacheIt != m_OpenFiles.end())
+ {
+ if (CacheIt != m_OpenFiles.begin())
+ {
+ auto CachedFile(std::move(CacheIt->second));
+ m_OpenFiles.erase(CacheIt);
+ m_OpenFiles.insert(m_OpenFiles.begin(), std::make_pair(SequenceIndex, std::move(CachedFile)));
+ }
+ CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
+ return Result;
+ }
+ const uint32_t LocalPathIndex = m_LocalLookup.SequenceIndexFirstPathIndex[SequenceIndex];
+ const std::filesystem::path LocalFilePath = (m_Path / m_LocalContent.Paths[LocalPathIndex]).make_preferred();
+ if (Size == m_LocalContent.RawSizes[LocalPathIndex])
+ {
+ IoBuffer Result = IoBufferBuilder::MakeFromFile(LocalFilePath);
+ return CompositeBuffer(SharedBuffer(Result));
+ }
+ if (m_OpenFiles.size() == m_OpenFiles.capacity())
+ {
+ m_OpenFiles.pop_back();
+ }
+ m_OpenFiles.insert(
+ m_OpenFiles.begin(),
+ std::make_pair(
+ SequenceIndex,
+ std::make_unique<BufferedOpenFile>(LocalFilePath, m_OpenReadCount, m_CurrentOpenFileCount, m_ReadCount, m_ReadByteCount)));
+ CompositeBuffer Result = m_OpenFiles.front().second->GetRange(Offset, Size);
+ return Result;
+ }
+
+private:
+ const std::filesystem::path m_Path;
+ const ChunkedFolderContent& m_LocalContent;
+ const ChunkedContentLookup& m_LocalLookup;
+ std::vector<std::pair<uint32_t, std::unique_ptr<BufferedOpenFile>>> m_OpenFiles;
+ std::atomic<uint64_t>& m_OpenReadCount;
+ std::atomic<uint64_t>& m_CurrentOpenFileCount;
+ std::atomic<uint64_t>& m_ReadCount;
+ std::atomic<uint64_t>& m_ReadByteCount;
+};
+
+BuildsOperationUploadFolder::BuildsOperationUploadFolder(LoggerRef Log,
+ ProgressBase& Progress,
+ StorageInstance& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ WorkerThreadPool& IOWorkerPool,
+ WorkerThreadPool& NetworkPool,
+ const Oid& BuildId,
+ const std::filesystem::path& Path,
+ bool CreateBuild,
+ const CbObject& MetaData,
+ const Options& Options)
+: m_Log(Log)
+, m_Progress(Progress)
+, m_Storage(Storage)
+, m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_IOWorkerPool(IOWorkerPool)
+, m_NetworkPool(NetworkPool)
+, m_BuildId(BuildId)
+, m_Path(Path)
+, m_CreateBuild(CreateBuild)
+, m_MetaData(MetaData)
+, m_Options(Options)
+{
+ m_NonCompressableExtensionHashes.reserve(Options.NonCompressableExtensions.size());
+ for (const std::string& Extension : Options.NonCompressableExtensions)
+ {
+ m_NonCompressableExtensionHashes.insert(HashStringAsLowerDjb2(Extension));
+ }
+}
+
+BuildsOperationUploadFolder::PrepareBuildResult
+BuildsOperationUploadFolder::PrepareBuild()
+{
+ ZEN_TRACE_CPU("PrepareBuild");
+
+ PrepareBuildResult Result;
+ Result.PreferredMultipartChunkSize = m_Options.PreferredMultipartChunkSize;
+ Stopwatch Timer;
+ if (m_CreateBuild)
+ {
+ ZEN_TRACE_CPU("CreateBuild");
+
+ Stopwatch PutBuildTimer;
+ CbObject PutBuildResult = m_Storage.BuildStorage->PutBuild(m_BuildId, m_MetaData);
+ Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs();
+ if (auto ChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+ {
+ Result.PreferredMultipartChunkSize = ChunkSize;
+ }
+ Result.PayloadSize = m_MetaData.GetSize();
+ }
+ else
+ {
+ ZEN_TRACE_CPU("PutBuild");
+ Stopwatch GetBuildTimer;
+ CbObject Build = m_Storage.BuildStorage->GetBuild(m_BuildId);
+ Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs();
+ Result.PayloadSize = Build.GetSize();
+ if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+ {
+ Result.PreferredMultipartChunkSize = ChunkSize;
+ }
+ else if (m_Options.AllowMultiparts)
+ {
+ ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", NiceBytes(Result.PreferredMultipartChunkSize));
+ }
+ }
+
+ if (!m_Options.IgnoreExistingBlocks)
+ {
+ ZEN_TRACE_CPU("FindBlocks");
+ Stopwatch KnownBlocksTimer;
+ CbObject BlockDescriptionList = m_Storage.BuildStorage->FindBlocks(m_BuildId, m_Options.FindBlockMaxCount);
+ if (BlockDescriptionList)
+ {
+ Result.KnownBlocks = ParseChunkBlockDescriptionList(BlockDescriptionList);
+ }
+ Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs();
+ }
+ Result.ElapsedTimeMs = Timer.GetElapsedTimeMs();
+ return Result;
+}
+
+std::vector<BuildsOperationUploadFolder::UploadPart>
+BuildsOperationUploadFolder::ReadFolder()
+{
+ std::vector<UploadPart> UploadParts;
+ std::filesystem::path ExcludeManifestPath = m_Path / m_Options.ZenExcludeManifestName;
+ tsl::robin_set<std::string> ExcludeAssetPaths;
+ if (IsFile(ExcludeManifestPath))
+ {
+ std::filesystem::path AbsoluteExcludeManifestPath =
+ MakeSafeAbsolutePath(ExcludeManifestPath.is_absolute() ? ExcludeManifestPath : m_Path / ExcludeManifestPath);
+ BuildManifest Manifest = ParseBuildManifest(AbsoluteExcludeManifestPath);
+ const std::vector<std::filesystem::path>& AssetPaths = Manifest.Parts.front().Files;
+ ExcludeAssetPaths.reserve(AssetPaths.size());
+ for (const std::filesystem::path& AssetPath : AssetPaths)
+ {
+ ExcludeAssetPaths.insert(AssetPath.generic_string());
+ }
+ }
+
+ UploadParts.resize(1);
+
+ UploadPart& Part = UploadParts.front();
+ GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
+
+ Part.Content = GetFolderContent(
+ Part.LocalFolderScanStats,
+ m_Path,
+ [this](const std::string_view& RelativePath) { return IsAcceptedFolder(RelativePath); },
+ [this, &ExcludeAssetPaths](const std::string_view& RelativePath, uint64_t Size, uint32_t Attributes) -> bool {
+ ZEN_UNUSED(Size, Attributes);
+ if (!IsAcceptedFile(RelativePath))
+ {
+ return false;
+ }
+ if (ExcludeAssetPaths.contains(std::filesystem::path(RelativePath).generic_string()))
+ {
+ return false;
+ }
+ return true;
+ },
+ m_IOWorkerPool,
+ m_Progress.GetProgressUpdateDelayMS(),
+ [&](bool, std::ptrdiff_t) { ZEN_INFO("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), m_Path); },
+ m_AbortFlag);
+ Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
+
+ return UploadParts;
+}
+
+std::vector<BuildsOperationUploadFolder::UploadPart>
+BuildsOperationUploadFolder::ReadManifestParts(const std::filesystem::path& ManifestPath)
+{
+ std::vector<UploadPart> UploadParts;
+ Stopwatch ManifestParseTimer;
+ std::filesystem::path AbsoluteManifestPath = MakeSafeAbsolutePath(ManifestPath.is_absolute() ? ManifestPath : m_Path / ManifestPath);
+ BuildManifest Manifest = ParseBuildManifest(AbsoluteManifestPath);
+ if (Manifest.Parts.empty())
+ {
+ throw std::runtime_error(fmt::format("Manifest file at '{}' is invalid", ManifestPath));
+ }
+
+ UploadParts.resize(Manifest.Parts.size());
+ for (size_t PartIndex = 0; PartIndex < Manifest.Parts.size(); PartIndex++)
+ {
+ BuildManifest::Part& PartManifest = Manifest.Parts[PartIndex];
+ if (ManifestPath.is_relative())
+ {
+ PartManifest.Files.push_back(ManifestPath);
+ }
+
+ UploadPart& Part = UploadParts[PartIndex];
+ FolderContent& Content = Part.Content;
+
+ GetFolderContentStatistics& LocalFolderScanStats = Part.LocalFolderScanStats;
+
+ const std::vector<std::filesystem::path>& AssetPaths = PartManifest.Files;
+ Content = GetValidFolderContent(
+ m_IOWorkerPool,
+ LocalFolderScanStats,
+ m_Path,
+ AssetPaths,
+ [](uint64_t PathCount, uint64_t CompletedPathCount) { ZEN_UNUSED(PathCount, CompletedPathCount); },
+ 1000,
+ m_AbortFlag,
+ m_PauseFlag);
+
+ if (Content.Paths.size() != AssetPaths.size())
+ {
+ const tsl::robin_set<std::filesystem::path> FoundPaths(Content.Paths.begin(), Content.Paths.end());
+ ExtendableStringBuilder<1024> SB;
+ for (const std::filesystem::path& AssetPath : AssetPaths)
+ {
+ if (!FoundPaths.contains(AssetPath))
+ {
+ SB << "\n " << AssetPath.generic_string();
+ }
+ }
+ throw std::runtime_error(
+ fmt::format("Manifest file at '{}' references files that does not exist{}", ManifestPath, SB.ToView()));
+ }
+
+ Part.PartId = PartManifest.PartId;
+ Part.PartName = PartManifest.PartName;
+ Part.TotalRawSize = std::accumulate(Part.Content.RawSizes.begin(), Part.Content.RawSizes.end(), std::uint64_t(0));
+ }
+
+ return UploadParts;
+}
+
+std::vector<std::pair<Oid, std::string>>
+BuildsOperationUploadFolder::Execute(const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const std::filesystem::path& ManifestPath,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache)
+{
+ ZEN_TRACE_CPU("BuildsOperationUploadFolder::Execute");
+ try
+ {
+ Stopwatch ReadPartsTimer;
+ std::vector<UploadPart> UploadParts = ManifestPath.empty() ? ReadFolder() : ReadManifestParts(ManifestPath);
+
+ for (UploadPart& Part : UploadParts)
+ {
+ if (Part.PartId == Oid::Zero)
+ {
+ if (UploadParts.size() != 1)
+ {
+ throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part id", ManifestPath));
+ }
+
+ if (BuildPartId == Oid::Zero)
+ {
+ Part.PartId = Oid::NewOid();
+ }
+ else
+ {
+ Part.PartId = BuildPartId;
+ }
+ }
+ if (Part.PartName.empty())
+ {
+ if (UploadParts.size() != 1)
+ {
+ throw std::runtime_error(fmt::format("Multi part upload manifest '{}' must contains build part name", ManifestPath));
+ }
+ if (BuildPartName.empty())
+ {
+ throw std::runtime_error("Build part name must be set");
+ }
+ Part.PartName = std::string(BuildPartName);
+ }
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Reading {} parts took {}", UploadParts.size(), NiceTimeSpanMs(ReadPartsTimer.GetElapsedTimeMs()));
+ }
+
+ const uint32_t PartsUploadStepCount = gsl::narrow<uint32_t>(uint32_t(PartTaskSteps::StepCount) * UploadParts.size());
+
+ const uint32_t PrepareBuildStep = 0;
+ const uint32_t UploadPartsStep = 1;
+ const uint32_t FinalizeBuildStep = UploadPartsStep + PartsUploadStepCount;
+ const uint32_t CleanupStep = FinalizeBuildStep + 1;
+ const uint32_t StepCount = CleanupStep + 1;
+
+ auto EndProgress = MakeGuard([&]() { m_Progress.SetLogOperationProgress(StepCount, StepCount); });
+
+ Stopwatch ProcessTimer;
+
+ CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir);
+ CreateDirectories(m_Options.TempDir);
+ auto _ = MakeGuard([&]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, m_Options.TempDir); });
+
+ m_Progress.SetLogOperationProgress(PrepareBuildStep, StepCount);
+
+ m_PrepBuildResultFuture = m_NetworkPool.EnqueueTask(std::packaged_task<PrepareBuildResult()>{[this] { return PrepareBuild(); }},
+ WorkerThreadPool::EMode::EnableBacklog);
+
+ for (uint32_t PartIndex = 0; PartIndex < UploadParts.size(); PartIndex++)
+ {
+ const uint32_t PartStepOffset = UploadPartsStep + (PartIndex * uint32_t(PartTaskSteps::StepCount));
+
+ const UploadPart& Part = UploadParts[PartIndex];
+ UploadBuildPart(ChunkController, ChunkCache, PartIndex, Part, PartStepOffset, StepCount);
+ if (m_AbortFlag)
+ {
+ return {};
+ }
+ }
+
+ m_Progress.SetLogOperationProgress(FinalizeBuildStep, StepCount);
+
+ if (m_CreateBuild && !m_AbortFlag)
+ {
+ Stopwatch FinalizeBuildTimer;
+ m_Storage.BuildStorage->FinalizeBuild(m_BuildId);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs()));
+ }
+ }
+
+ m_Progress.SetLogOperationProgress(CleanupStep, StepCount);
+
+ std::vector<std::pair<Oid, std::string>> Result;
+ Result.reserve(UploadParts.size());
+ for (UploadPart& Part : UploadParts)
+ {
+ Result.push_back(std::make_pair(Part.PartId, Part.PartName));
+ }
+ return Result;
+ }
+ catch (const std::exception&)
+ {
+ m_AbortFlag = true;
+ throw;
+ }
+}
+
+bool
+BuildsOperationUploadFolder::IsAcceptedFolder(const std::string_view& RelativePath) const
+{
+ for (const std::string& ExcludeFolder : m_Options.ExcludeFolders)
+ {
+ if (StrCaseStartsWith(RelativePath, ExcludeFolder))
+ {
+ if (RelativePath.length() == ExcludeFolder.length())
+ {
+ return false;
+ }
+ else if (RelativePath[ExcludeFolder.length()] == '/')
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+bool
+BuildsOperationUploadFolder::IsAcceptedFile(const std::string_view& RelativePath) const
+{
+ if (RelativePath == m_Options.ZenExcludeManifestName)
+ {
+ return false;
+ }
+ for (const std::string& ExcludeExtension : m_Options.ExcludeExtensions)
+ {
+ if (StrCaseEndsWith(RelativePath, ExcludeExtension))
+ {
+ return false;
+ }
+ }
+ return true;
+}
+
+void
+BuildsOperationUploadFolder::ArrangeChunksIntoBlocks(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ std::vector<uint32_t>& ChunkIndexes,
+ std::vector<std::vector<uint32_t>>& OutBlocks)
+{
+ ZEN_TRACE_CPU("ArrangeChunksIntoBlocks");
+ std::sort(ChunkIndexes.begin(), ChunkIndexes.end(), [&Content, &Lookup](uint32_t Lhs, uint32_t Rhs) {
+ const ChunkedContentLookup::ChunkSequenceLocation& LhsLocation = GetChunkSequenceLocations(Lookup, Lhs)[0];
+ const ChunkedContentLookup::ChunkSequenceLocation& RhsLocation = GetChunkSequenceLocations(Lookup, Rhs)[0];
+ if (LhsLocation.SequenceIndex < RhsLocation.SequenceIndex)
+ {
+ return true;
+ }
+ else if (LhsLocation.SequenceIndex > RhsLocation.SequenceIndex)
+ {
+ return false;
+ }
+ return LhsLocation.Offset < RhsLocation.Offset;
+ });
+
+ uint64_t MaxBlockSizeLowThreshold = m_Options.BlockParameters.MaxBlockSize - (m_Options.BlockParameters.MaxBlockSize / 16);
+
+ uint64_t BlockSize = 0;
+
+ uint32_t ChunkIndexStart = 0;
+ for (uint32_t ChunkIndexOffset = 0; ChunkIndexOffset < ChunkIndexes.size();)
+ {
+ const uint32_t ChunkIndex = ChunkIndexes[ChunkIndexOffset];
+ const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+
+ if (((BlockSize + ChunkSize) > m_Options.BlockParameters.MaxBlockSize) ||
+ (ChunkIndexOffset - ChunkIndexStart) > m_Options.BlockParameters.MaxChunksPerBlock)
+ {
+ // Within the span of MaxBlockSizeLowThreshold and MaxBlockSize, see if there is a break
+ // between source paths for chunks. Break the block at the last such break if any.
+ ZEN_ASSERT(ChunkIndexOffset > ChunkIndexStart);
+
+ const uint32_t ChunkSequenceIndex = Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[ChunkIndex]].SequenceIndex;
+
+ uint64_t ScanBlockSize = BlockSize;
+
+ uint32_t ScanChunkIndexOffset = ChunkIndexOffset - 1;
+ while (ScanChunkIndexOffset > (ChunkIndexStart + 2))
+ {
+ const uint32_t TestChunkIndex = ChunkIndexes[ScanChunkIndexOffset];
+ const uint64_t TestChunkSize = Content.ChunkedContent.ChunkRawSizes[TestChunkIndex];
+ if ((ScanBlockSize - TestChunkSize) < MaxBlockSizeLowThreshold)
+ {
+ break;
+ }
+
+ const uint32_t TestSequenceIndex =
+ Lookup.ChunkSequenceLocations[Lookup.ChunkSequenceLocationOffset[TestChunkIndex]].SequenceIndex;
+ if (ChunkSequenceIndex != TestSequenceIndex)
+ {
+ ChunkIndexOffset = ScanChunkIndexOffset + 1;
+ break;
+ }
+
+ ScanBlockSize -= TestChunkSize;
+ ScanChunkIndexOffset--;
+ }
+
+ std::vector<uint32_t> ChunksInBlock;
+ ChunksInBlock.reserve(ChunkIndexOffset - ChunkIndexStart);
+ for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexOffset; AddIndexOffset++)
+ {
+ const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset];
+ ChunksInBlock.push_back(AddChunkIndex);
+ }
+ OutBlocks.emplace_back(std::move(ChunksInBlock));
+ BlockSize = 0;
+ ChunkIndexStart = ChunkIndexOffset;
+ }
+ else
+ {
+ ChunkIndexOffset++;
+ BlockSize += ChunkSize;
+ }
+ }
+ if (ChunkIndexStart < ChunkIndexes.size())
+ {
+ std::vector<uint32_t> ChunksInBlock;
+ ChunksInBlock.reserve(ChunkIndexes.size() - ChunkIndexStart);
+ for (uint32_t AddIndexOffset = ChunkIndexStart; AddIndexOffset < ChunkIndexes.size(); AddIndexOffset++)
+ {
+ const uint32_t AddChunkIndex = ChunkIndexes[AddIndexOffset];
+ ChunksInBlock.push_back(AddChunkIndex);
+ }
+ OutBlocks.emplace_back(std::move(ChunksInBlock));
+ }
+}
+
+void
+BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const std::vector<std::vector<uint32_t>>& NewBlockChunks,
+ GeneratedBlocks& OutBlocks,
+ GenerateBlocksStatistics& GenerateBlocksStats,
+ UploadStatistics& UploadStats)
+{
+ ZEN_TRACE_CPU("GenerateBuildBlocks");
+ const std::size_t NewBlockCount = NewBlockChunks.size();
+ if (NewBlockCount == 0)
+ {
+ return;
+ }
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Generate Blocks");
+
+ OutBlocks.BlockDescriptions.resize(NewBlockCount);
+ OutBlocks.BlockSizes.resize(NewBlockCount);
+ OutBlocks.BlockMetaDatas.resize(NewBlockCount);
+ OutBlocks.BlockHeaders.resize(NewBlockCount);
+ OutBlocks.MetaDataHasBeenUploaded.resize(NewBlockCount, 0);
+ OutBlocks.BlockHashToBlockIndex.reserve(NewBlockCount);
+
+ RwLock Lock;
+ FilteredRate FilteredGeneratedBytesPerSecond;
+ FilteredRate FilteredUploadedBytesPerSecond;
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+ std::atomic<uint64_t> QueuedPendingBlocksForUpload = 0;
+
+ GenerateBuildBlocksContext Context{.Work = Work,
+ .GenerateBlobsPool = m_IOWorkerPool,
+ .UploadBlocksPool = m_NetworkPool,
+ .FilteredGeneratedBytesPerSecond = FilteredGeneratedBytesPerSecond,
+ .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond,
+ .QueuedPendingBlocksForUpload = QueuedPendingBlocksForUpload,
+ .Lock = Lock,
+ .OutBlocks = OutBlocks,
+ .GenerateBlocksStats = GenerateBlocksStats,
+ .UploadStats = UploadStats,
+ .NewBlockCount = NewBlockCount};
+
+ ScheduleBlockGeneration(Context, Content, Lookup, NewBlockChunks);
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+
+ FilteredGeneratedBytesPerSecond.Update(GenerateBlocksStats.GeneratedBlockByteCount.load());
+ FilteredUploadedBytesPerSecond.Update(UploadStats.BlocksBytes.load());
+
+ std::string Details = fmt::format("Generated {}/{} ({}, {}B/s). Uploaded {}/{} ({}, {}bits/s)",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NewBlockCount,
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(FilteredGeneratedBytesPerSecond.GetCurrent()),
+ UploadStats.BlockCount.load(),
+ NewBlockCount,
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceNum(FilteredUploadedBytesPerSecond.GetCurrent() * 8));
+
+ ProgressBar->UpdateState({.Task = "Generating blocks",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(NewBlockCount),
+ .RemainingCount = gsl::narrow<uint64_t>(NewBlockCount - GenerateBlocksStats.GeneratedBlockCount.load()),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+
+ ZEN_ASSERT(m_AbortFlag || QueuedPendingBlocksForUpload.load() == 0);
+
+ ProgressBar->Finish();
+
+ GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS = FilteredGeneratedBytesPerSecond.GetElapsedTimeUS();
+ UploadStats.ElapsedWallTimeUS = FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
+}
+
+void
+BuildsOperationUploadFolder::ScheduleBlockGeneration(GenerateBuildBlocksContext& Context,
+ const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const std::vector<std::vector<uint32_t>>& NewBlockChunks)
+{
+ for (size_t BlockIndex = 0; BlockIndex < Context.NewBlockCount; BlockIndex++)
+ {
+ if (Context.Work.IsAborted())
+ {
+ break;
+ }
+ const std::vector<uint32_t>& ChunksInBlock = NewBlockChunks[BlockIndex];
+ Context.Work.ScheduleWork(
+ Context.GenerateBlobsPool,
+ [this, &Context, &Content, &Lookup, ChunksInBlock, BlockIndex](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("GenerateBuildBlocks_Generate");
+
+ Context.FilteredGeneratedBytesPerSecond.Start();
+
+ Stopwatch GenerateTimer;
+ CompressedBuffer CompressedBlock =
+ GenerateBlock(Content, Lookup, ChunksInBlock, Context.OutBlocks.BlockDescriptions[BlockIndex]);
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Generated block {} ({}) containing {} chunks in {}",
+ Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
+ NiceBytes(CompressedBlock.GetCompressedSize()),
+ Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
+ NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
+ }
+
+ Context.OutBlocks.BlockSizes[BlockIndex] = CompressedBlock.GetCompressedSize();
+ {
+ CbObjectWriter Writer;
+ Writer.AddString("createdBy", "zen");
+ Context.OutBlocks.BlockMetaDatas[BlockIndex] = Writer.Save();
+ }
+ Context.GenerateBlocksStats.GeneratedBlockByteCount += Context.OutBlocks.BlockSizes[BlockIndex];
+ Context.GenerateBlocksStats.GeneratedBlockCount++;
+
+ Context.Lock.WithExclusiveLock([&]() {
+ Context.OutBlocks.BlockHashToBlockIndex.insert_or_assign(Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash,
+ BlockIndex);
+ });
+
+ {
+ std::span<const SharedBuffer> Segments = CompressedBlock.GetCompressed().GetSegments();
+ ZEN_ASSERT(Segments.size() >= 2);
+ Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
+ }
+
+ if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount)
+ {
+ Context.FilteredGeneratedBytesPerSecond.Stop();
+ }
+
+ if (Context.QueuedPendingBlocksForUpload.load() > 16)
+ {
+ std::span<const SharedBuffer> Segments = CompressedBlock.GetCompressed().GetSegments();
+ ZEN_ASSERT(Segments.size() >= 2);
+ Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
+ }
+ else
+ {
+ if (!m_AbortFlag)
+ {
+ Context.QueuedPendingBlocksForUpload++;
+ Context.Work.ScheduleWork(
+ Context.UploadBlocksPool,
+ [this, &Context, BlockIndex, Payload = std::move(CompressedBlock)](std::atomic<bool>&) mutable {
+ UploadGeneratedBlock(Context, BlockIndex, std::move(Payload));
+ });
+ }
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUploadFolder::UploadGeneratedBlock(GenerateBuildBlocksContext& Context, size_t BlockIndex, CompressedBuffer Payload)
+{
+ auto _ = MakeGuard([&Context] { Context.QueuedPendingBlocksForUpload--; });
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ if (Context.GenerateBlocksStats.GeneratedBlockCount == Context.NewBlockCount)
+ {
+ ZEN_TRACE_CPU("GenerateBuildBlocks_Save");
+
+ Context.FilteredUploadedBytesPerSecond.Stop();
+ std::span<const SharedBuffer> Segments = Payload.GetCompressed().GetSegments();
+ ZEN_ASSERT(Segments.size() >= 2);
+ Context.OutBlocks.BlockHeaders[BlockIndex] = CompositeBuffer(Segments[0], Segments[1]);
+ return;
+ }
+
+ ZEN_TRACE_CPU("GenerateBuildBlocks_Upload");
+
+ Context.FilteredUploadedBytesPerSecond.Start();
+
+ const CbObject BlockMetaData =
+ BuildChunkBlockDescription(Context.OutBlocks.BlockDescriptions[BlockIndex], Context.OutBlocks.BlockMetaDatas[BlockIndex]);
+
+ const IoHash& BlockHash = Context.OutBlocks.BlockDescriptions[BlockIndex].BlockHash;
+ const uint64_t CompressedBlockSize = Payload.GetCompressedSize();
+
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload.GetCompressed());
+ }
+
+ try
+ {
+ m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, std::move(Payload).GetCompressed());
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ Context.UploadStats.BlocksBytes += CompressedBlockSize;
+
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded block {} ({}) containing {} chunks",
+ BlockHash,
+ NiceBytes(CompressedBlockSize),
+ Context.OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
+ }
+
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, std::vector<IoHash>({BlockHash}), std::vector<CbObject>({BlockMetaData}));
+ }
+
+ bool MetadataSucceeded = false;
+ try
+ {
+ MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ if (MetadataSucceeded)
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize()));
+ }
+
+ Context.OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+ Context.UploadStats.BlocksBytes += BlockMetaData.GetSize();
+ }
+
+ Context.UploadStats.BlockCount++;
+ if (Context.UploadStats.BlockCount == Context.NewBlockCount)
+ {
+ Context.FilteredUploadedBytesPerSecond.Stop();
+ }
+}
+
+std::vector<uint32_t>
+BuildsOperationUploadFolder::CalculateAbsoluteChunkOrders(
+ const std::span<const IoHash> LocalChunkHashes,
+ const std::span<const uint32_t> LocalChunkOrder,
+ const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex,
+ const std::span<const uint32_t>& LooseChunkIndexes,
+ const std::span<const ChunkBlockDescription>& BlockDescriptions)
+{
+ ZEN_TRACE_CPU("CalculateAbsoluteChunkOrders");
+
+ std::vector<IoHash> TmpAbsoluteChunkHashes;
+ if (m_Options.DoExtraContentValidation)
+ {
+ TmpAbsoluteChunkHashes.reserve(LocalChunkHashes.size());
+ }
+ std::vector<uint32_t> LocalChunkIndexToAbsoluteChunkIndex;
+ LocalChunkIndexToAbsoluteChunkIndex.resize(LocalChunkHashes.size(), (uint32_t)-1);
+ std::uint32_t AbsoluteChunkCount = 0;
+ for (uint32_t ChunkIndex : LooseChunkIndexes)
+ {
+ LocalChunkIndexToAbsoluteChunkIndex[ChunkIndex] = AbsoluteChunkCount;
+ if (m_Options.DoExtraContentValidation)
+ {
+ TmpAbsoluteChunkHashes.push_back(LocalChunkHashes[ChunkIndex]);
+ }
+ AbsoluteChunkCount++;
+ }
+ for (const ChunkBlockDescription& Block : BlockDescriptions)
+ {
+ for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+ {
+ if (auto It = ChunkHashToLocalChunkIndex.find(ChunkHash); It != ChunkHashToLocalChunkIndex.end())
+ {
+ const uint32_t LocalChunkIndex = It->second;
+ ZEN_ASSERT_SLOW(LocalChunkHashes[LocalChunkIndex] == ChunkHash);
+ LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex] = AbsoluteChunkCount;
+ }
+ if (m_Options.DoExtraContentValidation)
+ {
+ TmpAbsoluteChunkHashes.push_back(ChunkHash);
+ }
+ AbsoluteChunkCount++;
+ }
+ }
+ std::vector<uint32_t> AbsoluteChunkOrder;
+ AbsoluteChunkOrder.reserve(LocalChunkHashes.size());
+ for (const uint32_t LocalChunkIndex : LocalChunkOrder)
+ {
+ const uint32_t AbsoluteChunkIndex = LocalChunkIndexToAbsoluteChunkIndex[LocalChunkIndex];
+ if (m_Options.DoExtraContentValidation)
+ {
+ ZEN_ASSERT(LocalChunkHashes[LocalChunkIndex] == TmpAbsoluteChunkHashes[AbsoluteChunkIndex]);
+ }
+ AbsoluteChunkOrder.push_back(AbsoluteChunkIndex);
+ }
+ if (m_Options.DoExtraContentValidation)
+ {
+ uint32_t OrderIndex = 0;
+ while (OrderIndex < LocalChunkOrder.size())
+ {
+ const uint32_t LocalChunkIndex = LocalChunkOrder[OrderIndex];
+ const IoHash& LocalChunkHash = LocalChunkHashes[LocalChunkIndex];
+ const uint32_t AbsoluteChunkIndex = AbsoluteChunkOrder[OrderIndex];
+ const IoHash& AbsoluteChunkHash = TmpAbsoluteChunkHashes[AbsoluteChunkIndex];
+ ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+ OrderIndex++;
+ }
+ }
+ return AbsoluteChunkOrder;
+}
+
+CompositeBuffer
+BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const IoHash& ChunkHash,
+ ReadFileCache& OpenFileCache)
+{
+ ZEN_TRACE_CPU("FetchChunk");
+ auto It = Lookup.ChunkHashToChunkIndex.find(ChunkHash);
+ ZEN_ASSERT(It != Lookup.ChunkHashToChunkIndex.end());
+ uint32_t ChunkIndex = It->second;
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex);
+ ZEN_ASSERT(!ChunkLocations.empty());
+ CompositeBuffer Chunk =
+ OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+ if (!Chunk)
+ {
+ throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'",
+ ChunkLocations[0].Offset,
+ Content.ChunkedContent.ChunkRawSizes[ChunkIndex],
+ Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]]));
+ }
+ ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash);
+ return Chunk;
+};
+
+CompressedBuffer
+BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const std::vector<uint32_t>& ChunksInBlock,
+ ChunkBlockDescription& OutBlockDescription)
+{
+ ZEN_TRACE_CPU("GenerateBlock");
+ ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
+ m_DiskStats.CurrentOpenFileCount,
+ m_DiskStats.ReadCount,
+ m_DiskStats.ReadByteCount,
+ m_Path,
+ Content,
+ Lookup,
+ 4);
+
+ std::vector<std::pair<IoHash, FetchChunkFunc>> BlockContent;
+ BlockContent.reserve(ChunksInBlock.size());
+ for (uint32_t ChunkIndex : ChunksInBlock)
+ {
+ BlockContent.emplace_back(std::make_pair(
+ Content.ChunkedContent.ChunkHashes[ChunkIndex],
+ [this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair<uint64_t, CompositeBuffer> {
+ CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache);
+ ZEN_ASSERT(Chunk);
+ uint64_t RawSize = Chunk.GetSize();
+
+ const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock &&
+ IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex);
+
+ const OodleCompressionLevel CompressionLevel =
+ ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+ return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel).GetCompressed()};
+ }));
+ }
+
+ return GenerateChunkBlock(std::move(BlockContent), OutBlockDescription);
+};
+
+CompressedBuffer
+BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ CompositeBuffer&& HeaderBuffer,
+ const std::vector<uint32_t>& ChunksInBlock)
+{
+ ZEN_TRACE_CPU("RebuildBlock");
+ ReadFileCache OpenFileCache(m_DiskStats.OpenReadCount,
+ m_DiskStats.CurrentOpenFileCount,
+ m_DiskStats.ReadCount,
+ m_DiskStats.ReadByteCount,
+ m_Path,
+ Content,
+ Lookup,
+ 4);
+
+ std::vector<SharedBuffer> ResultBuffers;
+ ResultBuffers.reserve(HeaderBuffer.GetSegments().size() + ChunksInBlock.size());
+ ResultBuffers.insert(ResultBuffers.end(), HeaderBuffer.GetSegments().begin(), HeaderBuffer.GetSegments().end());
+ for (uint32_t ChunkIndex : ChunksInBlock)
+ {
+ std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex);
+ ZEN_ASSERT(!ChunkLocations.empty());
+ CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex,
+ ChunkLocations[0].Offset,
+ Content.ChunkedContent.ChunkRawSizes[ChunkIndex]);
+ ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]);
+
+ const uint64_t RawSize = Chunk.GetSize();
+ const bool ShouldCompressChunk =
+ RawSize >= m_Options.MinimumSizeForCompressInBlock && IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex);
+
+ const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+
+ CompositeBuffer CompressedChunk =
+ CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, CompressionLevel).GetCompressed();
+ ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end());
+ }
+ return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers)));
+};
+
+void
+BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ uint32_t PartIndex,
+ const UploadPart& Part,
+ uint32_t PartStepOffset,
+ uint32_t StepCount)
+{
+ Stopwatch UploadTimer;
+
+ ChunkingStatistics ChunkingStats;
+ FindBlocksStatistics FindBlocksStats;
+ ReuseBlocksStatistics ReuseBlocksStats;
+ UploadStatistics UploadStats;
+ GenerateBlocksStatistics GenerateBlocksStats;
+ LooseChunksStatistics LooseChunksStats;
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::ChunkPartContent, StepCount);
+
+ ChunkedFolderContent LocalContent = ScanPartContent(Part, ChunkController, ChunkCache, ChunkingStats);
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalContent);
+
+ if (PartIndex == 0)
+ {
+ ConsumePrepareBuildResult();
+ }
+
+ ZEN_ASSERT(m_PreferredMultipartChunkSize != 0);
+ ZEN_ASSERT(m_LargeAttachmentSize != 0);
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::CalculateDelta, StepCount);
+
+ Stopwatch BlockArrangeTimer;
+
+ std::vector<uint32_t> LooseChunkIndexes;
+ std::vector<uint32_t> NewBlockChunkIndexes;
+ std::vector<size_t> ReuseBlockIndexes;
+ ClassifyChunksByBlockEligibility(LocalContent,
+ LooseChunkIndexes,
+ NewBlockChunkIndexes,
+ ReuseBlockIndexes,
+ LooseChunksStats,
+ FindBlocksStats,
+ ReuseBlocksStats);
+
+ std::vector<std::vector<uint32_t>> NewBlockChunks;
+ ArrangeChunksIntoBlocks(LocalContent, LocalLookup, NewBlockChunkIndexes, NewBlockChunks);
+
+ FindBlocksStats.NewBlocksCount += NewBlockChunks.size();
+ for (uint32_t ChunkIndex : NewBlockChunkIndexes)
+ {
+ FindBlocksStats.NewBlocksChunkByteCount += LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+ FindBlocksStats.NewBlocksChunkCount += NewBlockChunkIndexes.size();
+
+ const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0
+ ? (100.0 * ReuseBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount)
+ : 0.0;
+
+ const double AcceptedReduntantByteCountPercent =
+ ReuseBlocksStats.AcceptedByteCount > 0 ? (100.0 * ReuseBlocksStats.AcceptedReduntantByteCount) /
+ (ReuseBlocksStats.AcceptedByteCount + ReuseBlocksStats.AcceptedReduntantByteCount)
+ : 0.0;
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO(
+ "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n"
+ " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n"
+ " Accepting {} ({}) redundant chunks ({:.1f}%)\n"
+ " Rejected {} ({}) chunks in {} blocks\n"
+ " Arranged {} ({}) chunks in {} new blocks\n"
+ " Keeping {} ({}) chunks as loose chunks\n"
+ " Discovery completed in {}",
+ FindBlocksStats.FoundBlockChunkCount,
+ FindBlocksStats.FoundBlockCount,
+ NiceBytes(FindBlocksStats.FoundBlockByteCount),
+ NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS),
+
+ ReuseBlocksStats.AcceptedChunkCount,
+ NiceBytes(ReuseBlocksStats.AcceptedRawByteCount),
+ FindBlocksStats.AcceptedBlockCount,
+ AcceptedByteCountPercent,
+
+ ReuseBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(ReuseBlocksStats.AcceptedReduntantByteCount),
+ AcceptedReduntantByteCountPercent,
+
+ ReuseBlocksStats.RejectedChunkCount,
+ NiceBytes(ReuseBlocksStats.RejectedByteCount),
+ ReuseBlocksStats.RejectedBlockCount,
+
+ FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(FindBlocksStats.NewBlocksChunkByteCount),
+ FindBlocksStats.NewBlocksCount,
+
+ LooseChunksStats.ChunkCount,
+ NiceBytes(LooseChunksStats.ChunkByteCount),
+
+ NiceTimeSpanMs(BlockArrangeTimer.GetElapsedTimeMs()));
+ }
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::GenerateBlocks, StepCount);
+ GeneratedBlocks NewBlocks;
+
+ if (!NewBlockChunks.empty())
+ {
+ Stopwatch GenerateBuildBlocksTimer;
+ auto __ = MakeGuard([&]() {
+ uint64_t BlockGenerateTimeUs = GenerateBuildBlocksTimer.GetElapsedTimeUs();
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Generated {} ({}) and uploaded {} ({}) blocks in {}. Generate speed: {}B/sec. Transfer speed {}bits/sec.",
+ GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(GenerateBlocksStats.GeneratedBlockByteCount),
+ UploadStats.BlockCount.load(),
+ NiceBytes(UploadStats.BlocksBytes.load()),
+ NiceTimeSpanMs(BlockGenerateTimeUs / 1000),
+ NiceNum(GetBytesPerSecond(GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceNum(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.BlocksBytes * 8)));
+ }
+ });
+ GenerateBuildBlocks(LocalContent, LocalLookup, NewBlockChunks, NewBlocks, GenerateBlocksStats, UploadStats);
+ }
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::BuildPartManifest, StepCount);
+
+ BuiltPartManifest Manifest =
+ BuildPartManifestObject(LocalContent, LocalLookup, ChunkController, ReuseBlockIndexes, NewBlocks, LooseChunkIndexes);
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadBuildPart, StepCount);
+
+ Stopwatch PutBuildPartResultTimer;
+ std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult =
+ m_Storage.BuildStorage->PutBuildPart(m_BuildId, Part.PartId, Part.PartName, Manifest.PartManifest);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("PutBuildPart took {}, payload size {}. {} attachments are needed.",
+ NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()),
+ NiceBytes(Manifest.PartManifest.GetSize()),
+ PutBuildPartResult.second.size());
+ }
+ IoHash PartHash = PutBuildPartResult.first;
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::UploadAttachments, StepCount);
+
+ std::vector<IoHash> UnknownChunks;
+ if (m_Options.IgnoreExistingBlocks)
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("PutBuildPart uploading all attachments, needs are: {}", FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
+ }
+
+ std::vector<IoHash> ForceUploadChunkHashes;
+ ForceUploadChunkHashes.reserve(LooseChunkIndexes.size());
+
+ for (uint32_t ChunkIndex : LooseChunkIndexes)
+ {
+ ForceUploadChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ }
+
+ for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockHeaders.size(); BlockIndex++)
+ {
+ if (NewBlocks.BlockHeaders[BlockIndex])
+ {
+ // Block was not uploaded during generation
+ ForceUploadChunkHashes.push_back(NewBlocks.BlockDescriptions[BlockIndex].BlockHash);
+ }
+ }
+ UploadAttachmentBatch(ForceUploadChunkHashes,
+ UnknownChunks,
+ LocalContent,
+ LocalLookup,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ UploadStats,
+ LooseChunksStats);
+ }
+ else if (!PutBuildPartResult.second.empty())
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("PutBuildPart needs attachments: {}", FormatArray<IoHash>(PutBuildPartResult.second, "\n "sv));
+ }
+ UploadAttachmentBatch(PutBuildPartResult.second,
+ UnknownChunks,
+ LocalContent,
+ LocalLookup,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ UploadStats,
+ LooseChunksStats);
+ }
+
+ FinalizeBuildPartWithRetries(Part,
+ PartHash,
+ UnknownChunks,
+ LocalContent,
+ LocalLookup,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ UploadStats,
+ LooseChunksStats);
+
+ if (!NewBlocks.BlockDescriptions.empty() && !m_AbortFlag)
+ {
+ UploadMissingBlockMetadata(NewBlocks, UploadStats);
+ // The newly generated blocks are now known blocks so the next part upload can use those blocks as well
+ m_KnownBlocks.insert(m_KnownBlocks.end(), NewBlocks.BlockDescriptions.begin(), NewBlocks.BlockDescriptions.end());
+ }
+
+ m_Progress.SetLogOperationProgress(PartStepOffset + (uint32_t)PartTaskSteps::PutBuildPartStats, StepCount);
+
+ m_Storage.BuildStorage->PutBuildPartStats(
+ m_BuildId,
+ Part.PartId,
+ {{"totalSize", double(Part.LocalFolderScanStats.FoundFileByteCount.load())},
+ {"reusedRatio", AcceptedByteCountPercent / 100.0},
+ {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)},
+ {"reusedBlockByteCount", double(ReuseBlocksStats.AcceptedRawByteCount)},
+ {"newBlockCount", double(FindBlocksStats.NewBlocksCount)},
+ {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)},
+ {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())},
+ {"uploadedByteCount", double(UploadStats.BlocksBytes.load() + UploadStats.ChunksBytes.load())},
+ {"uploadedBytesPerSec",
+ double(GetBytesPerSecond(UploadStats.ElapsedWallTimeUS, UploadStats.ChunksBytes + UploadStats.BlocksBytes))},
+ {"elapsedTimeSec", double(UploadTimer.GetElapsedTimeMs() / 1000.0)}});
+
+ m_LocalFolderScanStats += Part.LocalFolderScanStats;
+ m_ChunkingStats += ChunkingStats;
+ m_FindBlocksStats += FindBlocksStats;
+ m_ReuseBlocksStats += ReuseBlocksStats;
+ m_UploadStats += UploadStats;
+ m_GenerateBlocksStats += GenerateBlocksStats;
+ m_LooseChunksStats += LooseChunksStats;
+}
+
+ChunkedFolderContent
+BuildsOperationUploadFolder::ScanPartContent(const UploadPart& Part,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ ChunkingStatistics& ChunkingStats)
+{
+ Stopwatch ScanTimer;
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Scan Folder");
+
+ FilteredRate FilteredBytesHashed;
+ FilteredBytesHashed.Start();
+ ChunkedFolderContent LocalContent = ChunkFolderContent(
+ ChunkingStats,
+ m_IOWorkerPool,
+ m_Path,
+ Part.Content,
+ ChunkController,
+ ChunkCache,
+ m_Progress.GetProgressUpdateDelayMS(),
+ [&](bool IsAborted, bool IsPaused, std::ptrdiff_t) {
+ FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load());
+ std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found",
+ ChunkingStats.FilesProcessed.load(),
+ Part.Content.Paths.size(),
+ NiceBytes(ChunkingStats.BytesHashed.load()),
+ NiceBytes(Part.TotalRawSize),
+ NiceNum(FilteredBytesHashed.GetCurrent()),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()));
+ ProgressBar->UpdateState({.Task = "Scanning files ",
+ .Details = Details,
+ .TotalCount = Part.TotalRawSize,
+ .RemainingCount = Part.TotalRawSize - ChunkingStats.BytesHashed.load(),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ },
+ m_AbortFlag,
+ m_PauseFlag);
+ FilteredBytesHashed.Stop();
+ ProgressBar->Finish();
+ if (m_AbortFlag)
+ {
+ return LocalContent;
+ }
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Found {} ({}) files divided into {} ({}) unique chunks in '{}' in {}. Average hash rate {}B/sec",
+ Part.Content.Paths.size(),
+ NiceBytes(Part.TotalRawSize),
+ ChunkingStats.UniqueChunksFound.load(),
+ NiceBytes(ChunkingStats.UniqueBytesFound.load()),
+ m_Path,
+ NiceTimeSpanMs(ScanTimer.GetElapsedTimeMs()),
+ NiceNum(GetBytesPerSecond(ChunkingStats.ElapsedWallTimeUS, ChunkingStats.BytesHashed)));
+ }
+
+ return LocalContent;
+}
+
+void
+BuildsOperationUploadFolder::ConsumePrepareBuildResult()
+{
+ const PrepareBuildResult PrepBuildResult = m_PrepBuildResultFuture.get();
+
+ m_FindBlocksStats.FindBlockTimeMS = PrepBuildResult.ElapsedTimeMs;
+ m_FindBlocksStats.FoundBlockCount = PrepBuildResult.KnownBlocks.size();
+
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Build prepare took {}. {} took {}, payload size {}{}",
+ NiceTimeSpanMs(PrepBuildResult.ElapsedTimeMs),
+ m_CreateBuild ? "PutBuild" : "GetBuild",
+ NiceTimeSpanMs(PrepBuildResult.PrepareBuildTimeMs),
+ NiceBytes(PrepBuildResult.PayloadSize),
+ m_Options.IgnoreExistingBlocks ? ""
+ : fmt::format(". Found {} blocks in {}",
+ PrepBuildResult.KnownBlocks.size(),
+ NiceTimeSpanMs(PrepBuildResult.FindBlocksTimeMs)));
+ }
+
+ m_PreferredMultipartChunkSize = PrepBuildResult.PreferredMultipartChunkSize;
+ m_LargeAttachmentSize = m_Options.AllowMultiparts ? m_PreferredMultipartChunkSize * 4u : (std::uint64_t)-1;
+ m_KnownBlocks = std::move(PrepBuildResult.KnownBlocks);
+}
+
+void
+BuildsOperationUploadFolder::ClassifyChunksByBlockEligibility(const ChunkedFolderContent& LocalContent,
+ std::vector<uint32_t>& OutLooseChunkIndexes,
+ std::vector<uint32_t>& OutNewBlockChunkIndexes,
+ std::vector<size_t>& OutReuseBlockIndexes,
+ LooseChunksStatistics& LooseChunksStats,
+ FindBlocksStatistics& FindBlocksStats,
+ ReuseBlocksStatistics& ReuseBlocksStats)
+{
+ const bool EnableBlocks = true;
+ std::vector<std::uint32_t> BlockChunkIndexes;
+ for (uint32_t ChunkIndex = 0; ChunkIndex < LocalContent.ChunkedContent.ChunkHashes.size(); ChunkIndex++)
+ {
+ const uint64_t ChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ if (!EnableBlocks || ChunkRawSize == 0 || ChunkRawSize > m_Options.BlockParameters.MaxChunkEmbedSize)
+ {
+ OutLooseChunkIndexes.push_back(ChunkIndex);
+ LooseChunksStats.ChunkByteCount += ChunkRawSize;
+ }
+ else
+ {
+ BlockChunkIndexes.push_back(ChunkIndex);
+ FindBlocksStats.PotentialChunkByteCount += ChunkRawSize;
+ }
+ }
+ FindBlocksStats.PotentialChunkCount += BlockChunkIndexes.size();
+ LooseChunksStats.ChunkCount = OutLooseChunkIndexes.size();
+
+ if (m_Options.IgnoreExistingBlocks)
+ {
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Ignoring any existing blocks in store");
+ }
+ OutNewBlockChunkIndexes = std::move(BlockChunkIndexes);
+ return;
+ }
+
+ OutReuseBlockIndexes = FindReuseBlocks(Log(),
+ m_Options.BlockReuseMinPercentLimit,
+ m_Options.IsVerbose,
+ ReuseBlocksStats,
+ m_KnownBlocks,
+ LocalContent.ChunkedContent.ChunkHashes,
+ BlockChunkIndexes,
+ OutNewBlockChunkIndexes);
+ FindBlocksStats.AcceptedBlockCount += OutReuseBlockIndexes.size();
+
+ for (const ChunkBlockDescription& Description : m_KnownBlocks)
+ {
+ for (uint32_t ChunkRawLength : Description.ChunkRawLengths)
+ {
+ FindBlocksStats.FoundBlockByteCount += ChunkRawLength;
+ }
+ FindBlocksStats.FoundBlockChunkCount += Description.ChunkRawHashes.size();
+ }
+}
+
+BuildsOperationUploadFolder::BuiltPartManifest
+BuildsOperationUploadFolder::BuildPartManifestObject(const ChunkedFolderContent& LocalContent,
+ const ChunkedContentLookup& LocalLookup,
+ ChunkingController& ChunkController,
+ std::span<const size_t> ReuseBlockIndexes,
+ const GeneratedBlocks& NewBlocks,
+ std::span<const uint32_t> LooseChunkIndexes)
+{
+ BuiltPartManifest Result;
+
+ CbObjectWriter PartManifestWriter;
+ Stopwatch ManifestGenerationTimer;
+ auto __ = MakeGuard([&]() {
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Generated build part manifest in {} ({})",
+ NiceTimeSpanMs(ManifestGenerationTimer.GetElapsedTimeMs()),
+ NiceBytes(PartManifestWriter.GetSaveSize()));
+ }
+ });
+
+ PartManifestWriter.BeginObject("chunker"sv);
+ {
+ PartManifestWriter.AddString("name"sv, ChunkController.GetName());
+ PartManifestWriter.AddObject("parameters"sv, ChunkController.GetParameters());
+ }
+ PartManifestWriter.EndObject(); // chunker
+
+ Result.AllChunkBlockHashes.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
+ Result.AllChunkBlockDescriptions.reserve(ReuseBlockIndexes.size() + NewBlocks.BlockDescriptions.size());
+ for (size_t ReuseBlockIndex : ReuseBlockIndexes)
+ {
+ Result.AllChunkBlockDescriptions.push_back(m_KnownBlocks[ReuseBlockIndex]);
+ Result.AllChunkBlockHashes.push_back(m_KnownBlocks[ReuseBlockIndex].BlockHash);
+ }
+ Result.AllChunkBlockDescriptions.insert(Result.AllChunkBlockDescriptions.end(),
+ NewBlocks.BlockDescriptions.begin(),
+ NewBlocks.BlockDescriptions.end());
+ for (const ChunkBlockDescription& BlockDescription : NewBlocks.BlockDescriptions)
+ {
+ Result.AllChunkBlockHashes.push_back(BlockDescription.BlockHash);
+ }
+
+ std::vector<IoHash> AbsoluteChunkHashes;
+ if (m_Options.DoExtraContentValidation)
+ {
+ tsl::robin_map<IoHash, size_t, IoHash::Hasher> ChunkHashToAbsoluteChunkIndex;
+ AbsoluteChunkHashes.reserve(LocalContent.ChunkedContent.ChunkHashes.size());
+ for (uint32_t ChunkIndex : LooseChunkIndexes)
+ {
+ ChunkHashToAbsoluteChunkIndex.insert({LocalContent.ChunkedContent.ChunkHashes[ChunkIndex], AbsoluteChunkHashes.size()});
+ AbsoluteChunkHashes.push_back(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ }
+ for (const ChunkBlockDescription& Block : Result.AllChunkBlockDescriptions)
+ {
+ for (const IoHash& ChunkHash : Block.ChunkRawHashes)
+ {
+ ChunkHashToAbsoluteChunkIndex.insert({ChunkHash, AbsoluteChunkHashes.size()});
+ AbsoluteChunkHashes.push_back(ChunkHash);
+ }
+ }
+ for (const IoHash& ChunkHash : LocalContent.ChunkedContent.ChunkHashes)
+ {
+ ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(ChunkHash)] == ChunkHash);
+ ZEN_ASSERT(LocalContent.ChunkedContent.ChunkHashes[LocalLookup.ChunkHashToChunkIndex.at(ChunkHash)] == ChunkHash);
+ }
+ for (const uint32_t ChunkIndex : LocalContent.ChunkedContent.ChunkOrders)
+ {
+ ZEN_ASSERT(AbsoluteChunkHashes[ChunkHashToAbsoluteChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex])] ==
+ LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]);
+ ZEN_ASSERT(LocalLookup.ChunkHashToChunkIndex.at(LocalContent.ChunkedContent.ChunkHashes[ChunkIndex]) == ChunkIndex);
+ }
+ }
+
+ std::vector<uint32_t> AbsoluteChunkOrders = CalculateAbsoluteChunkOrders(LocalContent.ChunkedContent.ChunkHashes,
+ LocalContent.ChunkedContent.ChunkOrders,
+ LocalLookup.ChunkHashToChunkIndex,
+ LooseChunkIndexes,
+ Result.AllChunkBlockDescriptions);
+
+ if (m_Options.DoExtraContentValidation)
+ {
+ for (uint32_t ChunkOrderIndex = 0; ChunkOrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); ChunkOrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[ChunkOrderIndex];
+ uint32_t AbsoluteChunkIndex = AbsoluteChunkOrders[ChunkOrderIndex];
+ const IoHash& LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+ const IoHash& AbsoluteChunkHash = AbsoluteChunkHashes[AbsoluteChunkIndex];
+ ZEN_ASSERT(LocalChunkHash == AbsoluteChunkHash);
+ }
+ }
+
+ WriteBuildContentToCompactBinary(PartManifestWriter,
+ LocalContent.Platform,
+ LocalContent.Paths,
+ LocalContent.RawHashes,
+ LocalContent.RawSizes,
+ LocalContent.Attributes,
+ LocalContent.ChunkedContent.SequenceRawHashes,
+ LocalContent.ChunkedContent.ChunkCounts,
+ LocalContent.ChunkedContent.ChunkHashes,
+ LocalContent.ChunkedContent.ChunkRawSizes,
+ AbsoluteChunkOrders,
+ LooseChunkIndexes,
+ Result.AllChunkBlockHashes);
+
+ if (m_Options.DoExtraContentValidation)
+ {
+ ChunkedFolderContent VerifyFolderContent;
+
+ std::vector<uint32_t> OutAbsoluteChunkOrders;
+ std::vector<IoHash> OutLooseChunkHashes;
+ std::vector<uint64_t> OutLooseChunkRawSizes;
+ std::vector<IoHash> OutBlockRawHashes;
+ ReadBuildContentFromCompactBinary(PartManifestWriter.Save(),
+ VerifyFolderContent.Platform,
+ VerifyFolderContent.Paths,
+ VerifyFolderContent.RawHashes,
+ VerifyFolderContent.RawSizes,
+ VerifyFolderContent.Attributes,
+ VerifyFolderContent.ChunkedContent.SequenceRawHashes,
+ VerifyFolderContent.ChunkedContent.ChunkCounts,
+ OutAbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ OutLooseChunkRawSizes,
+ OutBlockRawHashes);
+ ZEN_ASSERT(OutBlockRawHashes == Result.AllChunkBlockHashes);
+
+ for (uint32_t OrderIndex = 0; OrderIndex < OutAbsoluteChunkOrders.size(); OrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+
+ uint32_t VerifyChunkIndex = OutAbsoluteChunkOrders[OrderIndex];
+ const IoHash VerifyChunkHash = AbsoluteChunkHashes[VerifyChunkIndex];
+
+ ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
+ }
+
+ CalculateLocalChunkOrders(OutAbsoluteChunkOrders,
+ OutLooseChunkHashes,
+ OutLooseChunkRawSizes,
+ Result.AllChunkBlockDescriptions,
+ VerifyFolderContent.ChunkedContent.ChunkHashes,
+ VerifyFolderContent.ChunkedContent.ChunkRawSizes,
+ VerifyFolderContent.ChunkedContent.ChunkOrders,
+ m_Options.DoExtraContentValidation);
+
+ ZEN_ASSERT(LocalContent.Paths == VerifyFolderContent.Paths);
+ ZEN_ASSERT(LocalContent.RawHashes == VerifyFolderContent.RawHashes);
+ ZEN_ASSERT(LocalContent.RawSizes == VerifyFolderContent.RawSizes);
+ ZEN_ASSERT(LocalContent.Attributes == VerifyFolderContent.Attributes);
+ ZEN_ASSERT(LocalContent.ChunkedContent.SequenceRawHashes == VerifyFolderContent.ChunkedContent.SequenceRawHashes);
+ ZEN_ASSERT(LocalContent.ChunkedContent.ChunkCounts == VerifyFolderContent.ChunkedContent.ChunkCounts);
+
+ for (uint32_t OrderIndex = 0; OrderIndex < LocalContent.ChunkedContent.ChunkOrders.size(); OrderIndex++)
+ {
+ uint32_t LocalChunkIndex = LocalContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash LocalChunkHash = LocalContent.ChunkedContent.ChunkHashes[LocalChunkIndex];
+ uint64_t LocalChunkRawSize = LocalContent.ChunkedContent.ChunkRawSizes[LocalChunkIndex];
+
+ uint32_t VerifyChunkIndex = VerifyFolderContent.ChunkedContent.ChunkOrders[OrderIndex];
+ const IoHash VerifyChunkHash = VerifyFolderContent.ChunkedContent.ChunkHashes[VerifyChunkIndex];
+ uint64_t VerifyChunkRawSize = VerifyFolderContent.ChunkedContent.ChunkRawSizes[VerifyChunkIndex];
+
+ ZEN_ASSERT(LocalChunkHash == VerifyChunkHash);
+ ZEN_ASSERT(LocalChunkRawSize == VerifyChunkRawSize);
+ }
+ }
+
+ Result.PartManifest = PartManifestWriter.Save();
+ return Result;
+}
+
+void
+BuildsOperationUploadFolder::UploadAttachmentBatch(std::span<IoHash> RawHashes,
+ std::vector<IoHash>& OutUnknownChunks,
+ const ChunkedFolderContent& LocalContent,
+ const ChunkedContentLookup& LocalLookup,
+ const std::vector<std::vector<uint32_t>>& NewBlockChunks,
+ GeneratedBlocks& NewBlocks,
+ std::span<const uint32_t> LooseChunkIndexes,
+ UploadStatistics& UploadStats,
+ LooseChunksStatistics& LooseChunksStats)
+{
+ if (m_AbortFlag)
+ {
+ return;
+ }
+
+ UploadStatistics TempUploadStats;
+ LooseChunksStatistics TempLooseChunksStats;
+
+ Stopwatch TempUploadTimer;
+ auto __ = MakeGuard([&]() {
+ if (!m_Options.IsQuiet)
+ {
+ uint64_t TempChunkUploadTimeUs = TempUploadTimer.GetElapsedTimeUs();
+ ZEN_INFO(
+ "Uploaded {} ({}) blocks. "
+ "Compressed {} ({} {}B/s) and uploaded {} ({}) chunks. "
+ "Transferred {} ({}bits/s) in {}",
+ TempUploadStats.BlockCount.load(),
+ NiceBytes(TempUploadStats.BlocksBytes),
+
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)),
+ TempUploadStats.ChunkCount.load(),
+ NiceBytes(TempUploadStats.ChunksBytes),
+
+ NiceBytes(TempUploadStats.BlocksBytes + TempUploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(TempUploadStats.ElapsedWallTimeUS, TempUploadStats.ChunksBytes * 8)),
+ NiceTimeSpanMs(TempChunkUploadTimeUs / 1000));
+ }
+ });
+ UploadPartBlobs(LocalContent,
+ LocalLookup,
+ RawHashes,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ m_LargeAttachmentSize,
+ TempUploadStats,
+ TempLooseChunksStats,
+ OutUnknownChunks);
+ UploadStats += TempUploadStats;
+ LooseChunksStats += TempLooseChunksStats;
+}
+
+void
+BuildsOperationUploadFolder::FinalizeBuildPartWithRetries(const UploadPart& Part,
+ const IoHash& PartHash,
+ std::vector<IoHash>& InOutUnknownChunks,
+ const ChunkedFolderContent& LocalContent,
+ const ChunkedContentLookup& LocalLookup,
+ const std::vector<std::vector<uint32_t>>& NewBlockChunks,
+ GeneratedBlocks& NewBlocks,
+ std::span<const uint32_t> LooseChunkIndexes,
+ UploadStatistics& UploadStats,
+ LooseChunksStatistics& LooseChunksStats)
+{
+ auto BuildUnkownChunksResponse = [](const std::vector<IoHash>& UnknownChunks, bool WillRetry) {
+ return fmt::format(
+ "The following build blobs was reported as needed for upload but was reported as existing at the start of the "
+ "operation.{}{}",
+ WillRetry ? " Treating this as a transient inconsistency issue and will attempt to retry finalization."sv : ""sv,
+ FormatArray<IoHash>(UnknownChunks, "\n "sv));
+ };
+
+ if (!InOutUnknownChunks.empty())
+ {
+ ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ true));
+ }
+
+ uint32_t FinalizeBuildPartRetryCount = 5;
+ while (!m_AbortFlag && (FinalizeBuildPartRetryCount--) > 0)
+ {
+ Stopwatch FinalizeBuildPartTimer;
+ std::vector<IoHash> Needs = m_Storage.BuildStorage->FinalizeBuildPart(m_BuildId, Part.PartId, PartHash);
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("FinalizeBuildPart took {}. {} attachments are missing.",
+ NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()),
+ Needs.size());
+ }
+ if (Needs.empty())
+ {
+ break;
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("FinalizeBuildPart needs attachments: {}", FormatArray<IoHash>(Needs, "\n "sv));
+ }
+
+ std::vector<IoHash> RetryUnknownChunks;
+ UploadAttachmentBatch(Needs,
+ RetryUnknownChunks,
+ LocalContent,
+ LocalLookup,
+ NewBlockChunks,
+ NewBlocks,
+ LooseChunkIndexes,
+ UploadStats,
+ LooseChunksStats);
+ if (RetryUnknownChunks == InOutUnknownChunks)
+ {
+ if (FinalizeBuildPartRetryCount > 0)
+ {
+ // Back off a bit
+ Sleep(1000);
+ }
+ }
+ else
+ {
+ InOutUnknownChunks = RetryUnknownChunks;
+ ZEN_WARN("{}", BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ FinalizeBuildPartRetryCount != 0));
+ }
+ }
+
+ if (!InOutUnknownChunks.empty())
+ {
+ throw std::runtime_error(BuildUnkownChunksResponse(InOutUnknownChunks, /*WillRetry*/ false));
+ }
+}
+
+void
+BuildsOperationUploadFolder::UploadMissingBlockMetadata(GeneratedBlocks& NewBlocks, UploadStatistics& UploadStats)
+{
+ uint64_t UploadBlockMetadataCount = 0;
+ Stopwatch UploadBlockMetadataTimer;
+
+ uint32_t FailedMetadataUploadCount = 1;
+ int32_t MetadataUploadRetryCount = 3;
+ while ((MetadataUploadRetryCount-- > 0) && (FailedMetadataUploadCount > 0))
+ {
+ FailedMetadataUploadCount = 0;
+ for (size_t BlockIndex = 0; BlockIndex < NewBlocks.BlockDescriptions.size(); BlockIndex++)
+ {
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ const IoHash& BlockHash = NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
+ if (!NewBlocks.MetaDataHasBeenUploaded[BlockIndex])
+ {
+ const CbObject BlockMetaData =
+ BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]);
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId,
+ std::vector<IoHash>({BlockHash}),
+ std::vector<CbObject>({BlockMetaData}));
+ }
+ bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
+ if (MetadataSucceeded)
+ {
+ UploadStats.BlocksBytes += BlockMetaData.GetSize();
+ NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+ UploadBlockMetadataCount++;
+ }
+ else
+ {
+ FailedMetadataUploadCount++;
+ }
+ }
+ }
+ }
+ if (UploadBlockMetadataCount > 0)
+ {
+ uint64_t ElapsedUS = UploadBlockMetadataTimer.GetElapsedTimeUs();
+ UploadStats.ElapsedWallTimeUS += ElapsedUS;
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Uploaded metadata for {} blocks in {}", UploadBlockMetadataCount, NiceTimeSpanMs(ElapsedUS / 1000));
+ }
+ }
+}
+
+void
+BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ std::span<IoHash> RawHashes,
+ const std::vector<std::vector<uint32_t>>& NewBlockChunks,
+ GeneratedBlocks& NewBlocks,
+ std::span<const uint32_t> LooseChunkIndexes,
+ const std::uint64_t LargeAttachmentSize,
+ UploadStatistics& TempUploadStats,
+ LooseChunksStatistics& TempLooseChunksStats,
+ std::vector<IoHash>& OutUnknownChunks)
+{
+ ZEN_TRACE_CPU("UploadPartBlobs");
+
+ UploadPartClassification Classification =
+ ClassifyUploadRawHashes(RawHashes, Content, Lookup, NewBlocks, LooseChunkIndexes, OutUnknownChunks);
+
+ if (Classification.BlockIndexes.empty() && Classification.LooseChunkOrderIndexes.empty())
+ {
+ return;
+ }
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Upload Blobs");
+
+ FilteredRate FilteredGenerateBlockBytesPerSecond;
+ FilteredRate FilteredCompressedBytesPerSecond;
+ FilteredRate FilteredUploadedBytesPerSecond;
+
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ std::atomic<size_t> UploadedBlockSize = 0;
+ std::atomic<size_t> UploadedBlockCount = 0;
+ std::atomic<size_t> UploadedRawChunkSize = 0;
+ std::atomic<size_t> UploadedCompressedChunkSize = 0;
+ std::atomic<uint32_t> UploadedChunkCount = 0;
+ std::atomic<uint64_t> GeneratedBlockCount = 0;
+ std::atomic<uint64_t> GeneratedBlockByteCount = 0;
+ std::atomic<uint64_t> QueuedPendingInMemoryBlocksForUpload = 0;
+
+ const size_t UploadBlockCount = Classification.BlockIndexes.size();
+ const uint32_t UploadChunkCount = gsl::narrow<uint32_t>(Classification.LooseChunkOrderIndexes.size());
+ const uint64_t TotalRawSize = Classification.TotalLooseChunksSize + Classification.TotalBlocksSize;
+
+ UploadPartBlobsContext Context{.Work = Work,
+ .ReadChunkPool = m_IOWorkerPool,
+ .UploadChunkPool = m_NetworkPool,
+ .FilteredGenerateBlockBytesPerSecond = FilteredGenerateBlockBytesPerSecond,
+ .FilteredCompressedBytesPerSecond = FilteredCompressedBytesPerSecond,
+ .FilteredUploadedBytesPerSecond = FilteredUploadedBytesPerSecond,
+ .UploadedBlockSize = UploadedBlockSize,
+ .UploadedBlockCount = UploadedBlockCount,
+ .UploadedRawChunkSize = UploadedRawChunkSize,
+ .UploadedCompressedChunkSize = UploadedCompressedChunkSize,
+ .UploadedChunkCount = UploadedChunkCount,
+ .GeneratedBlockCount = GeneratedBlockCount,
+ .GeneratedBlockByteCount = GeneratedBlockByteCount,
+ .QueuedPendingInMemoryBlocksForUpload = QueuedPendingInMemoryBlocksForUpload,
+ .UploadBlockCount = UploadBlockCount,
+ .UploadChunkCount = UploadChunkCount,
+ .LargeAttachmentSize = LargeAttachmentSize,
+ .NewBlocks = NewBlocks,
+ .Content = Content,
+ .Lookup = Lookup,
+ .NewBlockChunks = NewBlockChunks,
+ .LooseChunkIndexes = LooseChunkIndexes,
+ .TempUploadStats = TempUploadStats,
+ .TempLooseChunksStats = TempLooseChunksStats};
+
+ ScheduleBlockGenerationAndUpload(Context, Classification.BlockIndexes);
+ ScheduleLooseChunkCompressionAndUpload(Context, Classification.LooseChunkOrderIndexes);
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+ FilteredCompressedBytesPerSecond.Update(TempLooseChunksStats.CompressedChunkRawBytes.load());
+ FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load());
+ FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load());
+ uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load();
+ uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load();
+
+ std::string Details = fmt::format(
+ "Compressed {}/{} ({}/{}{}) chunks. "
+ "Uploaded {}/{} ({}/{}) blobs "
+ "({}{})",
+ TempLooseChunksStats.CompressedChunkCount.load(),
+ Classification.LooseChunkOrderIndexes.size(),
+ NiceBytes(TempLooseChunksStats.CompressedChunkRawBytes),
+ NiceBytes(Classification.TotalLooseChunksSize),
+ (TempLooseChunksStats.CompressedChunkCount == Classification.LooseChunkOrderIndexes.size())
+ ? ""
+ : fmt::format(" {}B/s", NiceNum(FilteredCompressedBytesPerSecond.GetCurrent())),
+
+ UploadedBlockCount.load() + UploadedChunkCount.load(),
+ UploadBlockCount + UploadChunkCount,
+ NiceBytes(UploadedRawSize),
+ NiceBytes(TotalRawSize),
+
+ NiceBytes(UploadedCompressedSize),
+ (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount)
+ ? ""
+ : fmt::format(" {}bits/s", NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())));
+
+ ProgressBar->UpdateState({.Task = "Uploading blobs ",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(TotalRawSize),
+ .RemainingCount = gsl::narrow<uint64_t>(TotalRawSize - UploadedRawSize),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+
+ ZEN_ASSERT(m_AbortFlag || QueuedPendingInMemoryBlocksForUpload.load() == 0);
+
+ ProgressBar->Finish();
+
+ TempUploadStats.ElapsedWallTimeUS += FilteredUploadedBytesPerSecond.GetElapsedTimeUS();
+ TempLooseChunksStats.CompressChunksElapsedWallTimeUS += FilteredCompressedBytesPerSecond.GetElapsedTimeUS();
+}
+
+BuildsOperationUploadFolder::UploadPartClassification
+BuildsOperationUploadFolder::ClassifyUploadRawHashes(std::span<IoHash> RawHashes,
+ const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ const GeneratedBlocks& NewBlocks,
+ std::span<const uint32_t> LooseChunkIndexes,
+ std::vector<IoHash>& OutUnknownChunks)
+{
+ UploadPartClassification Result;
+
+ tsl::robin_map<uint32_t, uint32_t> ChunkIndexToLooseChunkOrderIndex;
+ ChunkIndexToLooseChunkOrderIndex.reserve(LooseChunkIndexes.size());
+ for (uint32_t OrderIndex = 0; OrderIndex < LooseChunkIndexes.size(); OrderIndex++)
+ {
+ ChunkIndexToLooseChunkOrderIndex.insert_or_assign(LooseChunkIndexes[OrderIndex], OrderIndex);
+ }
+
+ for (const IoHash& RawHash : RawHashes)
+ {
+ if (auto It = NewBlocks.BlockHashToBlockIndex.find(RawHash); It != NewBlocks.BlockHashToBlockIndex.end())
+ {
+ Result.BlockIndexes.push_back(It->second);
+ Result.TotalBlocksSize += NewBlocks.BlockSizes[It->second];
+ }
+ else if (auto ChunkIndexIt = Lookup.ChunkHashToChunkIndex.find(RawHash); ChunkIndexIt != Lookup.ChunkHashToChunkIndex.end())
+ {
+ const uint32_t ChunkIndex = ChunkIndexIt->second;
+ if (auto LooseOrderIndexIt = ChunkIndexToLooseChunkOrderIndex.find(ChunkIndex);
+ LooseOrderIndexIt != ChunkIndexToLooseChunkOrderIndex.end())
+ {
+ Result.LooseChunkOrderIndexes.push_back(LooseOrderIndexIt->second);
+ Result.TotalLooseChunksSize += Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ }
+ }
+ else
+ {
+ OutUnknownChunks.push_back(RawHash);
+ }
+ }
+ return Result;
+}
+
+void
+BuildsOperationUploadFolder::ScheduleBlockGenerationAndUpload(UploadPartBlobsContext& Context, std::span<const size_t> BlockIndexes)
+{
+ for (const size_t BlockIndex : BlockIndexes)
+ {
+ const IoHash& BlockHash = Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash;
+ if (m_AbortFlag)
+ {
+ break;
+ }
+ Context.Work.ScheduleWork(
+ Context.ReadChunkPool,
+ [this, &Context, BlockHash = IoHash(BlockHash), BlockIndex, GenerateBlockCount = BlockIndexes.size()](std::atomic<bool>&) {
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ ZEN_TRACE_CPU("UploadPartBlobs_GenerateBlock");
+
+ Context.FilteredGenerateBlockBytesPerSecond.Start();
+
+ Stopwatch GenerateTimer;
+ CompositeBuffer Payload;
+ if (Context.NewBlocks.BlockHeaders[BlockIndex])
+ {
+ Payload = RebuildBlock(Context.Content,
+ Context.Lookup,
+ std::move(Context.NewBlocks.BlockHeaders[BlockIndex]),
+ Context.NewBlockChunks[BlockIndex])
+ .GetCompressed();
+ }
+ else
+ {
+ ChunkBlockDescription BlockDescription;
+ CompressedBuffer CompressedBlock =
+ GenerateBlock(Context.Content, Context.Lookup, Context.NewBlockChunks[BlockIndex], BlockDescription);
+ if (!CompressedBlock)
+ {
+ throw std::runtime_error(fmt::format("Failed generating block {}", BlockHash));
+ }
+ ZEN_ASSERT(BlockDescription.BlockHash == BlockHash);
+ Payload = std::move(CompressedBlock).GetCompressed();
+ }
+
+ Context.GeneratedBlockByteCount += Context.NewBlocks.BlockSizes[BlockIndex];
+ if (Context.GeneratedBlockCount.fetch_add(1) + 1 == GenerateBlockCount)
+ {
+ Context.FilteredGenerateBlockBytesPerSecond.Stop();
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("{} block {} ({}) containing {} chunks in {}",
+ Context.NewBlocks.BlockHeaders[BlockIndex] ? "Regenerated" : "Generated",
+ Context.NewBlocks.BlockDescriptions[BlockIndex].BlockHash,
+ NiceBytes(Context.NewBlocks.BlockSizes[BlockIndex]),
+ Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size(),
+ NiceTimeSpanMs(GenerateTimer.GetElapsedTimeMs()));
+ }
+ if (!m_AbortFlag)
+ {
+ UploadBlockPayload(Context, BlockIndex, BlockHash, std::move(Payload));
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUploadFolder::UploadBlockPayload(UploadPartBlobsContext& Context,
+ size_t BlockIndex,
+ const IoHash& BlockHash,
+ CompositeBuffer Payload)
+{
+ bool IsInMemoryBlock = true;
+ if (Context.QueuedPendingInMemoryBlocksForUpload.load() > 16)
+ {
+ ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock");
+ std::filesystem::path TempFilePath = m_Options.TempDir / (BlockHash.ToHexString());
+ Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), TempFilePath));
+ IsInMemoryBlock = false;
+ }
+ else
+ {
+ Context.QueuedPendingInMemoryBlocksForUpload++;
+ }
+
+ Context.Work.ScheduleWork(
+ Context.UploadChunkPool,
+ [this, &Context, IsInMemoryBlock, BlockIndex, BlockHash = IoHash(BlockHash), Payload = CompositeBuffer(std::move(Payload))](
+ std::atomic<bool>&) {
+ auto _ = MakeGuard([IsInMemoryBlock, &Context] {
+ if (IsInMemoryBlock)
+ {
+ Context.QueuedPendingInMemoryBlocksForUpload--;
+ }
+ });
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ ZEN_TRACE_CPU("AsyncUploadBlock");
+
+ const uint64_t PayloadSize = Payload.GetSize();
+
+ Context.FilteredUploadedBytesPerSecond.Start();
+ const CbObject BlockMetaData =
+ BuildChunkBlockDescription(Context.NewBlocks.BlockDescriptions[BlockIndex], Context.NewBlocks.BlockMetaDatas[BlockIndex]);
+
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
+ }
+
+ try
+ {
+ m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded block {} ({}) containing {} chunks",
+ BlockHash,
+ NiceBytes(PayloadSize),
+ Context.NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size());
+ }
+ Context.UploadedBlockSize += PayloadSize;
+ Context.TempUploadStats.BlocksBytes += PayloadSize;
+
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId,
+ std::vector<IoHash>({BlockHash}),
+ std::vector<CbObject>({BlockMetaData}));
+ }
+
+ bool MetadataSucceeded = false;
+ try
+ {
+ MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ if (MetadataSucceeded)
+ {
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize()));
+ }
+ Context.NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true;
+ Context.TempUploadStats.BlocksBytes += BlockMetaData.GetSize();
+ }
+
+ Context.TempUploadStats.BlockCount++;
+
+ if (Context.UploadedBlockCount.fetch_add(1) + 1 == Context.UploadBlockCount &&
+ Context.UploadedChunkCount == Context.UploadChunkCount)
+ {
+ Context.FilteredUploadedBytesPerSecond.Stop();
+ }
+ });
+}
+
+void
+BuildsOperationUploadFolder::ScheduleLooseChunkCompressionAndUpload(UploadPartBlobsContext& Context,
+ std::span<const uint32_t> LooseChunkOrderIndexes)
+{
+ for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes)
+ {
+ const uint32_t ChunkIndex = Context.LooseChunkIndexes[LooseChunkOrderIndex];
+ Context.Work.ScheduleWork(Context.ReadChunkPool,
+ [this, &Context, LooseChunkOrderCount = LooseChunkOrderIndexes.size(), ChunkIndex](std::atomic<bool>&) {
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk");
+
+ Context.FilteredCompressedBytesPerSecond.Start();
+ Stopwatch CompressTimer;
+ CompositeBuffer Payload =
+ CompressChunk(Context.Content, Context.Lookup, ChunkIndex, Context.TempLooseChunksStats);
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Compressed chunk {} ({} -> {}) in {}",
+ Context.Content.ChunkedContent.ChunkHashes[ChunkIndex],
+ NiceBytes(Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex]),
+ NiceBytes(Payload.GetSize()),
+ NiceTimeSpanMs(CompressTimer.GetElapsedTimeMs()));
+ }
+ const uint64_t ChunkRawSize = Context.Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+ Context.TempUploadStats.ReadFromDiskBytes += ChunkRawSize;
+ if (Context.TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderCount)
+ {
+ Context.FilteredCompressedBytesPerSecond.Stop();
+ }
+ if (!m_AbortFlag)
+ {
+ UploadLooseChunkPayload(Context,
+ Context.Content.ChunkedContent.ChunkHashes[ChunkIndex],
+ ChunkRawSize,
+ std::move(Payload));
+ }
+ });
+ }
+}
+
+void
+BuildsOperationUploadFolder::UploadLooseChunkPayload(UploadPartBlobsContext& Context,
+ const IoHash& RawHash,
+ uint64_t RawSize,
+ CompositeBuffer Payload)
+{
+ Context.Work.ScheduleWork(
+ Context.UploadChunkPool,
+ [this, &Context, RawHash = IoHash(RawHash), RawSize, Payload = CompositeBuffer(std::move(Payload))](std::atomic<bool>&) mutable {
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ ZEN_TRACE_CPU("AsyncUploadLooseChunk");
+
+ const uint64_t PayloadSize = Payload.GetSize();
+
+ if (m_Storage.CacheStorage && m_Options.PopulateCache)
+ {
+ m_Storage.CacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
+ }
+
+ if (PayloadSize >= Context.LargeAttachmentSize)
+ {
+ ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart");
+ Context.TempUploadStats.MultipartAttachmentCount++;
+ try
+ {
+ std::vector<std::function<void()>> MultipartWork = m_Storage.BuildStorage->PutLargeBuildBlob(
+ m_BuildId,
+ RawHash,
+ ZenContentType::kCompressedBinary,
+ PayloadSize,
+ [Payload = std::move(Payload), &Context](uint64_t Offset, uint64_t Size) -> IoBuffer {
+ Context.FilteredUploadedBytesPerSecond.Start();
+
+ IoBuffer PartPayload = Payload.Mid(Offset, Size).Flatten().AsIoBuffer();
+ PartPayload.SetContentType(ZenContentType::kBinary);
+ return PartPayload;
+ },
+ [&Context, RawSize](uint64_t SentBytes, bool IsComplete) {
+ Context.TempUploadStats.ChunksBytes += SentBytes;
+ Context.UploadedCompressedChunkSize += SentBytes;
+ if (IsComplete)
+ {
+ Context.TempUploadStats.ChunkCount++;
+ if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount &&
+ Context.UploadedBlockCount == Context.UploadBlockCount)
+ {
+ Context.FilteredUploadedBytesPerSecond.Stop();
+ }
+ Context.UploadedRawChunkSize += RawSize;
+ }
+ });
+ for (auto& WorkPart : MultipartWork)
+ {
+ Context.Work.ScheduleWork(Context.UploadChunkPool, [Work = std::move(WorkPart)](std::atomic<bool>& AbortFlag) {
+ ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart_Work");
+ if (!AbortFlag)
+ {
+ Work();
+ }
+ });
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded multipart chunk {} ({})", RawHash, NiceBytes(PayloadSize));
+ }
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+ return;
+ }
+
+ ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart");
+ try
+ {
+ m_Storage.BuildStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload);
+ }
+ catch (const std::exception&)
+ {
+ // Silence http errors due to abort
+ if (!m_AbortFlag)
+ {
+ throw;
+ }
+ }
+ if (m_AbortFlag)
+ {
+ return;
+ }
+ if (m_Options.IsVerbose)
+ {
+ ZEN_INFO("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize));
+ }
+ Context.TempUploadStats.ChunksBytes += Payload.GetSize();
+ Context.TempUploadStats.ChunkCount++;
+ Context.UploadedCompressedChunkSize += Payload.GetSize();
+ Context.UploadedRawChunkSize += RawSize;
+ if (Context.UploadedChunkCount.fetch_add(1) + 1 == Context.UploadChunkCount &&
+ Context.UploadedBlockCount == Context.UploadBlockCount)
+ {
+ Context.FilteredUploadedBytesPerSecond.Stop();
+ }
+ });
+}
+
+CompositeBuffer
+BuildsOperationUploadFolder::CompressChunk(const ChunkedFolderContent& Content,
+ const ChunkedContentLookup& Lookup,
+ uint32_t ChunkIndex,
+ LooseChunksStatistics& TempLooseChunksStats)
+{
+ ZEN_TRACE_CPU("CompressChunk");
+ ZEN_ASSERT(!m_Options.TempDir.empty());
+ const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex];
+ const uint64_t ChunkSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex];
+
+ const ChunkedContentLookup::ChunkSequenceLocation& Source = GetChunkSequenceLocations(Lookup, ChunkIndex)[0];
+ const std::uint32_t PathIndex = Lookup.SequenceIndexFirstPathIndex[Source.SequenceIndex];
+ IoBuffer RawSource = IoBufferBuilder::MakeFromFile((m_Path / Content.Paths[PathIndex]).make_preferred(), Source.Offset, ChunkSize);
+ if (!RawSource)
+ {
+ throw std::runtime_error(fmt::format("Failed fetching chunk {}", ChunkHash));
+ }
+ if (RawSource.GetSize() != ChunkSize)
+ {
+ throw std::runtime_error(fmt::format("Fetched chunk {} has invalid size", ChunkHash));
+ }
+
+ const bool ShouldCompressChunk = IsChunkCompressable(m_NonCompressableExtensionHashes, Lookup, ChunkIndex);
+ const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None;
+
+ if (ShouldCompressChunk)
+ {
+ std::filesystem::path TempFilePath = m_Options.TempDir / ChunkHash.ToHexString();
+
+ BasicFile CompressedFile;
+ std::error_code Ec;
+ CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec);
+ if (Ec)
+ {
+ throw std::runtime_error(fmt::format("Failed creating temporary file for compressing blob {}, reason: ({}) {}",
+ ChunkHash,
+ Ec.value(),
+ Ec.message()));
+ }
+
+ uint64_t StreamRawBytes = 0;
+ uint64_t StreamCompressedBytes = 0;
+
+ bool CouldCompress = CompressedBuffer::CompressToStream(
+ CompositeBuffer(SharedBuffer(RawSource)),
+ [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) {
+ ZEN_UNUSED(SourceOffset);
+ TempLooseChunksStats.CompressedChunkRawBytes += SourceSize;
+ CompressedFile.Write(RangeBuffer, Offset);
+ TempLooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize();
+ StreamRawBytes += SourceSize;
+ StreamCompressedBytes += RangeBuffer.GetSize();
+ },
+ OodleCompressor::Mermaid,
+ CompressionLevel);
+ if (CouldCompress)
+ {
+ uint64_t CompressedSize = CompressedFile.FileSize();
+ void* FileHandle = CompressedFile.Detach();
+ IoBuffer TempPayload = IoBuffer(IoBuffer::File,
+ FileHandle,
+ 0,
+ CompressedSize,
+ /*IsWholeFile*/ true);
+ ZEN_ASSERT(TempPayload);
+ TempPayload.SetDeleteOnClose(true);
+ IoHash RawHash;
+ uint64_t RawSize;
+ CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(TempPayload), RawHash, RawSize);
+ ZEN_ASSERT(Compressed);
+ ZEN_ASSERT(RawHash == ChunkHash);
+ ZEN_ASSERT(RawSize == ChunkSize);
+
+ TempLooseChunksStats.CompressedChunkCount++;
+
+ return Compressed.GetCompressed();
+ }
+ else
+ {
+ TempLooseChunksStats.CompressedChunkRawBytes -= StreamRawBytes;
+ TempLooseChunksStats.CompressedChunkBytes -= StreamCompressedBytes;
+ }
+ CompressedFile.Close();
+ RemoveFile(TempFilePath, Ec);
+ ZEN_UNUSED(Ec);
+ }
+
+ CompressedBuffer CompressedBlob =
+ CompressedBuffer::Compress(SharedBuffer(std::move(RawSource)), OodleCompressor::Mermaid, CompressionLevel);
+ if (!CompressedBlob)
+ {
+ throw std::runtime_error(fmt::format("Failed to compress large blob {}", ChunkHash));
+ }
+ ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawHash() == ChunkHash);
+ ZEN_ASSERT_SLOW(CompressedBlob.DecodeRawSize() == ChunkSize);
+
+ TempLooseChunksStats.CompressedChunkRawBytes += ChunkSize;
+ TempLooseChunksStats.CompressedChunkBytes += CompressedBlob.GetCompressedSize();
+
+ // If we use none-compression, the compressed blob references the data and has 64 kb in memory so we don't need to write it to disk
+ if (ShouldCompressChunk)
+ {
+ std::filesystem::path TempFilePath = m_Options.TempDir / (ChunkHash.ToHexString());
+ IoBuffer TempPayload = WriteToTempFile(std::move(CompressedBlob).GetCompressed(), TempFilePath);
+ CompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(TempPayload));
+ }
+
+ TempLooseChunksStats.CompressedChunkCount++;
+ return std::move(CompressedBlob).GetCompressed();
+}
+
+std::vector<std::pair<Oid, std::string>>
+UploadFolder(LoggerRef Log,
+ ProgressBase& Progress,
+ TransferThreadWorkers& Workers,
+ StorageInstance& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ std::string_view BuildPartName,
+ const std::filesystem::path& Path,
+ const std::filesystem::path& ManifestPath,
+ const CbObject& MetaData,
+ ChunkingController& ChunkController,
+ ChunkingCache& ChunkCache,
+ const UploadFolderOptions& Options)
+{
+ Progress.SetLogOperationName("Upload Folder");
+
+ Stopwatch UploadTimer;
+
+ BuildsOperationUploadFolder UploadOp(
+ Log,
+ Progress,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ Workers.GetIOWorkerPool(),
+ Workers.GetNetworkPool(),
+ BuildId,
+ Path,
+ Options.CreateBuild,
+ std::move(MetaData),
+ BuildsOperationUploadFolder::Options{.IsQuiet = Options.IsQuiet,
+ .IsVerbose = Options.IsVerbose,
+ .DoExtraContentValidation = Options.DoExtraContentVerify,
+ .FindBlockMaxCount = Options.FindBlockMaxCount,
+ .BlockReuseMinPercentLimit = Options.BlockReuseMinPercentLimit,
+ .AllowMultiparts = Options.AllowMultiparts,
+ .IgnoreExistingBlocks = Options.IgnoreExistingBlocks,
+ .TempDir = Options.TempDir,
+ .ExcludeFolders = Options.ExcludeFolders,
+ .ExcludeExtensions = Options.ExcludeExtensions,
+ .NonCompressableExtensions = DefaultSplitOnlyExtensions,
+ .PopulateCache = Options.UploadToZenCache});
+
+ std::vector<std::pair<Oid, std::string>> UploadedParts =
+ UploadOp.Execute(BuildPartId, BuildPartName, ManifestPath, ChunkController, ChunkCache);
+ if (AbortFlag)
+ {
+ return {};
+ }
+
+ if (Options.IsVerbose)
+ {
+ ZEN_CONSOLE(
+ "Folder scanning stats:"
+ "\n FoundFileCount: {}"
+ "\n FoundFileByteCount: {}"
+ "\n AcceptedFileCount: {}"
+ "\n AcceptedFileByteCount: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
+ UploadOp.m_LocalFolderScanStats.AcceptedFileCount.load(),
+ NiceBytes(UploadOp.m_LocalFolderScanStats.AcceptedFileByteCount.load()),
+ NiceLatencyNs(UploadOp.m_LocalFolderScanStats.ElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE(
+ "Chunking stats:"
+ "\n FilesProcessed: {}"
+ "\n FilesChunked: {}"
+ "\n BytesHashed: {}"
+ "\n UniqueChunksFound: {}"
+ "\n UniqueSequencesFound: {}"
+ "\n UniqueBytesFound: {}"
+ "\n FilesFoundInCache: {}"
+ "\n ChunksFoundInCache: {}"
+ "\n FilesStoredInCache: {}"
+ "\n ChunksStoredInCache: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_ChunkingStats.FilesProcessed.load(),
+ UploadOp.m_ChunkingStats.FilesChunked.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.BytesHashed.load()),
+ UploadOp.m_ChunkingStats.UniqueChunksFound.load(),
+ UploadOp.m_ChunkingStats.UniqueSequencesFound.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.UniqueBytesFound.load()),
+ UploadOp.m_ChunkingStats.FilesFoundInCache.load(),
+ UploadOp.m_ChunkingStats.ChunksFoundInCache.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.BytesFoundInCache.load()),
+ UploadOp.m_ChunkingStats.FilesStoredInCache.load(),
+ UploadOp.m_ChunkingStats.ChunksStoredInCache.load(),
+ NiceBytes(UploadOp.m_ChunkingStats.BytesStoredInCache.load()),
+ NiceLatencyNs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE(
+ "Find block stats:"
+ "\n FindBlockTimeMS: {}"
+ "\n PotentialChunkCount: {}"
+ "\n PotentialChunkByteCount: {}"
+ "\n FoundBlockCount: {}"
+ "\n FoundBlockChunkCount: {}"
+ "\n FoundBlockByteCount: {}"
+ "\n AcceptedBlockCount: {}"
+ "\n NewBlocksCount: {}"
+ "\n NewBlocksChunkCount: {}"
+ "\n NewBlocksChunkByteCount: {}",
+ NiceTimeSpanMs(UploadOp.m_FindBlocksStats.FindBlockTimeMS),
+ UploadOp.m_FindBlocksStats.PotentialChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.PotentialChunkByteCount),
+ UploadOp.m_FindBlocksStats.FoundBlockCount,
+ UploadOp.m_FindBlocksStats.FoundBlockChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.FoundBlockByteCount),
+ UploadOp.m_FindBlocksStats.AcceptedBlockCount,
+ UploadOp.m_FindBlocksStats.NewBlocksCount,
+ UploadOp.m_FindBlocksStats.NewBlocksChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount));
+
+ ZEN_CONSOLE(
+ "Reuse block stats:"
+ "\n AcceptedChunkCount: {}"
+ "\n AcceptedByteCount: {}"
+ "\n AcceptedRawByteCount: {}"
+ "\n RejectedBlockCount: {}"
+ "\n RejectedChunkCount: {}"
+ "\n RejectedByteCount: {}"
+ "\n AcceptedReduntantChunkCount: {}"
+ "\n AcceptedReduntantByteCount: {}",
+ UploadOp.m_ReuseBlocksStats.AcceptedChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedByteCount),
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedRawByteCount),
+ UploadOp.m_ReuseBlocksStats.RejectedBlockCount,
+ UploadOp.m_ReuseBlocksStats.RejectedChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.RejectedByteCount),
+ UploadOp.m_ReuseBlocksStats.AcceptedReduntantChunkCount,
+ NiceBytes(UploadOp.m_ReuseBlocksStats.AcceptedReduntantByteCount));
+
+ ZEN_CONSOLE(
+ "Generate blocks stats:"
+ "\n GeneratedBlockByteCount: {}"
+ "\n GeneratedBlockCount: {}"
+ "\n GenerateBlocksElapsedWallTimeUS: {}",
+ NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceLatencyNs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE(
+ "Loose chunks stats:"
+ "\n ChunkCount: {}"
+ "\n ChunkByteCount: {}"
+ "\n CompressedChunkCount: {}"
+ "\n CompressChunksElapsedWallTimeUS: {}",
+ UploadOp.m_LooseChunksStats.ChunkCount,
+ NiceBytes(UploadOp.m_LooseChunksStats.ChunkByteCount),
+ UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
+ NiceLatencyNs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS * 1000));
+
+ ZEN_CONSOLE(
+ "Disk stats:"
+ "\n OpenReadCount: {}"
+ "\n OpenWriteCount: {}"
+ "\n ReadCount: {}"
+ "\n ReadByteCount: {}"
+ "\n WriteCount: {} ({} cloned)"
+ "\n WriteByteCount: {} ({} cloned)"
+ "\n CurrentOpenFileCount: {}",
+ UploadOp.m_DiskStats.OpenReadCount.load(),
+ UploadOp.m_DiskStats.OpenWriteCount.load(),
+ UploadOp.m_DiskStats.ReadCount.load(),
+ NiceBytes(UploadOp.m_DiskStats.ReadByteCount.load()),
+ UploadOp.m_DiskStats.WriteCount.load(),
+ UploadOp.m_DiskStats.CloneCount.load(),
+ NiceBytes(UploadOp.m_DiskStats.WriteByteCount.load()),
+ NiceBytes(UploadOp.m_DiskStats.CloneByteCount.load()),
+ UploadOp.m_DiskStats.CurrentOpenFileCount.load());
+
+ ZEN_CONSOLE(
+ "Upload stats:"
+ "\n BlockCount: {}"
+ "\n BlocksBytes: {}"
+ "\n ChunkCount: {}"
+ "\n ChunksBytes: {}"
+ "\n ReadFromDiskBytes: {}"
+ "\n MultipartAttachmentCount: {}"
+ "\n ElapsedWallTimeUS: {}",
+ UploadOp.m_UploadStats.BlockCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
+ UploadOp.m_UploadStats.ChunkCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
+ NiceBytes(UploadOp.m_UploadStats.ReadFromDiskBytes.load()),
+ UploadOp.m_UploadStats.MultipartAttachmentCount.load(),
+ NiceLatencyNs(UploadOp.m_UploadStats.ElapsedWallTimeUS * 1000));
+ }
+
+ const double DeltaByteCountPercent =
+ UploadOp.m_ChunkingStats.BytesHashed > 0
+ ? (100.0 * (UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes)) /
+ (UploadOp.m_ChunkingStats.BytesHashed)
+ : 0.0;
+
+ const std::string MultipartAttachmentStats =
+ Options.AllowMultiparts ? fmt::format(" ({} as multipart)", UploadOp.m_UploadStats.MultipartAttachmentCount.load()) : "";
+
+ if (!Options.IsQuiet)
+ {
+ ZEN_CONSOLE(
+ "Uploaded part {} ('{}') to build {}, {}\n"
+ " Scanned files: {:>8} ({}), {}B/sec, {}\n"
+ " New data: {:>8} ({}) {:.1f}%\n"
+ " New blocks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " New chunks: {:>8} ({} -> {}), {}B/sec, {}\n"
+ " Uploaded: {:>8} ({}), {}bits/sec, {}\n"
+ " Blocks: {:>8} ({})\n"
+ " Chunks: {:>8} ({}){}",
+ BuildPartId,
+ BuildPartName,
+ BuildId,
+ NiceTimeSpanMs(UploadTimer.GetElapsedTimeMs()),
+
+ UploadOp.m_LocalFolderScanStats.FoundFileCount.load(),
+ NiceBytes(UploadOp.m_LocalFolderScanStats.FoundFileByteCount.load()),
+ NiceNum(GetBytesPerSecond(UploadOp.m_ChunkingStats.ElapsedWallTimeUS, UploadOp.m_ChunkingStats.BytesHashed)),
+ NiceTimeSpanMs(UploadOp.m_ChunkingStats.ElapsedWallTimeUS / 1000),
+
+ UploadOp.m_FindBlocksStats.NewBlocksChunkCount + UploadOp.m_LooseChunksStats.CompressedChunkCount,
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount + UploadOp.m_LooseChunksStats.CompressedChunkBytes),
+ DeltaByteCountPercent,
+
+ UploadOp.m_GenerateBlocksStats.GeneratedBlockCount.load(),
+ NiceBytes(UploadOp.m_FindBlocksStats.NewBlocksChunkByteCount),
+ NiceBytes(UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount.load()),
+ NiceNum(GetBytesPerSecond(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS,
+ UploadOp.m_GenerateBlocksStats.GeneratedBlockByteCount)),
+ NiceTimeSpanMs(UploadOp.m_GenerateBlocksStats.GenerateBlocksElapsedWallTimeUS / 1000),
+
+ UploadOp.m_LooseChunksStats.CompressedChunkCount.load(),
+ NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkRawBytes),
+ NiceBytes(UploadOp.m_LooseChunksStats.CompressedChunkBytes.load()),
+ NiceNum(GetBytesPerSecond(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS,
+ UploadOp.m_LooseChunksStats.CompressedChunkRawBytes)),
+ NiceTimeSpanMs(UploadOp.m_LooseChunksStats.CompressChunksElapsedWallTimeUS / 1000),
+
+ UploadOp.m_UploadStats.BlockCount.load() + UploadOp.m_UploadStats.ChunkCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.BlocksBytes + UploadOp.m_UploadStats.ChunksBytes),
+ NiceNum(GetBytesPerSecond(UploadOp.m_UploadStats.ElapsedWallTimeUS,
+ (UploadOp.m_UploadStats.ChunksBytes + UploadOp.m_UploadStats.BlocksBytes) * 8)),
+ NiceTimeSpanMs(UploadOp.m_UploadStats.ElapsedWallTimeUS / 1000),
+
+ UploadOp.m_UploadStats.BlockCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.BlocksBytes.load()),
+
+ UploadOp.m_UploadStats.ChunkCount.load(),
+ NiceBytes(UploadOp.m_UploadStats.ChunksBytes.load()),
+ MultipartAttachmentStats);
+ }
+ return UploadedParts;
+}
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/buildvalidatebuildpart.cpp b/src/zenremotestore/builds/buildvalidatebuildpart.cpp
new file mode 100644
index 000000000..d06502683
--- /dev/null
+++ b/src/zenremotestore/builds/buildvalidatebuildpart.cpp
@@ -0,0 +1,374 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include <zenremotestore/builds/buildvalidatebuildpart.h>
+
+#include <zencore/fmtutils.h>
+#include <zencore/parallelwork.h>
+#include <zencore/scopeguard.h>
+#include <zencore/trace.h>
+#include <zenremotestore/builds/builduploadfolder.h>
+#include <zenremotestore/transferthreadworkers.h>
+#include <zenutil/filesystemutils.h>
+#include <zenutil/filteredrate.h>
+#include <zenutil/progress.h>
+
+namespace zen {
+
+using namespace std::literals;
+
+BuildsOperationValidateBuildPart::BuildsOperationValidateBuildPart(LoggerRef Log,
+ ProgressBase& Progress,
+ BuildStorageBase& Storage,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ WorkerThreadPool& IOWorkerPool,
+ WorkerThreadPool& NetworkPool,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ const std::string_view BuildPartName,
+ const Options& Options)
+
+: m_Log(Log)
+, m_Progress(Progress)
+, m_Storage(Storage)
+, m_AbortFlag(AbortFlag)
+, m_PauseFlag(PauseFlag)
+, m_IOWorkerPool(IOWorkerPool)
+, m_NetworkPool(NetworkPool)
+, m_BuildId(BuildId)
+, m_BuildPartId(BuildPartId)
+, m_BuildPartName(BuildPartName)
+, m_Options(Options)
+{
+}
+
+void
+BuildsOperationValidateBuildPart::Execute()
+{
+ ZEN_TRACE_CPU("ValidateBuildPart");
+ try
+ {
+ auto EndProgress =
+ MakeGuard([&]() { m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::StepCount, (uint32_t)TaskSteps::StepCount); });
+
+ Stopwatch Timer;
+ auto _ = MakeGuard([&]() {
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Validated build part {}/{} ('{}') in {}",
+ m_BuildId,
+ m_BuildPartId,
+ m_BuildPartName,
+ NiceTimeSpanMs(Timer.GetElapsedTimeMs()));
+ }
+ });
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuild, (uint32_t)TaskSteps::StepCount);
+
+ ResolvedBuildPart Resolved = ResolveBuildPart();
+
+ ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog);
+
+ const std::filesystem::path& TempFolder = m_Options.TempFolder;
+ ZEN_ASSERT(!TempFolder.empty());
+
+ CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder);
+ CreateDirectories(TempFolder);
+ auto __ = MakeGuard([this, TempFolder]() { CleanAndRemoveDirectory(m_IOWorkerPool, m_AbortFlag, m_PauseFlag, TempFolder); });
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::ValidateBlobs, (uint32_t)TaskSteps::StepCount);
+
+ std::unique_ptr<ProgressBase::ProgressBar> ProgressBar = m_Progress.CreateProgressBar("Validate Blobs");
+
+ const uint64_t AttachmentsToVerifyCount = Resolved.ChunkAttachments.size() + Resolved.BlockAttachments.size();
+ FilteredRate FilteredDownloadedBytesPerSecond;
+ FilteredRate FilteredVerifiedBytesPerSecond;
+
+ ValidateBlobsContext Context{.Work = Work,
+ .AttachmentsToVerifyCount = AttachmentsToVerifyCount,
+ .FilteredDownloadedBytesPerSecond = FilteredDownloadedBytesPerSecond,
+ .FilteredVerifiedBytesPerSecond = FilteredVerifiedBytesPerSecond};
+
+ ScheduleChunkAttachmentValidation(Context, Resolved.ChunkAttachments, TempFolder, Resolved.PreferredMultipartChunkSize);
+ ScheduleBlockAttachmentValidation(Context, Resolved.BlockAttachments);
+
+ Work.Wait(m_Progress.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) {
+ ZEN_UNUSED(PendingWork);
+
+ const uint64_t DownloadedAttachmentCount = m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount;
+ const uint64_t DownloadedByteCount = m_DownloadStats.DownloadedChunkByteCount + m_DownloadStats.DownloadedBlockByteCount;
+
+ FilteredDownloadedBytesPerSecond.Update(DownloadedByteCount);
+ FilteredVerifiedBytesPerSecond.Update(m_ValidateStats.VerifiedByteCount);
+
+ std::string Details = fmt::format("Downloaded {}/{} ({}, {}bits/s). Verified {}/{} ({}, {}B/s)",
+ DownloadedAttachmentCount,
+ AttachmentsToVerifyCount,
+ NiceBytes(DownloadedByteCount),
+ NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8),
+ m_ValidateStats.VerifiedAttachmentCount.load(),
+ AttachmentsToVerifyCount,
+ NiceBytes(m_ValidateStats.VerifiedByteCount.load()),
+ NiceNum(FilteredVerifiedBytesPerSecond.GetCurrent()));
+
+ ProgressBar->UpdateState(
+ {.Task = "Validating blobs ",
+ .Details = Details,
+ .TotalCount = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2),
+ .RemainingCount = gsl::narrow<uint64_t>(AttachmentsToVerifyCount * 2 -
+ (DownloadedAttachmentCount + m_ValidateStats.VerifiedAttachmentCount.load())),
+ .Status = ProgressBase::ProgressBar::State::CalculateStatus(IsAborted, IsPaused)},
+ false);
+ });
+
+ ProgressBar->Finish();
+ m_ValidateStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs();
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::Cleanup, (uint32_t)TaskSteps::StepCount);
+ }
+ catch (const std::exception&)
+ {
+ m_AbortFlag = true;
+ throw;
+ }
+}
+
+BuildsOperationValidateBuildPart::ResolvedBuildPart
+BuildsOperationValidateBuildPart::ResolveBuildPart()
+{
+ ResolvedBuildPart Result;
+ Result.PreferredMultipartChunkSize = 32u * 1024u * 1024u;
+
+ CbObject Build = m_Storage.GetBuild(m_BuildId);
+ if (!m_BuildPartName.empty())
+ {
+ m_BuildPartId = Build["parts"sv].AsObjectView()[m_BuildPartName].AsObjectId();
+ if (m_BuildPartId == Oid::Zero)
+ {
+ throw std::runtime_error(fmt::format("Build {} does not have a part named '{}'", m_BuildId, m_BuildPartName));
+ }
+ }
+ m_ValidateStats.BuildBlobSize = Build.GetSize();
+ if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0)
+ {
+ Result.PreferredMultipartChunkSize = ChunkSize;
+ }
+
+ m_Progress.SetLogOperationProgress((uint32_t)TaskSteps::FetchBuildPart, (uint32_t)TaskSteps::StepCount);
+
+ CbObject BuildPart = m_Storage.GetBuildPart(m_BuildId, m_BuildPartId);
+ m_ValidateStats.BuildPartSize = BuildPart.GetSize();
+ if (!m_Options.IsQuiet)
+ {
+ ZEN_INFO("Validating build part {}/{} ({})", m_BuildId, m_BuildPartId, NiceBytes(BuildPart.GetSize()));
+ }
+ if (const CbObjectView ChunkAttachmentsView = BuildPart["chunkAttachments"sv].AsObjectView())
+ {
+ for (CbFieldView LooseFileView : ChunkAttachmentsView["rawHashes"sv])
+ {
+ Result.ChunkAttachments.push_back(LooseFileView.AsBinaryAttachment());
+ }
+ }
+ m_ValidateStats.ChunkAttachmentCount = Result.ChunkAttachments.size();
+ if (const CbObjectView BlockAttachmentsView = BuildPart["blockAttachments"sv].AsObjectView())
+ {
+ for (CbFieldView BlocksView : BlockAttachmentsView["rawHashes"sv])
+ {
+ Result.BlockAttachments.push_back(BlocksView.AsBinaryAttachment());
+ }
+ }
+ m_ValidateStats.BlockAttachmentCount = Result.BlockAttachments.size();
+
+ std::vector<ChunkBlockDescription> VerifyBlockDescriptions =
+ ParseChunkBlockDescriptionList(m_Storage.GetBlockMetadatas(m_BuildId, Result.BlockAttachments));
+ if (VerifyBlockDescriptions.size() != Result.BlockAttachments.size())
+ {
+ throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing",
+ Result.BlockAttachments.size() - VerifyBlockDescriptions.size()));
+ }
+
+ return Result;
+}
+
+void
+BuildsOperationValidateBuildPart::ScheduleChunkAttachmentValidation(ValidateBlobsContext& Context,
+ std::span<const IoHash> ChunkAttachments,
+ const std::filesystem::path& TempFolder,
+ uint64_t PreferredMultipartChunkSize)
+{
+ for (const IoHash& ChunkAttachment : ChunkAttachments)
+ {
+ Context.Work.ScheduleWork(
+ m_NetworkPool,
+ [this, &Context, &TempFolder, PreferredMultipartChunkSize, ChunkAttachment = IoHash(ChunkAttachment)](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("ValidateBuildPart_GetChunk");
+
+ Context.FilteredDownloadedBytesPerSecond.Start();
+ DownloadLargeBlob(
+ m_Storage,
+ TempFolder,
+ m_BuildId,
+ ChunkAttachment,
+ PreferredMultipartChunkSize,
+ Context.Work,
+ m_NetworkPool,
+ m_DownloadStats.DownloadedChunkByteCount,
+ m_DownloadStats.MultipartAttachmentCount,
+ [this, &Context, ChunkHash = IoHash(ChunkAttachment)](IoBuffer&& Payload) {
+ m_DownloadStats.DownloadedChunkCount++;
+ Payload.SetContentType(ZenContentType::kCompressedBinary);
+ if (!m_AbortFlag)
+ {
+ Context.Work.ScheduleWork(
+ m_IOWorkerPool,
+ [this, &Context, Payload = IoBuffer(std::move(Payload)), ChunkHash](std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ ValidateDownloadedChunk(Context, ChunkHash, std::move(Payload));
+ }
+ });
+ }
+ });
+ }
+ });
+ }
+}
+
+void
+BuildsOperationValidateBuildPart::ScheduleBlockAttachmentValidation(ValidateBlobsContext& Context, std::span<const IoHash> BlockAttachments)
+{
+ for (const IoHash& BlockAttachment : BlockAttachments)
+ {
+ Context.Work.ScheduleWork(m_NetworkPool, [this, &Context, BlockAttachment = IoHash(BlockAttachment)](std::atomic<bool>&) {
+ if (!m_AbortFlag)
+ {
+ ZEN_TRACE_CPU("ValidateBuildPart_GetBlock");
+
+ Context.FilteredDownloadedBytesPerSecond.Start();
+ IoBuffer Payload = m_Storage.GetBuildBlob(m_BuildId, BlockAttachment);
+ m_DownloadStats.DownloadedBlockCount++;
+ m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize();
+ if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount)
+ {
+ Context.FilteredDownloadedBytesPerSecond.Stop();
+ }
+ if (!Payload)
+ {
+ throw std::runtime_error(fmt::format("Block attachment {} could not be found", BlockAttachment));
+ }
+ if (!m_AbortFlag)
+ {
+ Context.Work.ScheduleWork(m_IOWorkerPool,
+ [this, &Context, Payload = std::move(Payload), BlockAttachment](std::atomic<bool>&) mutable {
+ if (!m_AbortFlag)
+ {
+ ValidateDownloadedBlock(Context, BlockAttachment, std::move(Payload));
+ }
+ });
+ }
+ }
+ });
+ }
+}
+
+void
+BuildsOperationValidateBuildPart::ValidateDownloadedChunk(ValidateBlobsContext& Context, const IoHash& ChunkHash, IoBuffer Payload)
+{
+ ZEN_TRACE_CPU("ValidateBuildPart_Validate");
+
+ if (m_DownloadStats.DownloadedChunkCount + m_DownloadStats.DownloadedBlockCount == Context.AttachmentsToVerifyCount)
+ {
+ Context.FilteredDownloadedBytesPerSecond.Stop();
+ }
+
+ Context.FilteredVerifiedBytesPerSecond.Start();
+
+ uint64_t CompressedSize;
+ uint64_t DecompressedSize;
+ ValidateBlob(m_AbortFlag, std::move(Payload), ChunkHash, CompressedSize, DecompressedSize);
+ m_ValidateStats.VerifiedAttachmentCount++;
+ m_ValidateStats.VerifiedByteCount += DecompressedSize;
+ if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount)
+ {
+ Context.FilteredVerifiedBytesPerSecond.Stop();
+ }
+}
+
+void
+BuildsOperationValidateBuildPart::ValidateDownloadedBlock(ValidateBlobsContext& Context, const IoHash& BlockAttachment, IoBuffer Payload)
+{
+ ZEN_TRACE_CPU("ValidateBuildPart_ValidateBlock");
+
+ Context.FilteredVerifiedBytesPerSecond.Start();
+
+ uint64_t CompressedSize;
+ uint64_t DecompressedSize;
+ ValidateChunkBlock(std::move(Payload), BlockAttachment, CompressedSize, DecompressedSize);
+ m_ValidateStats.VerifiedAttachmentCount++;
+ m_ValidateStats.VerifiedByteCount += DecompressedSize;
+ if (m_ValidateStats.VerifiedAttachmentCount.load() == Context.AttachmentsToVerifyCount)
+ {
+ Context.FilteredVerifiedBytesPerSecond.Stop();
+ }
+}
+
+ChunkBlockDescription
+BuildsOperationValidateBuildPart::ValidateChunkBlock(IoBuffer&& Payload,
+ const IoHash& BlobHash,
+ uint64_t& OutCompressedSize,
+ uint64_t& OutDecompressedSize)
+{
+ CompositeBuffer BlockBuffer = ValidateBlob(m_AbortFlag, std::move(Payload), BlobHash, OutCompressedSize, OutDecompressedSize);
+ if (!BlockBuffer)
+ {
+ throw std::runtime_error(fmt::format("Chunk block blob {} is not compressed using 'None' compression level", BlobHash));
+ }
+ return GetChunkBlockDescription(BlockBuffer.Flatten(), BlobHash);
+}
+
+void
+ValidateBuildPart(LoggerRef Log,
+ ProgressBase& Progress,
+ std::atomic<bool>& AbortFlag,
+ std::atomic<bool>& PauseFlag,
+ bool IsQuiet,
+ bool IsVerbose,
+ TransferThreadWorkers& Workers,
+ BuildStorageBase& Storage,
+ const std::filesystem::path& TempFolder,
+ const Oid& BuildId,
+ const Oid& BuildPartId,
+ std::string_view BuildPartName)
+{
+ ZEN_TRACE_CPU("ValidateBuildPart");
+
+ Progress.SetLogOperationName("Validate Part");
+
+ BuildsOperationValidateBuildPart ValidateOp(
+ Log,
+ Progress,
+ Storage,
+ AbortFlag,
+ PauseFlag,
+ Workers.GetIOWorkerPool(),
+ Workers.GetNetworkPool(),
+ BuildId,
+ BuildPartId,
+ BuildPartName,
+ BuildsOperationValidateBuildPart::Options{.TempFolder = TempFolder, .IsQuiet = IsQuiet, .IsVerbose = IsVerbose});
+
+ ValidateOp.Execute();
+
+ const uint64_t DownloadedCount = ValidateOp.m_DownloadStats.DownloadedChunkCount + ValidateOp.m_DownloadStats.DownloadedBlockCount;
+ const uint64_t DownloadedByteCount =
+ ValidateOp.m_DownloadStats.DownloadedChunkByteCount + ValidateOp.m_DownloadStats.DownloadedBlockByteCount;
+ ZEN_CONSOLE("Verified: {:>8} ({}), {}B/sec, {}",
+ DownloadedCount,
+ NiceBytes(DownloadedByteCount),
+ NiceNum(GetBytesPerSecond(ValidateOp.m_ValidateStats.ElapsedWallTimeUS, DownloadedByteCount)),
+ NiceTimeSpanMs(ValidateOp.m_ValidateStats.ElapsedWallTimeUS / 1000));
+}
+
+} // namespace zen
diff --git a/src/zenremotestore/builds/filebuildstorage.cpp b/src/zenremotestore/builds/filebuildstorage.cpp
index 55e69de61..2f4904449 100644
--- a/src/zenremotestore/builds/filebuildstorage.cpp
+++ b/src/zenremotestore/builds/filebuildstorage.cpp
@@ -432,6 +432,45 @@ public:
return IoBuffer{};
}
+ virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId,
+ const IoHash& RawHash,
+ std::span<const std::pair<uint64_t, uint64_t>> Ranges) override
+ {
+ ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlobRanges");
+ ZEN_UNUSED(BuildId);
+ ZEN_ASSERT(!Ranges.empty());
+
+ uint64_t ReceivedBytes = 0;
+ uint64_t SentBytes = Ranges.size() * 2 * 8;
+
+ SimulateLatency(SentBytes, 0);
+ auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); });
+
+ Stopwatch ExecutionTimer;
+ auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); });
+
+ BuildBlobRanges Result;
+
+ const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash);
+ if (IsFile(BlockPath))
+ {
+ BasicFile File(BlockPath, BasicFile::Mode::kRead);
+
+ uint64_t RangeOffset = Ranges.front().first;
+ uint64_t RangeBytes = Ranges.back().first + Ranges.back().second - RangeOffset;
+ Result.PayloadBuffer = IoBufferBuilder::MakeFromFileHandle(File.Detach(), RangeOffset, RangeBytes);
+
+ Result.Ranges.reserve(Ranges.size());
+
+ for (const std::pair<uint64_t, uint64_t>& Range : Ranges)
+ {
+ Result.Ranges.push_back(std::make_pair(Range.first - RangeOffset, Range.second));
+ }
+ ReceivedBytes = Result.PayloadBuffer.GetSize();
+ }
+ return Result;
+ }
+
virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid& BuildId,
const IoHash& RawHash,
uint64_t ChunkSize,
diff --git a/src/zenremotestore/builds/jupiterbuildstorage.cpp b/src/zenremotestore/builds/jupiterbuildstorage.cpp
index 23d0ddd4c..c0cc16941 100644
--- a/src/zenremotestore/builds/jupiterbuildstorage.cpp
+++ b/src/zenremotestore/builds/jupiterbuildstorage.cpp
@@ -4,6 +4,7 @@
#include <zencore/compactbinarybuilder.h>
#include <zencore/compactbinaryutil.h>
+#include <zencore/compress.h>
#include <zencore/fmtutils.h>
#include <zencore/scopeguard.h>
#include <zencore/timer.h>
@@ -14,18 +15,19 @@ ZEN_THIRD_PARTY_INCLUDES_START
#include <tsl/robin_map.h>
ZEN_THIRD_PARTY_INCLUDES_END
-#include <regex>
+#include <string_view>
namespace zen {
using namespace std::literals;
namespace {
- void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix)
+ [[noreturn]] void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix)
{
- int Error = Result.ErrorCode < (int)HttpResponseCode::Continue ? Result.ErrorCode : 0;
- HttpResponseCode Status =
- Result.ErrorCode >= int(HttpResponseCode::Continue) ? HttpResponseCode(Result.ErrorCode) : HttpResponseCode::ImATeapot;
+ HttpClientErrorCode Error = Result.ErrorCode < static_cast<int>(HttpResponseCode::Continue) ? HttpClientErrorCode(Result.ErrorCode)
+ : HttpClientErrorCode::kOK;
+ HttpResponseCode Status = Result.ErrorCode >= static_cast<int>(HttpResponseCode::Continue) ? HttpResponseCode(Result.ErrorCode)
+ : HttpResponseCode::ImATeapot;
throw HttpClientError(fmt::format("{}: {} ({})", Prefix, Result.Reason, Result.ErrorCode), Error, Status);
}
} // namespace
@@ -262,7 +264,7 @@ public:
std::vector<std::function<void()>> WorkList;
for (auto& WorkItem : WorkItems)
{
- WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes]() {
+ WorkList.emplace_back([this, WorkItem = std::move(WorkItem), OnSentBytes = std::move(OnSentBytes)]() {
Stopwatch ExecutionTimer;
auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
bool IsComplete = false;
@@ -284,7 +286,10 @@ public:
Stopwatch ExecutionTimer;
auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
- CreateDirectories(m_TempFolderPath);
+ if (!m_TempFolderPath.empty())
+ {
+ CreateDirectories(m_TempFolderPath);
+ }
JupiterResult GetBuildBlobResult =
m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, RangeOffset, RangeBytes);
AddStatistic(GetBuildBlobResult);
@@ -295,6 +300,29 @@ public:
return std::move(GetBuildBlobResult.Response);
}
+ virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId,
+ const IoHash& RawHash,
+ std::span<const std::pair<uint64_t, uint64_t>> Ranges) override
+ {
+ ZEN_TRACE_CPU("Jupiter::GetBuildBlob");
+
+ Stopwatch ExecutionTimer;
+ auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); });
+ if (!m_TempFolderPath.empty())
+ {
+ CreateDirectories(m_TempFolderPath);
+ }
+
+ BuildBlobRangesResult GetBuildBlobResult =
+ m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, Ranges);
+ AddStatistic(GetBuildBlobResult);
+ if (!GetBuildBlobResult.Success)
+ {
+ ThrowFromJupiterResult(GetBuildBlobResult, "Failed fetching build blob ranges"sv);
+ }
+ return BuildBlobRanges{.PayloadBuffer = std::move(GetBuildBlobResult.Response), .Ranges = std::move(GetBuildBlobResult.Ranges)};
+ }
+
virtual std::vector<std::function<void()>> GetLargeBuildBlob(const Oid& BuildId,
const IoHash& RawHash,
uint64_t ChunkSize,
@@ -423,11 +451,13 @@ public:
virtual bool GetExtendedStatistics(ExtendedStatistics& OutStats) override
{
- OutStats.ReceivedBytesPerSource.reserve(m_ReceivedBytesPerSource.size());
- for (auto& It : m_ReceivedBytesPerSource)
- {
- OutStats.ReceivedBytesPerSource.insert_or_assign(It.first, m_SourceBytes[It.second]);
- }
+ m_SourceLock.WithSharedLock([this, &OutStats]() {
+ OutStats.ReceivedBytesPerSource.reserve(m_ReceivedBytesPerSource.size());
+ for (auto& It : m_ReceivedBytesPerSource)
+ {
+ OutStats.ReceivedBytesPerSource.insert_or_assign(It.first, m_SourceBytes[It.second].load(std::memory_order_relaxed));
+ }
+ });
return true;
}
@@ -500,15 +530,29 @@ private:
}
if (!Result.Source.empty())
{
- if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source);
- It != m_ReceivedBytesPerSource.end())
- {
- m_SourceBytes[It->second] += Result.ReceivedBytes;
- }
- else
+ if (!m_SourceLock.WithSharedLock([&]() {
+ if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source);
+ It != m_ReceivedBytesPerSource.end())
+ {
+ m_SourceBytes[It->second] += Result.ReceivedBytes;
+ return true;
+ }
+ return false;
+ }))
{
- m_ReceivedBytesPerSource.insert_or_assign(Result.Source, m_SourceBytes.size());
- m_SourceBytes.push_back(Result.ReceivedBytes);
+ m_SourceLock.WithExclusiveLock([&]() {
+ if (tsl::robin_map<std::string, uint32_t>::const_iterator It = m_ReceivedBytesPerSource.find(Result.Source);
+ It != m_ReceivedBytesPerSource.end())
+ {
+ m_SourceBytes[It->second] += Result.ReceivedBytes;
+ }
+ else if (m_SourceCount < MaxSourceCount)
+ {
+ size_t Index = m_SourceCount++;
+ m_ReceivedBytesPerSource.insert_or_assign(Result.Source, Index);
+ m_SourceBytes[Index] += Result.ReceivedBytes;
+ }
+ });
}
}
}
@@ -519,8 +563,11 @@ private:
const std::string m_Bucket;
const std::filesystem::path m_TempFolderPath;
- tsl::robin_map<std::string, uint32_t> m_ReceivedBytesPerSource;
- std::vector<uint64_t> m_SourceBytes;
+ RwLock m_SourceLock;
+ tsl::robin_map<std::string, uint32_t> m_ReceivedBytesPerSource;
+ static constexpr size_t MaxSourceCount = 8u;
+ std::array<std::atomic<uint64_t>, MaxSourceCount> m_SourceBytes;
+ size_t m_SourceCount = 0;
};
std::unique_ptr<BuildStorageBase>
@@ -551,35 +598,135 @@ ParseBuildStorageUrl(std::string_view InUrl,
Url.erase(ApiString, ExtendedApiString.length());
}
- const std::string ArtifactURLRegExString = R"((http[s]?:\/\/.*?)\/(.*?)\/(.*?)\/(.*))";
- const std::regex ArtifactURLRegEx(ArtifactURLRegExString, std::regex::ECMAScript | std::regex::icase);
- std::match_results<std::string_view::const_iterator> MatchResults;
- std::string_view UrlToParse(Url);
- if (regex_match(begin(UrlToParse), end(UrlToParse), MatchResults, ArtifactURLRegEx) && MatchResults.size() == 5)
- {
- auto GetMatch = [&MatchResults](uint32_t Index) -> std::string_view {
- ZEN_ASSERT(Index < MatchResults.size());
+ // Parse URL of the form: http[s]://host/namespace/bucket/buildid
+ std::string_view Remaining(Url);
- const auto& Match = MatchResults[Index];
+ // Find the end of the scheme (e.g. "http://" or "https://")
+ size_t SchemeEnd = Remaining.find("://");
+ if (SchemeEnd == std::string_view::npos)
+ {
+ return false;
+ }
+ SchemeEnd += 3; // skip past "://"
- return std::string_view(&*Match.first, Match.second - Match.first);
- };
+ // Find the first '/' after the host
+ size_t HostEnd = Remaining.find('/', SchemeEnd);
+ if (HostEnd == std::string_view::npos)
+ {
+ return false;
+ }
- const std::string_view Host = GetMatch(1);
- const std::string_view Namespace = GetMatch(2);
- const std::string_view Bucket = GetMatch(3);
- const std::string_view BuildId = GetMatch(4);
+ // Find the '/' after namespace
+ size_t NamespaceEnd = Remaining.find('/', HostEnd + 1);
+ if (NamespaceEnd == std::string_view::npos)
+ {
+ return false;
+ }
- OutHost = Host;
- OutNamespace = Namespace;
- OutBucket = Bucket;
- OutBuildId = BuildId;
- return true;
+ // Find the '/' after bucket
+ size_t BucketEnd = Remaining.find('/', NamespaceEnd + 1);
+ if (BucketEnd == std::string_view::npos)
+ {
+ return false;
}
- else
+
+ // BuildId must be non-empty
+ if (BucketEnd + 1 >= Remaining.size())
{
return false;
}
+
+ OutHost = Remaining.substr(0, HostEnd);
+ OutNamespace = Remaining.substr(HostEnd + 1, NamespaceEnd - HostEnd - 1);
+ OutBucket = Remaining.substr(NamespaceEnd + 1, BucketEnd - NamespaceEnd - 1);
+ OutBuildId = Remaining.substr(BucketEnd + 1);
+ return true;
}
} // namespace zen
+
+#if ZEN_WITH_TESTS
+
+# include <zencore/testing.h>
+
+namespace zen {
+
+void
+jupiterbuildstorage_forcelink()
+{
+}
+
+} // namespace zen
+
+TEST_SUITE_BEGIN("remotestore.jupiterbuildstorage");
+
+TEST_CASE("ParseBuildStorageUrl.ValidUrl")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ bool Result =
+ zen::ParseBuildStorageUrl("https://horde.devtools.epicgames.com/mynamespace/mybucket/mybuildid", Host, Namespace, Bucket, BuildId);
+ CHECK(Result);
+ CHECK(Host == "https://horde.devtools.epicgames.com");
+ CHECK(Namespace == "mynamespace");
+ CHECK(Bucket == "mybucket");
+ CHECK(BuildId == "mybuildid");
+}
+
+TEST_CASE("ParseBuildStorageUrl.ValidUrlWithApiPrefix")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ bool Result = zen::ParseBuildStorageUrl("https://horde.devtools.epicgames.com/api/v2/builds/mynamespace/mybucket/mybuildid",
+ Host,
+ Namespace,
+ Bucket,
+ BuildId);
+ CHECK(Result);
+ CHECK(Host == "https://horde.devtools.epicgames.com");
+ CHECK(Namespace == "mynamespace");
+ CHECK(Bucket == "mybucket");
+ CHECK(BuildId == "mybuildid");
+}
+
+TEST_CASE("ParseBuildStorageUrl.HttpScheme")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ bool Result = zen::ParseBuildStorageUrl("http://localhost/ns/bucket/build123", Host, Namespace, Bucket, BuildId);
+ CHECK(Result);
+ CHECK(Host == "http://localhost");
+ CHECK(Namespace == "ns");
+ CHECK(Bucket == "bucket");
+ CHECK(BuildId == "build123");
+}
+
+TEST_CASE("ParseBuildStorageUrl.BuildIdWithSlashes")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ bool Result = zen::ParseBuildStorageUrl("https://host/ns/bucket/build/with/slashes", Host, Namespace, Bucket, BuildId);
+ CHECK(Result);
+ CHECK(Host == "https://host");
+ CHECK(Namespace == "ns");
+ CHECK(Bucket == "bucket");
+ CHECK(BuildId == "build/with/slashes");
+}
+
+TEST_CASE("ParseBuildStorageUrl.MissingBuildId")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ CHECK_FALSE(zen::ParseBuildStorageUrl("https://host/ns/bucket/", Host, Namespace, Bucket, BuildId));
+}
+
+TEST_CASE("ParseBuildStorageUrl.MissingBucket")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ CHECK_FALSE(zen::ParseBuildStorageUrl("https://host/ns", Host, Namespace, Bucket, BuildId));
+}
+
+TEST_CASE("ParseBuildStorageUrl.NoScheme")
+{
+ std::string Host, Namespace, Bucket, BuildId;
+ CHECK_FALSE(zen::ParseBuildStorageUrl("host/ns/bucket/buildid", Host, Namespace, Bucket, BuildId));
+}
+
+TEST_SUITE_END();
+
+#endif // ZEN_WITH_TESTS
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-28 02:17:22 +0000