From 8878156f30e375b93ebe99f02d94c581bcbbf43c Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:53:59 -0600 Subject: Add CookPackageArtifacts attachment to web ui --- src/zenserver/frontend/html/pages/entry.js | 69 ++++++++++++++++++++++++++++- src/zenserver/projectstore/projectstore.cpp | 43 ++++++++++++++++-- 2 files changed, 106 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 65a3ef39b..3891239f0 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -59,6 +59,63 @@ export class Page extends ZenPage } } + _find_iohash_field(container, name) + { + const found_field = container.find(name); + if (found_field != undefined) + { + var found_value = found_field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + return ret; + } + } + return null; + } + + async _build_meta(section, entry) + { + var tree = {} + const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); + if (cookart != null) + { + tree["cook"] = { CookPackageArtifacts: cookart}; + } + + if (Object.keys(tree).length == 0) + return; + + const sub_section = section.add_section("meta"); + + for (const cat_name in tree) + { + const cat_section = sub_section.add_section(cat_name); + const table = cat_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + Object.entries(tree[cat_name]).forEach(([key, value]) => + { + const row = table.add_row(key); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + ); + + const do_nothing = () => void(0); + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); + }); + } + } + async _build_page() { var entry = await this._entry; @@ -78,8 +135,16 @@ export class Page extends ZenPage delete tree["$id"]; - const sub_section = section.add_section("deps"); - this._build_deps(sub_section, tree); + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("deps"); + this._build_deps(sub_section, tree); + } + } + + // meta + { + this._build_meta(section, entry); } // data diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 86791e29a..1132e458e 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4812,11 +4812,46 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || + AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(ZenContentType::kBinary); + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; + } + + if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) + { + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + ExtendableStringBuilder<1024> Sb; + if (AcceptType == HttpContentType::kJSON) + { + ContainerObject.ToJson(Sb); + } + else if (AcceptType == HttpContentType::kYAML) + { + ContainerObject.ToYaml(Sb); + } + IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); + OutChunk = SerializedBuffer; + } + else + { + OutChunk = DecompressedBuffer; + } + OutChunk.SetContentType(AcceptType); + } + else + { + OutChunk = DecompressedBuffer; + OutChunk.SetContentType(ZenContentType::kBinary); + } } else { -- cgit v1.2.3 From cc4d4751201bbf83531ea9919f242bfac3add8d3 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:14:47 -0600 Subject: Moved AcceptType Handling From ProjectStore To HttpProjectStore --- src/zenserver/projectstore/httpprojectstore.cpp | 34 +++++++++++++- src/zenserver/projectstore/projectstore.cpp | 61 ++----------------------- src/zenserver/projectstore/projectstore.h | 1 - 3 files changed, 35 insertions(+), 61 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 47748dd90..f9a13220a 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -885,10 +886,39 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) case HttpVerb::kGet: { IoBuffer Value; - std::pair Result = - m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr); + std::pair Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr); if (Result.first == HttpResponseCode::OK) { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("chunk - '{}' WRONGTYPE", Cid)); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } m_ProjectStats.ChunkHitCount++; return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); } diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 1132e458e..ea3f2aad9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -4770,7 +4770,6 @@ std::pair ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag) { @@ -4812,51 +4811,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kJSON || - AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || AcceptType == ZenContentType::kCbObject) - { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) - { - return {HttpResponseCode::NotAcceptable, fmt::format("chunk - '{}' WRONGTYPE", Cid)}; - } - - if (AcceptType == HttpContentType::kJSON || AcceptType == HttpContentType::kYAML) - { - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - ExtendableStringBuilder<1024> Sb; - if (AcceptType == HttpContentType::kJSON) - { - ContainerObject.ToJson(Sb); - } - else if (AcceptType == HttpContentType::kYAML) - { - ContainerObject.ToYaml(Sb); - } - IoBuffer SerializedBuffer(IoBuffer::Clone, Sb.Data(), Sb.Size()); - OutChunk = SerializedBuffer; - } - else - { - OutChunk = DecompressedBuffer; - } - OutChunk.SetContentType(AcceptType); - } - else - { - OutChunk = DecompressedBuffer; - OutChunk.SetContentType(ZenContentType::kBinary); - } - } - else - { - OutChunk.SetContentType(ZenContentType::kCompressedBinary); - } + OutChunk.SetContentType(ZenContentType::kCompressedBinary); return {HttpResponseCode::OK, {}}; } @@ -8551,12 +8506,7 @@ TEST_CASE("project.store.partial.read") uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; @@ -8565,12 +8515,7 @@ TEST_CASE("project.store.partial.read") CHECK(ModificationTag != 0); CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::NotModified); } diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8f2d3ce0d..368da5ea4 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -449,7 +449,6 @@ public: std::pair GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag); -- cgit v1.2.3 From 44c61d6d310a605074df0baa2af27410c589daae Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:18:13 -0600 Subject: Removed do_nothing from entry.js --- src/zenserver/frontend/html/pages/entry.js | 2 -- 1 file changed, 2 deletions(-) (limited to 'src') diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 3891239f0..f127cb0a3 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -107,7 +107,6 @@ export class Page extends ZenPage "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); @@ -193,7 +192,6 @@ export class Page extends ZenPage ); link.first_child().attr("download", `${io_hash}_${base_name}`); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); -- cgit v1.2.3 From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Mar 2025 17:06:23 +0100 Subject: zen build cache service (#318) - **EXPERIMENTAL** `zen builds` - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata - Feature: New `/builds` endpoint for caching build blobs and blob metadata - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs --- src/zen/cmds/builds_cmd.cpp | 1883 +++++++++++--------- src/zen/cmds/builds_cmd.h | 3 + src/zenhttp/httpclient.cpp | 26 +- src/zenhttp/include/zenhttp/formatters.h | 46 +- src/zenhttp/packageformat.cpp | 45 +- src/zenserver/admin/admin.cpp | 16 + src/zenserver/admin/admin.h | 3 + src/zenserver/buildstore/httpbuildstore.cpp | 526 ++++++ src/zenserver/buildstore/httpbuildstore.h | 65 + src/zenserver/config.cpp | 10 + src/zenserver/config.h | 12 + src/zenserver/workspaces/httpworkspaces.cpp | 2 +- src/zenserver/zenserver.cpp | 19 + src/zenserver/zenserver.h | 4 + src/zenstore-test/zenstore-test.cpp | 2 + src/zenstore/blockstore.cpp | 2 +- src/zenstore/buildstore/buildstore.cpp | 1475 +++++++++++++++ src/zenstore/compactcas.cpp | 4 +- src/zenstore/compactcas.h | 4 +- src/zenstore/gc.cpp | 21 +- src/zenstore/include/zenstore/accesstime.h | 47 + src/zenstore/include/zenstore/blockstore.h | 2 +- .../include/zenstore/buildstore/buildstore.h | 186 ++ .../include/zenstore/cache/cachedisklayer.h | 1 + src/zenstore/include/zenstore/cache/cacheshared.h | 38 - src/zenstore/include/zenstore/gc.h | 4 + src/zenutil/buildstoragecache.cpp | 362 ++++ src/zenutil/chunkblock.cpp | 2 +- src/zenutil/filebuildstorage.cpp | 27 +- src/zenutil/include/zenutil/buildstorage.h | 6 +- src/zenutil/include/zenutil/buildstoragecache.h | 52 + .../include/zenutil/logging/rotatingfilesink.h | 1 - src/zenutil/jupiter/jupiterbuildstorage.cpp | 35 +- 33 files changed, 3984 insertions(+), 947 deletions(-) create mode 100644 src/zenserver/buildstore/httpbuildstore.cpp create mode 100644 src/zenserver/buildstore/httpbuildstore.h create mode 100644 src/zenstore/buildstore/buildstore.cpp create mode 100644 src/zenstore/include/zenstore/accesstime.h create mode 100644 src/zenstore/include/zenstore/buildstore/buildstore.h create mode 100644 src/zenutil/buildstoragecache.cpp create mode 100644 src/zenutil/include/zenutil/buildstoragecache.h (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 889ccef0b..b2ad579f1 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,8 @@ ZEN_THIRD_PARTY_INCLUDES_END #define EXTRA_VERIFY 0 +#define ZEN_CLOUD_STORAGE "Cloud Storage" + namespace zen { namespace { static std::atomic AbortFlag = false; @@ -87,6 +90,8 @@ namespace { const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; + const bool SingleThreaded = false; + const std::string ZenFolderName = ".zen"; const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); @@ -204,22 +209,27 @@ namespace { { try { - std::filesystem::remove(LocalFilePath); - } - catch (const std::exception&) - { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Sleep(200); - try - { - std::filesystem::remove(LocalFilePath); - } - catch (const std::exception& Ex) + std::error_code Ec; + std::filesystem::remove(LocalFilePath, Ec); + if (Ec) { - ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); - CleanWipe = false; + // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time + Ec.clear(); + if (std::filesystem::exists(LocalFilePath, Ec) || Ec) + { + Sleep(200); + if (std::filesystem::exists(LocalFilePath)) + { + std::filesystem::remove(LocalFilePath); + } + } } } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); + CleanWipe = false; + } } for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) @@ -425,7 +435,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, [](bool, std::ptrdiff_t) {}, AbortFlag); @@ -439,7 +449,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent FolderContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, Content, ChunkController, @@ -603,11 +613,9 @@ namespace { struct WriteChunkStatistics { - std::atomic ChunkCountWritten = 0; - std::atomic ChunkBytesWritten = 0; - uint64_t DownloadTimeUs = 0; - uint64_t WriteTimeUs = 0; - uint64_t WriteChunksElapsedWallTimeUs = 0; + uint64_t DownloadTimeUs = 0; + uint64_t WriteTimeUs = 0; + uint64_t WriteChunksElapsedWallTimeUs = 0; }; struct RebuildFolderStateStatistics @@ -626,6 +634,15 @@ namespace { uint64_t VerifyElapsedWallTimeUs = 0; }; + struct StorageInstance + { + std::unique_ptr BuildStorageHttp; + std::unique_ptr BuildStorage; + std::string StorageName; + std::unique_ptr CacheHttp; + std::unique_ptr BuildCacheStorage; + }; + std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, const std::span LocalChunkOrder, const tsl::robin_map& ChunkHashToLocalChunkIndex, @@ -772,7 +789,7 @@ namespace { std::span ChunkCounts, std::span LocalChunkHashes, std::span LocalChunkRawSizes, - std::vector AbsoluteChunkOrders, + const std::vector& AbsoluteChunkOrders, const std::span LooseLocalChunkIndexes, const std::span BlockHashes) { @@ -1444,7 +1461,7 @@ namespace { for (auto& WorkItem : WorkItems) { Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(),// + NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic&) { ZEN_TRACE_CPU("DownloadLargeBlob_Work"); if (!AbortFlag) @@ -1513,15 +1530,16 @@ namespace { } ValidateStats.BlockAttachmentCount = BlockAttachments.size(); - std::vector VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); + std::vector VerifyBlockDescriptions = + ParseChunkBlockDescriptionList(Storage.GetBlockMetadatas(BuildId, BlockAttachments)); if (VerifyBlockDescriptions.size() != BlockAttachments.size()) { throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", BlockAttachments.size() - VerifyBlockDescriptions.size())); } - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ParallellWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1881,7 +1899,7 @@ namespace { void GenerateBuildBlocks(const std::filesystem::path& Path, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, - BuildStorage& Storage, + StorageInstance& Storage, const Oid& BuildId, const std::vector>& NewBlockChunks, GeneratedBlocks& OutBlocks, @@ -1904,9 +1922,8 @@ namespace { RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = - GetMediumWorkerPool(EWorkloadType::Burst); // GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// - WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& GenerateBlobsPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& UploadBlocksPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; @@ -2005,21 +2022,35 @@ namespace { const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - Storage.PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload).GetCompressed()); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); + } + + Storage.BuildStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); UploadStats.BlocksBytes += CompressedBlockSize; + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(CompressedBlockSize), OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - Storage.PutBlockMetadata(BuildId, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(BlockMetaData.GetSize())); OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2074,7 +2105,7 @@ namespace { } } - void UploadPartBlobs(BuildStorage& Storage, + void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, const ChunkedFolderContent& Content, @@ -2092,8 +2123,8 @@ namespace { { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& ReadChunkPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& UploadChunkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& ReadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& UploadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); FilteredRate FilteredGenerateBlockBytesPerSecond; FilteredRate FilteredCompressedBytesPerSecond; @@ -2177,18 +2208,26 @@ namespace { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + } + Storage.BuildStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(PayloadSize), NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); UploadedBlockSize += PayloadSize; UploadStats.BlocksBytes += PayloadSize; - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(BlockMetaData.GetSize())); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2214,12 +2253,17 @@ namespace { ZEN_TRACE_CPU("AsyncUploadLooseChunk"); const uint64_t PayloadSize = Payload.GetSize(); - ; + + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + } + if (PayloadSize >= LargeAttachmentSize) { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); UploadStats.MultipartAttachmentCount++; - std::vector> MultipartWork = Storage.PutLargeBuildBlob( + std::vector> MultipartWork = Storage.BuildStorage->PutLargeBuildBlob( BuildId, RawHash, ZenContentType::kCompressedBinary, @@ -2232,7 +2276,7 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&, RawSize](uint64_t SentBytes, bool IsComplete) { + [&, Payload, RawSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; UploadedCompressedChunkSize += SentBytes; if (IsComplete) @@ -2264,7 +2308,7 @@ namespace { else { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); - Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + Storage.BuildStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); UploadStats.ChunksBytes += Payload.GetSize(); UploadStats.ChunkCount++; @@ -2303,7 +2347,7 @@ namespace { if (!AbortFlag) { Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool() + SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -2362,7 +2406,7 @@ namespace { { const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool(),// ReadChunkPool, + SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { @@ -2598,7 +2642,7 @@ namespace { return FilteredReuseBlockIndexes; }; - void UploadFolder(BuildStorage& Storage, + void UploadFolder(StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, const std::string_view BuildPartName, @@ -2654,7 +2698,7 @@ namespace { ZEN_TRACE_CPU("CreateBuild"); Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); + CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); Result.PayloadSize = MetaData.GetSize(); @@ -2663,7 +2707,7 @@ namespace { { ZEN_TRACE_CPU("PutBuild"); Stopwatch GetBuildTimer; - CbObject Build = Storage.GetBuild(BuildId); + CbObject Build = Storage.BuildStorage->GetBuild(BuildId); Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); Result.PayloadSize = Build.GetSize(); if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) @@ -2681,7 +2725,7 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = Storage.FindBlocks(BuildId); + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -2796,10 +2840,10 @@ namespace { } return true; }, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { - ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, AbortFlag); } @@ -2855,7 +2899,7 @@ namespace { FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, Content, *ChunkController, @@ -2985,7 +3029,7 @@ namespace { (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) : 0.0; ZEN_CONSOLE( - "Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n" + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" " Accepting {} ({}) redundant chunks ({:.1f}%)\n" " Rejected {} ({}) chunks in {} blocks\n" @@ -3204,7 +3248,8 @@ namespace { } Stopwatch PutBuildPartResultTimer; - std::pair> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); + std::pair> PutBuildPartResult = + Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), @@ -3289,7 +3334,7 @@ namespace { while (!AbortFlag) { Stopwatch FinalizeBuildPartTimer; - std::vector Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); + std::vector Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), Needs.size()); @@ -3304,7 +3349,7 @@ namespace { if (CreateBuild && !AbortFlag) { Stopwatch FinalizeBuildTimer; - Storage.FinalizeBuild(BuildId); + Storage.BuildStorage->FinalizeBuild(BuildId); ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); } @@ -3321,7 +3366,13 @@ namespace { { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); UploadStats.BlocksBytes += BlockMetaData.GetSize(); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; UploadBlockMetadataCount++; @@ -3340,7 +3391,7 @@ namespace { DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { - ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } ZEN_CONSOLE_VERBOSE( @@ -3570,7 +3621,7 @@ namespace { ValidateInfo); - Storage.PutBuildPartStats( + Storage.BuildStorage->PutBuildPartStats( BuildId, BuildPartId, {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, @@ -3597,7 +3648,7 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ParallellWork Work(AbortFlag); @@ -3873,6 +3924,22 @@ namespace { return ChunkTargetPtrs; }; + uint64_t GetChunkWriteCount(std::span> SequenceIndexChunksLeftToWriteCounters, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) + { + uint64_t WriteCount = 0; + std::span ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) + { + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) + { + WriteCount++; + } + } + return WriteCount; + }; + void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); @@ -3892,8 +3959,39 @@ namespace { } } + void VerifySequence(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + uint32_t RemoteSequenceIndex) + { + ZEN_TRACE_CPU("VerifySequence"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const std::uint32_t RemotePathIndex = Lookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ExpectedSize = RemoteContent.RawSizes[RemotePathIndex]; + IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + const uint64_t VerifySize = VerifyBuffer.GetSize(); + if (VerifySize != ExpectedSize) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}", + SequenceRawHash, + VerifySize, + ExpectedSize)); + } + ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer)); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + } + } + void VerifyAndCompleteChunkSequencesAsync(const std::filesystem::path& TargetFolder, const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, std::span RemoteSequenceIndexes, ParallellWork& Work, WorkerThreadPool& VerifyPool) @@ -3908,40 +4006,24 @@ namespace { const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; Work.ScheduleWork( VerifyPool, - [&RemoteContent, TargetFolder, RemoteSequenceIndex](std::atomic&) { + [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic&) { if (!AbortFlag) { ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); + if (!AbortFlag) { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = IoHash::HashBuffer( - IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(TargetFolder, SequenceRawHash); } - FinalizeChunkSequence(TargetFolder, SequenceRawHash); } }, Work.DefaultErrorFunction()); } const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); - } - } FinalizeChunkSequence(TargetFolder, SequenceRawHash); } @@ -3985,8 +4067,7 @@ namespace { const BlockWriteOps& Ops, ParallellWork& Work, WorkerThreadPool& VerifyPool, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockChunkOps"); { @@ -4017,12 +4098,6 @@ namespace { FileOffset, RemoteContent.RawSizes[PathIndex]); } - WriteChunkStats.ChunkCountWritten += gsl::narrow(Ops.ChunkBuffers.size()); - WriteChunkStats.ChunkBytesWritten += - std::accumulate(Ops.ChunkBuffers.begin(), - Ops.ChunkBuffers.end(), - uint64_t(0), - [](uint64_t Current, const CompositeBuffer& Buffer) -> uint64_t { return Current + Buffer.GetSize(); }); } if (!AbortFlag) { @@ -4036,7 +4111,7 @@ namespace { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, CompletedChunkSequences, Work, VerifyPool); + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, Lookup, CompletedChunkSequences, Work, VerifyPool); } } @@ -4162,8 +4237,7 @@ namespace { CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockToDisk"); @@ -4197,8 +4271,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4222,8 +4295,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4240,8 +4312,7 @@ namespace { uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); @@ -4267,8 +4338,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } else @@ -4355,8 +4425,7 @@ namespace { void StreamDecompress(const std::filesystem::path& CacheFolderPath, const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("StreamDecompress"); const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); @@ -4390,7 +4459,6 @@ namespace { DiskStats.ReadByteCount += SourceSize; if (!AbortFlag) { - WriteChunkStats.ChunkBytesWritten += RangeBuffer.GetSize(); DecompressedTemp.Write(RangeBuffer, Offset); for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) { @@ -4424,7 +4492,7 @@ namespace { throw std::runtime_error( fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); } - WriteChunkStats.ChunkCountWritten++; + // WriteChunkStats.ChunkCountWritten++; } bool WriteCompressedChunk(const std::filesystem::path& TargetFolder, @@ -4433,8 +4501,7 @@ namespace { const IoHash& ChunkHash, const std::vector& ChunkTargetPtrs, IoBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); @@ -4444,7 +4511,7 @@ namespace { { const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; - StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats, WriteChunkStats); + StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats); } else { @@ -4459,8 +4526,6 @@ namespace { ChunkTargetPtrs, CompositeBuffer(std::move(Chunk)), OpenFileCache); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += ChunkRawSize; return true; } } @@ -4479,8 +4544,7 @@ namespace { std::atomic& WritePartsComplete, const uint64_t TotalPartWriteCount, FilteredRate& FilteredWrittenBytesPerSecond, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); @@ -4527,7 +4591,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, SequenceIndexChunksLeftToWriteCounters, CompressedChunkPath, @@ -4565,8 +4629,7 @@ namespace { ChunkHash, ChunkTargetPtrs, std::move(CompressedPart), - DiskStats, - WriteChunkStats); + DiskStats); if (!AbortFlag) { WritePartsComplete++; @@ -4581,7 +4644,12 @@ namespace { CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); if (NeedHashVerify) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, RemoteContent, CompletedSequences, Work, WritePool); + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); } else { @@ -4593,7 +4661,7 @@ namespace { Work.DefaultErrorFunction()); }; - void UpdateFolder(BuildStorage& Storage, + void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, const std::uint64_t LargeAttachmentSize, @@ -4667,8 +4735,8 @@ namespace { if (SequenceSize == CacheDirContent.FileSizes[Index]) { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CacheMappingStats.CacheSequenceHashesCount += SequenceSize; - CacheMappingStats.CacheSequenceHashesByteCount++; + CacheMappingStats.CacheSequenceHashesCount++; + CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; continue; } } @@ -4869,21 +4937,17 @@ namespace { NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); } - uint32_t ChunkCountToWrite = 0; + uint64_t BytesToWrite = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) { - if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + if (ChunkWriteCount > 0) { - ChunkCountToWrite++; - } - else - { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); - if (!ChunkTargetPtrs.empty()) + BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; + if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; - ChunkCountToWrite++; } } } @@ -4900,8 +4964,8 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -4922,7 +4986,7 @@ namespace { const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); continue; } bool NeedsCopy = true; @@ -4933,7 +4997,7 @@ namespace { if (ChunkTargetPtrs.empty()) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); } else { @@ -5025,6 +5089,10 @@ namespace { uint32_t CurrentOffset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) @@ -5064,6 +5132,7 @@ namespace { } ZEN_ASSERT(!BlockRanges.empty()); + std::vector CollapsedBlockRanges; auto It = BlockRanges.begin(); CollapsedBlockRanges.push_back(*It++); @@ -5085,10 +5154,87 @@ namespace { ++It; } - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); + const std::uint64_t WantedSize = std::accumulate( + CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + ZEN_ASSERT(WantedSize <= TotalBlockSize); + if (WantedSize > ((TotalBlockSize * 95) / 100)) + { + ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) + { + ZEN_CONSOLE_VERBOSE("Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) + { + ZEN_CONSOLE_VERBOSE("Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) + { + ZEN_CONSOLE_VERBOSE("Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + { + ZEN_CONSOLE_VERBOSE("Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else + { + if (WantedSize > ((TotalBlockSize * 5) / 10)) + { + ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", + (WantedSize * 100) / TotalBlockSize, + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + } + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); - BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + } } else { @@ -5101,10 +5247,69 @@ namespace { } else { - ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); } } + struct BlobsExistsResult + { + tsl::robin_set ExistingBlobs; + uint64_t ElapsedTimeMs = 0; + }; + + BlobsExistsResult ExistsResult; + + if (Storage.BuildCacheStorage) + { + ZEN_TRACE_CPU("BlobCacheExistCheck"); + Stopwatch Timer; + + tsl::robin_set BlobHashesSet; + + BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size()); + for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks) + { + BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); + } + for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks) + { + const uint32_t BlockIndex = BlockRange.BlockIndex; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + for (uint32_t BlockIndex : FullBlockWorks) + { + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + + if (!BlobHashesSet.empty()) + { + const std::vector BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end()); + const std::vector CacheExistsResult = + Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes); + + if (CacheExistsResult.size() == BlobHashes.size()) + { + ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + { + if (CacheExistsResult[BlobIndex].HasBody) + { + ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]); + } + } + } + ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!ExistsResult.ExistingBlobs.empty()) + { + ZEN_CONSOLE("Found {} out of {} needed blobs in remote cache in {}", + ExistsResult.ExistingBlobs.size(), + BlobHashes.size(), + NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); + } + } + } for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) { if (AbortFlag) @@ -5119,7 +5324,7 @@ namespace { const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; Work.ScheduleWork( - WritePool, // NetworkPool, // GetSyncWorkerPool(),// + WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { if (!AbortFlag) { @@ -5152,151 +5357,202 @@ namespace { } } } - if (!ExistingCompressedChunkPath.empty()) + if (!AbortFlag) + { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Path, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + WritePool, + [&Path, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - FilteredWrittenBytesPerSecond.Start(); + FilteredWrittenBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats, - WriteChunkStats); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]; - WritePartsComplete++; + std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + DiskStats); + WritePartsComplete++; - if (!AbortFlag) - { - if (WritePartsComplete == TotalPartWriteCount) + if (!AbortFlag) { - FilteredWrittenBytesPerSecond.Stop(); - } + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } - std::filesystem::remove(CompressedChunkPath); + std::filesystem::remove(CompressedChunkPath); - std::vector CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - if (NeedHashVerify) + std::vector CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } + } + } + }, + Work.DefaultErrorFunction()); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&Path, + &Storage, + BuildId, + &RemoteContent, + &RemoteLookup, + &ExistsResult, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + LargeAttachmentSize, + PreferredMultipartChunkSize, + RemoteChunkIndex, + ChunkTargetPtrs, + &DownloadStats](std::atomic&) mutable { + if (!AbortFlag) + { + FilteredDownloadedBytesPerSecond.Start(); + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - CompletedSequences, - Work, - WritePool); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob(*Storage.BuildStorage, + Path / ZenTempDownloadFolderName, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (!AbortFlag) + { + AsyncWriteDownloadedChunk( + Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + }); } else { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + IoBuffer BuildBlob; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash)) + { + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + } + if (!BuildBlob) + { + BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + if (BuildBlob && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + BuildBlob.GetContentType(), + CompositeBuffer(SharedBuffer(BuildBlob))); + } + } + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } } } - } - }, - Work.DefaultErrorFunction()); - } - else - { - FilteredDownloadedBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) - { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(Storage, - Path / ZenTempDownloadFolderName, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); - }); - } - else - { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - - IoBuffer BuildBlob = Storage.GetBuildBlob(BuildId, ChunkHash); - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); + }, + Work.DefaultErrorFunction()); } } } @@ -5312,7 +5568,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, CopyDataIndex](std::atomic&) { if (!AbortFlag) { @@ -5439,16 +5695,6 @@ namespace { ChunkSource, Op.Target->Offset, RemoteContent.RawSizes[RemotePathIndex]); - for (size_t WrittenOpIndex = WriteOpIndex; WrittenOpIndex < WriteOpIndex + WriteCount; WrittenOpIndex++) - { - const WriteOp& WrittenOp = WriteOps[WrittenOpIndex]; - if (ChunkIndexesWritten.insert(WrittenOp.ChunkIndex).second) - { - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[WrittenOp.ChunkIndex]; - } - } CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? @@ -5469,10 +5715,13 @@ namespace { } VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, + RemoteLookup, CompletedChunkSequences, Work, WritePool); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", + NiceBytes(CacheLocalFileBytesRead), + LocalContent.Paths[LocalPathIndex]); } WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) @@ -5492,7 +5741,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(), // WritePool, + WritePool, [&, BlockIndex](std::atomic&) mutable { if (!AbortFlag) { @@ -5509,27 +5758,29 @@ namespace { fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); } - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); - if (WritePartsComplete == TotalPartWriteCount) + if (!AbortFlag) { - FilteredWrittenBytesPerSecond.Stop(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + std::filesystem::remove(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } } }, @@ -5546,7 +5797,7 @@ namespace { ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); const uint32_t BlockIndex = BlockRange.BlockIndex; Work.ScheduleWork( - NetworkPool, // NetworkPool, // GetSyncWorkerPool() + NetworkPool, [&, BlockIndex, BlockRange](std::atomic&) { if (!AbortFlag) { @@ -5555,131 +5806,148 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = - Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } - - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!AbortFlag) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + FilteredDownloadedBytesPerSecond.Stop(); + } - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = - Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / + fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, // GetSyncWorkerPool(), - [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( - std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( + std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else - { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -5693,7 +5961,7 @@ namespace { break; } Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(), // NetworkPool, + NetworkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -5702,133 +5970,152 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); + + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (BlockBuffer && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlockBuffer))); + } } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } + if (!AbortFlag) + { + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - std::filesystem::path BlockChunkPath; + std::filesystem::path BlockChunkPath; - // Check if the dowloaded block is file based and we can move it directly without rewriting it - { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + // Check if the dowloaded block is file based and we can move it directly without rewriting it { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; + WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -5840,27 +6127,28 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= WriteChunkStats.ChunkCountWritten.load()); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load()); FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); - std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", + std::string DownloadRateString = + (DownloadStats.RequestsCompleteCount == TotalRequestCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); + std::string Details = fmt::format("{}/{} ({}{}) downloaded. {}/{} ({}B/s) written.", DownloadStats.RequestsCompleteCount.load(), TotalRequestCount, NiceBytes(DownloadedBytes), - NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - WriteChunkStats.ChunkCountWritten.load(), - ChunkCountToWrite, + DownloadRateString, NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState( - {.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(ChunkCountToWrite), - .RemainingCount = gsl::narrow(ChunkCountToWrite - WriteChunkStats.ChunkCountWritten.load())}, - false); + WriteProgressBar.UpdateState({.Task = "Writing chunks ", + .Details = Details, + .TotalCount = gsl::narrow(BytesToWrite), + .RemainingCount = gsl::narrow(BytesToWrite - DiskStats.WriteByteCount.load())}, + false); }); } @@ -5981,7 +6269,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); Stopwatch Timer; - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); ProgressBar RebuildProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -6019,7 +6307,7 @@ namespace { } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic&) { if (!AbortFlag) { @@ -6070,6 +6358,10 @@ namespace { TargetsComplete++; while (TargetOffset < (BaseTargetOffset + TargetCount)) { + if (AbortFlag) + { + break; + } ZEN_TRACE_CPU("FinalizeTree_Copy"); ZEN_ASSERT(Targets[TargetOffset].first == RawHash); @@ -6140,17 +6432,15 @@ namespace { std::vector> Result; { Stopwatch GetBuildTimer; - - std::vector> AvailableParts; - - CbObject BuildObject = Storage.GetBuild(BuildId); - - ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", + CbObject BuildObject = Storage.GetBuild(BuildId); + ZEN_CONSOLE("GetBuild took {}. Name: '{}' ({}, {}), Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["BuildName"sv].AsString(), + BuildObject["name"sv].AsString(), + BuildObject["type"sv].AsString(), + BuildObject["Configuration"sv].AsString(), NiceBytes(BuildObject.GetSize())); - ZEN_DEBUG("Build object: {}", BuildObject); + ZEN_CONSOLE_VERBOSE("Build object: {}", BuildObject); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -6160,6 +6450,8 @@ namespace { OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + std::vector> AvailableParts; + for (CbFieldView PartView : PartsObject) { const std::string BuildPartName = std::string(PartView.GetName()); @@ -6221,7 +6513,7 @@ namespace { return Result; } - ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, + ChunkedFolderContent GetRemoteContent(StorageInstance& Storage, const Oid& BuildId, const std::vector>& BuildParts, std::unique_ptr& OutChunkController, @@ -6234,7 +6526,7 @@ namespace { Stopwatch GetBuildPartTimer; const Oid BuildPartId = BuildParts[0].first; const std::string_view BuildPartName = BuildParts[0].second; - CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildPartId); + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", BuildPartId, BuildPartName, @@ -6248,7 +6540,7 @@ namespace { OutChunkController = CreateChunkingController(ChunkerName, Parameters); } - auto ParseBuildPartManifest = [](BuildStorage& Storage, + auto ParseBuildPartManifest = [](StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, CbObject BuildPartManifest, @@ -6274,12 +6566,103 @@ namespace { // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them - Stopwatch GetBlockMetadataTimer; - OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes); - ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); + { + Stopwatch GetBlockMetadataTimer; + + std::vector UnorderedList; + tsl::robin_map BlockDescriptionLookup; + if (Storage.BuildCacheStorage) + { + std::vector CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes); + UnorderedList.reserve(CacheBlockMetadatas.size()); + for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size(); + CacheBlockMetadataIndex++) + { + const CbObject& CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex]; + ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block"); + } + else + { + UnorderedList.emplace_back(std::move(Description)); + } + } + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); + } + } + + if (UnorderedList.size() < BlockRawHashes.size()) + { + std::vector RemainingBlockHashes; + RemainingBlockHashes.reserve(BlockRawHashes.size() - UnorderedList.size()); + for (const IoHash& BlockRawHash : BlockRawHashes) + { + if (!BlockDescriptionLookup.contains(BlockRawHash)) + { + RemainingBlockHashes.push_back(BlockRawHash); + } + } + CbObject BlockMetadatas = Storage.BuildStorage->GetBlockMetadatas(BuildId, RemainingBlockHashes); + std::vector RemainingList; + { + CbArrayView BlocksArray = BlockMetadatas["blocks"sv].AsArrayView(); + std::vector FoundBlockHashes; + std::vector FoundBlockMetadatas; + for (CbFieldView Block : BlocksArray) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); + + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block"); + } + else + { + if (Storage.BuildCacheStorage) + { + UniqueBuffer MetaBuffer = UniqueBuffer::Alloc(Block.GetSize()); + Block.CopyTo(MetaBuffer.GetMutableView()); + CbObject BlockMetadata(MetaBuffer.MoveToShared()); + + FoundBlockHashes.push_back(Description.BlockHash); + FoundBlockMetadatas.push_back(BlockMetadata); + } + RemainingList.emplace_back(std::move(Description)); + } + } + if (Storage.BuildCacheStorage && !FoundBlockHashes.empty()) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, FoundBlockHashes, FoundBlockMetadatas); + } + } + + for (size_t DescriptionIndex = 0; DescriptionIndex < RemainingList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = RemainingList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, UnorderedList.size() + DescriptionIndex); + } + UnorderedList.insert(UnorderedList.end(), RemainingList.begin(), RemainingList.end()); + } + + OutBlockDescriptions.reserve(BlockDescriptionLookup.size()); + for (const IoHash& BlockHash : BlockRawHashes) + { + if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) + { + OutBlockDescriptions.push_back(std::move(UnorderedList[It->second])); + } + } + + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } if (OutBlockDescriptions.size() != BlockRawHashes.size()) { @@ -6292,7 +6675,8 @@ namespace { ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); std::vector AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); - std::vector FoundBlocks = Storage.FindBlocks(BuildId); + std::vector FoundBlocks = + ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); for (const IoHash& BlockHash : BlockRawHashes) { @@ -6315,7 +6699,7 @@ namespace { } else { - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockHash); + IoBuffer BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockHash); if (!BlockBuffer) { throw std::runtime_error(fmt::format("Block {} could not be found", BlockHash)); @@ -6381,7 +6765,7 @@ namespace { const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; Stopwatch GetOverlayBuildPartTimer; - CbObject OverlayBuildPartManifest = Storage.GetBuildPart(BuildId, OverlayBuildPartId); + CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", OverlayBuildPartId, OverlayBuildPartName, @@ -6486,9 +6870,11 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, + [&](bool, std::ptrdiff_t) { + ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + }, AbortFlag); if (AbortFlag) { @@ -6557,7 +6943,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, UpdatedContent, ChunkController, @@ -6609,8 +6995,6 @@ namespace { { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); } - - ZEN_CONSOLE("Using cached local state"); } ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); ScanContent = false; @@ -6636,7 +7020,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), Path, CurrentLocalFolderContent, ChunkController, @@ -6670,7 +7054,7 @@ namespace { return LocalContent; } - void DownloadFolder(BuildStorage& Storage, + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector& BuildPartIds, std::span BuildPartNames, @@ -6694,7 +7078,7 @@ namespace { std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; std::vector> AllBuildParts = - ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); std::vector PartContents; @@ -6786,7 +7170,7 @@ namespace { { BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, BuildPartString.ToView()); + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path); FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -7131,6 +7515,10 @@ BuildsCommand::BuildsCommand() ""); }; + auto AddCacheOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), ""); + }; + auto AddOutputOptions = [this](cxxopts::Options& Ops) { Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), ""); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); @@ -7169,6 +7557,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_UploadOptions); AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); + AddCacheOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7232,6 +7621,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_DownloadOptions); AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); + AddCacheOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7284,6 +7674,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); + AddCacheOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_TestOptions.add_option("", @@ -7304,6 +7695,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); + AddCacheOptions(m_FetchBlobOptions); m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_FetchBlobOptions .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); @@ -7333,6 +7725,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); + AddCacheOptions(m_MultiTestDownloadOptions); m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); @@ -7385,6 +7778,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); } } + else if (m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } }; std::unique_ptr Auth; @@ -7393,7 +7790,9 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : StringToPath(m_SystemRootDir); + std::filesystem::path DataRoot = m_SystemRootDir.empty() + ? PickDefaultSystemRootDirectory() + : std::filesystem::absolute(StringToPath(m_SystemRootDir)).make_preferred(); if (m_EncryptionKey.empty()) { @@ -7448,7 +7847,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(StringToPath(m_AccessTokenPath)); + std::string ResolvedAccessToken = + ReadAccessTokenFromFile(std::filesystem::absolute(StringToPath(m_AccessTokenPath)).make_preferred()); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -7486,15 +7886,63 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseOutputOptions(); + auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + const std::filesystem::path& TempPath) -> StorageInstance { + ParseStorageOptions(); + + StorageInstance Result; + + if (!m_BuildsUrl.empty()) + { + ParseAuthOptions(); + Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Using cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); + Result.BuildStorage = + CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); + Result.StorageName = ZEN_CLOUD_STORAGE; + } + else if (!m_StoragePath.empty()) + { + std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + ZEN_CONSOLE("Using folder {}", StoragePath); + Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + if (!m_ZenCacheHost.empty()) + { + Result.CacheHttp = std::make_unique(m_ZenCacheHost, + HttpClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = false, + .AllowResume = true, + .RetryCount = 0}); + if (Result.CacheHttp->Get("/health").IsSuccess()) + { + Result.BuildCacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); + } + else + { + Result.CacheHttp.reset(); + } + } + return Result; + }; + try { if (SubOption == &m_ListOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - CbObject QueryObject; if (m_ListQueryPath.empty()) { @@ -7505,7 +7953,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = StringToPath(m_ListQueryPath); + std::filesystem::path ListQueryPath = std::filesystem::absolute(StringToPath(m_ListQueryPath)).make_preferred(); if (ToLower(ListQueryPath.extension().string()) == ".cbo") { QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); @@ -7525,28 +7973,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE_VERBOSE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE_VERBOSE("Querying builds in folder '{}'.", StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; - CbObject Response = Storage->ListBuilds(QueryObject); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderName); + + CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); if (m_ListResultPath.empty()) { @@ -7556,7 +7987,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = StringToPath(m_ListResultPath); + std::filesystem::path ListResultPath = std::filesystem::absolute(StringToPath(m_ListResultPath)).make_preferred(); if (ToLower(ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); @@ -7575,11 +8006,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); @@ -7610,7 +8036,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)); if (m_BuildPartName.empty()) { @@ -7645,48 +8071,20 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); } + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", - m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", - m_BuildPartName, - Path, - StoragePath, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); CbObject MetaData; if (m_CreateBuild) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath = StringToPath(m_BuildMetadataPath); + std::filesystem::path MetadataPath = std::filesystem::absolute(StringToPath(m_BuildMetadataPath)); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -7713,12 +8111,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, Path, - StringToPath(m_ManifestPath), + std::filesystem::absolute(StringToPath(m_ManifestPath)).make_preferred(), m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -7735,7 +8133,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7751,11 +8149,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -7786,37 +8179,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - BuildId, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, @@ -7835,7 +8205,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7859,8 +8229,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); - std::filesystem::path DiffPath = StringToPath(m_DiffPath); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path DiffPath = std::filesystem::absolute(StringToPath(m_DiffPath)).make_preferred(); DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -7872,10 +8242,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); // m_StoragePath = "D:\\buildstorage"; // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; // std::vector BuildIdStrings{"07d3942f0e7f4ca1b13b0587", @@ -7886,34 +8252,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - FormatArray(m_BuildIds, " "sv), - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray(m_BuildIds, " "sv), Path, StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -7923,7 +8267,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); } - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId, {}, {}, @@ -7945,36 +8289,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr Storage; - std::string StorageName; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = StringToPath(m_StoragePath); + std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); if (m_BuildsUrl.empty() && StoragePath.empty()) { StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); CleanDirectory(StoragePath, {}); + m_StoragePath = StoragePath.generic_string(); } auto _ = MakeGuard([&]() { if (m_BuildsUrl.empty() && StoragePath.empty()) @@ -7983,33 +8320,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } }); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!StoragePath.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - StoragePath, - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -8032,7 +8346,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, @@ -8052,7 +8366,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -8064,7 +8378,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -8115,7 +8429,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SourceSize > 256) { Work.ScheduleWork( - GetMediumWorkerPool(EWorkloadType::Burst), + SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), [SourceSize, FilePath](std::atomic&) { if (!AbortFlag) { @@ -8168,7 +8482,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -8187,7 +8501,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId2, BuildPartId2, m_BuildPartName, @@ -8206,7 +8520,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8214,7 +8528,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, @@ -8230,7 +8544,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, @@ -8250,11 +8564,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -8266,44 +8575,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); } - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } + const Oid BuildId = Oid::FromHexString(m_BuildId); - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - std::unique_ptr Storage; - std::string StorageName; + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); - std::filesystem::path Path = StringToPath(m_Path); + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); + ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); if (AbortFlag) { return 11; @@ -8317,10 +8600,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); + // HttpClient Http(m_BuildsUrl, ClientSettings); if (m_BuildsUrl.empty() && m_StoragePath.empty()) { @@ -8342,39 +8622,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + BuildStorage::Statistics StorageStats; - std::unique_ptr Storage; - std::string StorageName; + BuildStorageCache::Statistics StorageCacheStats; - std::filesystem::path Path = StringToPath(m_Path); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); ValidateStatistics ValidateStats; DownloadStatistics DownloadStats; - ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); return AbortFlag ? 13 : 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 1634975c1..b5af236e1 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -40,6 +40,9 @@ private: std::string m_StoragePath; bool m_WriteMetadataAsJson = false; + // cache + std::string m_ZenCacheHost; + std::string m_BuildId; bool m_CreateBuild = false; std::string m_BuildMetadataPath; diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 30711a432..fe5232d89 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -45,6 +45,7 @@ namespace detail { TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {} ~TempPayloadFile() { + ZEN_TRACE_CPU("TempPayloadFile::Close"); try { if (m_FileHandle) @@ -87,6 +88,7 @@ namespace detail { std::error_code Open(const std::filesystem::path& TempFolderPath) { + ZEN_TRACE_CPU("TempPayloadFile::Open"); ZEN_ASSERT(m_FileHandle == nullptr); std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) | @@ -131,6 +133,7 @@ namespace detail { std::error_code Write(std::string_view DataString) { + ZEN_TRACE_CPU("TempPayloadFile::Write"); const uint8_t* DataPtr = (const uint8_t*)DataString.data(); size_t DataSize = DataString.size(); if (DataSize >= CacheBufferSize) @@ -165,6 +168,7 @@ namespace detail { IoBuffer DetachToIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::DetachToIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -180,6 +184,7 @@ namespace detail { IoBuffer BorrowIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::BorrowIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -193,6 +198,7 @@ namespace detail { uint64_t GetSize() const { return m_WriteOffset; } void ResetWritePos(uint64_t WriteOffset) { + ZEN_TRACE_CPU("TempPayloadFile::ResetWritePos"); Flush(); m_WriteOffset = WriteOffset; } @@ -200,6 +206,7 @@ namespace detail { private: std::error_code Flush() { + ZEN_TRACE_CPU("TempPayloadFile::Flush"); if (m_CacheBufferOffset == 0) { return {}; @@ -211,6 +218,7 @@ namespace detail { std::error_code AppendData(const void* Data, uint64_t Size) { + ZEN_TRACE_CPU("TempPayloadFile::AppendData"); ZEN_ASSERT(m_FileHandle != nullptr); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; @@ -314,7 +322,11 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) { - ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT && + HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED) + { + ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + } // Client side failure code return HttpClient::Response{ @@ -376,6 +388,7 @@ ShouldRetry(const cpr::Response& Response) static bool ValidatePayload(cpr::Response& Response, std::unique_ptr& PayloadFile) { + ZEN_TRACE_CPU("ValidatePayload"); IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile) ? PayloadFile->BorrowIoBuffer() : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size()); @@ -535,12 +548,14 @@ struct HttpClient::Impl : public RefCounted inline cpr::Session* operator->() const { return CprSession; } inline cpr::Response Get() { + ZEN_TRACE_CPU("HttpClient::Impl::Get"); cpr::Response Result = CprSession->Get(); ZEN_TRACE("GET {}", Result); return Result; } inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional&& Header = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Download"); if (Header) { CprSession->SetHeaderCallback(std::move(Header.value())); @@ -553,12 +568,14 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Head() { + ZEN_TRACE_CPU("HttpClient::Impl::Head"); cpr::Response Result = CprSession->Head(); ZEN_TRACE("HEAD {}", Result); return Result; } inline cpr::Response Put(std::optional&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Put"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -570,6 +587,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Post(std::optional&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Post"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -581,6 +599,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Delete() { + ZEN_TRACE_CPU("HttpClient::Impl::Delete"); cpr::Response Result = CprSession->Delete(); ZEN_TRACE("DELETE {}", Result); return Result; @@ -620,6 +639,7 @@ HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log) HttpClient::Impl::~Impl() { + ZEN_TRACE_CPU("HttpClient::Impl::~Impl"); m_SessionLock.WithExclusiveLock([&] { for (auto CprSession : m_Sessions) { @@ -638,6 +658,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, const std::string_view SessionId, std::optional AccessToken) { + ZEN_TRACE_CPU("HttpClient::Impl::AllocSession"); cpr::Session* CprSession = nullptr; m_SessionLock.WithExclusiveLock([&] { if (!m_Sessions.empty()) @@ -694,6 +715,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, void HttpClient::Impl::ReleaseSession(cpr::Session* CprSession) { + ZEN_TRACE_CPU("HttpClient::Impl::ReleaseSession"); CprSession->SetUrl({}); CprSession->SetHeader({}); CprSession->SetBody({}); @@ -718,6 +740,7 @@ HttpClient::~HttpClient() bool HttpClient::Authenticate() { + ZEN_TRACE_CPU("HttpClient::Authenticate"); std::optional Token = GetAccessToken(); if (!Token) { @@ -729,6 +752,7 @@ HttpClient::Authenticate() const std::optional HttpClient::GetAccessToken() { + ZEN_TRACE_CPU("HttpClient::GetAccessToken"); if (!m_ConnectionSettings.AccessTokenProvider.has_value()) { return {}; diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index 74da9ab05..05a23d675 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -73,9 +73,11 @@ struct fmt::formatter if (zen::IsHttpSuccessCode(Response.status_code)) { return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}", + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}", Response.url.str(), Response.status_code, + Response.error.message, + int(Response.error.code), Response.uploaded_bytes, Response.downloaded_bytes, NiceResponseTime.c_str()); @@ -92,29 +94,35 @@ struct fmt::formatter zen::ExtendableStringBuilder<256> Sb; std::string_view Json = Obj.ToJson(Sb).ToView(); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Json, - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}). Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Json, + Response.reason); } else { zen::BodyLogFormatter Body(Response.text); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Body.GetText(), - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Body.GetText(), + Response.reason); } } } diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index ae80851e4..9d423ecbc 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -279,11 +279,10 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar { IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); ZEN_ASSERT(ObjIoBuffer.GetSize() > 0); - ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); - *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Attachment.GetHash()}; + ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); } else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary()) { @@ -500,30 +499,25 @@ ParsePackageMessage(IoBuffer Payload, std::function 0) diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 2888f5450..0da6e31ad 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include "config.h" @@ -105,6 +106,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) @@ -112,6 +114,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, , m_CacheStore(CacheStore) , m_CidStore(CidStore) , m_ProjectStore(ProjectStore) +, m_BuildStore(BuildStore) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { @@ -306,6 +309,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "Interval" << ToTimeSpan(State.Config.Interval); Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); @@ -401,6 +405,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } } + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt(Param)) @@ -782,6 +794,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, { m_ProjectStore->Flush(); } + if (m_BuildStore) + { + m_BuildStore->Flush(); + } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index 563c4f536..e7821dead 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -12,6 +12,7 @@ class JobQueue; class ZenCacheStore; class CidStore; class ProjectStore; +class BuildStore; struct ZenServerOptions; class HttpAdminService : public zen::HttpService @@ -28,6 +29,7 @@ public: ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions); ~HttpAdminService(); @@ -42,6 +44,7 @@ private: ZenCacheStore* m_CacheStore; CidStore* m_CidStore; ProjectStore* m_ProjectStore; + BuildStore* m_BuildStore; LogPaths m_LogPaths; const ZenServerOptions& m_ServerOptions; }; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..06bfea423 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -0,0 +1,526 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_StatsService.RegisterHandler("builds", *this); + + m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize >= BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector BlobHashes; + std::vector MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector Attachments; + tsl::robin_set AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h new file mode 100644 index 000000000..a59aa882a --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 809092378..0da98210c 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -377,6 +377,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); @@ -1031,6 +1034,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value>(BucketConfigs), ""); + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("stats", "", "statsd", diff --git a/src/zenserver/config.h b/src/zenserver/config.h index c7781aada..a87b6f8b3 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -59,11 +59,17 @@ struct ZenProjectStoreEvictionPolicy int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; }; +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + struct ZenGcConfig { // ZenCasEvictionPolicy Cas; ZenCacheEvictionPolicy Cache; ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; int32_t MonitorIntervalSeconds = 30; int32_t IntervalSeconds = 0; bool CollectSmallObjects = true; @@ -130,6 +136,11 @@ struct ZenProjectStoreConfig bool StoreProjectAttachmentMetaData = false; }; +struct ZenBuildStoreConfig +{ + bool Enabled = false; +}; + struct ZenWorkspacesConfig { bool Enabled = false; @@ -145,6 +156,7 @@ struct ZenServerOptions zen::HttpServerConfig HttpServerConfig; ZenStructuredCacheConfig StructuredCacheConfig; ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; std::filesystem::path SystemRootDir; // System root directory (used for machine level config) diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 8a4b977ad..0b7fd0400 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -84,7 +84,7 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con HttpWorkspacesService::~HttpWorkspacesService() { - m_StatsService.UnregisterHandler("prj", *this); + m_StatsService.UnregisterHandler("ws", *this); } const char* diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..03e269d49 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_Workspaces)); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + BuildStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "builds"; + m_BuildStore = std::make_unique(std::move(ObjCfg), m_GcManager); + } + if (ServerOptions.StructuredCacheConfig.Enabled) { InitializeStructuredCache(ServerOptions); @@ -310,6 +318,12 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_ObjStoreService); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_Http->RegisterService(*m_BuildStoreService); + } + #if ZEN_WITH_VFS m_VfsService = std::make_unique(); m_VfsService->AddService(Ref(m_ProjectStore)); @@ -327,6 +341,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, .Enabled = ServerOptions.GcConfig.Enabled, .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, @@ -347,6 +362,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_CacheStore.Get(), m_CidStore.get(), m_ProjectStore, + m_BuildStore.get(), HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, @@ -801,6 +817,9 @@ ZenServer::Cleanup() m_ObjStoreService.reset(); m_FrontendService.reset(); + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_StructuredCacheService.reset(); m_UpstreamService.reset(); m_UpstreamCache.reset(); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 80054dc35..5cfa04ba1 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include #include #include "admin/admin.h" +#include "buildstore/httpbuildstore.h" #include "cache/httpstructuredcache.h" #include "diag/diagsvcs.h" #include "frontend/frontend.h" @@ -127,6 +128,8 @@ private: Ref m_CacheStore; std::unique_ptr m_OpenProcessCache; HttpTestService m_TestService; + std::unique_ptr m_BuildStore; + #if ZEN_WITH_TESTS HttpTestingService m_TestingService; #endif @@ -140,6 +143,7 @@ private: HttpHealthService m_HealthService; std::unique_ptr m_FrontendService; std::unique_ptr m_ObjStoreService; + std::unique_ptr m_BuildStoreService; std::unique_ptr m_VfsService; std::unique_ptr m_JobQueue; std::unique_ptr m_AdminService; diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index e5b312984..c56971520 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -18,6 +19,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) zen::zenstore_forcelinktests(); zen::logging::InitializeLogging(); + zen::buildstore_forcelink(); zen::MaximizeOpenFileCount(); return ZEN_RUN_TESTS(argc, argv); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e976c061d..63c0388fa 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -578,7 +578,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons } void -BlockStore::WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback) +BlockStore::WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback) { ZEN_MEMSCOPE(GetBlocksTag()); ZEN_TRACE_CPU("BlockStore::WriteChunks"); diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp new file mode 100644 index 000000000..8674aab75 --- /dev/null +++ b/src/zenstore/buildstore/buildstore.cpp @@ -0,0 +1,1475 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_WITH_TESTS +# include +# include +# include +# include +#endif // ZEN_WITH_TESTS + +namespace zen { +const FLLMTag& +GetBuildstoreTag() +{ + static FLLMTag _("store", FLLMTag("builds")); + + return _; +} + +using namespace std::literals; + +namespace blobstore::impl { + + const std::string BaseName = "builds"; + const char* IndexExtension = ".uidx"; + const char* LogExtension = ".slog"; + + std::filesystem::path GetBlobIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + IndexExtension); + } + + std::filesystem::path GetBlobLogPath(const std::filesystem::path& RootDirectory) { return RootDirectory / (BaseName + LogExtension); } + + std::filesystem::path GetMetaIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + IndexExtension); + } + + std::filesystem::path GetMetaLogPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + LogExtension); + } +} // namespace blobstore::impl + +BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) +: m_Config(Config) +, m_Gc(Gc) +, m_LargeBlobStore(m_Gc) +, m_SmallBlobStore(Gc) +, m_MetadataBlockStore() +{ + ZEN_TRACE_CPU("BuildStore::BuildStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + try + { + std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); + std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); + bool IsNew = !(std::filesystem::exists(BlobLogPath) && std::filesystem::exists(MetaLogPath)); + + if (!IsNew) + { + m_BlobLogFlushPosition = ReadPayloadLog(RwLock::ExclusiveLockScope(m_Lock), BlobLogPath, 0); + m_MetaLogFlushPosition = ReadMetadataLog(RwLock::ExclusiveLockScope(m_Lock), MetaLogPath, 0); + } + m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew); + m_SmallBlobStore.Initialize(Config.RootDirectory, + "blob_cas", + m_Config.SmallBlobBlockStoreMaxBlockSize, + m_Config.SmallBlobBlockStoreAlignement, + IsNew); + m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20); + { + BlockStore::BlockIndexSet KnownBlocks; + for (const BlobEntry& Blob : m_BlobEntries) + { + if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) + { + const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; + KnownBlocks.insert(Metadata.Location.BlockIndex); + } + } + m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + } + + m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite); + m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite); + + m_Gc.AddGcReferencer(*this); + m_Gc.AddGcReferenceLocker(*this); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what()); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + } +} + +BuildStore::~BuildStore() +{ + try + { + ZEN_TRACE_CPU("BuildStore::~BuildStore"); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + Flush(); + m_MetadatalogFile.Close(); + m_PayloadlogFile.Close(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BuildStore() threw exception: {}", Ex.what()); + } +} + +void +BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) +{ + ZEN_TRACE_CPU("BuildStore::PutBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCompressedBinary); + { + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex BlobIndex = It->second; + if (m_BlobEntries[BlobIndex].Payload) + { + return; + } + } + } + + PayloadEntry Entry; + if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize) + { + CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = PayloadEntry::kStandalone}; + } + else + { + CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = 0}; + } + m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Payload) + { + m_PayloadEntries[Blob.Payload] = Entry; + } + else + { + Blob.Payload = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + // we only remove during GC and compact this then... + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } +} + +IoBuffer +BuildStore::GetBlob(const IoHash& BlobHash) +{ + ZEN_TRACE_CPU("BuildStore::GetBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + RwLock::SharedLockScope Lock(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + Blob.LastAccessTime = GcClock::TickCount(); + if (Blob.Payload) + { + const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload]; + const bool IsStandalone = (Entry.Flags & PayloadEntry::kStandalone) != 0; + Lock.ReleaseNow(); + + IoBuffer Chunk; + if (IsStandalone) + { + ZEN_TRACE_CPU("GetLarge"); + Chunk = m_LargeBlobStore.FindChunk(BlobHash); + } + else + { + ZEN_TRACE_CPU("GetSmall"); + Chunk = m_SmallBlobStore.FindChunk(BlobHash); + } + if (Chunk) + { + Chunk.SetContentType(ZenContentType::kCompressedBinary); + return Chunk; + } + else + { + ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block"); + } + } + } + return {}; +} + +std::vector +BuildStore::BlobsExists(std::span BlobHashes) +{ + ZEN_TRACE_CPU("BuildStore::BlobsExists"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector Result; + Result.reserve(BlobHashes.size()); + RwLock::SharedLockScope _(m_Lock); + for (const IoHash& BlobHash : BlobHashes) + { + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + bool HasPayload = !!Blob.Payload; + bool HasMetadata = !!Blob.Metadata; + Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata}); + } + else + { + Result.push_back({}); + } + } + return Result; +} + +void +BuildStore::PutMetadatas(std::span BlobHashes, std::span MetaDatas) +{ + ZEN_TRACE_CPU("BuildStore::PutMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + size_t WriteBlobIndex = 0; + m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span Locations) { + RwLock::ExclusiveLockScope _(m_Lock); + for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++) + { + const IoBuffer& Data = MetaDatas[WriteBlobIndex]; + const IoHash& BlobHash = BlobHashes[WriteBlobIndex]; + const BlockStoreLocation& Location = Locations[LocationIndex]; + + MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0}; + m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Metadata) + { + m_MetadataEntries[Blob.Metadata] = Entry; + } + else + { + Blob.Metadata = MetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + MetadataIndex NewMetadataIndex = MetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } + WriteBlobIndex++; + if (m_TrackedCacheKeys) + { + m_TrackedCacheKeys->insert(BlobHash); + } + } + }); +} + +std::vector +BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* OptionalWorkerPool) +{ + ZEN_TRACE_CPU("BuildStore::GetMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector MetaLocations; + std::vector MetaLocationResultIndexes; + MetaLocations.reserve(BlobHashes.size()); + MetaLocationResultIndexes.reserve(BlobHashes.size()); + tsl::robin_set ReferencedBlocks; + + std::vector Result; + std::vector ResultContentTypes; + Result.resize(BlobHashes.size()); + ResultContentTypes.resize(BlobHashes.size(), ZenContentType::kUnknownContentType); + { + RwLock::SharedLockScope _(m_Lock); + for (size_t Index = 0; Index < BlobHashes.size(); Index++) + { + const IoHash& BlobHash = BlobHashes[Index]; + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlobEntry = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlobEntry.Metadata) + { + const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata]; + MetaLocations.push_back(ExistingMetadataEntry.Location); + MetaLocationResultIndexes.push_back(Index); + ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); + ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; + } + ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::Tick()); + } + } + } + + auto DoOneBlock = [&](std::span ChunkIndexes) { + if (ChunkIndexes.size() < 4) + { + for (size_t ChunkIndex : ChunkIndexes) + { + IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]); + if (Chunk) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = std::move(Chunk); + } + } + return true; + } + return m_MetadataBlockStore.IterateBlock( + MetaLocations, + ChunkIndexes, + [&](size_t ChunkIndex, const void* Data, uint64_t Size) { + if (Data != nullptr) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size); + } + return true; + }, + [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = File.GetChunk(Offset, Size); + return true; + }, + 8u * 1024u); + }; + + if (!MetaLocations.empty()) + { + Latch WorkLatch(1); + + m_MetadataBlockStore.IterateChunks(MetaLocations, [&](uint32_t BlockIndex, std::span ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + { + return DoOneBlock(ChunkIndexes); + } + else + { + ZEN_ASSERT(OptionalWorkerPool != nullptr); + WorkLatch.AddCount(1); + try + { + OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector(ChunkIndexes.begin(), ChunkIndexes.end())]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + try + { + DoOneBlock(ChunkIndexes); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + } + catch (const std::exception& Ex) + { + WorkLatch.CountDown(); + ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + return true; + } + }); + + WorkLatch.CountDown(); + WorkLatch.Wait(); + } + for (size_t Index = 0; Index < Result.size(); Index++) + { + if (Result[Index]) + { + Result[Index].SetContentType(ResultContentTypes[Index]); + } + } + return Result; +} + +void +BuildStore::Flush() +{ + ZEN_TRACE_CPU("BuildStore::Flush"); + try + { + m_LargeBlobStore.Flush(); + m_SmallBlobStore.Flush(); + m_MetadataBlockStore.Flush(false); + + m_PayloadlogFile.Flush(); + m_MetadatalogFile.Flush(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("BuildStore::Flush failed. Reason: {}", Ex.what()); + } +} + +void +BuildStore::CompactState() +{ + ZEN_TRACE_CPU("BuildStore::CompactState"); + + std::vector BlobEntries; + std::vector PayloadEntries; + std::vector MetadataEntries; + + tsl::robin_map BlobLookup; + + RwLock::ExclusiveLockScope _(m_Lock); + const size_t EntryCount = m_BlobLookup.size(); + BlobLookup.reserve(EntryCount); + const size_t PayloadCount = m_PayloadEntries.size(); + PayloadEntries.reserve(PayloadCount); + const size_t MetadataCount = m_MetadataEntries.size(); + MetadataEntries.reserve(MetadataCount); + + for (auto LookupIt : m_BlobLookup) + { + const IoHash& BlobHash = LookupIt.first; + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const BlobIndex WriteBlobIndex(gsl::narrow(BlobEntries.size())); + BlobEntries.push_back(ReadBlobEntry); + BlobEntry& WriteBlobEntry = BlobEntries.back(); + + if (WriteBlobEntry.Payload) + { + const PayloadEntry& ReadPayloadEntry = m_PayloadEntries[ReadBlobEntry.Payload]; + WriteBlobEntry.Payload = PayloadIndex(gsl::narrow(PayloadEntries.size())); + PayloadEntries.push_back(ReadPayloadEntry); + } + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_MetadataEntries[ReadBlobEntry.Metadata]; + WriteBlobEntry.Metadata = MetadataIndex(gsl::narrow(MetadataEntries.size())); + MetadataEntries.push_back(ReadMetadataEntry); + } + + BlobLookup.insert({BlobHash, WriteBlobIndex}); + } + m_BlobEntries.swap(BlobEntries); + m_PayloadEntries.swap(PayloadEntries); + m_MetadataEntries.swap(MetadataEntries); + m_BlobLookup.swap(BlobLookup); +} + +uint64_t +BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); + if (!std::filesystem::is_regular_file(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' payload log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile CasLog; + if (!CasLog.IsValid(LogPath)) + { + std::filesystem::remove(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full payload log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const PayloadDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & PayloadEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + m_BlobLookup.erase(Record.BlobHash); + return; + } + + if (!ValidatePayloadDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid payload entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Payload) + { + const PayloadIndex ExistingPayloadIndex = ExistingBlob.Payload; + m_PayloadEntries[ExistingPayloadIndex] = Record.Entry; + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + ExistingBlob.Payload = NewPayloadIndex; + } + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid payload entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +uint64_t +BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); + if (!std::filesystem::is_regular_file(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' metadata log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile CasLog; + if (!CasLog.IsValid(LogPath)) + { + std::filesystem::remove(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full metadata log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const MetadataDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & MetadataEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + m_BlobLookup.erase(Record.BlobHash); + return; + } + + if (!ValidateMetadataDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid metadata entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Metadata) + { + const MetadataIndex ExistingMetadataIndex = ExistingBlob.Metadata; + m_MetadataEntries[ExistingMetadataIndex] = Record.Entry; + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + ExistingBlob.Metadata = NewMetadataIndex; + } + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid metadata entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +bool +BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) + { + OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.Flags, Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & PayloadEntry::kTombStone) + { + return true; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0 || Entry.Entry.Reserved3 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +bool +BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Location.Size == 0) + { + OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size); + return false; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & MetadataEntry::kTombStone) + { + return true; + } + if (Entry.Entry.ContentType == ZenContentType::kCOUNT) + { + OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +class BuildStoreGcReferenceChecker : public GcReferenceChecker +{ +public: + BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {} + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + + virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); } + + virtual void UpdateLockedState(GcCtx& Ctx) override + { + ZEN_TRACE_CPU("Builds::UpdateLockedState"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + m_References.reserve(m_Store.m_BlobLookup.size()); + for (const auto& It : m_Store.m_BlobLookup) + { + const BuildStore::BlobIndex ExistingBlobIndex = It.second; + if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload) + { + m_References.push_back(It.first); + } + } + FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References); + } + + virtual std::span GetUnusedReferences(GcCtx& Ctx, std::span IoCids) override + { + ZEN_UNUSED(Ctx); + ZEN_TRACE_CPU("Builds::GetUnusedReferences"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + size_t InitialCount = IoCids.size(); + size_t UsedCount = InitialCount; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}", + "buildstore", + UsedCount, + InitialCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + std::span UnusedReferences = KeepUnusedReferences(m_References, IoCids); + UsedCount = IoCids.size() - UnusedReferences.size(); + return UnusedReferences; + } + +private: + BuildStore& m_Store; + std::vector m_References; +}; + +std::string +BuildStore::GetGcName(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string()); +} + +class BuildStoreGcCompator : public GcStoreCompactor +{ + using BlobEntry = BuildStore::BlobEntry; + using PayloadEntry = BuildStore::PayloadEntry; + using MetadataEntry = BuildStore::MetadataEntry; + using MetadataDiskEntry = BuildStore::MetadataDiskEntry; + using BlobIndex = BuildStore::BlobIndex; + using PayloadIndex = BuildStore::PayloadIndex; + using MetadataIndex = BuildStore::MetadataIndex; + +public: + BuildStoreGcCompator(BuildStore& Store, std::vector&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {} + + virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function& ClaimDiskReserveCallback) override + { + ZEN_UNUSED(ClaimDiskReserveCallback); + ZEN_TRACE_CPU("Builds::CompactStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}", + m_Store.m_Config.RootDirectory, + NiceBytes(Stats.RemovedDisk), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + if (!m_RemovedBlobs.empty()) + { + if (Ctx.Settings.CollectSmallObjects) + { + m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique(); }); + auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); }); + + BlockStore::BlockUsageMap BlockUsage; + { + RwLock::SharedLockScope __(m_Store.m_Lock); + + for (auto LookupIt : m_Store.m_BlobLookup) + { + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex]; + + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata]; + + uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex; + uint64_t ChunkSize = RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement); + + if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end()) + { + BlockStore::BlockUsageInfo& Info = BlockUsageIt.value(); + Info.EntryCount++; + Info.DiskUsage += ChunkSize; + } + else + { + BlockUsage.insert_or_assign(BlockIndex, + BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1}); + } + } + } + } + + BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90); + BlockStoreCompactState BlockCompactState; + std::vector BlockCompactStateKeys; + BlockCompactState.IncludeBlocks(BlocksToCompact); + + if (BlocksToCompact.size() > 0) + { + { + RwLock::SharedLockScope ___(m_Store.m_Lock); + for (const auto& Entry : m_Store.m_BlobLookup) + { + BlobIndex Index = Entry.second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location)) + { + BlockCompactStateKeys.push_back(Entry.first); + } + } + } + } + + if (Ctx.Settings.IsDeleteMode) + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + + m_Store.m_MetadataBlockStore.CompactBlocks( + BlockCompactState, + m_Store.m_Config.MetadataBlockStoreAlignement, + [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + std::vector MovedEntries; + MovedEntries.reserve(MovedArray.size()); + RwLock::ExclusiveLockScope _(m_Store.m_Lock); + for (const std::pair& Moved : MovedArray) + { + size_t ChunkIndex = Moved.first; + const IoHash& Key = BlockCompactStateKeys[ChunkIndex]; + + ZEN_ASSERT(m_Store.m_TrackedCacheKeys); + if (m_Store.m_TrackedCacheKeys->contains(Key)) + { + continue; + } + + if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + m_Store.m_MetadataEntries[Meta].Location = Moved.second; + MovedEntries.push_back( + MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); + } + } + } + m_Store.m_MetadatalogFile.Append(MovedEntries); + Stats.RemovedDisk += FreedDiskSpace; + if (Ctx.IsCancelledFlag.load()) + { + return false; + } + return true; + }, + ClaimDiskReserveCallback, + fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory)); + } + else + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + } + } + } + } + } + + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + +private: + BuildStore& m_Store; + const std::vector m_RemovedBlobs; +}; + +GcStoreCompactor* +BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) +{ + ZEN_TRACE_CPU("Builds::RemoveExpiredData"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}", + m_Config.RootDirectory, + Stats.CheckedCount, + Stats.FoundCount, + Stats.DeletedCount, + NiceBytes(Stats.FreedMemory), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + }); + + const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count(); + + std::vector ExpiredBlobs; + { + RwLock::SharedLockScope __(m_Lock); + for (const auto& It : m_BlobLookup) + { + const BlobIndex ReadBlobIndex = It.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + if (AccessTick < ExpireTicks) + { + ExpiredBlobs.push_back(It.first); + } + } + Stats.CheckedCount += m_BlobLookup.size(); + Stats.FoundCount += ExpiredBlobs.size(); + } + + std::vector RemovedBlobs; + if (!ExpiredBlobs.empty()) + { + if (Ctx.Settings.IsDeleteMode) + { + RemovedBlobs.reserve(ExpiredBlobs.size()); + + std::vector RemovedPayloads; + std::vector RemoveMetadatas; + + RwLock::ExclusiveLockScope __(m_Lock); + if (Ctx.IsCancelledFlag.load()) + { + return nullptr; + } + + for (const IoHash& ExpiredBlob : ExpiredBlobs) + { + if (auto It = m_BlobLookup.find(ExpiredBlob); It != m_BlobLookup.end()) + { + const BlobIndex ReadBlobIndex = It->second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + + if (AccessTick < ExpireTicks) + { + if (ReadBlobEntry.Payload) + { + RemovedPayloads.push_back( + PayloadDiskEntry{.Entry = m_PayloadEntries[ReadBlobEntry.Payload], .BlobHash = ExpiredBlob}); + RemovedPayloads.back().Entry.Flags |= PayloadEntry::kTombStone; + m_PayloadEntries[ReadBlobEntry.Payload] = {}; + m_BlobEntries[ReadBlobIndex].Payload = {}; + } + if (ReadBlobEntry.Metadata) + { + RemoveMetadatas.push_back( + MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob}); + RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; + m_MetadataEntries[ReadBlobEntry.Metadata] = {}; + m_BlobEntries[ReadBlobIndex].Metadata = {}; + } + + m_BlobLookup.erase(It); + + RemovedBlobs.push_back(ExpiredBlob); + Stats.DeletedCount++; + } + } + } + if (!RemovedPayloads.empty()) + { + m_PayloadlogFile.Append(RemovedPayloads); + } + if (!RemoveMetadatas.empty()) + { + m_MetadatalogFile.Append(RemoveMetadatas); + } + } + } + + if (!RemovedBlobs.empty()) + { + CompactState(); + } + + return new BuildStoreGcCompator(*this, std::move(RemovedBlobs)); +} + +std::vector +BuildStore::CreateReferenceCheckers(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + return {new BuildStoreGcReferenceChecker(*this)}; +} + +std::vector +BuildStore::CreateReferenceValidators(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + return {}; +} + +std::vector +BuildStore::LockState(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + std::vector Locks; + Locks.emplace_back(RwLock::SharedLockScope(m_Lock)); + return Locks; +} + +/* + ___________ __ + \__ ___/___ _______/ |_ ______ + | |_/ __ \ / ___/\ __\/ ___/ + | |\ ___/ \___ \ | | \___ \ + |____| \___ >____ > |__| /____ > + \/ \/ \/ +*/ + +#if ZEN_WITH_TESTS + +TEST_CASE("BuildStore.Blobs") +{ + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } +} + +namespace blockstore::testing { + IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector>& KeyValues) + { + CbObjectWriter Writer; + Writer.AddHash("rawHash"sv, BlobHash); + Writer.BeginObject("values"); + { + for (const auto& V : KeyValues) + { + Writer.AddString(V.first, V.second); + } + } + Writer.EndObject(); // values + return Writer.Save().GetBuffer().AsIoBuffer(); + }; + +} // namespace blockstore::testing + +TEST_CASE("BuildStore.Metadata") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector BlobHashes; + std::vector MetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I))); + MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); + MetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(BlobHashes, MetaPayloads); + + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + std::vector ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + for (const IoHash& BlobHash : BlobHashes) + { + CHECK(!Store.GetBlob(BlobHash)); + } + } + std::vector CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + for (const auto& MetadataIt : MetadataPayloads) + { + CHECK(!MetadataIt); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } + + std::vector BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK_EQ(IoHash::HashBuffer(MetadataPayload), IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + + BlobMetaPayloads.clear(); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back( + MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } +} + +TEST_CASE("BuildStore.GC") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector CompressedBlobsHashes; + std::vector BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), + .CollectSmallObjects = false, + .IsDeleteMode = false, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash); + } + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() + std::chrono::hours(1), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(!Blob); + } + + std::vector MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(!MetadataPayload); + } + } + } +} + +void +buildstore_forcelink() +{ +} + +#endif + +} // namespace zen diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2be0542db..b64bc26dd 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -226,7 +226,7 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) } std::vector -CasContainerStrategy::InsertChunks(std::span Chunks, std::span ChunkHashes) +CasContainerStrategy::InsertChunks(std::span Chunks, std::span ChunkHashes) { ZEN_MEMSCOPE(GetCasContainerTag()); @@ -323,7 +323,7 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks) } bool -CasContainerStrategy::IterateChunks(std::span ChunkHashes, +CasContainerStrategy::IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit) diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 07e620086..2eb4c233a 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -52,11 +52,11 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore ~CasContainerStrategy(); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); - std::vector InsertChunks(std::span Chunks, std::span ChunkHashes); + std::vector InsertChunks(std::span Chunks, std::span ChunkHashes); IoBuffer FindChunk(const IoHash& ChunkHash); bool HaveChunk(const IoHash& ChunkHash); void FilterChunks(HashKeySet& InOutChunks); - bool IterateChunks(std::span ChunkHashes, + bool IterateChunks(std::span ChunkHashes, const std::function& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 7ac10d613..fe5ae284b 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1081,7 +1081,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size()); { ZEN_TRACE_CPU("GcV2::LockReferencers"); - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceLockers Latch WorkLeft(1); { @@ -1108,7 +1108,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::UpdateLockedState"); // Locking all references checkers so we have a steady state of which references are used - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceCheckers Latch WorkLeft(1); @@ -1739,6 +1739,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons { Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime); Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime); + Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime); Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects; Writer << "IsDeleteMode"sv << Settings.IsDeleteMode; Writer << "SkipCidDelete"sv << Settings.SkipCidDelete; @@ -1940,6 +1941,7 @@ GcScheduler::SchedulerThread() std::chrono::seconds LightweightGcInterval = m_Config.LightweightInterval; std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; + std::chrono::seconds MaxBuildStoreDuration = m_Config.MaxBuildStoreDuration; uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; bool SkipCid = false; GcVersion UseGCVersion = m_Config.UseGCVersion; @@ -1975,6 +1977,10 @@ GcScheduler::SchedulerThread() { MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration; } + if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max()) + { + MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration; + } if (TriggerParams.DiskSizeSoftLimit != 0) { DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit; @@ -2046,6 +2052,8 @@ GcScheduler::SchedulerThread() MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration; GcClock::TimePoint ProjectStoreExpireTime = MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration; + GcClock::TimePoint BuildStoreExpireTime = + MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration; const GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); @@ -2102,6 +2110,10 @@ GcScheduler::SchedulerThread() { ProjectStoreExpireTime = SizeBasedExpireTime; } + if (SizeBasedExpireTime > BuildStoreExpireTime) + { + BuildStoreExpireTime = SizeBasedExpireTime; + } } std::chrono::seconds RemainingTimeUntilGc = @@ -2227,6 +2239,7 @@ GcScheduler::SchedulerThread() bool GcSuccess = CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, + BuildStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, @@ -2333,6 +2346,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time bool GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, @@ -2416,6 +2430,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcSettings Settings = {.CacheExpireTime = CacheExpireTime, .ProjectStoreExpireTime = ProjectStoreExpireTime, + .BuildStoreExpireTime = BuildStoreExpireTime, .CollectSmallObjects = CollectSmallObjects, .IsDeleteMode = Delete, .SkipCidDelete = SkipCid, @@ -2447,6 +2462,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, } SB.Append(fmt::format(" Cache cutoff time: {}\n", Settings.CacheExpireTime)); SB.Append(fmt::format(" Project store cutoff time: {}\n", Settings.ProjectStoreExpireTime)); + SB.Append(fmt::format(" Build store cutoff time: {}\n", Settings.BuildStoreExpireTime)); }; { @@ -2552,6 +2568,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, if (Delete) { GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime); + KeepRangeStart = Min(KeepRangeStart, BuildStoreExpireTime); m_LastGcExpireTime = KeepRangeStart; std::unique_lock Lock(m_GcMutex); m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count()); diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h new file mode 100644 index 000000000..a28dc908b --- /dev/null +++ b/src/zenstore/include/zenstore/accesstime.h @@ -0,0 +1,47 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen { + +// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch +struct AccessTime +{ + explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} + AccessTime& operator=(GcClock::Tick Tick) noexcept + { + SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); + return *this; + } + operator GcClock::Tick() const noexcept + { + return std::chrono::duration_cast(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) + .count(); + } + + AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime& operator=(AccessTime&& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + AccessTime& operator=(const AccessTime& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + +private: + static uint32_t ToSeconds(GcClock::Tick Tick) + { + return gsl::narrow(std::chrono::duration_cast(GcClock::Duration(Tick)).count()); + } + std::atomic_uint32_t SecondsSinceEpoch; +}; + +} // namespace zen diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 97357e5cb..0c72a13aa 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -156,7 +156,7 @@ public: void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback); typedef std::function Locations)> WriteChunksCallback; - void WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback); + void WriteChunks(std::span Datas, uint32_t Alignment, const WriteChunksCallback& Callback); IoBuffer TryGetChunk(const BlockStoreLocation& Location) const; void Flush(bool ForceNewBlock); diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h new file mode 100644 index 000000000..302af5f9c --- /dev/null +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -0,0 +1,186 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include "../compactcas.h" +#include "../filecas.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +struct BuildStoreConfig +{ + std::filesystem::path RootDirectory; + uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024; + uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024; + uint32_t SmallBlobBlockStoreAlignement = 16; + uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024; + uint32_t MetadataBlockStoreAlignement = 8; +}; + +class BuildStore : public GcReferencer, public GcReferenceLocker //, public GcStorage +{ +public: + explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc); + virtual ~BuildStore(); + + void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload); + IoBuffer GetBlob(const IoHash& BlobHashes); + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + std::vector BlobsExists(std::span BlobHashes); + + void PutMetadatas(std::span BlobHashes, std::span MetaDatas); + std::vector GetMetadatas(std::span BlobHashes, WorkerThreadPool* OptionalWorkerPool); + + void Flush(); + +private: + void CompactState(); + + uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + + //////// GcReferencer + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual std::vector CreateReferenceCheckers(GcCtx& Ctx) override; + virtual std::vector CreateReferenceValidators(GcCtx& Ctx) override; + + //////// GcReferenceLocker + virtual std::vector LockState(GcCtx& Ctx) override; + +#pragma pack(push) +#pragma pack(1) + struct PayloadEntry + { + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value + + uint8_t Flags = 0; + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + }; + static_assert(sizeof(PayloadEntry) == 4); + + struct PayloadDiskEntry + { + PayloadEntry Entry; // 4 bytes + IoHash BlobHash; // 20 bytes + }; + static_assert(sizeof(PayloadDiskEntry) == 24); + + struct MetadataEntry + { + BlockStoreLocation Location; // 12 bytes + + ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + uint8_t Flags = 0; // 1 byte + + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + }; + static_assert(sizeof(MetadataEntry) == 16); + + struct MetadataDiskEntry + { + MetadataEntry Entry; // 16 bytes + IoHash BlobHash; // 20 bytes + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + uint8_t Reserved4 = 0; + }; + static_assert(sizeof(MetadataDiskEntry) == 40); + +#pragma pack(pop) + + static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason); + static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason); + + struct PayloadIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + PayloadIndex() = default; + explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const PayloadIndex& Other) const = default; + }; + + struct MetadataIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + MetadataIndex() = default; + explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const MetadataIndex& Other) const = default; + }; + + struct BlobIndex + { + uint32_t Index = std::numeric_limits::max(); + + operator bool() const { return Index != std::numeric_limits::max(); }; + BlobIndex() = default; + explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const BlobIndex& Other) const = default; + }; + + struct BlobEntry + { + PayloadIndex Payload; + MetadataIndex Metadata; + AccessTime LastAccessTime; + }; + static_assert(sizeof(BlobEntry) == 12); + + const BuildStoreConfig m_Config; + GcManager& m_Gc; + + RwLock m_Lock; + + std::vector m_PayloadEntries; + std::vector m_MetadataEntries; + + std::vector m_BlobEntries; + tsl::robin_map m_BlobLookup; + + FileCasStrategy m_LargeBlobStore; + CasContainerStrategy m_SmallBlobStore; + BlockStore m_MetadataBlockStore; + + TCasLogFile m_PayloadlogFile; + TCasLogFile m_MetadatalogFile; + uint64_t m_BlobLogFlushPosition = 0; + uint64_t m_MetaLogFlushPosition = 0; + + std::unique_ptr m_TrackedCacheKeys; + + friend class BuildStoreGcReferenceChecker; + friend class BuildStoreGcReferencePruner; + friend class BuildStoreGcCompator; +}; + +void buildstore_forcelink(); + +} // namespace zen diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 05400c784..5a51718d3 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -5,6 +5,7 @@ #include "cacheshared.h" #include +#include #include #include diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 521c78bb1..ef1b803de 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -72,42 +72,4 @@ struct CacheContentStats bool IsKnownBadBucketName(std::string_view BucketName); bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer); -////////////////////////////////////////////////////////////////////////// - -// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch -struct AccessTime -{ - explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} - AccessTime& operator=(GcClock::Tick Tick) noexcept - { - SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); - return *this; - } - operator GcClock::Tick() const noexcept - { - return std::chrono::duration_cast(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) - .count(); - } - - AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime& operator=(AccessTime&& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - AccessTime& operator=(const AccessTime& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - -private: - static uint32_t ToSeconds(GcClock::Tick Tick) - { - return gsl::narrow(std::chrono::duration_cast(GcClock::Duration(Tick)).count()); - } - std::atomic_uint32_t SecondsSinceEpoch; -}; - } // namespace zen diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 3daae0a93..67aadef71 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -55,6 +55,7 @@ struct GcSettings { GcClock::TimePoint CacheExpireTime = GcClock::Now(); GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now(); + GcClock::TimePoint BuildStoreExpireTime = GcClock::Now(); bool CollectSmallObjects = false; bool IsDeleteMode = false; bool SkipCidDelete = false; @@ -412,6 +413,7 @@ struct GcSchedulerConfig std::chrono::seconds Interval{}; std::chrono::seconds MaxCacheDuration{86400}; std::chrono::seconds MaxProjectStoreDuration{604800}; + std::chrono::seconds MaxBuildStoreDuration{604800}; bool CollectSmallObjects = true; bool Enabled = true; uint64_t DiskReserveSize = 1ul << 28; @@ -496,6 +498,7 @@ public: bool CollectSmallObjects = false; std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max(); std::chrono::seconds MaxProjectStoreDuration = std::chrono::seconds::max(); + std::chrono::seconds MaxBuildStoreDuration = std::chrono::seconds::max(); uint64_t DiskSizeSoftLimit = 0; bool SkipCid = false; bool SkipDelete = false; @@ -528,6 +531,7 @@ private: void SchedulerThread(); bool CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp new file mode 100644 index 000000000..c95215889 --- /dev/null +++ b/src/zenutil/buildstoragecache.cpp @@ -0,0 +1,362 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +class ZenBuildStorageCache : public BuildStorageCache +{ +public: + explicit ZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) + : m_HttpClient(HttpClient) + , m_Stats(Stats) + , m_Namespace(Namespace.empty() ? "none" : Namespace) + , m_Bucket(Bucket.empty() ? "none" : Bucket) + , m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred()) + , m_BackgroundWorkPool(1) + , m_PendingBackgroundWorkCount(1) + , m_CancelBackgroundWork(false) + { + } + + virtual ~ZenBuildStorageCache() + { + try + { + m_CancelBackgroundWork.store(true); + m_PendingBackgroundWorkCount.CountDown(); + m_PendingBackgroundWorkCount.Wait(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~ZenBuildStorageCache() failed with: {}", Ex.what()); + } + } + + void ScheduleBackgroundWork(std::function&& Work) + { + m_PendingBackgroundWorkCount.AddCount(1); + try + { + m_BackgroundWorkPool.ScheduleWork([this, Work = std::move(Work)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::BackgroundWork"); + auto _ = MakeGuard([this]() { m_PendingBackgroundWorkCount.CountDown(); }); + if (!m_CancelBackgroundWork) + { + try + { + Work(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed executing background upload to build cache. Reason: {}", Ex.what()); + } + } + }); + } + catch (const std::exception& Ex) + { + m_PendingBackgroundWorkCount.CountDown(); + ZEN_ERROR("Failed scheduling background upload to build cache. Reason: {}", Ex.what()); + } + } + + virtual void PutBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + const CompositeBuffer& Payload) override + { + ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); + ScheduleBackgroundWork( + [this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBuildBlob"); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::Response CacheResponse = + m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + Payload, + ContentType); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlob"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::KeyValueMap Headers; + if (RangeOffset != 0 || RangeBytes != (uint64_t)-1) + { + Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)}); + } + CreateDirectories(m_TempFolderPath); + HttpClient::Response CacheResponse = + m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + m_TempFolderPath, + Headers); + AddStatistic(CacheResponse); + if (CacheResponse.IsSuccess()) + { + return CacheResponse.ResponsePayload; + } + return {}; + } + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) override + { + ScheduleBackgroundWork([this, + BuildId = Oid(BuildId), + BlobRawHashes = std::vector(BlobHashes.begin(), BlobHashes.end()), + MetaDatas = std::vector(MetaDatas.begin(), MetaDatas.end())]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + const uint64_t BlobCount = BlobRawHashes.size(); + + CbPackage RequestPackage; + std::vector Attachments; + tsl::robin_set AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter RequestWriter; + RequestWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + RequestWriter.AddHash(BlobRawHashes[BlockHashIndex]); + } + RequestWriter.EndArray(); // blobHashes + + RequestWriter.BeginArray("metadatas"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + const IoHash ObjectHash = MetaDatas[BlockHashIndex].GetHash(); + RequestWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(MetaDatas[BlockHashIndex], ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + + RequestWriter.EndArray(); // metadatas + + RequestPackage.SetObject(RequestWriter.Save()); + } + RequestPackage.AddAttachments(Attachments); + + CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage); + + HttpClient::Response CacheResponse = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/putBlobMetadata", m_Namespace, m_Bucket, BuildId), + RpcRequestBuffer, + ZenContentType::kCbPackage); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob metadata to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/getBlobMetadata", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + std::vector Result; + + CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload); + CbObject ResponseObject = ResponsePackage.GetObject(); + + CbArrayView BlobHashArray = ResponseObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView(); + Result.reserve(MetadatasArray.Num()); + auto BlobHashesIt = BlobHashes.begin(); + auto BlobHashArrayIt = begin(BlobHashArray); + auto MetadataArrayIt = begin(MetadatasArray); + while (MetadataArrayIt != end(MetadatasArray)) + { + const IoHash BlobHash = (*BlobHashArrayIt).AsHash(); + while (BlobHash != *BlobHashesIt) + { + ZEN_ASSERT(BlobHashesIt != BlobHashes.end()); + BlobHashesIt++; + } + + ZEN_ASSERT(BlobHash == *BlobHashesIt); + + const IoHash MetaHash = (*MetadataArrayIt).AsAttachment(); + const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash); + ZEN_ASSERT(MetaAttachment); + + CbObject Metadata = MetaAttachment->AsObject(); + Result.emplace_back(std::move(Metadata)); + + BlobHashArrayIt++; + MetadataArrayIt++; + BlobHashesIt++; + } + return Result; + } + return {}; + } + + virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::BlobsExists"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/exists", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + CbObject ResponseObject = LoadCompactBinaryObject(Response.ResponsePayload); + if (!ResponseObject) + { + throw std::runtime_error("BlobExists reponse is invalid, failed to load payload as compact binary object"); + } + CbArrayView BlobsExistsArray = ResponseObject["blobExists"sv].AsArrayView(); + if (!BlobsExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'blobExists' array is missing"); + } + if (BlobsExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error(fmt::format("BlobExists reponse is invalid, 'blobExists' array contains {} entries, expected {}", + BlobsExistsArray.Num(), + BlobHashes.size())); + } + + CbArrayView MetadatasExistsArray = ResponseObject["metadataExists"sv].AsArrayView(); + if (!MetadatasExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'metadataExists' array is missing"); + } + if (MetadatasExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error( + fmt::format("BlobExists reponse is invalid, 'metadataExists' array contains {} entries, expected {}", + MetadatasExistsArray.Num(), + BlobHashes.size())); + } + + std::vector Result; + Result.reserve(BlobHashes.size()); + auto BlobExistsIt = begin(BlobsExistsArray); + auto MetadataExistsIt = begin(MetadatasExistsArray); + while (BlobExistsIt != end(BlobsExistsArray)) + { + ZEN_ASSERT(MetadataExistsIt != end(MetadatasExistsArray)); + + const bool HasBody = (*BlobExistsIt).AsBool(); + const bool HasMetadata = (*MetadataExistsIt).AsBool(); + + Result.push_back({.HasBody = HasBody, .HasMetadata = HasMetadata}); + + BlobExistsIt++; + MetadataExistsIt++; + } + return Result; + } + return {}; + } + +private: + void AddStatistic(const HttpClient::Response& Result) + { + m_Stats.TotalBytesWritten += Result.UploadedBytes; + m_Stats.TotalBytesRead += Result.DownloadedBytes; + m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); + m_Stats.TotalRequestCount++; + } + + HttpClient& m_HttpClient; + BuildStorageCache::Statistics& m_Stats; + const std::string m_Namespace; + const std::string m_Bucket; + const std::filesystem::path m_TempFolderPath; + + WorkerThreadPool m_BackgroundWorkPool; + Latch m_PendingBackgroundWorkCount; + std::atomic m_CancelBackgroundWork; +}; + +std::unique_ptr +CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath) +{ + return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath); +} + +} // namespace zen diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index f3c14edc4..abfc0fb63 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -52,7 +52,7 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) return {}; } std::vector Result; - CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); + CbArrayView Blocks = BlocksObject["blocks"sv].AsArrayView(); Result.reserve(Blocks.Num()); for (CbFieldView BlockView : Blocks) { diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 130fec355..f040e9ece 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,18 +442,19 @@ public: SimulateLatency(0, 0); } - virtual std::vector FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(BuildId), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); for (const std::filesystem::path& MetaDataFile : Content.Files) { IoHash ChunkHash; @@ -467,24 +468,28 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } } - SimulateLatency(0, sizeof(IoHash) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) override { ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); + for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); @@ -495,10 +500,12 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } - SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 2ebd65a00..f8c7c012c 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -54,9 +54,9 @@ public: uint64_t ChunkSize, std::function&& Receiver) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual std::vector FindBlocks(const Oid& BuildId) = 0; - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) = 0; + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; }; diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h new file mode 100644 index 000000000..08c936bf5 --- /dev/null +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include + +namespace zen { + +class HttpClient; + +class BuildStorageCache +{ +public: + struct Statistics + { + std::atomic TotalBytesRead = 0; + std::atomic TotalBytesWritten = 0; + std::atomic TotalRequestCount = 0; + std::atomic TotalRequestTimeUs = 0; + std::atomic TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorageCache() {} + + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) = 0; + virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) = 0; + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; +}; + +std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath); +} // namespace zen diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 758722156..cd28bdcb2 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -27,7 +27,6 @@ public: { ZEN_MEMSCOPE(ELLMTag::Logging); - ZEN_MEMSCOPE(ELLMTag::Logging); std::error_code Ec; if (RotateOnOpen) { diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index d70fd8c00..b6d9e3990 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -49,7 +49,7 @@ public: { throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode)); } - return PayloadToJson("Failed listing builds"sv, ListResult.Response); + return PayloadToCbObject("Failed listing builds"sv, ListResult.Response); } virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override @@ -66,7 +66,7 @@ public: { throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); + return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); } virtual CbObject GetBuild(const Oid& BuildId) override @@ -81,7 +81,7 @@ public: { throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); } virtual void FinalizeBuild(const Oid& BuildId) override @@ -134,7 +134,7 @@ public: GetBuildPartResult.Reason, GetBuildPartResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); } virtual std::vector FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override @@ -289,7 +289,7 @@ public: } } - virtual std::vector FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); @@ -301,10 +301,10 @@ public: { throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode)); } - return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response)); + return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response); } - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) override { ZEN_TRACE_CPU("Jupiter::GetBlockMetadata"); @@ -328,24 +328,7 @@ public: throw std::runtime_error( fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); } - std::vector UnorderedList = - ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response)); - tsl::robin_map BlockDescriptionLookup; - for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) - { - const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; - BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); - } - std::vector SortedBlockDescriptions; - SortedBlockDescriptions.reserve(BlockDescriptionLookup.size()); - for (const IoHash& BlockHash : BlockHashes) - { - if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) - { - SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second])); - } - } - return SortedBlockDescriptions; + return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response); } virtual void PutBuildPartStats(const Oid& BuildId, @@ -373,7 +356,7 @@ public: } private: - static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload) + static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload) { if (Payload.GetContentType() == ZenContentType::kJSON) { -- cgit v1.2.3 From df9bbea4fb1dad9e8c853f14600259a4af12d334 Mon Sep 17 00:00:00 2001 From: zousar <2936246+zousar@users.noreply.github.com> Date: Wed, 26 Mar 2025 16:57:48 -0600 Subject: Descriptive type conversion messages Handling decompression or validation errors with more descriptive messages --- src/zenserver/projectstore/httpprojectstore.cpp | 54 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index f9a13220a..6313fd69e 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -896,27 +896,51 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); - if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || - AcceptType == ZenContentType::kCbObject) + if (DecompressedBuffer) { - CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); - if (!!CbErr) + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) { - m_ProjectStats.BadRequestCount++; - ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE", ProjectId, OplogId, Cid); - return HttpReq.WriteResponse(HttpResponseCode::NotAcceptable, - HttpContentType::kText, - fmt::format("chunk - '{}' WRONGTYPE", Cid)); - } + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } - m_ProjectStats.ChunkHitCount++; - CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); - return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } } else { - Value = DecompressedBuffer; - Value.SetContentType(ZenContentType::kBinary); + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); } } m_ProjectStats.ChunkHitCount++; -- cgit v1.2.3 From a0a0dba13317533f882a85b7f4087588cfa09066 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 14:09:01 +0100 Subject: optional compress of block chunks (#326) - Feature: zenserver: Add command line option `--gc-buildstore-duration-seconds` to control GC life time of build store data - Improvement: ELF and MachO executable files are no longer chunked - Improvement: Compress chunks in blocks that encloses a full file (such as small executables) - Bugfix: Strip path delimiter at end of string in StringToPath --- src/zen/cmds/builds_cmd.cpp | 187 ++++++++++++++++------- src/zencore/filesystem.cpp | 9 +- src/zenhttp/httpclient.cpp | 2 +- src/zenserver/config.cpp | 10 ++ src/zenstore/buildstore/buildstore.cpp | 31 +++- src/zenutil/chunkingcontroller.cpp | 39 ++++- src/zenutil/include/zenutil/chunkingcontroller.h | 4 + 7 files changed, 216 insertions(+), 66 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index b2ad579f1..08d30948b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -82,9 +82,10 @@ namespace { size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize; }; - const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, - .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; - + const ChunksBlockParameters DefaultChunksBlockParams{ + .MaxBlockSize = 32u * 1024u * 1024u, + .MaxChunkEmbedSize = 2u * 1024u * 1024u // DefaultChunkedParams.MaxSize + }; const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u; const double DefaultLatency = 0; // .0010; @@ -92,6 +93,8 @@ namespace { const bool SingleThreaded = false; + const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; + const std::string ZenFolderName = ".zen"; const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); @@ -511,6 +514,7 @@ namespace { uint64_t AcceptedBlockCount = 0; uint64_t AcceptedChunkCount = 0; uint64_t AcceptedByteCount = 0; + uint64_t AcceptedRawByteCount = 0; uint64_t RejectedBlockCount = 0; uint64_t RejectedChunkCount = 0; uint64_t RejectedByteCount = 0; @@ -549,6 +553,7 @@ namespace { uint64_t ChunkCount = 0; uint64_t ChunkByteCount = 0; std::atomic CompressedChunkCount = 0; + std::atomic CompressedChunkRawBytes = 0; std::atomic CompressedChunkBytes = 0; uint64_t CompressChunksElapsedWallTimeUS = 0; @@ -557,6 +562,7 @@ namespace { ChunkCount += Rhs.ChunkCount; ChunkByteCount += Rhs.ChunkByteCount; CompressedChunkCount += Rhs.CompressedChunkCount; + CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes; CompressedChunkBytes += Rhs.CompressedChunkBytes; CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; return *this; @@ -1369,7 +1375,15 @@ namespace { ZEN_ASSERT(false); } uint64_t RawSize = Chunk.GetSize(); - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + // Standalone chunk, not part of a sequence + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid)}; + } + else + { + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + } })); } @@ -1393,13 +1407,24 @@ namespace { { std::span ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); - CompositeBuffer CompressedChunk = - CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + + const uint64_t RawSize = Chunk.GetSize(); + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + CompositeBuffer CompressedChunk = CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } + else + { + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } } return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); }; @@ -1805,7 +1830,6 @@ namespace { const ChunkedContentLookup& Lookup, uint32_t ChunkIndex, const std::filesystem::path& TempFolderPath, - std::atomic& ReadRawBytes, LooseChunksStatistics& LooseChunksStats) { ZEN_TRACE_CPU("CompressChunk"); @@ -1841,7 +1865,7 @@ namespace { CompositeBuffer(SharedBuffer(RawSource)), [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { ZEN_UNUSED(SourceOffset); - ReadRawBytes += SourceSize; + LooseChunksStats.CompressedChunkRawBytes += SourceSize; CompressedFile.Write(RangeBuffer, Offset); LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); }); @@ -2330,16 +2354,8 @@ namespace { std::atomic GeneratedBlockCount = 0; std::atomic GeneratedBlockByteCount = 0; - std::vector CompressLooseChunkOrderIndexes; - std::atomic QueuedPendingInMemoryBlocksForUpload = 0; - // Start upload of any pre-compressed loose chunks - for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) - { - CompressLooseChunkOrderIndexes.push_back(LooseChunkOrderIndex); - } - // Start generation of any non-prebuilt blocks and schedule upload for (const size_t BlockIndex : BlockIndexes) { @@ -2399,12 +2415,10 @@ namespace { } } - std::atomic RawLooseChunkByteCount = 0; - // Start compression of any non-precompressed loose chunks and schedule upload - for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) + for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { - const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; + const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; Work.ScheduleWork( SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, [&, ChunkIndex](std::atomic&) { @@ -2413,20 +2427,15 @@ namespace { ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); FilteredCompressedBytesPerSecond.Start(); - CompositeBuffer Payload = CompressChunk(Path, - Content, - Lookup, - ChunkIndex, - Path / ZenTempChunkFolderName, - RawLooseChunkByteCount, - LooseChunksStats); + CompositeBuffer Payload = + CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName, LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), NiceBytes(Payload.GetSize())); const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; UploadStats.ReadFromDiskBytes += ChunkRawSize; - if (LooseChunksStats.CompressedChunkCount == CompressLooseChunkOrderIndexes.size()) + if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) { FilteredCompressedBytesPerSecond.Stop(); } @@ -2441,7 +2450,7 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkBytes.load()); + FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); @@ -2452,8 +2461,8 @@ namespace { "Uploaded {}/{} ({}/{}) blobs " "({} {}bits/s)", LooseChunksStats.CompressedChunkCount.load(), - CompressLooseChunkOrderIndexes.size(), - NiceBytes(RawLooseChunkByteCount), + LooseChunkOrderIndexes.size(), + NiceBytes(LooseChunksStats.CompressedChunkRawBytes), NiceBytes(TotalLooseChunksSize), NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()), @@ -2582,9 +2591,10 @@ namespace { for (size_t KnownBlockIndex : ReuseBlockIndexes) { std::vector FoundChunkIndexes; - size_t BlockSize = 0; - size_t AdjustedReuseSize = 0; - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + size_t BlockSize = 0; + size_t AdjustedReuseSize = 0; + size_t AdjustedRawReuseSize = 0; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; for (size_t BlockChunkIndex = 0; BlockChunkIndex < KnownBlock.ChunkRawHashes.size(); BlockChunkIndex++) { const IoHash& BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex]; @@ -2597,6 +2607,7 @@ namespace { { FoundChunkIndexes.push_back(ChunkIndex); AdjustedReuseSize += KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + AdjustedRawReuseSize += KnownBlock.ChunkRawLengths[BlockChunkIndex]; } } } @@ -2617,6 +2628,7 @@ namespace { } FindBlocksStats.AcceptedChunkCount += FoundChunkIndexes.size(); FindBlocksStats.AcceptedByteCount += AdjustedReuseSize; + FindBlocksStats.AcceptedRawByteCount += AdjustedRawReuseSize; FindBlocksStats.AcceptedReduntantChunkCount += KnownBlock.ChunkRawHashes.size() - FoundChunkIndexes.size(); FindBlocksStats.AcceptedReduntantByteCount += BlockSize - AdjustedReuseSize; } @@ -3020,9 +3032,10 @@ namespace { } FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size(); - const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 - ? (100.0 * FindBlocksStats.AcceptedByteCount / FindBlocksStats.PotentialChunkByteCount) - : 0.0; + const double AcceptedByteCountPercent = + FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * FindBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; const double AcceptedReduntantByteCountPercent = FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / @@ -3042,7 +3055,7 @@ namespace { NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), FindBlocksStats.AcceptedChunkCount, - NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.AcceptedBlockCount, AcceptedByteCountPercent, @@ -3276,8 +3289,8 @@ namespace { TempLooseChunksStats.CompressedChunkCount.load(), NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, - TempLooseChunksStats.CompressedChunkBytes)), + NiceNum( + GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), TempUploadStats.ChunkCount.load(), NiceBytes(TempUploadStats.ChunksBytes), @@ -3435,6 +3448,7 @@ namespace { "\n AcceptedBlockCount: {}" "\n AcceptedChunkCount: {}" "\n AcceptedByteCount: {}" + "\n AcceptedRawByteCount: {}" "\n RejectedBlockCount: {}" "\n RejectedChunkCount: {}" "\n RejectedByteCount: {}" @@ -3452,6 +3466,7 @@ namespace { FindBlocksStats.AcceptedBlockCount, FindBlocksStats.AcceptedChunkCount, NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.RejectedBlockCount, FindBlocksStats.RejectedChunkCount, NiceBytes(FindBlocksStats.RejectedByteCount), @@ -3627,7 +3642,7 @@ namespace { {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, {"reusedRatio", AcceptedByteCountPercent / 100.0}, {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, - {"reusedBlockByteCount", double(FindBlocksStats.AcceptedByteCount)}, + {"reusedBlockByteCount", double(FindBlocksStats.AcceptedRawByteCount)}, {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, @@ -4192,11 +4207,36 @@ namespace { bool NeedsWrite = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) { - MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock + CompressedBuffer::GetHeaderSizeForNoneEncoder(), - ChunkCompressedSize - CompressedBuffer::GetHeaderSizeForNoneEncoder()); - IoBuffer Decompressed(IoBuffer::Wrap, ChunkMemoryView.GetData(), ChunkMemoryView.GetSize()); - ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize); + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); + ZEN_ASSERT(CompressedChunk); + ZEN_ASSERT(VerifyChunkHash == ChunkHash); + ZEN_ASSERT(VerifyChunkSize == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + OodleCompressor ChunkCompressor; + OodleCompressionLevel ChunkCompressionLevel; + uint64_t ChunkBlockSize; + + bool GetCompressParametersSuccess = + CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize); + ZEN_ASSERT(GetCompressParametersSuccess); + + IoBuffer Decompressed; + if (ChunkCompressionLevel == OodleCompressionLevel::None) + { + MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()); + Decompressed = + IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize()); + } + else + { + Decompressed = CompressedChunk.Decompress().AsIoBuffer(); + } ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { OutOps.WriteOps.push_back( @@ -6208,7 +6248,17 @@ namespace { ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); SetFileReadOnly(LocalFilePath, false); ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); - std::filesystem::rename(LocalFilePath, CacheFilePath); + std::error_code Ec; + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); } ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", @@ -6343,7 +6393,17 @@ namespace { { SetFileReadOnly(FirstTargetFilePath, false); } - std::filesystem::rename(CacheFilePath, FirstTargetFilePath); + std::error_code Ec; + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } @@ -7893,15 +7953,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Result; + std::string StorageDescription; + std::string CacheDescription; + if (!m_BuildsUrl.empty()) { ParseAuthOptions(); Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); - ZEN_CONSOLE("Using cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); + StorageDescription = fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + m_BuildsUrl, + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); Result.StorageName = ZEN_CLOUD_STORAGE; @@ -7909,7 +7972,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) else if (!m_StoragePath.empty()) { std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); - ZEN_CONSOLE("Using folder {}", StoragePath); + StorageDescription = fmt::format("folder {}", StoragePath); Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); } @@ -7930,12 +7993,24 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); + CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(" {}.", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" {}.", m_Bucket); + } } else { Result.CacheHttp.reset(); } } + ZEN_CONSOLE("Remote: {}.{}", + StorageDescription, + CacheDescription.empty() ? std::string("") : fmt::format(" Cache: {}", CacheDescription)); return Result; }; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 05e2bf049..8a369f02e 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -2046,14 +2046,17 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) std::filesystem::path StringToPath(const std::string_view& Path) { + std::string_view UnquotedPath = Path; + if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') { - return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred(); + UnquotedPath = Path.substr(1, Path.length() - 2); } - else + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) { - return std::filesystem::path(Path).make_preferred(); + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); } + return std::filesystem::path(UnquotedPath).make_preferred(); } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index fe5232d89..f3baf37ce 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -325,7 +325,7 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT && HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED) { - ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + ZEN_WARN("HttpClient client failure (session: {}): {}", SessionId, HttpResponse); } // Client side failure code diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 0da98210c..d3af0c6a6 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -495,6 +495,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); ////// security LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); @@ -961,6 +964,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), ""); + options.add_option("gc", + "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + options.add_option("gc", "", "disk-reserve-size", diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 8674aab75..eb36be049 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -340,7 +340,7 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; } - ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::Tick()); + ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount()); } } } @@ -543,7 +543,17 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys if (Record.Entry.Flags & PayloadEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation - m_BlobLookup.erase(Record.BlobHash); + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Metadata) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Payload = {}; + } + } return; } @@ -575,7 +585,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys m_PayloadEntries.push_back(Record.Entry); const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); - m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); } }, @@ -635,7 +645,18 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy if (Record.Entry.Flags & MetadataEntry::kTombStone) { // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation - m_BlobLookup.erase(Record.BlobHash); + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Payload) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Metadata = {}; + } + } return; } @@ -667,7 +688,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy m_MetadataEntries.push_back(Record.Entry); const BlobIndex NewBlobIndex(gsl::narrow(m_BlobEntries.size())); - m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::Tick())}); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); } }, diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index 2a7057a46..a5ebce193 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -41,9 +41,13 @@ class BasicChunkingController : public ChunkingController { public: BasicChunkingController(std::span ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) : m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end()) + , m_ExcludeElfFiles(ExcludeElfFiles) + , m_ExcludeMachOFiles(ExcludeMachOFiles) , m_ChunkFileSizeLimit(ChunkFileSizeLimit) , m_ChunkingParams(ChunkingParams) { @@ -51,6 +55,8 @@ public: BasicChunkingController(CbObjectView Parameters) : m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView())) + , m_ExcludeElfFiles(Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles)) + , m_ExcludeMachOFiles(Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles)) , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) { @@ -73,6 +79,25 @@ public: } BasicFile Buffer(InputPath, BasicFile::Mode::kRead); + if (m_ExcludeElfFiles && Buffer.FileSize() > 4) + { + uint32_t ElfCheck = 0; + Buffer.Read(&ElfCheck, 4, 0); + if (ElfCheck == 0x464c457f) + { + return false; + } + } + if (m_ExcludeMachOFiles && Buffer.FileSize() > 4) + { + uint32_t MachOCheck = 0; + Buffer.Read(&MachOCheck, 4, 0); + if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe)) + { + return false; + } + } + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag); return true; } @@ -90,6 +115,10 @@ public: } } Writer.EndArray(); // ChunkExcludeExtensions + + Writer.AddBool("ExcludeElfFiles"sv, m_ExcludeElfFiles); + Writer.AddBool("ExcludeMachOFiles"sv, m_ExcludeMachOFiles); + Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); Writer.BeginObject("ChunkingParams"sv); { @@ -106,6 +135,8 @@ public: protected: const std::vector m_ChunkExcludeExtensions; + const bool m_ExcludeElfFiles = false; + const bool m_ExcludeMachOFiles = false; const uint64_t m_ChunkFileSizeLimit; const ChunkedParams m_ChunkingParams; }; @@ -230,10 +261,16 @@ protected: std::unique_ptr CreateBasicChunkingController(std::span ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) { - return std::make_unique(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams); + return std::make_unique(ExcludeExtensions, + ExcludeElfFiles, + ExcludeMachOFiles, + ChunkFileSizeLimit, + ChunkingParams); } std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters) diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 246f4498a..970917fb0 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -12,6 +12,8 @@ namespace zen { const std::vector DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -40,6 +42,8 @@ public: std::unique_ptr CreateBasicChunkingController( std::span ExcludeExtensions = DefaultChunkingExcludeExtensions, + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, const ChunkedParams& ChunkingParams = DefaultChunkedParams); std::unique_ptr CreateBasicChunkingController(CbObjectView Parameters); -- cgit v1.2.3 From 013ac818cd09c1d31bf9411e00b2bbbf02defa3f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 27 Mar 2025 16:08:47 +0100 Subject: build cache prime (#327) - Feature: zen `--boost-workers` option to builds `upload`, `download` and `validate-part` that will increase the number of worker threads, may cause computer to be less responsive - Feature: zen `--cache-prime-only` that uploads referenced data from a part to `--zen-cache-host` if it is not already present. Target folder will be untouched. --- src/zen/cmds/builds_cmd.cpp | 1087 ++++++++++++++--------- src/zen/cmds/builds_cmd.h | 6 +- src/zenutil/buildstoragecache.cpp | 59 +- src/zenutil/include/zenutil/buildstoragecache.h | 7 +- src/zenutil/include/zenutil/workerpools.h | 3 + src/zenutil/workerpools.cpp | 14 +- 6 files changed, 722 insertions(+), 454 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 08d30948b..3a54de935 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -91,7 +91,16 @@ namespace { const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const bool SingleThreaded = false; + const bool SingleThreaded = false; + bool BoostWorkerThreads = false; + + WorkerThreadPool& GetIOWorkerPool() + { + return SingleThreaded ? GetSyncWorkerPool() + : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst) + : GetMediumWorkerPool(EWorkloadType::Burst); + } + WorkerThreadPool& GetNetworkPool() { return SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); } const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; @@ -438,7 +447,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [](bool, std::ptrdiff_t) {}, AbortFlag); @@ -452,7 +461,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent FolderContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, ChunkController, @@ -1563,8 +1572,8 @@ namespace { BlockAttachments.size() - VerifyBlockDescriptions.size())); } - WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1946,8 +1955,8 @@ namespace { RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& UploadBlocksPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& GenerateBlobsPool = GetIOWorkerPool(); + WorkerThreadPool& UploadBlocksPool = GetNetworkPool(); FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; @@ -2147,8 +2156,8 @@ namespace { { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& ReadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& UploadChunkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& ReadChunkPool = GetIOWorkerPool(); + WorkerThreadPool& UploadChunkPool = GetNetworkPool(); FilteredRate FilteredGenerateBlockBytesPerSecond; FilteredRate FilteredCompressedBytesPerSecond; @@ -2363,7 +2372,7 @@ namespace { if (!AbortFlag) { Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, + ReadChunkPool, [&, BlockIndex](std::atomic&) { if (!AbortFlag) { @@ -2420,7 +2429,7 @@ namespace { { const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : ReadChunkPool, + ReadChunkPool, [&, ChunkIndex](std::atomic&) { if (!AbortFlag) { @@ -2697,54 +2706,52 @@ namespace { FindBlocksStatistics FindBlocksStats; - std::future PrepBuildResultFuture = - GetSmallWorkerPool(EWorkloadType::Burst) - .EnqueueTask(std::packaged_task{ - [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { - ZEN_TRACE_CPU("PrepareBuild"); + std::future PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task{ + [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + ZEN_TRACE_CPU("PrepareBuild"); - PrepareBuildResult Result; - Stopwatch Timer; - if (CreateBuild) - { - ZEN_TRACE_CPU("CreateBuild"); + PrepareBuildResult Result; + Stopwatch Timer; + if (CreateBuild) + { + ZEN_TRACE_CPU("CreateBuild"); - Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); - Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); - Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); - Result.PayloadSize = MetaData.GetSize(); - } - else - { - ZEN_TRACE_CPU("PutBuild"); - Stopwatch GetBuildTimer; - CbObject Build = Storage.BuildStorage->GetBuild(BuildId); - Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); - Result.PayloadSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - else if (AllowMultiparts) - { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", - NiceBytes(Result.PreferredMultipartChunkSize)); - } - } + Stopwatch PutBuildTimer; + CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); + Result.PayloadSize = MetaData.GetSize(); + } + else + { + ZEN_TRACE_CPU("PutBuild"); + Stopwatch GetBuildTimer; + CbObject Build = Storage.BuildStorage->GetBuild(BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); + } + } - if (!IgnoreExistingBlocks) - { - ZEN_TRACE_CPU("FindBlocks"); - Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); - FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); - FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); - Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - return Result; - }}); + if (!IgnoreExistingBlocks) + { + ZEN_TRACE_CPU("FindBlocks"); + Stopwatch KnownBlocksTimer; + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); + FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; + }}); ChunkedFolderContent LocalContent; @@ -2852,7 +2859,7 @@ namespace { } return true; }, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); @@ -2911,7 +2918,7 @@ namespace { FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, *ChunkController, @@ -3663,7 +3670,7 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& VerifyPool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); @@ -4712,6 +4719,7 @@ namespace { const std::vector& LooseChunkHashes, bool AllowPartialBlockRequests, bool WipeTargetFolder, + bool PrimeCacheOnly, FolderContent& OutLocalFolderState, DiskStatistics& DiskStats, CacheMappingStatistics& CacheMappingStats, @@ -4721,7 +4729,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder"); - ZEN_UNUSED(WipeTargetFolder); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); Stopwatch IndexTimer; @@ -4741,6 +4749,7 @@ namespace { tsl::robin_map CachedChunkHashesFound; tsl::robin_map CachedSequenceHashesFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); @@ -4786,6 +4795,7 @@ namespace { } tsl::robin_map CachedBlocksFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); @@ -4831,8 +4841,10 @@ namespace { } std::vector LocalPathIndexesMatchingSequenceIndexes; - // Pick up all whole files we can use from current local state + + if (!PrimeCacheOnly) { + // Pick up all whole files we can use from current local state ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) @@ -4870,6 +4882,15 @@ namespace { } } } + else + { + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + } + } // Pick up all chunks in current local state struct CacheCopyData { @@ -4887,6 +4908,7 @@ namespace { tsl::robin_map RawHashToCacheCopyDataIndex; std::vector CacheCopyDatas; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); @@ -5004,8 +5026,8 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; - WorkerThreadPool& NetworkPool = SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); - WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -5097,191 +5119,205 @@ namespace { const std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); if (!BlockChunkIndexNeeded.empty()) { - bool UsingCachedBlock = false; - if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + if (PrimeCacheOnly) { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); - + TotalRequestCount++; TotalPartWriteCount++; - std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) + FullBlockWorks.push_back(BlockIndex); + } + else + { + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) { - CachedChunkBlockIndexes.push_back(BlockIndex); - UsingCachedBlock = true; + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); + + TotalPartWriteCount++; + + std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + if (std::filesystem::exists(BlockPath)) + { + CachedChunkBlockIndexes.push_back(BlockIndex); + UsingCachedBlock = true; + } } - } - if (!UsingCachedBlock) - { - bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); - bool CanDoPartialBlockDownload = - (BlockDescription.HeaderSize > 0) && - (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + if (!UsingCachedBlock) { - std::vector BlockRanges; + bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = + (BlockDescription.HeaderSize > 0) && + (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); + if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + { + std::vector BlockRanges; - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); - uint32_t NeedBlockChunkIndexOffset = 0; - uint32_t ChunkBlockIndex = 0; - uint32_t CurrentOffset = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + uint32_t NeedBlockChunkIndexOffset = 0; + uint32_t ChunkBlockIndex = 0; + uint32_t CurrentOffset = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); - const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(CurrentOffset)); + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); - BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; - while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && - ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) - { - const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && + ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) { - if (NextRange.RangeLength > 0) + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + NextRange = {.BlockIndex = BlockIndex}; + } + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - BlockRanges.push_back(NextRange); - NextRange = {.BlockIndex = BlockIndex}; + if (NextRange.RangeLength == 0) + { + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; } - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - } - else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) - { - if (NextRange.RangeLength == 0) + else { - NextRange.RangeStart = CurrentOffset; - NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + ZEN_ASSERT(false); } - NextRange.RangeLength += ChunkCompressedLength; - NextRange.ChunkBlockIndexCount++; - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - NeedBlockChunkIndexOffset++; } - else + if (NextRange.RangeLength > 0) { - ZEN_ASSERT(false); + BlockRanges.push_back(NextRange); } - } - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - } - ZEN_ASSERT(!BlockRanges.empty()); + ZEN_ASSERT(!BlockRanges.empty()); - std::vector CollapsedBlockRanges; - auto It = BlockRanges.begin(); - CollapsedBlockRanges.push_back(*It++); - while (It != BlockRanges.end()) - { - BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); - uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); - uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; - if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out - { - LastRange.ChunkBlockIndexCount = - (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; - LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; - } - else + std::vector CollapsedBlockRanges; + auto It = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*It++); + while (It != BlockRanges.end()) { - CollapsedBlockRanges.push_back(*It); + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; + if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + { + LastRange.ChunkBlockIndexCount = + (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + } + else + { + CollapsedBlockRanges.push_back(*It); + } + ++It; } - ++It; - } - - const std::uint64_t WantedSize = std::accumulate( - CollapsedBlockRanges.begin(), - CollapsedBlockRanges.end(), - uint64_t(0), - [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - ZEN_ASSERT(WantedSize <= TotalBlockSize); - if (WantedSize > ((TotalBlockSize * 95) / 100)) - { - ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize)); - TotalRequestCount++; - TotalPartWriteCount++; - - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) - { - ZEN_CONSOLE_VERBOSE("Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) - { - ZEN_CONSOLE_VERBOSE("Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + const std::uint64_t WantedSize = std::accumulate( + CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + ZEN_ASSERT(WantedSize <= TotalBlockSize); + if (WantedSize > ((TotalBlockSize * 95) / 100)) + { + ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + TotalRequestCount++; + TotalPartWriteCount++; - FullBlockWorks.push_back(BlockIndex); - } - else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) - { - ZEN_CONSOLE_VERBOSE("Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - TotalRequestCount++; - TotalPartWriteCount++; + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else + { + if (WantedSize > ((TotalBlockSize * 5) / 10)) + { + ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", + (WantedSize * 100) / TotalBlockSize, + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + } + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); - FullBlockWorks.push_back(BlockIndex); + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); + } } - else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) + else { - ZEN_CONSOLE_VERBOSE("Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); TotalRequestCount++; TotalPartWriteCount++; FullBlockWorks.push_back(BlockIndex); } - else - { - if (WantedSize > ((TotalBlockSize * 5) / 10)) - { - ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", - (WantedSize * 100) / TotalBlockSize, - NiceBytes(WantedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - CollapsedBlockRanges.size()); - } - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); - - BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); - } - } - else - { - TotalRequestCount++; - TotalPartWriteCount++; - - FullBlockWorks.push_back(BlockIndex); } } } @@ -5363,6 +5399,12 @@ namespace { std::move(LooseChunkHashWork.ChunkTargetPtrs); const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic&) mutable { @@ -5370,6 +5412,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; + if (!PrimeCacheOnly) { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); @@ -5481,6 +5524,7 @@ namespace { NetworkPool, [&Path, &Storage, + PrimeCacheOnly, BuildId, &RemoteContent, &RemoteLookup, @@ -5503,54 +5547,67 @@ namespace { &DownloadStats](std::atomic&) mutable { if (!AbortFlag) { - FilteredDownloadedBytesPerSecond.Start(); const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(*Storage.BuildStorage, - Path / ZenTempDownloadFolderName, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - if (!AbortFlag) - { - AsyncWriteDownloadedChunk( - Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); - } - }); + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - else + if (!BuildBlob) { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - IoBuffer BuildBlob; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash)) + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { - BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + *Storage.BuildStorage, + Path / ZenTempDownloadFolderName, + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); } - if (!BuildBlob) + else { + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); if (BuildBlob && Storage.BuildCacheStorage) { @@ -5560,34 +5617,37 @@ namespace { BuildBlob.GetContentType(), CompositeBuffer(SharedBuffer(BuildBlob))); } - } - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - if (!AbortFlag) - { - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BuildBlob) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(Path, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats); } } } @@ -5602,6 +5662,7 @@ namespace { for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -5775,6 +5836,7 @@ namespace { for (uint32_t BlockIndex : CachedChunkBlockIndexes) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -5829,6 +5891,7 @@ namespace { for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -6000,6 +6063,13 @@ namespace { { break; } + + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( NetworkPool, [&, BlockIndex](std::atomic&) { @@ -6011,8 +6081,10 @@ namespace { FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer; - if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + IoBuffer BlockBuffer; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) { BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); } @@ -6042,119 +6114,123 @@ namespace { FilteredDownloadedBytesPerSecond.Stop(); } - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!PrimeCacheOnly) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else - { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + std::filesystem::remove(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + std::filesystem::remove(BlockChunkPath); + } - WritePartsComplete++; + WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } } @@ -6176,19 +6252,24 @@ namespace { (DownloadStats.RequestsCompleteCount == TotalRequestCount) ? "" : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string Details = fmt::format("{}/{} ({}{}) downloaded. {}/{} ({}B/s) written.", - DownloadStats.RequestsCompleteCount.load(), - TotalRequestCount, - NiceBytes(DownloadedBytes), - DownloadRateString, - NiceBytes(DiskStats.WriteByteCount.load()), - NiceBytes(BytesToWrite), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); - WriteProgressBar.UpdateState({.Task = "Writing chunks ", - .Details = Details, - .TotalCount = gsl::narrow(BytesToWrite), - .RemainingCount = gsl::narrow(BytesToWrite - DiskStats.WriteByteCount.load())}, - false); + std::string WriteDetails = PrimeCacheOnly ? "" + : fmt::format(" {}/{} ({}B/s) written.", + NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", + DownloadStats.RequestsCompleteCount.load(), + TotalRequestCount, + NiceBytes(DownloadedBytes), + DownloadRateString, + WriteDetails); + WriteProgressBar.UpdateState( + {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ", + .Details = Details, + .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, + .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) + : (BytesToWrite - DiskStats.WriteByteCount.load())}, + false); }); } @@ -6202,21 +6283,24 @@ namespace { WriteProgressBar.Finish(); - uint32_t RawSequencesMissingWriteCount = 0; - for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) + if (!PrimeCacheOnly) { - const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; - if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + uint32_t RawSequencesMissingWriteCount = 0; + for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) { - RawSequencesMissingWriteCount++; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; - ZEN_ASSERT(!IncompletePath.empty()); - const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; + if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + { + RawSequencesMissingWriteCount++; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; + ZEN_ASSERT(!IncompletePath.empty()); + const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + } } + ZEN_ASSERT(RawSequencesMissingWriteCount == 0); } - ZEN_ASSERT(RawSequencesMissingWriteCount == 0); const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); @@ -6234,6 +6318,11 @@ namespace { WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } + if (PrimeCacheOnly) + { + return; + } + // Move all files we will reuse to cache folder // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place if (!LocalPathIndexesMatchingSequenceIndexes.empty()) @@ -6319,7 +6408,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); Stopwatch Timer; - WorkerThreadPool& WritePool = SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst); + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar RebuildProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -6930,7 +7019,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); @@ -7003,7 +7092,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, UpdatedContent, ChunkController, @@ -7080,7 +7169,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( ChunkingStats, - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, CurrentLocalFolderContent, ChunkController, @@ -7122,10 +7211,13 @@ namespace { bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, - bool PostDownloadVerify) + bool PostDownloadVerify, + bool PrimeCacheOnly) { ZEN_TRACE_CPU("DownloadFolder"); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + Stopwatch DownloadTimer; const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; @@ -7150,27 +7242,30 @@ namespace { ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - if (!ChunkController) - { - ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); - ChunkController = CreateBasicChunkingController(); - } - + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; - if (std::filesystem::is_directory(Path)) + if (!PrimeCacheOnly) { - if (!WipeTargetFolder) + if (std::filesystem::is_directory(Path)) + { + if (!WipeTargetFolder) + { + if (!ChunkController) + { + ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ChunkController = CreateBasicChunkingController(); + } + + LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + } + } + else { - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + CreateDirectories(Path); } } - else - { - CreateDirectories(Path); - } if (AbortFlag) { return; @@ -7251,6 +7346,7 @@ namespace { LooseChunkHashes, AllowPartialBlockRequests, WipeTargetFolder, + PrimeCacheOnly, LocalFolderState, DiskStats, CacheMappingStats, @@ -7260,20 +7356,23 @@ namespace { if (!AbortFlag) { - VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); + if (!PrimeCacheOnly) + { + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); - Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + CreateDirectories((Path / ZenStateFilePath).parent_path()); + TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 + } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + DownloadStats.DownloadedPartialBlockCount.load(); const uint64_t DownloadByteCount = DownloadStats.DownloadedChunkByteCount.load() + @@ -7307,6 +7406,23 @@ namespace { NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); } } + if (PrimeCacheOnly) + { + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + if (Remaining == 0) + { + ZEN_CONSOLE("Build cache upload complete"); + } + else + { + ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + } + return !AbortFlag; + }); + } + } if (CleanDirectory(ZenTempFolder, {})) { std::filesystem::remove(ZenTempFolder); @@ -7584,6 +7700,15 @@ BuildsCommand::BuildsCommand() Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), ""); }; + auto AddWorkerOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "boost-workers", + "Increase the number of worker threads - may cause computer to less responsive", + cxxopts::value(m_BoostWorkerThreads), + ""); + }; + m_Options.add_option("", "v", "verb", @@ -7618,6 +7743,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); AddCacheOptions(m_UploadOptions); + AddWorkerOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7682,6 +7808,15 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); AddCacheOptions(m_DownloadOptions); + + m_DownloadOptions.add_option("cache", + "", + "cache-prime-only", + "Only download blobs missing in cache and upload to cache", + cxxopts::value(m_PrimeCacheOnly), + ""); + + AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7719,6 +7854,7 @@ BuildsCommand::BuildsCommand() m_DownloadOptions.positional_help("local-path build-id build-part-name"); AddOutputOptions(m_DiffOptions); + AddWorkerOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), ""); @@ -7735,6 +7871,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); AddCacheOptions(m_TestOptions); + AddWorkerOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_TestOptions.add_option("", @@ -7765,6 +7902,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); + AddWorkerOptions(m_ValidateBuildPartOptions); m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_ValidateBuildPartOptions.add_option("", "", @@ -7786,6 +7924,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); AddCacheOptions(m_MultiTestDownloadOptions); + AddWorkerOptions(m_MultiTestDownloadOptions); m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), ""); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), ""); @@ -7991,9 +8130,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .RetryCount = 0}); if (Result.CacheHttp->Get("/health").IsSuccess()) { - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, m_Bucket, TempPath / "zencache"); - CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); + Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + m_PrimeCacheOnly); + CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); if (!m_Namespace.empty()) { CacheDescription += fmt::format(" {}.", m_Namespace); @@ -8014,6 +8157,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return Result; }; + BoostWorkerThreads = m_BoostWorkerThreads; + try { if (SubOption == &m_ListOptions) @@ -8243,6 +8388,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + if (m_PostDownloadVerify && m_PrimeCacheOnly) + { + throw zen::OptionParseException( + fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", m_DownloadOptions.help())); + } + + if (m_Clean && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled"); + } + + if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + } + std::vector BuildPartIds; for (const std::string& BuildPartId : m_BuildPartIds) { @@ -8267,9 +8428,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_BuildPartNames, Path, m_AllowMultiparts, - m_AllowPartialBlockRequests, + m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, - m_PostDownloadVerify); + m_PostDownloadVerify, + m_PrimeCacheOnly); if (false) { @@ -8350,7 +8512,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); @@ -8441,7 +8604,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + true, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -8453,7 +8625,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -8504,7 +8685,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SourceSize > 256) { Work.ScheduleWork( - SingleThreaded ? GetSyncWorkerPool() : GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), [SourceSize, FilePath](std::atomic&) { if (!AbortFlag) { @@ -8557,7 +8738,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -8595,7 +8785,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8611,7 +8810,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8627,7 +8827,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index b5af236e1..46257a567 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -27,8 +27,9 @@ private: std::string m_SystemRootDir; - bool m_PlainProgress = false; - bool m_Verbose = false; + bool m_PlainProgress = false; + bool m_Verbose = false; + bool m_BoostWorkerThreads = false; // cloud builds std::string m_BuildsUrl; @@ -42,6 +43,7 @@ private: // cache std::string m_ZenCacheHost; + bool m_PrimeCacheOnly = false; std::string m_BuildId; bool m_CreateBuild = false; diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp index c95215889..f273ac699 100644 --- a/src/zenutil/buildstoragecache.cpp +++ b/src/zenutil/buildstoragecache.cpp @@ -11,6 +11,7 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -27,13 +28,16 @@ public: BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath) + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) : m_HttpClient(HttpClient) , m_Stats(Stats) , m_Namespace(Namespace.empty() ? "none" : Namespace) , m_Bucket(Bucket.empty() ? "none" : Bucket) , m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred()) - , m_BackgroundWorkPool(1) + , m_BoostBackgroundThreadCount(BoostBackgroundThreadCount) + , m_BackgroundWorkPool(m_BoostBackgroundThreadCount ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)) , m_PendingBackgroundWorkCount(1) , m_CancelBackgroundWork(false) { @@ -44,8 +48,11 @@ public: try { m_CancelBackgroundWork.store(true); - m_PendingBackgroundWorkCount.CountDown(); - m_PendingBackgroundWorkCount.Wait(); + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + m_PendingBackgroundWorkCount.Wait(); + } } catch (const std::exception& Ex) { @@ -86,6 +93,7 @@ public: ZenContentType ContentType, const CompositeBuffer& Payload) override { + ZEN_ASSERT(!IsFlushed); ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); ScheduleBackgroundWork( [this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() { @@ -132,6 +140,7 @@ public: virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) override { + ZEN_ASSERT(!IsFlushed); ScheduleBackgroundWork([this, BuildId = Oid(BuildId), BlobRawHashes = std::vector(BlobHashes.begin(), BlobHashes.end()), @@ -329,6 +338,39 @@ public: return {}; } + virtual void Flush(int32_t UpdateInteralMS, std::function&& UpdateCallback) override + { + if (IsFlushed) + { + return; + } + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + IsFlushed = true; + } + if (m_PendingBackgroundWorkCount.Wait(100)) + { + return; + } + while (true) + { + intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining(); + if (UpdateCallback(Remaining)) + { + if (m_PendingBackgroundWorkCount.Wait(UpdateInteralMS)) + { + UpdateCallback(0); + return; + } + } + else + { + m_CancelBackgroundWork.store(true); + } + } + } + private: void AddStatistic(const HttpClient::Response& Result) { @@ -343,8 +385,10 @@ private: const std::string m_Namespace; const std::string m_Bucket; const std::filesystem::path m_TempFolderPath; + const bool m_BoostBackgroundThreadCount; + bool IsFlushed = false; - WorkerThreadPool m_BackgroundWorkPool; + WorkerThreadPool& m_BackgroundWorkPool; Latch m_PendingBackgroundWorkCount; std::atomic m_CancelBackgroundWork; }; @@ -354,9 +398,10 @@ CreateZenBuildStorageCache(HttpClient& HttpClient, BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath) + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) { - return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath); + return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount); } } // namespace zen diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h index 08c936bf5..cab35328d 100644 --- a/src/zenutil/include/zenutil/buildstoragecache.h +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -42,11 +42,16 @@ public: }; virtual std::vector BlobsExists(const Oid& BuildId, std::span BlobHashes) = 0; + + virtual void Flush( + int32_t UpdateInteralMS, + std::function&& UpdateCallback = [](intptr_t) { return true; }) = 0; }; std::unique_ptr CreateZenBuildStorageCache(HttpClient& HttpClient, BuildStorageCache::Statistics& Stats, std::string_view Namespace, std::string_view Bucket, - const std::filesystem::path& TempFolderPath); + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount); } // namespace zen diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h index 9683ad720..df2033bca 100644 --- a/src/zenutil/include/zenutil/workerpools.h +++ b/src/zenutil/include/zenutil/workerpools.h @@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType); // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType); +// Worker pool with minimum number of worker threads, but at least one thread +WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType); + // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread // This is useful for debugging when multiple async thread can make stepping in debugger complicated WorkerThreadPool& GetSyncWorkerPool(); diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp index e3165e838..797034978 100644 --- a/src/zenutil/workerpools.cpp +++ b/src/zenutil/workerpools.cpp @@ -11,9 +11,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace { - const int LargeWorkerThreadPoolTreadCount = gsl::narrow(std::thread::hardware_concurrency()); + const int LargeWorkerThreadPoolTreadCount = gsl::narrow(Max(std::thread::hardware_concurrency() - 1u, 2u)); const int MediumWorkerThreadPoolTreadCount = gsl::narrow(Max((std::thread::hardware_concurrency() / 4u), 2u)); const int SmallWorkerThreadPoolTreadCount = gsl::narrow(Max((std::thread::hardware_concurrency() / 8u), 1u)); + const int TinyWorkerThreadPoolTreadCount = 1; static bool IsShutDown = false; @@ -35,6 +36,9 @@ namespace { WorkerPool BurstSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(burst)"}; WorkerPool BackgroundSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(bkg)"}; + WorkerPool BurstTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(burst)"}; + WorkerPool BackgroundTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(bkg)"}; + WorkerPool SyncWorkerPool = {.TreadCount = 0, .Name = "SyncThreadPool"}; WorkerThreadPool& EnsurePoolPtr(WorkerPool& Pool) @@ -74,6 +78,12 @@ GetSmallWorkerPool(EWorkloadType WorkloadType) return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstSmallWorkerPool : BackgroundSmallWorkerPool); } +WorkerThreadPool& +GetTinyWorkerPool(EWorkloadType WorkloadType) +{ + return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstTinyWorkerPool : BackgroundTinyWorkerPool); +} + WorkerThreadPool& GetSyncWorkerPool() { @@ -91,6 +101,8 @@ ShutdownWorkerPools() BackgroundMediumWorkerPool.Pool.reset(); BurstSmallWorkerPool.Pool.reset(); BackgroundSmallWorkerPool.Pool.reset(); + BurstTinyWorkerPool.Pool.reset(); + BackgroundTinyWorkerPool.Pool.reset(); SyncWorkerPool.Pool.reset(); } } // namespace zen -- cgit v1.2.3 From 8f192ab154ff9de41d4c063138478400fe2aef24 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 28 Mar 2025 14:12:42 +0100 Subject: temp path options and reduced scanning of target folder (#328) - Feature: zen: `--zen-folder-path` added to `builds` command, `list`, `upload`, `download`, `fetch-blob`, `validate-part` to control where `.zen` folder is placed and named - Improvement: Only check known files from remote state when downloading to a target folder with no local state file - Improvement: Don't move existing local to cache and back if they are untouched --- src/zen/cmds/builds_cmd.cpp | 812 ++++++++++++++++++++++++++++------------- src/zen/cmds/builds_cmd.h | 2 + src/zenutil/chunkedcontent.cpp | 18 +- 3 files changed, 580 insertions(+), 252 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 3a54de935..d4add0e04 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -104,22 +104,34 @@ namespace { const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; - const std::string ZenFolderName = ".zen"; - const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); - const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); - const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + const std::string ZenFolderName = ".zen"; + std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } + // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; + // } + std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "tmp"; } - const std::string ZenTempCacheFolderName = - fmt::format("{}/cache", ZenTempFolderName); // Decompressed and verified data - chunks & sequences - const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); // Temp storage for whole and partial blocks - const std::string ZenTempChunkFolderName = - fmt::format("{}/chunks", ZenTempFolderName); // Temp storage for decompressed and validated chunks + std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences + } + std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks + } + std::filesystem::path ZenTempChunkFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "chunks"; // Temp storage for decompressed and validated chunks + } - const std::string ZenTempDownloadFolderName = - fmt::format("{}/download", ZenTempFolderName); // Temp storage for unverfied downloaded blobs + std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks + } - const std::string ZenTempStorageFolderName = - fmt::format("{}/storage", ZenTempFolderName); // Temp storage folder for BuildStorage implementations + // std::filesystem::path ZenTempStorageFolderPath(const std::filesystem::path& ZenFolderPath) + // { + // return ZenTempFolderPath(ZenFolderPath) / "storage"; // Temp storage folder for BuildStorage implementations + // } const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; @@ -150,6 +162,23 @@ namespace { ); + std::filesystem::path MakeSafeAbsolutePath(const std::string Path) + { + std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); +#if ZEN_PLATFORM_WINDOWS && \ + 0 // TODO: We need UNC for long file names but we need to stop using std::filesystem for those paths - std::filesystem::* functions + const std::string_view Prefix = "\\\\?\\"; + const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); + std::u8string PathString = AbsolutePath.u8string(); + if (!PathString.starts_with(PrefixU8)) + { + PathString.insert(0, PrefixU8); + return std::filesystem::path(PathString); + } +#endif + return AbsolutePath; + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -2141,6 +2170,7 @@ namespace { void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span RawHashes, @@ -2214,7 +2244,7 @@ namespace { if (QueuedPendingInMemoryBlocksForUpload.load() > 16) { ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); - Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), Path / ZenTempBlockFolderName, BlockHash)); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), ZenTempBlockFolderPath(ZenFolderPath), BlockHash)); IsInMemoryBlock = false; } else @@ -2437,7 +2467,7 @@ namespace { FilteredCompressedBytesPerSecond.Start(); CompositeBuffer Payload = - CompressChunk(Path, Content, Lookup, ChunkIndex, Path / ZenTempChunkFolderName, LooseChunksStats); + CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), @@ -2668,6 +2698,7 @@ namespace { const Oid& BuildPartId, const std::string_view BuildPartName, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::filesystem::path& ManifestPath, const uint8_t BlockReuseMinPercentLimit, bool AllowMultiparts, @@ -2678,7 +2709,7 @@ namespace { { Stopwatch ProcessTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); CleanDirectory(ZenTempFolder, {}); auto _ = MakeGuard([&]() { @@ -2687,8 +2718,8 @@ namespace { std::filesystem::remove(ZenTempFolder); } }); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempChunkFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); std::uint64_t TotalRawSize = 0; @@ -3308,6 +3339,7 @@ namespace { UploadPartBlobs(Storage, BuildId, Path, + ZenFolderPath, LocalContent, LocalLookup, RawHashes, @@ -4579,7 +4611,7 @@ namespace { return false; } - void AsyncWriteDownloadedChunk(const std::filesystem::path& Path, + void AsyncWriteDownloadedChunk(const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, uint32_t RemoteChunkIndex, @@ -4613,7 +4645,7 @@ namespace { { Payload.SetDeleteOnClose(false); Payload = {}; - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { @@ -4632,7 +4664,7 @@ namespace { { ZEN_TRACE_CPU("WriteTempChunk"); // Could not be moved and rather large, lets store it on disk - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload); Payload = {}; } @@ -4668,7 +4700,7 @@ namespace { } } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, @@ -4711,6 +4743,7 @@ namespace { void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::uint64_t LargeAttachmentSize, const std::uint64_t PreferredMultipartChunkSize, const ChunkedFolderContent& LocalContent, @@ -4739,7 +4772,7 @@ namespace { ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName; + const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); Stopwatch CacheMappingTimer; @@ -4786,6 +4819,12 @@ namespace { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); CacheMappingStats.CacheSequenceHashesCount++; CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + continue; } } @@ -4808,7 +4847,7 @@ namespace { } DirectoryContent BlockDirContent; - GetDirectoryContent(Path / ZenTempBlockFolderName, + GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath), DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, BlockDirContent); CachedBlocksFound.reserve(BlockDirContent.Files.size()); @@ -4856,6 +4895,8 @@ namespace { // const uint32_t RemoteSequenceIndex = CacheSequenceIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4863,13 +4904,16 @@ namespace { // const uint32_t RemoteChunkIndex = CacheChunkIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; CacheMappingStats.LocalPathsMatchingSequencesByteCount += RawSize; @@ -5135,7 +5179,8 @@ namespace { TotalPartWriteCount++; - std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + std::filesystem::path BlockPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); if (std::filesystem::exists(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); @@ -5414,8 +5459,9 @@ namespace { std::filesystem::path ExistingCompressedChunkPath; if (!PrimeCacheOnly) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = + ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); if (std::filesystem::exists(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); @@ -5448,6 +5494,7 @@ namespace { Work.ScheduleWork( WritePool, [&Path, + &ZenFolderPath, &RemoteContent, &RemoteLookup, &CacheFolderPath, @@ -5479,7 +5526,7 @@ namespace { CompressedChunkPath)); } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, RemoteLookup, @@ -5523,9 +5570,10 @@ namespace { Work.ScheduleWork( NetworkPool, [&Path, + &ZenFolderPath, &Storage, - PrimeCacheOnly, BuildId, + &PrimeCacheOnly, &RemoteContent, &RemoteLookup, &ExistsResult, @@ -5563,7 +5611,7 @@ namespace { ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); DownloadLargeBlob( *Storage.BuildStorage, - Path / ZenTempDownloadFolderName, + ZenTempDownloadFolderPath(ZenFolderPath), BuildId, ChunkHash, PreferredMultipartChunkSize, @@ -5588,7 +5636,7 @@ namespace { { if (!AbortFlag) { - AsyncWriteDownloadedChunk(Path, + AsyncWriteDownloadedChunk(ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -5633,7 +5681,7 @@ namespace { { FilteredDownloadedBytesPerSecond.Stop(); } - AsyncWriteDownloadedChunk(Path, + AsyncWriteDownloadedChunk(ZenFolderPath, RemoteContent, RemoteLookup, RemoteChunkIndex, @@ -5852,8 +5900,9 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredWrittenBytesPerSecond.Start(); - std::filesystem::path BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + std::filesystem::path BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); if (!BlockBuffer) { throw std::runtime_error( @@ -5955,11 +6004,10 @@ namespace { { BlockBuffer.SetDeleteOnClose(false); BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -5978,11 +6026,10 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -6130,8 +6177,9 @@ namespace { if (!Ec) { BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -6150,7 +6198,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); BlockBuffer = {}; } @@ -6323,36 +6371,106 @@ namespace { return; } - // Move all files we will reuse to cache folder - // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place - if (!LocalPathIndexesMatchingSequenceIndexes.empty()) + tsl::robin_map RemotePathIndexToLocalPathIndex; + RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size()); + + tsl::robin_map SequenceHashToLocalPathIndex; + std::vector RemoveLocalPathIndexes; + + if (!WipeTargetFolder) { - ZEN_TRACE_CPU("UpdateFolder_CacheReused"); - uint64_t TotalFullFileSizeCached = 0; - for (uint32_t LocalPathIndex : LocalPathIndexesMatchingSequenceIndexes) + tsl::robin_set CachedRemoteSequences; + tsl::robin_map RemotePathToRemoteIndex; + RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); + for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); - SetFileReadOnly(LocalFilePath, false); - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); - std::error_code Ec; - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); + } + + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + uint64_t CachedCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; + for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + + ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + + if (!WipeTargetFolder) { - Sleep(100); - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); + RemotePathIt != RemotePathToRemoteIndex.end()) + { + const uint32_t RemotePathIndex = RemotePathIt->second; + if (RemoteContent.RawHashes[RemotePathIndex] == RawHash) + { + // It is already in it's desired place + RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; + SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); + MatchCount++; + continue; + } + else + { + HashMismatchCount++; + } + } + else + { + PathMismatchCount++; + } } - if (Ec) + if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + { + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + if (!CachedRemoteSequences.contains(RawHash)) + { + // We need it + ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + + std::error_code Ec; + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + CachedRemoteSequences.insert(RawHash); + CachedCount++; + } + else + { + // We already have it + ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + SkippedCount++; + } + } + else if (!WipeTargetFolder) { - zen::ThrowSystemError(Ec.value(), Ec.message()); + // We don't need it + RemoveLocalPathIndexes.push_back(LocalPathIndex); + DeleteCount++; } - TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); } - ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(TotalFullFileSizeCached)); + + ZEN_DEBUG( + "Local state prep: MatchCount: {}, PathMismatchCount: {}, HashMismatchCount: {}, CachedCount: {}, SkippedCount: {}, " + "DeleteCount: {}", + MatchCount, + PathMismatchCount, + HashMismatchCount, + CachedCount, + SkippedCount, + DeleteCount); } if (WipeTargetFolder) @@ -6374,34 +6492,17 @@ namespace { Stopwatch Timer; // Remove unused tracked files - tsl::robin_map RemotePathToRemoteIndex; - RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); - for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) - { - RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); - } - std::vector LocalFilesToRemove; - for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) + + if (!RemoveLocalPathIndexes.empty()) { - if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string())) + ZEN_CONSOLE("Cleaning {} removed files from {}", RemoveLocalPathIndexes.size(), Path); + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) - { - LocalFilesToRemove.emplace_back(std::move(LocalFilePath)); - } - } - } - if (!LocalFilesToRemove.empty()) - { - ZEN_CONSOLE("Cleaning {} removed files from {}", LocalFilesToRemove.size(), Path); - for (const std::filesystem::path& LocalFilePath : LocalFilesToRemove) - { SetFileReadOnly(LocalFilePath, false); std::filesystem::remove(LocalFilePath); } } - RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } { @@ -6420,14 +6521,28 @@ namespace { std::atomic TargetsComplete = 0; - std::vector> Targets; + struct FinalizeTarget + { + IoHash RawHash; + uint32_t RemotePathIndex; + }; + + std::vector Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { - Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); } - std::sort(Targets.begin(), Targets.end(), [](const std::pair& Lhs, const std::pair& Rhs) { - return Lhs.first < Rhs.first; + std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { + if (Lhs.RawHash < Rhs.RawHash) + { + return true; + } + else if (Lhs.RawHash > Rhs.RawHash) + { + return false; + } + return Lhs.RemotePathIndex < Rhs.RemotePathIndex; }); size_t TargetOffset = 0; @@ -6438,9 +6553,8 @@ namespace { break; } - size_t TargetCount = 1; - const IoHash& RawHash = Targets[TargetOffset].first; - while (Targets[TargetOffset + TargetCount].first == RawHash) + size_t TargetCount = 1; + while (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash) { TargetCount++; } @@ -6452,93 +6566,157 @@ namespace { { ZEN_TRACE_CPU("FinalizeTree_Work"); - size_t TargetOffset = BaseTargetOffset; - const IoHash& RawHash = Targets[TargetOffset].first; - const uint32_t FirstTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstTargetPathIndex]; - OutLocalFolderState.Paths[FirstTargetPathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstTargetPathIndex] = RemoteContent.RawSizes[FirstTargetPathIndex]; - const std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; + if (RawHash == IoHash::Zero) { - if (std::filesystem::exists(FirstTargetFilePath)) + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - SetFileReadOnly(FirstTargetFilePath, false); - } - CreateDirectories(FirstTargetFilePath.parent_path()); - { - BasicFile OutputFile; - OutputFile.Open(FirstTargetFilePath, BasicFile::Mode::kTruncate); + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) + { + if (std::filesystem::exists(TargetFilePath)) + { + SetFileReadOnly(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + } + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; } } else { - ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace"); + ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); - CreateDirectories(FirstTargetFilePath.parent_path()); - if (std::filesystem::exists(FirstTargetFilePath)) - { - SetFileReadOnly(FirstTargetFilePath, false); - } - std::error_code Ec; - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - Sleep(100); - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); } - if (Ec) + else { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } + if (std::filesystem::exists(FirstTargetFilePath)) + { + SetFileReadOnly(FirstTargetFilePath, false); + } + else + { + CreateDirectories(FirstTargetFilePath.parent_path()); + } - OutLocalFolderState.Attributes[FirstTargetPathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstTargetPathIndex]); - OutLocalFolderState.ModificationTicks[FirstTargetPathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); + InplaceIt != SequenceHashToLocalPathIndex.end()) + { + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(std::filesystem::exists(SourceFilePath)); - TargetOffset++; - TargetsComplete++; - while (TargetOffset < (BaseTargetOffset + TargetCount)) - { - if (AbortFlag) - { - break; - } - ZEN_TRACE_CPU("FinalizeTree_Copy"); - - ZEN_ASSERT(Targets[TargetOffset].first == RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); - const uint32_t ExtraTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& ExtraTargetPath = RemoteContent.Paths[ExtraTargetPathIndex]; - const std::filesystem::path ExtraTargetFilePath = (Path / ExtraTargetPath).make_preferred(); - OutLocalFolderState.Paths[ExtraTargetPathIndex] = ExtraTargetPath; - OutLocalFolderState.RawSizes[ExtraTargetPathIndex] = RemoteContent.RawSizes[ExtraTargetPathIndex]; - CreateDirectories(ExtraTargetFilePath.parent_path()); - if (std::filesystem::exists(ExtraTargetFilePath)) - { - SetFileReadOnly(ExtraTargetFilePath, false); + CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + + std::error_code Ec; + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100); + std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + } } - CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false}); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - OutLocalFolderState.Attributes[ExtraTargetPathIndex] = + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + + OutLocalFolderState.Attributes[FirstRemotePathIndex] = RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(ExtraTargetFilePath) - : SetNativeFileAttributes(ExtraTargetFilePath, + ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, RemoteContent.Platform, - RemoteContent.Attributes[ExtraTargetPathIndex]); - OutLocalFolderState.ModificationTicks[ExtraTargetPathIndex] = - GetModificationTickFromPath(ExtraTargetFilePath); + RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = + GetModificationTickFromPath(FirstTargetFilePath); TargetOffset++; TargetsComplete++; + + while (TargetOffset < (BaseTargetOffset + TargetCount)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(std::filesystem::exists(TargetFilePath)); + } + else + { + if (std::filesystem::exists(TargetFilePath)) + { + SetFileReadOnly(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + + ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } } } }, @@ -6980,66 +7158,111 @@ namespace { ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, - ChunkingController& ChunkController) + const std::filesystem::path& ZenFolderPath, + ChunkingController& ChunkController, + const ChunkedFolderContent& ReferenceContent, + FolderContent& OutLocalFolderContent) { - ChunkedFolderContent LocalContent; + { + const uint32_t PathCount = gsl::narrow(ReferenceContent.Paths.size()); + OutLocalFolderContent.Paths.resize(PathCount); + OutLocalFolderContent.RawSizes.resize(PathCount); + OutLocalFolderContent.Attributes.resize(PathCount); + OutLocalFolderContent.ModificationTicks.resize(PathCount); - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string_view& ExcludeFolder : ExcludeFolders) { - if (RelativePath.starts_with(ExcludeFolder)) + Stopwatch Timer; + auto _ = + MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + ProgressBar ProgressBar(UsePlainProgress); + + ParallellWork Work(AbortFlag); + std::atomic CompletedPathCount = 0; + uint32_t PathIndex = 0; + + while (PathIndex < PathCount) { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } + uint32_t PathRangeCount = Min(1024u, PathCount - PathIndex); + Work.ScheduleWork( + GetIOWorkerPool(), + [PathIndex, + PathRangeCount, + &ReferenceContent, + &Path, + &OutLocalFolderContent, + &CompletedPathCount, + &LocalFolderScanStats](std::atomic&) { + for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) + { + const std::filesystem::path& FilePath = ReferenceContent.Paths[PathRangeIndex]; + std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); + if (std::filesystem::exists(LocalFilePath)) + { + const uint64_t FileSize = std::filesystem::file_size(LocalFilePath); + OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; + OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; + OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); + OutLocalFolderContent.ModificationTicks[PathRangeIndex] = GetModificationTickFromPath(LocalFilePath); + LocalFolderScanStats.FoundFileCount++; + LocalFolderScanStats.FoundFileByteCount += FileSize; + LocalFolderScanStats.AcceptedFileCount++; + LocalFolderScanStats.AcceptedFileByteCount += FileSize; + } + CompletedPathCount++; + } + }, + Work.DefaultErrorFunction()); + PathIndex += PathRangeCount; } + Work.Wait(200, [&](bool, ptrdiff_t) { + // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} checked, {} found", + CompletedPathCount.load(), + PathCount, + LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount.load()}, + false); + }); + ProgressBar.Finish(); } - return true; - }; - auto IsAcceptedFile = [ExcludeExtensions = - DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string_view& ExcludeExtension : ExcludeExtensions) + uint32_t WritePathIndex = 0; + for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++) { - if (RelativePath.ends_with(ExcludeExtension)) + if (!OutLocalFolderContent.Paths[ReadPathIndex].empty()) { - return false; + if (WritePathIndex < ReadPathIndex) + { + OutLocalFolderContent.Paths[WritePathIndex] = std::move(OutLocalFolderContent.Paths[ReadPathIndex]); + OutLocalFolderContent.RawSizes[WritePathIndex] = OutLocalFolderContent.RawSizes[ReadPathIndex]; + OutLocalFolderContent.Attributes[WritePathIndex] = OutLocalFolderContent.Attributes[ReadPathIndex]; + OutLocalFolderContent.ModificationTicks[WritePathIndex] = OutLocalFolderContent.ModificationTicks[ReadPathIndex]; + } + WritePathIndex++; } } - return true; - }; - FolderContent CurrentLocalFolderContent = GetFolderContent( - LocalFolderScanStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - GetIOWorkerPool(), - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); - }, - AbortFlag); - if (AbortFlag) - { - return {}; + OutLocalFolderContent.Paths.resize(WritePathIndex); + OutLocalFolderContent.RawSizes.resize(WritePathIndex); + OutLocalFolderContent.Attributes.resize(WritePathIndex); + OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); } - FolderContent LocalFolderState; + FolderContent LocalFolderState; + ChunkedFolderContent LocalContent; bool ScanContent = true; std::vector PathIndexesOufOfDate; - if (std::filesystem::is_regular_file(Path / ZenStateFilePath)) + if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) { try { Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object; + CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; if (CurrentStateObject) { Oid CurrentBuildId; @@ -7066,11 +7289,11 @@ namespace { std::span(SavedPartContents).subspan(1)); } - if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) + if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { const size_t LocaStatePathCount = LocalFolderState.Paths.size(); std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); if (!DeletedPaths.empty()) { LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); @@ -7160,27 +7383,27 @@ namespace { if (ScanContent) { uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : CurrentLocalFolderContent.RawSizes) + for (const uint64_t RawSize : OutLocalFolderContent.RawSizes) { ByteCountToScan += RawSize; } ProgressBar ProgressBar(false); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + LocalContent = ChunkFolderContent( ChunkingStats, GetIOWorkerPool(), Path, - CurrentLocalFolderContent, + OutLocalFolderContent, ChunkController, UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), - CurrentLocalFolderContent.Paths.size(), + OutLocalFolderContent.Paths.size(), NiceBytes(ChunkingStats.BytesHashed.load()), - ByteCountToScan, + NiceBytes(ByteCountToScan), NiceNum(FilteredBytesHashed.GetCurrent()), ChunkingStats.UniqueChunksFound.load(), NiceBytes(ChunkingStats.UniqueBytesFound.load())); @@ -7208,6 +7431,7 @@ namespace { const std::vector& BuildPartIds, std::span BuildPartNames, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, @@ -7220,12 +7444,12 @@ namespace { Stopwatch DownloadTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempCacheFolderName); - CreateDirectories(Path / ZenTempDownloadFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath)); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; @@ -7246,6 +7470,7 @@ namespace { GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; + FolderContent LocalFolderContent; if (!PrimeCacheOnly) { if (std::filesystem::is_directory(Path)) @@ -7258,7 +7483,13 @@ namespace { ChunkController = CreateBasicChunkingController(); } - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + LocalContent = GetLocalContent(LocalFolderScanStats, + ChunkingStats, + Path, + ZenFolderPath, + *ChunkController, + RemoteContent, + LocalFolderContent); } } else @@ -7317,6 +7548,12 @@ namespace { { ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); } else { @@ -7338,6 +7575,7 @@ namespace { UpdateFolder(Storage, BuildId, Path, + ZenFolderPath, LargeAttachmentSize, PreferredMultipartChunkSize, LocalContent, @@ -7363,14 +7601,14 @@ namespace { Stopwatch WriteStateTimer; CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 ExtendableStringBuilder<1024> SB; CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + @@ -7709,6 +7947,14 @@ BuildsCommand::BuildsCommand() ""); }; + auto AddZenFolderOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "zen-folder-path", + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), + cxxopts::value(m_ZenFolderPath), + ""); + }; m_Options.add_option("", "v", "verb", @@ -7722,6 +7968,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ListOptions); AddFileOptions(m_ListOptions); AddOutputOptions(m_ListOptions); + AddZenFolderOptions(m_ListOptions); m_ListOptions.add_options()("h,help", "Print help"); m_ListOptions.add_option("", "", @@ -7744,6 +7991,7 @@ BuildsCommand::BuildsCommand() AddOutputOptions(m_UploadOptions); AddCacheOptions(m_UploadOptions); AddWorkerOptions(m_UploadOptions); + AddZenFolderOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_UploadOptions.add_option("", @@ -7808,7 +8056,8 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); AddCacheOptions(m_DownloadOptions); - + AddZenFolderOptions(m_DownloadOptions); + AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_option("cache", "", "cache-prime-only", @@ -7816,7 +8065,6 @@ BuildsCommand::BuildsCommand() cxxopts::value(m_PrimeCacheOnly), ""); - AddWorkerOptions(m_DownloadOptions); m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), ""); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); @@ -7848,6 +8096,7 @@ BuildsCommand::BuildsCommand() "Allow request for partial chunk blocks. Defaults to true.", cxxopts::value(m_AllowPartialBlockRequests), ""); + m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), ""); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); @@ -7893,6 +8142,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); AddCacheOptions(m_FetchBlobOptions); + AddZenFolderOptions(m_FetchBlobOptions); m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_FetchBlobOptions .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), ""); @@ -7903,6 +8153,7 @@ BuildsCommand::BuildsCommand() AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); AddWorkerOptions(m_ValidateBuildPartOptions); + AddZenFolderOptions(m_ValidateBuildPartOptions); m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), ""); m_ValidateBuildPartOptions.add_option("", "", @@ -7989,10 +8240,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() - ? PickDefaultSystemRootDirectory() - : std::filesystem::absolute(StringToPath(m_SystemRootDir)).make_preferred(); - + std::filesystem::path DataRoot = + m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; @@ -8046,8 +8295,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = - ReadAccessTokenFromFile(std::filesystem::absolute(StringToPath(m_AccessTokenPath)).make_preferred()); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath)); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -8110,7 +8358,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_StoragePath.empty()) { - std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); StorageDescription = fmt::format("folder {}", StoragePath); Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); @@ -8173,7 +8421,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = std::filesystem::absolute(StringToPath(m_ListQueryPath)).make_preferred(); + std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath); if (ToLower(ListQueryPath.extension().string()) == ".cbo") { QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); @@ -8195,7 +8443,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); @@ -8207,7 +8466,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = std::filesystem::absolute(StringToPath(m_ListResultPath)).make_preferred(); + std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); if (ToLower(ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); @@ -8256,7 +8515,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); if (m_BuildPartName.empty()) { @@ -8297,14 +8556,25 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); CbObject MetaData; if (m_CreateBuild) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath = std::filesystem::absolute(StringToPath(m_BuildMetadataPath)); + std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -8336,7 +8606,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, Path, - std::filesystem::absolute(StringToPath(m_ManifestPath)).make_preferred(), + ZenFolderPath, + MakeSafeAbsolutePath(m_ManifestPath), m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -8415,18 +8686,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, Path, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, @@ -8466,8 +8741,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); - std::filesystem::path DiffPath = std::filesystem::absolute(StringToPath(m_DiffPath)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath); DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -8489,12 +8764,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -8509,6 +8787,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {}, {}, Path, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), @@ -8532,7 +8811,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = Path.filename().string(); @@ -8542,7 +8821,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const Oid BuildId = Oid::FromHexString(m_BuildId); const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = std::filesystem::absolute(StringToPath(m_StoragePath)).make_preferred(); + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); if (m_BuildsUrl.empty() && StoragePath.empty()) { @@ -8561,7 +8840,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? (DownloadPath / ZenFolderName) : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -8589,6 +8872,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, Path, + ZenFolderPath, {}, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -8602,13 +8886,13 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 11; } - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); DownloadFolder(Storage, BuildId, {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, @@ -8630,6 +8914,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8743,6 +9028,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8771,6 +9057,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId2, m_BuildPartName, DownloadPath, + ZenFolderPath, {}, m_BlockReuseMinPercentLimit, m_AllowMultiparts, @@ -8790,6 +9077,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8807,6 +9095,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8824,6 +9113,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, @@ -8853,12 +9143,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const Oid BuildId = Oid::FromHexString(m_BuildId); - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); uint64_t CompressedSize; uint64_t DecompressedSize; @@ -8898,12 +9199,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = std::filesystem::absolute(StringToPath(m_Path)).make_preferred(); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; - StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, Path / ZenTempFolderName); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + std::filesystem::remove(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 46257a567..4a77f8bd7 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -31,6 +31,8 @@ private: bool m_Verbose = false; bool m_BoostWorkerThreads = false; + std::string m_ZenFolderPath; + // cloud builds std::string m_BuildsUrl; bool m_AssumeHttp2 = false; diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index bb1ee5183..1e8447a57 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -304,14 +304,22 @@ FolderContent GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) { ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); - FolderContent Result = {.Platform = Old.Platform}; + + const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); + + FolderContent Result = {.Platform = Old.Platform}; + Result.Paths.reserve(NewPathCount); + Result.RawSizes.reserve(NewPathCount); + Result.Attributes.reserve(NewPathCount); + Result.ModificationTicks.reserve(NewPathCount); + tsl::robin_map NewPathToIndex; - const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); NewPathToIndex.reserve(NewPathCount); for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++) { NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex}); } + uint32_t OldPathCount = gsl::narrow(Old.Paths.size()); for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++) { @@ -667,6 +675,12 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); tsl::robin_map ChunkHashToChunkIndex; + const size_t ExpectedCount = BaseContent.Paths.size() - DeletedPaths.size(); + Result.Paths.reserve(ExpectedCount); + Result.RawSizes.reserve(ExpectedCount); + Result.Attributes.reserve(ExpectedCount); + Result.RawHashes.reserve(ExpectedCount); + tsl::robin_map RawHashToSequenceRawHashIndex; for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) { -- cgit v1.2.3 From 173f722fe773beae525bfca9657f64497cc16b67 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Sun, 30 Mar 2025 10:35:01 +0200 Subject: check file from local track state during download (#329) --- src/zen/cmds/builds_cmd.cpp | 265 ++++++++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 118 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d4add0e04..e3dc20621 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -7163,8 +7163,79 @@ namespace { const ChunkedFolderContent& ReferenceContent, FolderContent& OutLocalFolderContent) { + FolderContent LocalFolderState; + ChunkedFolderContent LocalContent; + + bool HasLocalState = false; + if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + { + try + { + Stopwatch ReadStateTimer; + CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; + if (CurrentStateObject) + { + Oid CurrentBuildId; + std::vector SavedBuildPartIds; + std::vector SavedBuildPartsNames; + std::vector SavedPartContents; + if (ReadStateObject(CurrentStateObject, + CurrentBuildId, + SavedBuildPartIds, + SavedBuildPartsNames, + SavedPartContents, + LocalFolderState)) + { + if (!SavedPartContents.empty()) + { + if (SavedPartContents.size() == 1) + { + LocalContent = std::move(SavedPartContents[0]); + } + else + { + LocalContent = + MergeChunkedFolderContents(SavedPartContents[0], + std::span(SavedPartContents).subspan(1)); + } + HasLocalState = true; + } + } + } + ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + } + } + { - const uint32_t PathCount = gsl::narrow(ReferenceContent.Paths.size()); + const uint32_t LocalPathCount = gsl::narrow(ReferenceContent.Paths.size()); + const uint32_t RemotePathCount = gsl::narrow(LocalFolderState.Paths.size()); + + std::vector PathsToCheck; + PathsToCheck.reserve(LocalPathCount + RemotePathCount); + + tsl::robin_set FileSet; + FileSet.reserve(LocalPathCount + RemotePathCount); + + for (const std::filesystem::path& LocalPath : LocalFolderState.Paths) + { + FileSet.insert(LocalPath.generic_string()); + PathsToCheck.push_back(LocalPath); + } + + for (const std::filesystem::path& RemotePath : ReferenceContent.Paths) + { + if (FileSet.insert(RemotePath.generic_string()).second) + { + PathsToCheck.push_back(RemotePath); + } + } + + const uint32_t PathCount = gsl::narrow(PathsToCheck.size()); + OutLocalFolderContent.Paths.resize(PathCount); OutLocalFolderContent.RawSizes.resize(PathCount); OutLocalFolderContent.Attributes.resize(PathCount); @@ -7188,14 +7259,14 @@ namespace { GetIOWorkerPool(), [PathIndex, PathRangeCount, - &ReferenceContent, + &PathsToCheck, &Path, &OutLocalFolderContent, &CompletedPathCount, &LocalFolderScanStats](std::atomic&) { for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) { - const std::filesystem::path& FilePath = ReferenceContent.Paths[PathRangeIndex]; + const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); if (std::filesystem::exists(LocalFilePath)) { @@ -7252,132 +7323,90 @@ namespace { OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); } - FolderContent LocalFolderState; - ChunkedFolderContent LocalContent; - bool ScanContent = true; std::vector PathIndexesOufOfDate; - if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + if (HasLocalState) { - try + if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { - Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; - if (CurrentStateObject) + const size_t LocaStatePathCount = LocalFolderState.Paths.size(); + std::vector DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); + if (!DeletedPaths.empty()) { - Oid CurrentBuildId; - std::vector SavedBuildPartIds; - std::vector SavedBuildPartsNames; - std::vector SavedPartContents; - if (ReadStateObject(CurrentStateObject, - CurrentBuildId, - SavedBuildPartIds, - SavedBuildPartsNames, - SavedPartContents, - LocalFolderState)) - { - if (!SavedPartContents.empty()) - { - if (SavedPartContents.size() == 1) - { - LocalContent = std::move(SavedPartContents[0]); - } - else - { - LocalContent = - MergeChunkedFolderContents(SavedPartContents[0], - std::span(SavedPartContents).subspan(1)); - } - - if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) - { - const size_t LocaStatePathCount = LocalFolderState.Paths.size(); - std::vector DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocaStatePathCount); - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(false); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - ChunkingStats, - GetIOWorkerPool(), - Path, - UpdatedContent, - ChunkController, - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, - false); - }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); - } - } - else - { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set LocalFolderPaths; - LocalFolderPaths.reserve(LocalFolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector DeletedPaths; - for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) - { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } - } - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } - } - ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - ScanContent = false; - } + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocaStatePathCount); + if (UpdatedContent.Paths.size() > 0) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : UpdatedContent.RawSizes) + { + ByteCountToScan += RawSize; } + ProgressBar ProgressBar(false); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetIOWorkerPool(), + Path, + UpdatedContent, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + if (AbortFlag) + { + return {}; + } + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); } } - catch (const std::exception& Ex) + else { - ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + // Remove files from LocalContent no longer in LocalFolderState + tsl::robin_set LocalFolderPaths; + LocalFolderPaths.reserve(LocalFolderState.Paths.size()); + for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) + { + LocalFolderPaths.insert(LocalFolderPath.generic_string()); + } + std::vector DeletedPaths; + for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) + { + if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) + { + DeletedPaths.push_back(LocalContentPath); + } + } + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } } + ScanContent = false; } if (ScanContent) -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zen/cmds/admin_cmd.cpp | 4 +- src/zen/cmds/builds_cmd.cpp | 309 +++++--- src/zen/cmds/cache_cmd.cpp | 2 +- src/zen/cmds/copy_cmd.cpp | 20 +- src/zen/cmds/projectstore_cmd.cpp | 4 +- src/zen/cmds/rpcreplay_cmd.cpp | 2 +- src/zen/cmds/run_cmd.cpp | 2 +- src/zen/cmds/serve_cmd.cpp | 2 +- src/zen/cmds/status_cmd.cpp | 2 +- src/zen/cmds/up_cmd.cpp | 4 +- src/zencore/basicfile.cpp | 12 +- src/zencore/except.cpp | 2 +- src/zencore/filesystem.cpp | 875 ++++++++++++++++++--- src/zencore/include/zencore/filesystem.h | 85 +- src/zencore/process.cpp | 2 +- src/zencore/testutils.cpp | 11 +- src/zenhttp/auth/authmgr.cpp | 2 +- src/zenserver-test/zenserver-test.cpp | 18 +- src/zenserver/admin/admin.cpp | 2 +- src/zenserver/config.cpp | 2 +- src/zenserver/frontend/frontend.cpp | 4 +- src/zenserver/main.cpp | 6 +- src/zenserver/objectstore/objectstore.cpp | 10 +- .../projectstore/fileremoteprojectstore.cpp | 10 +- src/zenserver/projectstore/projectstore.cpp | 106 +-- src/zenserver/projectstore/remoteprojectstore.cpp | 6 +- src/zenserver/workspaces/httpworkspaces.cpp | 2 +- src/zenserver/zenserver.cpp | 6 +- src/zenstore/blockstore.cpp | 18 +- src/zenstore/buildstore/buildstore.cpp | 10 +- src/zenstore/cache/cachedisklayer.cpp | 48 +- src/zenstore/cas.cpp | 2 +- src/zenstore/caslog.cpp | 2 +- src/zenstore/compactcas.cpp | 30 +- src/zenstore/filecas.cpp | 78 +- src/zenstore/gc.cpp | 20 +- src/zenstore/workspaces.cpp | 26 +- src/zenutil/cache/rpcrecording.cpp | 8 +- src/zenutil/filebuildstorage.cpp | 14 +- src/zenutil/zenserverprocess.cpp | 8 +- src/zenvfs/vfsprovider.cpp | 13 +- 41 files changed, 1270 insertions(+), 519 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 835e01151..573639c2d 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -688,7 +688,7 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) static bool TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target) { - if (!std::filesystem::is_regular_file(Source)) + if (!IsFile(Source)) { return false; } @@ -717,7 +717,7 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path DataPath = StringToPath(m_DataPath); std::filesystem::path TargetPath = StringToPath(m_TargetPath); - if (!std::filesystem::is_directory(DataPath)) + if (!IsDir(DataPath)) { throw OptionParseException("data path must exist"); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e3dc20621..d2ba20e78 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -165,12 +165,11 @@ namespace { std::filesystem::path MakeSafeAbsolutePath(const std::string Path) { std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); -#if ZEN_PLATFORM_WINDOWS && \ - 0 // TODO: We need UNC for long file names but we need to stop using std::filesystem for those paths - std::filesystem::* functions +#if ZEN_PLATFORM_WINDOWS && 1 const std::string_view Prefix = "\\\\?\\"; const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); std::u8string PathString = AbsolutePath.u8string(); - if (!PathString.starts_with(PrefixU8)) + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) { PathString.insert(0, PrefixU8); return std::filesystem::path(PathString); @@ -179,6 +178,82 @@ namespace { return AbsolutePath; } + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) + { + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + RenameFile(SourcePath, TargetPath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -250,21 +325,8 @@ namespace { { try { - std::error_code Ec; - std::filesystem::remove(LocalFilePath, Ec); - if (Ec) - { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Ec.clear(); - if (std::filesystem::exists(LocalFilePath, Ec) || Ec) - { - Sleep(200); - if (std::filesystem::exists(LocalFilePath)) - { - std::filesystem::remove(LocalFilePath); - } - } - } + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } catch (const std::exception& Ex) { @@ -273,7 +335,7 @@ namespace { } } - for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; for (const std::string_view ExcludeDirectory : ExcludeDirectories) @@ -288,23 +350,27 @@ namespace { { try { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); - } - catch (const std::exception&) - { - Sleep(200); - try + std::error_code Ec; + zen::CleanDirectory(LocalDirPath, true, Ec); + if (Ec) { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); + Sleep(200); + zen::CleanDirectory(LocalDirPath, true); + Ec.clear(); } - catch (const std::exception& Ex) + + RemoveDir(LocalDirPath, Ec); + if (Ec) { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; + Sleep(200); + RemoveDir(LocalDirPath); } } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); + CleanWipe = false; + } } } return CleanWipe; @@ -312,7 +378,7 @@ namespace { std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -355,7 +421,7 @@ namespace { const IoHash& Hash, const std::string& Suffix = {}) { - std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / (Hash.ToHexString() + Suffix); return WriteToTempFile(std::move(Buffer), TempFilePath); } @@ -453,12 +519,12 @@ namespace { std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred(); + return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); } std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + return CacheFolderPath / RawHash.ToHexString(); } ChunkedFolderContent ScanAndChunkFolder( @@ -1611,7 +1677,8 @@ namespace { auto __ = MakeGuard([&TempFolder]() { if (CleanDirectory(TempFolder, {})) { - std::filesystem::remove(TempFolder); + std::error_code DummyEc; + RemoveDir(TempFolder, DummyEc); } }); @@ -1888,7 +1955,7 @@ namespace { } ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); { - std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString(); BasicFile CompressedFile; std::error_code Ec; @@ -1930,7 +1997,7 @@ namespace { return Compressed.GetCompressed(); } CompressedFile.Close(); - std::filesystem::remove(TempFilePath, Ec); + RemoveFile(TempFilePath, Ec); ZEN_UNUSED(Ec); } @@ -2715,7 +2782,8 @@ namespace { auto _ = MakeGuard([&]() { if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + std::error_code DummyEc; + RemoveDir(ZenTempFolder, DummyEc); } }); CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); @@ -2861,7 +2929,7 @@ namespace { { std::filesystem::path ExcludeManifestPath = Path / ZenExcludeManifestName; tsl::robin_set ExcludeAssetPaths; - if (std::filesystem::is_regular_file(ExcludeManifestPath)) + if (IsFile(ExcludeManifestPath)) { std::vector AssetPaths = ParseManifest(Path, ExcludeManifestPath); ExcludeAssetPaths.reserve(AssetPaths.size()); @@ -2904,12 +2972,13 @@ namespace { for (const std::filesystem::path& AssetPath : AssetPaths) { Content.Paths.push_back(AssetPath); - Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath)); + const std::filesystem::path AssetFilePath = (Path / AssetPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(Path / AssetPath)); + Content.Attributes.push_back(GetFileAttributes(AssetFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(Path / AssetPath)); + Content.Attributes.push_back(GetFileMode(AssetFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); LocalFolderScanStats.AcceptedFileCount++; @@ -2917,12 +2986,13 @@ namespace { if (ManifestPath.is_relative()) { Content.Paths.push_back(ManifestPath); - Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath)); + const std::filesystem::path ManifestFilePath = (Path / ManifestPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(ManifestPath)); + Content.Attributes.push_back(GetFileAttributes(ManifestFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(ManifestPath)); + Content.Attributes.push_back(GetFileMode(ManifestFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); @@ -3751,7 +3821,7 @@ namespace { if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) { const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!std::filesystem::exists(TargetPath)) + if (!IsFile(TargetPath)) { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); @@ -3761,7 +3831,7 @@ namespace { else { std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow(std::filesystem::file_size(TargetPath, Ec)); + uint64_t SizeOnDisk = gsl::narrow(FileSizeFromPath(TargetPath, Ec)); if (Ec) { ErrorLock.WithExclusiveLock([&]() { @@ -3997,9 +4067,15 @@ namespace { void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), - GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::error_code Ec; + RenameFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash), + Ec); + if (Ec) + { + throw std::system_error(Ec); + } } void FinalizeChunkSequences(const std::filesystem::path& TargetFolder, @@ -4646,7 +4722,7 @@ namespace { Payload.SetDeleteOnClose(false); Payload = {}; CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + RenameFile(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { CompressedChunkPath = std::filesystem::path{}; @@ -4717,7 +4793,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -4823,13 +4899,13 @@ namespace { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); continue; } } } - std::filesystem::remove(CacheDirContent.Files[Index]); + RemoveFileWithRetry(CacheDirContent.Files[Index]); } } @@ -4875,7 +4951,7 @@ namespace { } } } - std::filesystem::remove(BlockDirContent.Files[Index]); + RemoveFileWithRetry(BlockDirContent.Files[Index]); } } @@ -4896,7 +4972,7 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4905,14 +4981,14 @@ namespace { // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; @@ -5181,7 +5257,7 @@ namespace { std::filesystem::path BlockPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) + if (IsFile(BlockPath)) { CachedChunkBlockIndexes.push_back(BlockIndex); UsingCachedBlock = true; @@ -5462,7 +5538,7 @@ namespace { const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; std::filesystem::path CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); - if (std::filesystem::exists(CompressedChunkPath)) + if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); if (ExistingCompressedPart) @@ -5481,7 +5557,7 @@ namespace { else { std::error_code DummyEc; - std::filesystem::remove(CompressedChunkPath, DummyEc); + RemoveFile(CompressedChunkPath, DummyEc); } } } @@ -5543,7 +5619,7 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); @@ -5923,11 +5999,11 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); @@ -6008,7 +6084,7 @@ namespace { BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6079,14 +6155,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6180,7 +6256,7 @@ namespace { BlockBuffer = {}; BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); + RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { BlockChunkPath = std::filesystem::path{}; @@ -6259,14 +6335,14 @@ namespace { DiskStats)) { std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); + RemoveFile(BlockChunkPath, DummyEc); throw std::runtime_error( fmt::format("Block {} is malformed", BlockDescription.BlockHash)); } if (!BlockChunkPath.empty()) { - std::filesystem::remove(BlockChunkPath); + RemoveFile(BlockChunkPath); } WritePartsComplete++; @@ -6398,7 +6474,7 @@ namespace { const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; - ZEN_ASSERT_SLOW(std::filesystem::is_regular_file(Path / LocalContent.Paths[LocalPathIndex])); + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); if (!WipeTargetFolder) { @@ -6430,27 +6506,18 @@ namespace { if (!CachedRemoteSequences.contains(RawHash)) { // We need it - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - std::error_code Ec; - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(LocalFilePath, CacheFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedRemoteSequences.insert(RawHash); CachedCount++; } else { // We already have it - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); SkippedCount++; } } @@ -6499,8 +6566,8 @@ namespace { for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) { const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnly(LocalFilePath, false); - std::filesystem::remove(LocalFilePath); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); } } } @@ -6579,9 +6646,9 @@ namespace { std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { @@ -6615,13 +6682,13 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); } else { - if (std::filesystem::exists(FirstTargetFilePath)) + if (IsFile(FirstTargetFilePath)) { - SetFileReadOnly(FirstTargetFilePath, false); + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); } else { @@ -6634,7 +6701,7 @@ namespace { const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); - ZEN_ASSERT_SLOW(std::filesystem::exists(SourceFilePath)); + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; @@ -6643,19 +6710,10 @@ namespace { { const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); - std::error_code Ec; - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - for (size_t Retries = 0; Ec && Retries < 3; Retries++) - { - Sleep(100); - std::filesystem::rename(CacheFilePath, FirstTargetFilePath, Ec); - } - if (Ec) - { - zen::ThrowSystemError(Ec.value(), Ec.message()); - } RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; } } @@ -6685,20 +6743,20 @@ namespace { if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - ZEN_ASSERT_SLOW(std::filesystem::exists(TargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(TargetFilePath)); } else { - if (std::filesystem::exists(TargetFilePath)) + if (IsFile(TargetFilePath)) { - SetFileReadOnly(TargetFilePath, false); + SetFileReadOnlyWithRetry(TargetFilePath, false); } else { CreateDirectories(TargetFilePath.parent_path()); } - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } @@ -7167,7 +7225,7 @@ namespace { ChunkedFolderContent LocalContent; bool HasLocalState = false; - if (std::filesystem::is_regular_file(ZenStateFilePath(ZenFolderPath))) + if (IsFile(ZenStateFilePath(ZenFolderPath))) { try { @@ -7268,9 +7326,9 @@ namespace { { const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) + if (IsFile(LocalFilePath)) { - const uint64_t FileSize = std::filesystem::file_size(LocalFilePath); + const uint64_t FileSize = FileSizeFromPath(LocalFilePath); OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); @@ -7502,7 +7560,7 @@ namespace { FolderContent LocalFolderContent; if (!PrimeCacheOnly) { - if (std::filesystem::is_directory(Path)) + if (IsDir(Path)) { if (!WipeTargetFolder) { @@ -7692,7 +7750,7 @@ namespace { } if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + RemoveDirWithRetry(ZenTempFolder); } } @@ -8479,7 +8537,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8592,7 +8650,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -8721,7 +8779,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8799,7 +8857,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageCache::Statistics StorageCacheStats; const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (Path / ZenFolderName).make_preferred() : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -8871,7 +8929,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); const std::filesystem::path ZenFolderPath = - m_ZenFolderPath.empty() ? (DownloadPath / ZenFolderName) : MakeSafeAbsolutePath(m_ZenFolderPath); + m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); @@ -9000,10 +9058,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { Work.ScheduleWork( GetIOWorkerPool(), - [SourceSize, FilePath](std::atomic&) { + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic&) { if (!AbortFlag) { - bool IsReadOnly = SetFileReadOnly(FilePath, false); + bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); { BasicFile Source(FilePath, BasicFile::Mode::kWrite); uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); @@ -9030,7 +9088,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } break; case 1: - std::filesystem::remove(FilePath); + { + (void)SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + } break; default: break; @@ -9184,7 +9245,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); @@ -9240,7 +9301,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (CleanDirectory(ZenFolderPath, {})) { std::error_code DummyEc; - std::filesystem::remove(ZenFolderPath, DummyEc); + RemoveDir(ZenFolderPath, DummyEc); } }); diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 6ec6a80db..185edc35d 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -629,7 +629,7 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_OutputPath.empty()) { TargetPath = std::filesystem::path(m_OutputPath); - if (std::filesystem::is_directory(TargetPath)) + if (IsDir(TargetPath)) { TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index cc6ddd505..53e80c896 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -64,8 +64,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); - const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); + const bool IsFileCopy = IsFile(m_CopySource); + const bool IsDirCopy = IsDir(m_CopySource); if (!IsFileCopy && !IsDirCopy) { @@ -79,20 +79,14 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsDirCopy) { - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - const bool IsTargetDir = std::filesystem::is_directory(ToPath); - if (!IsTargetDir) - { - if (std::filesystem::is_regular_file(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - } + throw std::runtime_error("Attempted copy of directory into file"); } - else + + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 6bc499f03..13c7c4b23 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,7 +41,7 @@ namespace { std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -2185,7 +2185,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - std::filesystem::remove_all(TmpPath); + DeleteDirectories(TmpPath); ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 5b88a1f73..4fc38d92a 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -196,7 +196,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("Rpc replay command requires a path"); } - if (!std::filesystem::exists(m_RecordingPath) || !std::filesystem::is_directory(m_RecordingPath)) + if (!IsDir(m_RecordingPath)) { throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); } diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp index a99ba9704..309b8996a 100644 --- a/src/zen/cmds/run_cmd.cpp +++ b/src/zen/cmds/run_cmd.cpp @@ -100,7 +100,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - CleanDirectory(BaseDirectory); + CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); } } diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index f87725e36..64039e4c9 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -67,7 +67,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("command requires a root path"); } - if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + if (!IsDir(m_RootPath)) { throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); } diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 4d1534e05..88c0b22a2 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -33,7 +33,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 44a41146c..aacc115a0 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -159,7 +159,7 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; @@ -229,7 +229,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index a181bbd66..ea526399c 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -590,7 +590,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: // deleting the temporary file BasicFile::Close(); - std::filesystem::rename(m_TempPath, FinalFileName, Ec); + RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { @@ -984,9 +984,9 @@ TEST_CASE("TemporaryFile") TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); - CHECK(std::filesystem::exists(Path)); + CHECK(IsFile(Path)); } - CHECK(std::filesystem::exists(Path) == false); + CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") @@ -997,11 +997,11 @@ TEST_CASE("TemporaryFile") CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); + CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); + CHECK(IsFile(TempPath) == false); + CHECK(IsFile(FinalPath)); } } diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp index d5eabea9d..610b0ced5 100644 --- a/src/zencore/except.cpp +++ b/src/zencore/except.cpp @@ -47,7 +47,7 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string { if (HRESULT_FACILITY(hRes) == FACILITY_WIN32) { - throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message)); + throw std::system_error(std::error_code(HRESULT_CODE(hRes), std::system_category()), std::string(Message)); } else { diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8a369f02e..6ff4dd053 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -86,16 +86,9 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) } bool -CreateDirectories(const wchar_t* Dir) +CreateDirectories(const wchar_t* Path) { - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); + return CreateDirectories(std::filesystem::path(Path)); } // Erase all files and directories in a given directory, leaving an empty directory @@ -212,75 +205,326 @@ DeleteDirectoriesInternal(const wchar_t* DirPath) bool CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles) { - if (std::filesystem::exists(DirPath)) + if (IsDir(DirPath)) { return WipeDirectory(DirPath, KeepDotFiles); } return CreateDirectories(DirPath); } + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; +#endif // ZEN_PLATFORM_WINDOWS + +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; #endif // ZEN_PLATFORM_WINDOWS +} bool -CreateDirectories(const std::filesystem::path& Dir) +IsFileModeReadOnly(uint32_t FileMode) { - if (Dir.string().ends_with(":")) + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + +bool +RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::RemoveDirectory(Path.native().c_str()); + if (!Success) { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } return false; } - while (!std::filesystem::is_directory(Dir)) + return true; +#else + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::DeleteFile(Path.native().c_str()); + if (!Success) { - if (Dir.has_parent_path()) + if (ForceRemoveReadOnlyFiles) { - CreateDirectories(Dir.parent_path()); + DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str()); + if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) + { + ::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(Path.native().c_str()); + } } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) + if (!Success) { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; } - return true; } - return false; + return true; +#else + if (!ForceRemoveReadOnlyFiles) + { + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + const uint32_t Mode = (uint32_t)Stat.st_mode; + if (IsFileModeReadOnly(Mode)) + { + Ec = MakeErrorCode(EACCES); + return false; + } + } + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +static void +WipeDirectoryContentInternal(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsFile(LocalFilePath)) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + } + } + if (Ec) + { + return; + } + } + + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + { + WipeDirectoryContentInternal(LocalDirPath, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + return; + } + + RemoveDirNative(LocalDirPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(LocalDirPath)) + { + RemoveDirNative(LocalDirPath, Ec); + } + } + if (Ec) + { + return; + } + } +} + +bool +CreateDirectory(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::CreateDirectory(Path.native().c_str(), nullptr); + if (!Success) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return Success; +#else + return std::filesystem::create_directory(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS } bool -DeleteDirectories(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path) { - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); + std::error_code Ec; + bool Success = CreateDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to create directories for '{}'", Path.string())); + } + return Success; } bool -CleanDirectory(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec) { - if (std::filesystem::exists(Dir)) + if (Path.string().ends_with(":")) { - bool Success = true; + return false; + } + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return false; + } + if (Exists) + { + return false; + } - for (const auto& Item : std::filesystem::directory_iterator(Dir)) + if (Path.has_parent_path()) + { + bool Result = CreateDirectories(Path.parent_path(), Ec); + if (Ec) { - std::error_code ErrorCode; - const uintmax_t RemovedCount = std::filesystem::remove_all(Item, ErrorCode); - - Success = Success && !ErrorCode && RemovedCount; + return Result; } + } + return CreateDirectory(Path, Ec); +} - return Success; +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles) +{ + std::error_code Ec; + bool Result = CleanDirectory(Path, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to clean directory for '{}'", Path.string())); } + return Result; +} - return CreateDirectories(Dir); +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + if (Exists) + { + WipeDirectoryContentInternal(Path, ForceRemoveReadOnlyFiles, Ec); + return false; + } + return CreateDirectory(Path, Ec); +} + +bool +DeleteDirectories(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = DeleteDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to delete directories for '{}'", Path.string())); + } + return Result; } bool -CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir) +DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + + if (Exists) + { + WipeDirectoryContentInternal(Path, false, Ec); + if (Ec) + { + return false; + } + bool Result = RemoveDirNative(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(Path)) + { + Result = RemoveDirNative(Path, Ec); + } + } + return Result; + } + return false; +} + +bool +CleanDirectoryExceptDotFiles(const std::filesystem::path& Path) { #if ZEN_PLATFORM_WINDOWS const bool KeepDotFiles = true; - return CleanDirectory(Dir.c_str(), KeepDotFiles); + return CleanDirectory(Path.c_str(), KeepDotFiles); #else - ZEN_UNUSED(Dir); + ZEN_UNUSED(Path); ZEN_NOT_IMPLEMENTED(); #endif @@ -637,7 +881,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { // Validate arguments - if (FromPath.empty() || !std::filesystem::is_directory(FromPath)) + if (FromPath.empty() || !IsDir(FromPath)) throw std::runtime_error("invalid CopyTree source directory specified"); if (ToPath.empty()) @@ -646,16 +890,13 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop if (Options.MustClone && !SupportsBlockRefCounting(FromPath)) throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath)); - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - if (!std::filesystem::is_directory(ToPath)) - { - throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); - } + throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); } - else + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } if (Options.MustClone && !SupportsBlockRefCounting(ToPath)) @@ -811,7 +1052,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { Outfile.Close(); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else @@ -819,7 +1060,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { close(Fd); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); } #endif // ZEN_PLATFORM_WINDOWS @@ -1172,7 +1413,7 @@ void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { #if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; + std::vector FileInfoBuffer(8 * 1024); FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; @@ -1183,7 +1424,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr if (FAILED(hRes)) { - if (hRes == ERROR_FILE_NOT_FOUND || hRes == ERROR_PATH_NOT_FOUND) + if (HRESULT_CODE(hRes) == ERROR_FILE_NOT_FOUND || HRESULT_CODE(hRes) == ERROR_PATH_NOT_FOUND) { // Directory no longer exist, treat it as empty return; @@ -1193,8 +1434,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr while (Continue) { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + BOOL Success = + GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer.data(), (DWORD)(FileInfoBuffer.size() * sizeof(uint64_t))); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration uint64_t EntryOffset = 0; @@ -1213,7 +1455,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr for (;;) { const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast(reinterpret_cast(FileInfoBuffer) + EntryOffset); + reinterpret_cast(reinterpret_cast(FileInfoBuffer.data()) + EntryOffset); std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); @@ -1338,6 +1580,172 @@ CanonicalPath(std::filesystem::path InPath, std::error_code& Ec) #endif } +bool +IsFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a file", Path.string())); + } + return Result; +} + +bool +IsFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISREG(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a directory", Path.string())); + } + return Result; +} + +bool +IsDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISDIR(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove file '{}'", Path.string())); + } + return Success; +} + +bool +RemoveFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveFileNative(Path, false, Ec); +#else + bool IsDirectory = std::filesystem::is_directory(Path, Ec); + if (IsDirectory) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveFileNative(Path, false, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove directory '{}'", Path.string())); + } + return Success; +} + +bool +RemoveDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveDirNative(Path, Ec); +#else + bool IsFile = std::filesystem::is_regular_file(Path, Ec); + if (IsFile) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveDirNative(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec) { @@ -1434,6 +1842,49 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec) #endif // ZEN_PLATFORM_WINDOWS } +uint64_t +FileSizeFromPath(const std::filesystem::path& Path) +{ + std::error_code Ec; + uint64_t Size = FileSizeFromPath(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get file size for path '{}'", Path.string())); + } + return Size; +} + +uint64_t +FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + void* Handle = ::CreateFile(Path.native().c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + DWORD LastError = GetLastError(); + Ec = MakeErrorCode(LastError); + return 0; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + LARGE_INTEGER FileSize; + BOOL Success = GetFileSizeEx(Handle, &FileSize); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + return 0; + } + return FileSize.QuadPart; +#else + return std::filesystem::file_size(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + uint64_t FileSizeFromHandle(void* NativeHandle) { @@ -1483,7 +1934,13 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) // PathFromHandle void* Handle; #if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + Handle = CreateFileW(Filename.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); if (Handle == INVALID_HANDLE_VALUE) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); @@ -1493,7 +1950,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) { - ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(Ec, fmt::format("Failed to get modification tick for path '{}'", Filename.string())); } return ModificatonTick; #else @@ -1507,6 +1964,56 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename path from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFileEx(SourcePath.native().c_str(), TargetPath.native().c_str(), MOVEFILE_REPLACE_EXISTING); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameDirectory(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename directory from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFile(SourcePath.native().c_str(), TargetPath.native().c_str()); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path GetRunningExecutablePath() { @@ -1793,7 +2300,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) }; auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool { - bool Exists = std::filesystem::exists(Path, Ec); + bool Exists = IsFile(Path, Ec); if (Ec) { return false; @@ -1802,7 +2309,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) { return true; } - uintmax_t Size = std::filesystem::file_size(Path, Ec); + uintmax_t Size = FileSizeFromPath(Path, Ec); if (Ec) { return false; @@ -1821,17 +2328,17 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) for (auto i = MaxFiles; i > 0; i--) { std::filesystem::path src = GetFileName(i - 1); - if (!std::filesystem::exists(src)) + if (!IsFile(src)) { continue; } std::error_code DummyEc; std::filesystem::path target = GetFileName(i); - if (std::filesystem::exists(target, DummyEc)) + if (IsFile(target, DummyEc)) { - std::filesystem::remove(target, DummyEc); + RemoveFile(target, DummyEc); } - std::filesystem::rename(src, target, DummyEc); + RenameFile(src, target, DummyEc); } } @@ -1868,16 +2375,16 @@ RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDir { const std::filesystem::path SourcePath = GetPathForIndex(i - 1); - if (std::filesystem::exists(SourcePath)) + if (IsDir(SourcePath)) { std::filesystem::path TargetPath = GetPathForIndex(i); std::error_code DummyEc; - if (std::filesystem::exists(TargetPath, DummyEc)) + if (IsDir(TargetPath, DummyEc)) { - std::filesystem::remove_all(TargetPath, DummyEc); + DeleteDirectories(TargetPath, DummyEc); } - std::filesystem::rename(SourcePath, TargetPath, DummyEc); + RenameDirectory(SourcePath, TargetPath, DummyEc); } } @@ -1936,22 +2443,46 @@ PickDefaultSystemRootDirectory() #if ZEN_PLATFORM_WINDOWS uint32_t -GetFileAttributes(const std::filesystem::path& Filename) +GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec) { DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); if (Attributes == INVALID_FILE_ATTRIBUTES) { - ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Attributes; } +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = zen::GetFileAttributes(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to get attributes of file '{}'", Filename.string())); + } + return Result; +} + void -SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec) { if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) { - ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + } +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + std::error_code Ec; + zen::SetFileAttributes(Filename, Attributes, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set attributes of file {}", Filename.string())); } } @@ -1961,88 +2492,104 @@ SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) uint32_t GetFileMode(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = GetFileMode(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get mode of file {}", Filename)); + } + return Result; +} + +uint32_t +GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec) { struct stat Stat; int err = stat(Filename.native().c_str(), &Stat); if (err) { - ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Stat.st_mode; } void -SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode) { - int err = chmod(Filename.native().c_str(), (mode_t)Attributes); - if (err) + std::error_code Ec; + SetFileMode(Filename, Mode, Ec); + if (Ec) { - ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + throw std::system_error(Ec, fmt::format("Failed to set mode of file {}", Filename)); } } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -#if ZEN_PLATFORM_WINDOWS -const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; -#else -const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; -#endif // ZEN_PLATFORM_WINDOWS - -const uint32_t FileModeWriteEnableFlags = 0222; - -bool -IsFileAttributeReadOnly(uint32_t FileAttributes) -{ -#if ZEN_PLATFORM_WINDOWS - return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; -#else - return (FileAttributes & 0x00000001) != 0; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -IsFileModeReadOnly(uint32_t FileMode) -{ - return (FileMode & FileModeWriteEnableFlags) == 0; -} - -uint32_t -MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec) { - return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); + int err = chmod(Filename.native().c_str(), (mode_t)Mode); + if (err) + { + Ec = MakeErrorCodeFromLastError(); + } } -uint32_t -MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) -{ - return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); -} +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool -SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - uint32_t CurrentAttributes = GetFileAttributes(Filename); - uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + uint32_t CurrentAttributes = GetFileAttributes(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); if (CurrentAttributes != NewAttributes) { - SetFileAttributes(Filename, NewAttributes); + SetFileAttributes(Filename, NewAttributes, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - uint32_t CurrentMode = GetFileMode(Filename); - uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + uint32_t CurrentMode = GetFileMode(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); if (CurrentMode != NewMode) { - SetFileMode(Filename, NewMode); + SetFileMode(Filename, NewMode, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC return false; } +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ + std::error_code Ec; + bool Result = SetFileReadOnly(Filename, ReadOnly, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set read only mode of file '{}'", Filename.string())); + } + return Result; +} + std::filesystem::path StringToPath(const std::string_view& Path) { @@ -2079,7 +2626,7 @@ TEST_CASE("filesystem") path BinPath = GetRunningExecutablePath(); const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); + CHECK(IsFile(BinPath)); // PathFromHandle void* Handle; @@ -2132,6 +2679,80 @@ TEST_CASE("filesystem") CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); } +TEST_CASE("Filesystem.Basics") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; + CleanDirectory(TestBaseDir, true); + DeleteDirectories(TestBaseDir); + CHECK(!IsDir(TestBaseDir)); + CHECK(CleanDirectory(TestBaseDir, false)); + CHECK(IsDir(TestBaseDir)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(!IsDir(TestBaseDir / "no_such_thing")); + CHECK(!IsDir("hgjda/cev_/q12")); + CHECK(!IsFile(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "no_such_thing")); + CHECK(!IsFile("hgjda/cev_/q12")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir) == 0); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "no_such_file")); + CHECK(!CreateDirectories(TestBaseDir)); + CHECK(CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(!CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep" / "no")); + CHECK_THROWS(WriteFile(TestBaseDir / "nested" / "a", IoBuffer(20))); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "a" / "yo", IoBuffer(20))); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "nested" / "a" / "yo") == 20); + CHECK(!IsFile(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsDir(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested")); + CHECK(!IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(CreateDirectories(TestBaseDir / "nested" / "deeper")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "deeper" / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK(IsDir(TestBaseDir / "new_place")); + CHECK(!IsFile(TestBaseDir / "new_place")); + CHECK_THROWS(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(!RemoveDir(TestBaseDir / "nested" / "deeper")); + CHECK(RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "new_place" / "yo")); + CHECK(!RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested")); + CHECK_THROWS(RemoveDir(TestBaseDir)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameFile(TestBaseDir / "yo", TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "yo")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK_THROWS(RemoveDir(TestBaseDir / "new_place" / "yo")); + CHECK(DeleteDirectories(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsDir(TestBaseDir)); + CHECK(!IsDir(TestBaseDir / "nested")); + CHECK(CreateDirectories(TestBaseDir / "nested")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK_THROWS(CleanDirectory(TestBaseDir, false)); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", false)); + CHECK(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK(!CleanDirectory(TestBaseDir / "nested", true)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(RemoveDir(TestBaseDir)); +} + TEST_CASE("WriteFile") { std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); @@ -2166,7 +2787,7 @@ TEST_CASE("WriteFile") CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); } - std::filesystem::remove(TempFile); + RemoveFile(TempFile); } TEST_CASE("DiskSpaceInfo") @@ -2223,7 +2844,7 @@ TEST_CASE("PathBuilder") TEST_CASE("RotateDirectories") { std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; - CleanDirectory(TestBaseDir); + CleanDirectory(TestBaseDir, false); std::filesystem::path RotateDir = TestBaseDir / "rotate_dir" / "dir_to_rotate"; IoBuffer DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5); @@ -2237,16 +2858,16 @@ TEST_CASE("RotateDirectories") const int RotateMax = 10; NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); - CHECK(std::filesystem::exists(DirWithSuffix(2))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); + CHECK(IsDir(DirWithSuffix(2))); for (int i = 0; i < RotateMax; ++i) { @@ -2256,16 +2877,16 @@ TEST_CASE("RotateDirectories") CHECK_EQ(IsError, false); } - CHECK(!std::filesystem::exists(RotateDir)); + CHECK(!IsDir(RotateDir)); for (int i = 0; i < RotateMax; ++i) { - CHECK(std::filesystem::exists(DirWithSuffix(i + 1))); + CHECK(IsDir(DirWithSuffix(i + 1))); } for (int i = RotateMax; i < RotateMax + 5; ++i) { - CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1))); + CHECK(!IsDir(DirWithSuffix(RotateMax + i + 1))); } } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 9a2b15d1d..c23f16d03 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -20,21 +20,35 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists. + + Will also create any required parent direCleanDirectoryctories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ @@ -44,6 +58,46 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_ */ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); @@ -56,6 +110,22 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -277,12 +347,16 @@ std::filesystem::path PickDefaultSystemRootDirectory(); #if ZEN_PLATFORM_WINDOWS uint32_t GetFileAttributes(const std::filesystem::path& Filename); +uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec); void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC uint32_t GetFileMode(const std::filesystem::path& Filename); -void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool IsFileAttributeReadOnly(uint32_t FileAttributes); @@ -290,6 +364,7 @@ bool IsFileModeReadOnly(uint32_t FileMode); uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); std::filesystem::path StringToPath(const std::string_view& Path); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 0761521dc..2fe5b8948 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -60,7 +60,7 @@ GetPidStatus(int Pid, std::error_code& OutEc) { std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid); std::filesystem::path StatPath = EntryPath / "stat"; - if (std::filesystem::is_regular_file(StatPath)) + if (IsFile(StatPath)) { FILE* StatFile = fopen(StatPath.c_str(), "r"); if (StatFile) diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 641d5508a..9f50de032 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -4,6 +4,7 @@ #if ZEN_WITH_TESTS +# include # include # include "zencore/string.h" @@ -19,8 +20,8 @@ CreateTemporaryDirectory() std::error_code Ec; std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); - std::filesystem::remove_all(DirPath, Ec); - std::filesystem::create_directories(DirPath); + DeleteDirectories(DirPath, Ec); + CreateDirectories(DirPath); return DirPath; } @@ -32,14 +33,14 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory) { std::error_code Ec; - std::filesystem::remove_all(Directory, Ec); - std::filesystem::create_directories(Directory); + DeleteDirectories(Directory, Ec); + CreateDirectories(Directory); } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { std::error_code Ec; - std::filesystem::remove_all(m_RootPath, Ec); + DeleteDirectories(m_RootPath, Ec); } IoBuffer diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 1a9892d5c..8f7befc80 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -379,7 +379,7 @@ private: AuthState.EndArray(); } - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); std::optional Reason; diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 027a35998..78a735ea0 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -3306,18 +3306,18 @@ GenerateFolderContent(const std::filesystem::path& RootPath) std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -3473,7 +3473,7 @@ TEST_CASE("workspaces.create") while (true) { std::error_code Ec; - std::filesystem::remove_all(Root2Path / Share2Path, Ec); + DeleteDirectories(Root2Path / Share2Path, Ec); if (!Ec) break; } @@ -3630,7 +3630,7 @@ TEST_CASE("workspaces.lifetimes") } // Wipe system config - std::filesystem::remove_all(SystemRootPath); + DeleteDirectories(SystemRootPath); // Restart @@ -3696,8 +3696,8 @@ TEST_CASE("workspaces.share") uint64_t Size = FileObject["size"sv].AsUInt64(); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); - CHECK(std::filesystem::file_size(AbsFilePath) == Size); + CHECK(IsFile(AbsFilePath)); + CHECK(FileSizeFromPath(AbsFilePath) == Size); Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size)); } } @@ -3720,7 +3720,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } } @@ -3740,7 +3740,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 0da6e31ad..73166e608 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -40,7 +40,7 @@ struct DirStats DirStats GetStatsForDirectory(std::filesystem::path Dir) { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) return {}; struct StatsTraversal : public GetDirectoryContentVisitor diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index d3af0c6a6..52f539dcd 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -1116,7 +1116,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (DataDir.empty()) throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); - if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) + if (!IsDir(ServerOptions.BaseSnapshotDir)) throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); } diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 31d9e1c94..104b26954 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -50,7 +50,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { break; } - if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode)) + if (IsFile(ParentPath / "xmake.lua", ErrorCode)) { if (ErrorCode) { @@ -59,7 +59,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html"; - if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + if (IsDir(HtmlDir, ErrorCode)) { m_Directory = HtmlDir; } diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index d5419d342..78ddd39a0 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -406,17 +406,17 @@ main(int argc, char* argv[]) if (!DeleteReason.empty()) { - if (std::filesystem::exists(ServerOptions.DataDir)) + if (IsDir(ServerOptions.DataDir)) { ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } - if (!std::filesystem::exists(ServerOptions.DataDir)) + if (!IsDir(ServerOptions.DataDir)) { ServerOptions.IsFirstRun = true; - std::filesystem::create_directories(ServerOptions.DataDir); + CreateDirectories(ServerOptions.DataDir); } if (!ServerOptions.BaseSnapshotDir.empty()) diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index e757ef84e..5af803617 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -251,7 +251,7 @@ HttpObjectStoreService::Inititalize() ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!fs::exists(BucketsPath)) + if (!IsDir(BucketsPath)) { CreateDirectories(BucketsPath); } @@ -324,7 +324,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { std::lock_guard _(BucketsMutex); - if (!fs::exists(BucketPath)) + if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); @@ -406,7 +406,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); FileSystemTraversal Traversal; - if (std::filesystem::exists(FullPath)) + if (IsDir(FullPath)) { std::lock_guard _(BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); @@ -475,7 +475,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st } const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!fs::exists(FilePath)) + if (!IsFile(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -576,7 +576,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { std::lock_guard _(BucketsMutex); - if (!fs::exists(FileDirectory)) + if (!IsDir(FileDirectory)) { CreateDirectories(FileDirectory); } diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 98e292d91..375e44e59 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -73,7 +73,7 @@ public: ContainerObject.IterateAttachments([&](CbFieldView FieldView) { IoHash AttachmentHash = FieldView.AsBinaryAttachment(); std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash); - if (!std::filesystem::exists(AttachmentPath)) + if (!IsFile(AttachmentPath)) { Result.Needs.insert(AttachmentHash); } @@ -111,7 +111,7 @@ public: Stopwatch Timer; SaveAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::exists(ChunkPath)) + if (!IsFile(ChunkPath)) { try { @@ -182,7 +182,7 @@ public: for (const IoHash& RawHash : BlockHashes) { std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (std::filesystem::is_regular_file(ChunkPath)) + if (IsFile(ChunkPath)) { ExistingBlockHashes.push_back(RawHash); } @@ -203,7 +203,7 @@ public: Stopwatch Timer; LoadAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::is_regular_file(ChunkPath)) + if (!IsFile(ChunkPath)) { Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); @@ -246,7 +246,7 @@ private: LoadContainerResult Result; std::filesystem::path SourcePath = m_OutputPath; SourcePath.append(Name); - if (!std::filesystem::is_regular_file(SourcePath)) + if (!IsFile(SourcePath)) { Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index ea3f2aad9..1966eeef9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -58,7 +58,7 @@ namespace { std::filesystem::path DroppedBucketPath; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return true; } @@ -68,7 +68,7 @@ namespace { std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId); DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { if (!DeleteDirectories(DroppedBucketPath)) { @@ -77,7 +77,7 @@ namespace { Dir); continue; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir); continue; @@ -88,13 +88,13 @@ namespace { do { std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { OutDeleteDir = DroppedBucketPath; return true; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.", Dir, @@ -486,7 +486,7 @@ struct ProjectStore::OplogStorage : public RefCounted [[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); } [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath)); + return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath)); } [[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); } [[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath) @@ -496,13 +496,13 @@ struct ProjectStore::OplogStorage : public RefCounted void WipeState() const { std::error_code Ec; - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); } static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); } - uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); } + uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); } uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); } static uint64_t OpsSize(const std::filesystem::path& BasePath) @@ -510,7 +510,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (Exists(BasePath)) { std::error_code DummyEc; - return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath)); + return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath)); } return 0; } @@ -689,7 +689,7 @@ struct ProjectStore::OplogStorage : public RefCounted m_OpBlobs.Close(); Oplog.Close(); - std::filesystem::rename(OplogPath, GetLogPath(), Ec); + RenameFile(OplogPath, GetLogPath(), Ec); if (Ec) { throw std::system_error( @@ -702,9 +702,9 @@ struct ProjectStore::OplogStorage : public RefCounted if (Ec) { // We failed late - clean everything up as best we can - std::filesystem::remove(OpBlobs.GetPath(), Ec); - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); throw std::system_error(Ec, fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'", OpBlobs.GetPath(), @@ -739,7 +739,7 @@ struct ProjectStore::OplogStorage : public RefCounted } catch (const std::exception& /*Ex*/) { - std::filesystem::remove(OpBlobs.GetPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); throw; } } @@ -1108,7 +1108,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath); m_Storage->WipeState(); std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } } m_Storage->Open(/* IsCreate */ !StoreExists); @@ -1116,7 +1116,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, m_MetaPath = m_BasePath / "ops.meta"sv; m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath()); - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); } ProjectStore::Oplog::~Oplog() @@ -1142,7 +1142,7 @@ ProjectStore::Oplog::Flush() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } uint64_t LogCount = m_Storage->LogCount(); @@ -1238,19 +1238,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = OplogStorage::OpsSize(BasePath); std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::exists(StateFilePath)) + if (IsFile(StateFilePath)) { - Size += std::filesystem::file_size(StateFilePath); + Size += FileSizeFromPath(StateFilePath); } std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv; - if (std::filesystem::exists(MetaFilePath)) + if (IsFile(MetaFilePath)) { - Size += std::filesystem::file_size(MetaFilePath); + Size += FileSizeFromPath(MetaFilePath); } std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv; - if (std::filesystem::exists(IndexFilePath)) + if (IsFile(IndexFilePath)) { - Size += std::filesystem::file_size(IndexFilePath); + Size += FileSizeFromPath(IndexFilePath); } return Size; @@ -1303,7 +1303,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath) using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - return std::filesystem::is_regular_file(StateFilePath); + return IsFile(StateFilePath); } bool @@ -1337,7 +1337,7 @@ ProjectStore::Oplog::Read() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } ReadIndexSnapshot(); @@ -1438,7 +1438,7 @@ ProjectStore::Oplog::Reset() m_Storage = new OplogStorage(this, m_BasePath); m_Storage->Open(true); m_MetaValid = false; - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); Write(); } // Erase content on disk @@ -1457,7 +1457,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::is_regular_file(StateFilePath)) + if (IsFile(StateFilePath)) { // ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath); @@ -1536,7 +1536,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo if (File.Hash == IoHash::Zero) { std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); HasMissingEntries = true; @@ -1625,10 +1625,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp"; // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'", GetOuterProject()->Identifier, @@ -1641,9 +1641,9 @@ ProjectStore::Oplog::WriteIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -1778,11 +1778,11 @@ ProjectStore::Oplog::WriteIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'", @@ -1793,10 +1793,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'", m_OuterProject->Identifier, @@ -1814,7 +1814,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot"); std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint64_t EntryCount = 0; Stopwatch Timer; @@ -3133,7 +3133,7 @@ ProjectStore::Project::~Project() bool ProjectStore::Project::Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(BasePath / "Project.zcb"); + return IsFile(BasePath / "Project.zcb"); } void @@ -3207,7 +3207,7 @@ ProjectStore::Project::ReadAccessTimes() using namespace std::literals; std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - if (!std::filesystem::exists(ProjectAccessTimesFilePath)) + if (!IsFile(ProjectAccessTimesFilePath)) { return; } @@ -3598,14 +3598,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = 0; std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv; - if (std::filesystem::exists(AccessTimesFilePath)) + if (IsFile(AccessTimesFilePath)) { - Size += std::filesystem::file_size(AccessTimesFilePath); + Size += FileSizeFromPath(AccessTimesFilePath); } std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectFilePath)) + if (IsFile(ProjectFilePath)) { - Size += std::filesystem::file_size(ProjectFilePath); + Size += FileSizeFromPath(ProjectFilePath); } return Size; @@ -3717,7 +3717,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName, if (!MarkerPath.empty()) { std::error_code Ec; - if (std::filesystem::exists(MarkerPath, Ec)) + if (IsFile(MarkerPath, Ec)) { if (Ec) { @@ -3870,7 +3870,7 @@ void ProjectStore::DiscoverProjects() { ZEN_MEMSCOPE(GetProjectstoreTag()); - if (!std::filesystem::exists(m_ProjectBasePath)) + if (!IsDir(m_ProjectBasePath)) { return; } @@ -3979,7 +3979,7 @@ ProjectStore::StorageSize() const GcStorageSize Result; { - if (std::filesystem::exists(m_ProjectBasePath)) + if (IsDir(m_ProjectBasePath)) { DirectoryContent ProjectsFolderContent; GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent); @@ -3987,7 +3987,7 @@ ProjectStore::StorageSize() const for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) { std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectStateFilePath)) + if (IsFile(ProjectStateFilePath)) { Result.DiskSize += Project::TotalSize(ProjectBasePath); DirectoryContent DirContent; @@ -7243,7 +7243,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project1FilePath); + RemoveFile(Project1FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7272,7 +7272,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2Oplog1Path); + RemoveFile(Project2Oplog1Path); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7301,7 +7301,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2FilePath); + RemoveFile(Project2FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), @@ -8025,7 +8025,7 @@ TEST_CASE("project.store.rpc.getchunks") CompositeBuffer Buffer = Attachment->AsCompositeBinary(); CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), IoHash::HashBuffer(Buffer)); - CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second)); CHECK(!Chunk.FindView("RawSize")); } { diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index a7263da83..f96b3e185 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -1212,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore, { std::string_view ServerPath = View["serverpath"sv].AsString(); std::filesystem::path FilePath = Project.RootDir / ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { remotestore_impl::ReportMessage( OptionalContext, @@ -3083,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore, OptionalContext]() { auto _ = MakeGuard([&DechunkLatch, &TempFileName] { std::error_code Ec; - if (std::filesystem::exists(TempFileName, Ec)) + if (IsFile(TempFileName, Ec)) { - std::filesystem::remove(TempFileName, Ec); + RemoveFile(TempFileName, Ec); if (Ec) { ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message()); diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 0b7fd0400..ac0aaef8e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -1100,7 +1100,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace } } - if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath)) + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) { return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 03e269d49..45c91d691 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -434,7 +434,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -483,7 +483,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -914,7 +914,7 @@ ZenServer::CheckStateMarker() std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 63c0388fa..7cc09be15 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -85,7 +85,7 @@ BlockStoreFile::Create(uint64_t InitialSize) ZEN_TRACE_CPU("BlockStoreFile::Create"); auto ParentPath = m_Path.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + if (!IsDir(ParentPath)) { CreateDirectories(ParentPath); } @@ -215,7 +215,7 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p } if (MetaWriteTime < BlockWriteTime) { - std::filesystem::remove(MetaPath, Ec); + RemoveFile(MetaPath, Ec); return false; } return true; @@ -239,7 +239,7 @@ BlockStoreFile::MetaSize() const if (IsMetaDataValid(m_Path, MetaPath)) { std::error_code DummyEc; - if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc) + if (uint64_t Size = FileSizeFromPath(MetaPath, DummyEc); !DummyEc) { return Size; } @@ -252,7 +252,7 @@ BlockStoreFile::RemoveMeta() { std::filesystem::path MetaPath = GetMetaPath(); std::error_code DummyEc; - std::filesystem::remove(MetaPath, DummyEc); + RemoveFile(MetaPath, DummyEc); } std::filesystem::path @@ -291,7 +291,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max m_MaxBlockSize = MaxBlockSize; m_MaxBlockCount = MaxBlockCount; - if (std::filesystem::is_directory(m_BlocksBasePath)) + if (IsDir(m_BlocksBasePath)) { uint32_t NextBlockIndex = 0; std::vector FoldersToScan; @@ -500,7 +500,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&, { OutBlockPath = GetBlockPath(m_BlocksBasePath, ProbeIndex); std::error_code Ec; - bool Exists = std::filesystem::exists(OutBlockPath, Ec); + bool Exists = IsFile(OutBlockPath, Ec); if (Ec) { ZEN_WARN("Failed to probe existence of file '{}' when trying to allocate a new block. Reason: '{}'", @@ -1375,14 +1375,14 @@ TEST_CASE("blockstore.blockfile") BoopChunk = File1.GetChunk(5, 5); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); const char* Data = static_cast(DataChunk.GetData()); CHECK(std::string(Data) == "data"); const char* Boop = static_cast(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); { IoBuffer DataChunk; @@ -1401,7 +1401,7 @@ TEST_CASE("blockstore.blockfile") const char* Boop = static_cast(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(!std::filesystem::exists(RootDirectory / "1")); + CHECK(!IsFile(RootDirectory / "1")); } namespace blockstore::impl { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index eb36be049..f26901458 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -71,7 +71,7 @@ BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) { std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); - bool IsNew = !(std::filesystem::exists(BlobLogPath) && std::filesystem::exists(MetaLogPath)); + bool IsNew = !(IsFile(BlobLogPath) && IsFile(MetaLogPath)); if (!IsNew) { @@ -501,7 +501,7 @@ uint64_t BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -518,7 +518,7 @@ BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesys TCasLogFile CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); @@ -603,7 +603,7 @@ uint64_t BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) { ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -620,7 +620,7 @@ BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesy TCasLogFile CasLog; if (!CasLog.IsValid(LogPath)) { - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } CasLog.Open(LogPath, CasLogFile::Mode::kRead); diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..b2d2416be 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -200,21 +200,21 @@ namespace cache::impl { int DropIndex = 0; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return false; } std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { DropIndex++; continue; } std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { DeleteDirectories(DroppedBucketPath); @@ -909,16 +909,16 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, { std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { - if (!std::filesystem::remove(LogPath, Ec) || Ec) + if (!RemoveFile(LogPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); std::error_code RemoveIndexEc; - std::filesystem::remove(IndexPath, RemoveIndexEc); + RemoveFile(IndexPath, RemoveIndexEc); } } } @@ -939,7 +939,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const { ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile"); - if (!std::filesystem::is_regular_file(IndexPath)) + if (!IsFile(IndexPath)) { return 0; } @@ -1023,7 +1023,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std:: { ZEN_TRACE_CPU("Z$::Bucket::ReadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -1103,37 +1103,37 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco if (IsNew) { - fs::remove(LogPath); - fs::remove(IndexPath); - fs::remove_all(m_BlocksBasePath); + RemoveFile(LogPath); + RemoveFile(IndexPath); + DeleteDirectories(m_BlocksBasePath); } CreateDirectories(m_BucketDir); m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexLock, IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition); } - else if (fs::is_regular_file(LogPath)) + else if (IsFile(LogPath)) { ZEN_WARN("removing invalid log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -2146,7 +2146,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); std::error_code Ec; - uintmax_t size = std::filesystem::file_size(DataFilePath.ToPath(), Ec); + uintmax_t size = FileSizeFromPath(DataFilePath.ToPath(), Ec); if (Ec) { ReportBadKey(HashKey); @@ -2287,11 +2287,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) BuildPath(Path, Entry.Key); fs::path FilePath = Path.ToPath(); RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key)); - if (fs::is_regular_file(FilePath)) + if (IsFile(FilePath)) { ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8()); std::error_code Ec; - fs::remove(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... + RemoveFile(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... } } } @@ -2424,7 +2424,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (CleanUpTempFile) { std::error_code Ec; - std::filesystem::remove(DataFile.GetPath(), Ec); + RemoveFile(DataFile.GetPath(), Ec); if (Ec) { ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'", @@ -2452,7 +2452,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); // We do a speculative remove of the file instead of probing with a exists call and check the error code instead - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec) { if (Ec.value() != ENOENT) @@ -2460,7 +2460,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message()); Sleep(100); Ec.clear(); - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir)); @@ -2791,7 +2791,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - if (!fs::remove(FilePath, Ec)) + if (!RemoveFile(FilePath, Ec)) { continue; } @@ -2812,7 +2812,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(FilePath, Ec); + bool Existed = IsFile(FilePath, Ec); if (Ec) { ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'", diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..ed42f254e 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -118,7 +118,7 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Ensure root directory exists - create if it doesn't exist already - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); // Open or create manifest diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..492ce9317 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -37,7 +37,7 @@ CasLogFile::~CasLogFile() bool CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) { - if (!std::filesystem::is_regular_file(FileName)) + if (!IsFile(FileName)) { return false; } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index b64bc26dd..184251da7 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -927,10 +927,10 @@ CasContainerStrategy::MakeIndexSnapshot() fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); return; @@ -939,9 +939,9 @@ CasContainerStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -992,21 +992,21 @@ CasContainerStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); } @@ -1092,7 +1092,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski if (!TCasLogFile::IsValid(LogPath)) { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } @@ -1155,7 +1155,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore) { - std::filesystem::remove_all(BasePath); + DeleteDirectories(BasePath); } CreateDirectories(BasePath); @@ -1165,19 +1165,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); std::filesystem::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { @@ -1186,7 +1186,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..14bdc41f0 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -176,10 +176,10 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN if (IsNewStore) { - std::filesystem::remove(LogPath); - std::filesystem::remove(IndexPath); + RemoveFile(LogPath); + RemoveFile(IndexPath); - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { // We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside // in this folder as well @@ -211,24 +211,24 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor); for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots) { - std::filesystem::remove_all(SharededRoot); + DeleteDirectories(SharededRoot); } } } - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile::IsValid(LogPath)) { @@ -237,7 +237,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -327,7 +327,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); std::error_code Ec; - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("unable to move existing CAS file {} to {}", ChunkPath, TempPath)); @@ -452,7 +452,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { PayloadFile.Close(); std::error_code DummyEc; - std::filesystem::remove(ChunkPath, DummyEc); + RemoveFile(ChunkPath, DummyEc); throw; } bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); @@ -503,7 +503,7 @@ FileCasStrategy::SafeOpenChunk(const IoHash& ChunkHash, uint64 ExpectedSize) { std::error_code Ec; std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { Chunk.SetDeleteOnClose(true); @@ -574,7 +574,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) ShardingHelper Name(m_RootDirectory, ChunkHash); const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath(); - uint64_t FileSize = static_cast(std::filesystem::file_size(ChunkPath, Ec)); + uint64_t FileSize = static_cast(FileSizeFromPath(ChunkPath, Ec)); if (Ec) { ZEN_WARN("get file size FAILED, file cas '{}'", ChunkPath); @@ -582,9 +582,9 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) } ZEN_DEBUG("deleting CAS payload file '{}' {}", ChunkPath, NiceBytes(FileSize)); - std::filesystem::remove(ChunkPath, Ec); + RemoveFile(ChunkPath, Ec); - if (!Ec || !std::filesystem::exists(ChunkPath)) + if (!Ec || !IsFile(ChunkPath)) { { RwLock::ExclusiveLockScope _(m_Lock); @@ -941,10 +941,10 @@ FileCasStrategy::MakeIndexSnapshot() fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message()); return; @@ -953,9 +953,9 @@ FileCasStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, STmpIndexPath); + RenameFile(IndexPath, STmpIndexPath); } // Write the current state of the location map to a new index state @@ -1004,21 +1004,21 @@ FileCasStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(STmpIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(STmpIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message()); } } } - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message()); } @@ -1032,7 +1032,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; std::vector Entries; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1077,7 +1077,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& return 0; } - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { ZEN_INFO("missing index for file cas, scanning for cas files in {}", m_RootDirectory); TCasLogFile CasLog; @@ -1116,7 +1116,7 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr using namespace filecas::impl; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { uint64_t LogEntryCount = 0; Stopwatch Timer; @@ -1274,12 +1274,12 @@ public: ChunkPath); } std::error_code Ec; - uint64_t SizeOnDisk = std::filesystem::file_size(ChunkPath, Ec); + uint64_t SizeOnDisk = FileSizeFromPath(ChunkPath, Ec); if (Ec) { SizeOnDisk = 0; } - bool Existed = std::filesystem::remove(ChunkPath, Ec); + bool Existed = RemoveFile(ChunkPath, Ec); if (Ec) { // Target file may be open for read, attempt to move it to a temp file and mark it delete on close @@ -1290,7 +1290,7 @@ public: if (OldChunk) { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { OldChunk.SetDeleteOnClose(true); @@ -1317,7 +1317,7 @@ public: else { std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(ChunkPath, Ec); + bool Existed = IsFile(ChunkPath, Ec); if (Ec) { if (Ctx.Settings.Verbose) @@ -1516,7 +1516,7 @@ TEST_CASE("cas.chunk.moveoverwrite") Payload1.SetDeleteOnClose(true); CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, true); - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1526,9 +1526,9 @@ TEST_CASE("cas.chunk.moveoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1554,7 +1554,7 @@ TEST_CASE("cas.chunk.moveoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; @@ -1598,9 +1598,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, true); - CHECK(std::filesystem::exists(Payload1Path)); + CHECK(IsFile(Payload1Path)); Payload1 = {}; - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1611,9 +1611,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1640,7 +1640,7 @@ TEST_CASE("cas.chunk.copyoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index fe5ae284b..ac4dda83f 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -62,11 +62,11 @@ namespace { { if (Size == 0) { - std::filesystem::remove(Path); + RemoveFile(Path); return std::error_code{}; } CreateDirectories(Path.parent_path()); - if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) + if (IsFile(Path) && FileSizeFromPath(Path) == Size) { return std::error_code(); } @@ -1262,12 +1262,12 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::CompactStores"); auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(Settings.DiskReservePath)) + if (!IsFile(Settings.DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath); - if (std::filesystem::remove(Settings.DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath); + if (RemoveFile(Settings.DiskReservePath)) { return ReclaimedSize; } @@ -1557,7 +1557,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_Config.LightweightInterval = m_Config.MonitorInterval; } - std::filesystem::create_directories(Config.RootDirectory); + CreateDirectories(Config.RootDirectory); std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) @@ -1850,7 +1850,7 @@ GcScheduler::GetState() const if (Result.Config.DiskReserveSize != 0) { Ec.clear(); - Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; + Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; } if (Result.Status != GcSchedulerStatus::kRunning) @@ -2389,12 +2389,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { // We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(DiskReservePath)) + if (!IsFile(DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath); - if (std::filesystem::remove(DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath); + if (RemoveFile(DiskReservePath)) { return ReclaimedSize; } diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..0ca2adab2 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -444,7 +444,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& RootPath = Workspace->GetConfig().RootPath; std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::string Error; std::vector WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); @@ -458,7 +458,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& SharePath = Configuration.SharePath; - if (std::filesystem::is_directory(RootPath / SharePath)) + if (IsDir(RootPath / SharePath)) { DeletedShares.erase(Configuration.Id); @@ -808,7 +808,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work ZEN_DEBUG("Reading workspaces state from {}", WorkspaceStatePath); const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -847,7 +847,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p ZEN_DEBUG("Reading workspace state from {}", WorkspaceRoot); std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -886,7 +886,7 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work { throw std::invalid_argument(fmt::format("invalid root path '{}' for workspace {}", Configuration.RootPath, Configuration.Id)); } - if (!std::filesystem::is_directory(Configuration.RootPath)) + if (!IsDir(Configuration.RootPath)) { throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); @@ -965,7 +965,7 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, throw std::invalid_argument( fmt::format("workspace share path '{}' is not a sub-path of workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); } - if (!std::filesystem::is_directory(WorkspaceRoot / Configuration.SharePath)) + if (!IsDir(WorkspaceRoot / Configuration.SharePath)) { throw std::invalid_argument( fmt::format("workspace share path '{}' does not exist in workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); @@ -1244,7 +1244,7 @@ Workspaces::FindWorkspaceShare(const Oid& WorkspaceId, const Oid& ShareId, bool const Workspaces::WorkspaceConfiguration& WorkspaceConfig = Workspace->GetConfig(); const Workspaces::WorkspaceShareConfiguration& ShareConfig = Share->GetConfig(); std::filesystem::path FullSharePath = WorkspaceConfig.RootPath / ShareConfig.SharePath; - if (std::filesystem::is_directory(FullSharePath)) + if (IsDir(FullSharePath)) { if (ForceRefresh || !Share->IsInitialized()) { @@ -1306,18 +1306,18 @@ namespace { std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -1365,13 +1365,13 @@ TEST_CASE("workspaces.scanfolder") Structure->IterateEntries([&](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::filesystem::path AbsPath = RootPath / Entry.RelativePath; - CHECK(std::filesystem::is_regular_file(AbsPath)); - CHECK(std::filesystem::file_size(AbsPath) == Entry.Size); + CHECK(IsFile(AbsPath)); + CHECK(FileSizeFromPath(AbsPath) == Entry.Size); const FolderStructure::FileEntry* FindEntry = Structure->FindEntry(Id); CHECK(FindEntry); std::filesystem::path Path = RootPath / FindEntry->RelativePath; CHECK(AbsPath == Path); - CHECK(std::filesystem::file_size(AbsPath) == FindEntry->Size); + CHECK(FileSizeFromPath(AbsPath) == FindEntry->Size); }); } diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 1f951167d..380c182b2 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -46,7 +46,7 @@ struct RecordedRequestsWriter void BeginWrite(const std::filesystem::path& BasePath) { m_BasePath = BasePath; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void EndWrite() @@ -426,7 +426,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath, m_BasePath = BasePath; m_SegmentIndex = SegmentIndex; m_RequestBaseIndex = RequestBaseIndex; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void @@ -1051,14 +1051,14 @@ public: static bool IsCompatible(const std::filesystem::path& BasePath) { - if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb")) + if (IsFile(BasePath / "rpc_recording_info.zcb")) { return true; } const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0); - if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin")) + if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin")) { // top-level metadata is missing, possibly because of premature exit // on the recording side diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f040e9ece..0fa05194f 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -235,7 +235,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); @@ -260,7 +260,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); @@ -346,7 +346,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); IoBuffer Payload; @@ -383,7 +383,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { struct WorkloadData { @@ -461,7 +461,7 @@ public: if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) { std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -493,7 +493,7 @@ public: for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); - if (std::filesystem::is_regular_file(MetaDataFile)) + if (IsFile(MetaDataFile)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -623,7 +623,7 @@ protected: BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { NeededAttachments.push_back(AttachmentHash); } diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b36f11741..c0c2a754a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -534,7 +534,7 @@ ZenServerEnvironment::CreateNewTestDir() TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1)); std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); - ZEN_ASSERT(!std::filesystem::exists(TestPath)); + ZEN_ASSERT(!IsDir(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); @@ -568,7 +568,7 @@ ZenServerInstance::~ZenServerInstance() { Shutdown(); std::error_code DummyEc; - std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); + RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); } catch (const std::exception& Err) { @@ -1033,7 +1033,7 @@ std::string ZenServerInstance::GetLogOutput() const { std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - if (std::filesystem::is_regular_file(OutputPath)) + if (IsFile(OutputPath)) { FileContents Contents = ReadFile(OutputPath); if (!Contents.ErrorCode) @@ -1123,7 +1123,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; } - if (!std::filesystem::is_directory(Info.DataDir)) + if (!IsDir(Info.DataDir)) { OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir); return false; diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp index a3cfe9d15..9cec5372a 100644 --- a/src/zenvfs/vfsprovider.cpp +++ b/src/zenvfs/vfsprovider.cpp @@ -373,13 +373,12 @@ VfsProvider::Initialize() std::filesystem::path ManifestPath = Root / ".zen_vfs"; bool HaveManifest = false; - if (std::filesystem::exists(Root)) + if (IsFile(Root)) + { + throw std::runtime_error("specified VFS root exists but is not a directory"); + } + if (IsDir(Root)) { - if (!std::filesystem::is_directory(Root)) - { - throw std::runtime_error("specified VFS root exists but is not a directory"); - } - std::error_code Ec; m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str()); @@ -388,7 +387,7 @@ VfsProvider::Initialize() throw std::system_error(Ec); } - if (std::filesystem::exists(ManifestPath)) + if (IsFile(ManifestPath)) { FileContents ManifestData = zen::ReadFile(ManifestPath); CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten()); -- cgit v1.2.3 From ebe13120c030f8d24c5f05c068d79b2f72fc3c0b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 11:30:28 +0200 Subject: multithreaded clean (#331) - Improvement: Faster cleaning of directories - Improvement: Faster initial scanning of local state --- src/zen/cmds/builds_cmd.cpp | 425 ++++++++++++++++++++++++------- src/zencore/filesystem.cpp | 55 +++- src/zencore/include/zencore/filesystem.h | 5 + 3 files changed, 393 insertions(+), 92 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index d2ba20e78..2bbb21012 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -316,25 +316,108 @@ namespace { bool CleanDirectory(const std::filesystem::path& Path, std::span ExcludeDirectories) { ZEN_TRACE_CPU("CleanDirectory"); + Stopwatch Timer; - bool CleanWipe = true; + ProgressBar Progress(UsePlainProgress); - DirectoryContent LocalDirectoryContent; - GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); - for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + std::atomic CleanWipe = true; + std::atomic DiscoveredItemCount = 0; + std::atomic DeletedItemCount = 0; + std::atomic DeletedByteCount = 0; + ParallellWork Work(AbortFlag); + + struct AsyncVisitor : public GetDirectoryContentVisitor { - try + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic& InCleanWipe, + std::atomic& InDiscoveredItemCount, + std::atomic& InDeletedItemCount, + std::atomic& InDeletedByteCount, + std::span InExcludeDirectories) + : Path(InPath) + , CleanWipe(InCleanWipe) + , DiscoveredItemCount(InDiscoveredItemCount) + , DeletedItemCount(InDeletedItemCount) + , DeletedByteCount(InDeletedByteCount) + , ExcludeDirectories(InExcludeDirectories) { - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); } - catch (const std::exception& Ex) + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); - CleanWipe = false; + ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); + if (!AbortFlag) + { + if (!Content.FileNames.empty()) + { + DiscoveredItemCount += Content.FileNames.size(); + + const std::string RelativeRootString = RelativeRoot.generic_string(); + bool RemoveContent = true; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (RelativeRootString.starts_with(ExcludeDirectory)) + { + if (RelativeRootString.length() > ExcludeDirectory.length()) + { + const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; + if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || + MaybePathDelimiter == std::filesystem::path::preferred_separator) + { + RemoveContent = false; + break; + } + } + else + { + RemoveContent = false; + break; + } + } + } + if (RemoveContent) + { + ZEN_TRACE_CPU("DeleteFiles"); + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); + try + { + SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + CleanWipe = false; + } + } + } + } + } } - } + const std::filesystem::path& Path; + std::atomic& CleanWipe; + std::atomic& DiscoveredItemCount; + std::atomic& DeletedItemCount; + std::atomic& DeletedByteCount; + std::span ExcludeDirectories; + } Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories); + GetDirectoryContent( + Path, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, + Visitor, + GetIOWorkerPool(), + Work.PendingWork()); + + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + DiscoveredItemCount += LocalDirectoryContent.Directories.size(); + std::vector DirectoriesToDelete; + DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size()); for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; @@ -348,31 +431,78 @@ namespace { } if (!Leave) { - try - { - std::error_code Ec; - zen::CleanDirectory(LocalDirPath, true, Ec); - if (Ec) - { - Sleep(200); - zen::CleanDirectory(LocalDirPath, true); - Ec.clear(); - } + DirectoriesToDelete.emplace_back(std::move(LocalDirPath)); + DiscoveredItemCount++; + } + } - RemoveDir(LocalDirPath, Ec); - if (Ec) - { - Sleep(200); - RemoveDir(LocalDirPath); - } - } - catch (const std::exception& Ex) + uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); + }); + + for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) + { + ZEN_TRACE_CPU("DeleteDirs"); + try + { + std::error_code Ec; + zen::CleanDirectory(DirectoryToDelete, true, Ec); + if (Ec) { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; + Sleep(200); + zen::CleanDirectory(DirectoryToDelete, true); + Ec.clear(); } + + RemoveDirWithRetry(DirectoryToDelete); + + DeletedItemCount++; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + CleanWipe = false; + } + + uint64_t NowMs = Timer.GetElapsedTimeMs(); + if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + { + LastUpdateTimeMs = NowMs; + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); } } + + Progress.Finish(); + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (ElapsedTimeMs >= 200) + { + ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}", + Path, + DiscoveredItemCount.load(), + NiceBytes(DeletedByteCount.load()), + NiceTimeSpanMs(ElapsedTimeMs)); + } return CleanWipe; } @@ -1959,7 +2089,7 @@ namespace { BasicFile CompressedFile; std::error_code Ec; - CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec); + CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec); if (Ec) { throw std::runtime_error( @@ -2406,7 +2536,7 @@ namespace { PartPayload.SetContentType(ZenContentType::kBinary); return PartPayload; }, - [&, Payload, RawSize](uint64_t SentBytes, bool IsComplete) { + [&, RawSize](uint64_t SentBytes, bool IsComplete) { UploadStats.ChunksBytes += SentBytes; UploadedCompressedChunkSize += SentBytes; if (IsComplete) @@ -4159,7 +4289,10 @@ namespace { bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span> SequenceIndexChunksLeftToWriteCounters) { - return SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1; + uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1); + ZEN_ASSERT(PreviousValue >= 1); + ZEN_ASSERT(PreviousValue != (uint32_t)-1); + return PreviousValue == 1; } std::vector CompleteChunkTargets(const std::vector& ChunkTargetPtrs, @@ -5680,7 +5813,31 @@ namespace { { BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - if (!BuildBlob) + if (BuildBlob) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + else { if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) { @@ -6420,6 +6577,10 @@ namespace { const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; ZEN_ASSERT(!IncompletePath.empty()); const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_CONSOLE("{}: Max count {}, Current count {}", + IncompletePath, + ExpectedSequenceCount, + SequenceIndexChunksLeftToWriteCounter.load()); ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); } } @@ -6453,8 +6614,14 @@ namespace { tsl::robin_map SequenceHashToLocalPathIndex; std::vector RemoveLocalPathIndexes; - if (!WipeTargetFolder) + if (AbortFlag) + { + return; + } + { + ZEN_TRACE_CPU("UpdateFolder_PrepareTarget"); + tsl::robin_set CachedRemoteSequences; tsl::robin_map RemotePathToRemoteIndex; RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); @@ -6463,14 +6630,21 @@ namespace { RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); } - uint64_t MatchCount = 0; - uint64_t PathMismatchCount = 0; - uint64_t HashMismatchCount = 0; - uint64_t CachedCount = 0; - uint64_t SkippedCount = 0; - uint64_t DeleteCount = 0; + std::vector FilesToCache; + + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + std::atomic CachedCount = 0; + std::atomic CachedByteCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { + if (AbortFlag) + { + break; + } const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; @@ -6502,22 +6676,16 @@ namespace { } if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) { - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); if (!CachedRemoteSequences.contains(RawHash)) { + ZEN_TRACE_CPU("MoveToCache"); // We need it - ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); - const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); - - RenameFileWithRetry(LocalFilePath, CacheFilePath); - + FilesToCache.push_back(LocalPathIndex); CachedRemoteSequences.insert(RawHash); - CachedCount++; } else { // We already have it - ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); SkippedCount++; } } @@ -6529,15 +6697,80 @@ namespace { } } - ZEN_DEBUG( - "Local state prep: MatchCount: {}, PathMismatchCount: {}, HashMismatchCount: {}, CachedCount: {}, SkippedCount: {}, " - "DeleteCount: {}", - MatchCount, - PathMismatchCount, - HashMismatchCount, - CachedCount, - SkippedCount, - DeleteCount); + if (AbortFlag) + { + return; + } + + { + ZEN_TRACE_CPU("UpdateFolder_CopyToCache"); + + Stopwatch Timer; + + WorkerThreadPool& WritePool = GetIOWorkerPool(); + + ProgressBar CacheLocalProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); + + for (uint32_t LocalPathIndex : FilesToCache) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; + } + }, + Work.DefaultErrorFunction()); + } + + { + ZEN_TRACE_CPU("CacheLocal_Wait"); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + const uint64_t WorkTotal = FilesToCache.size(); + const uint64_t WorkComplete = CachedCount.load(); + std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount)); + CacheLocalProgressBar.UpdateState({.Task = "Caching local ", + .Details = Details, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, + false); + }); + } + + if (AbortFlag) + { + return; + } + + CacheLocalProgressBar.Finish(); + + ZEN_DEBUG( + "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " + "Delete: {}", + MatchCount, + PathMismatchCount, + HashMismatchCount, + CachedCount.load(), + NiceBytes(CachedByteCount.load()), + SkippedCount, + DeleteCount); + } } if (WipeTargetFolder) @@ -6546,30 +6779,16 @@ namespace { Stopwatch Timer; // Clean target folder - ZEN_CONSOLE("Wiping {}", Path); if (!CleanDirectory(Path, DefaultExcludeFolders)) { ZEN_WARN("Some files in {} could not be removed", Path); } RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } - else - { - ZEN_TRACE_CPU("UpdateFolder_RemoveUnused"); - Stopwatch Timer; - - // Remove unused tracked files - if (!RemoveLocalPathIndexes.empty()) - { - ZEN_CONSOLE("Cleaning {} removed files from {}", RemoveLocalPathIndexes.size(), Path); - for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) - { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - SetFileReadOnlyWithRetry(LocalFilePath, false); - RemoveFileWithRetry(LocalFilePath); - } - } + if (AbortFlag) + { + return; } { @@ -6586,6 +6805,28 @@ namespace { OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size()); OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size()); + std::atomic DeletedCount = 0; + + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic&) { + if (!AbortFlag) + { + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); + DeletedCount++; + } + }, + Work.DefaultErrorFunction()); + } + std::atomic TargetsComplete = 0; struct FinalizeTarget @@ -6638,6 +6879,7 @@ namespace { if (RawHash == IoHash::Zero) { + ZEN_TRACE_CPU("ZeroSize"); while (TargetOffset < (BaseTargetOffset + TargetCount)) { const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; @@ -6674,6 +6916,7 @@ namespace { } else { + ZEN_TRACE_CPU("Files"); ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; @@ -6698,16 +6941,19 @@ namespace { if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); InplaceIt != SequenceHashToLocalPathIndex.end()) { + ZEN_TRACE_CPU("Copy"); const uint32_t LocalPathIndex = InplaceIt->second; const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } else { + ZEN_TRACE_CPU("Rename"); const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); @@ -6747,6 +6993,7 @@ namespace { } else { + ZEN_TRACE_CPU("Copy"); if (IsFile(TargetFilePath)) { SetFileReadOnlyWithRetry(TargetFilePath, false); @@ -6757,6 +7004,7 @@ namespace { } ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; } @@ -6788,11 +7036,13 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); + const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); + const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); + std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", .Details = Details, - .TotalCount = gsl::narrow(Targets.size()), - .RemainingCount = gsl::narrow(Targets.size() - TargetsComplete.load())}, + .TotalCount = gsl::narrow(WorkTotal), + .RemainingCount = gsl::narrow(WorkTotal - WorkComplete)}, false); }); } @@ -7312,7 +7562,7 @@ namespace { while (PathIndex < PathCount) { - uint32_t PathRangeCount = Min(1024u, PathCount - PathIndex); + uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); Work.ScheduleWork( GetIOWorkerPool(), [PathIndex, @@ -7326,17 +7576,16 @@ namespace { { const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); - if (IsFile(LocalFilePath)) + if (TryGetFileProperties(LocalFilePath, + OutLocalFolderContent.RawSizes[PathRangeIndex], + OutLocalFolderContent.ModificationTicks[PathRangeIndex], + OutLocalFolderContent.Attributes[PathRangeIndex])) { - const uint64_t FileSize = FileSizeFromPath(LocalFilePath); - OutLocalFolderContent.Paths[PathRangeIndex] = FilePath; - OutLocalFolderContent.RawSizes[PathRangeIndex] = FileSize; - OutLocalFolderContent.Attributes[PathRangeIndex] = GetNativeFileAttributes(LocalFilePath); - OutLocalFolderContent.ModificationTicks[PathRangeIndex] = GetModificationTickFromPath(LocalFilePath); + OutLocalFolderContent.Paths[PathRangeIndex] = std::move(FilePath); LocalFolderScanStats.FoundFileCount++; - LocalFolderScanStats.FoundFileByteCount += FileSize; + LocalFolderScanStats.FoundFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; LocalFolderScanStats.AcceptedFileCount++; - LocalFolderScanStats.AcceptedFileByteCount += FileSize; + LocalFolderScanStats.AcceptedFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; } CompletedPathCount++; } diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 6ff4dd053..4ec563ba3 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -280,16 +280,17 @@ bool RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - BOOL Success = ::DeleteFile(Path.native().c_str()); + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + BOOL Success = ::DeleteFile(NativePath); if (!Success) { if (ForceRemoveReadOnlyFiles) { - DWORD FileAttributes = ::GetFileAttributes(Path.native().c_str()); + DWORD FileAttributes = ::GetFileAttributes(NativePath); if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) { - ::SetFileAttributes(Path.native().c_str(), MakeFileAttributeReadOnly(FileAttributes, false)); - Success = ::DeleteFile(Path.native().c_str()); + ::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(NativePath); } } if (!Success) @@ -1964,6 +1965,52 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +bool +TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + { + void* Handle = CreateFileW(NativePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + return false; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + + BY_HANDLE_FILE_INFORMATION Bhfh = {}; + if (!GetFileInformationByHandle(Handle, &Bhfh)) + { + return false; + } + OutSize = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow; + OutModificationTick = ((uint64_t(Bhfh.ftLastWriteTime.dwHighDateTime) << 32) | Bhfh.ftLastWriteTime.dwLowDateTime); + OutNativeModeOrAttributes = Bhfh.dwFileAttributes; + return true; + } +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err) + { + return false; + } + OutModificationTick = gsl::narrow(Stat.st_mtime); + OutSize = size_t(Stat.st_size); + OutNativeModeOrAttributes = (uint32_t)Stat.st_mode; + return true; +#endif +} + void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) { diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index c23f16d03..66deffa6f 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -110,6 +110,11 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes); + /** Move a file, if the files are not on the same drive the function will fail */ ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -- cgit v1.2.3 From 2103b5d1f20f1b46a30eced4130c22a7cf7ae694 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 13:53:01 +0200 Subject: logging improvement (#332) * Cache -> Download cache * cleaned up info regarding local cache/state and remote cache --- src/zen/cmds/builds_cmd.cpp | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 2bbb21012..eebb6d82b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -5235,21 +5235,24 @@ namespace { } } - if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() || - !LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) - { - ZEN_CONSOLE( - "Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks", - CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), - CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), - CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) + { + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks.", + CachedSequenceHashesFound.size(), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + CachedChunkHashesFound.size(), + NiceBytes(CacheMappingStats.CacheChunkByteCount), + CachedBlocksFound.size(), + NiceBytes(CacheMappingStats.CacheBlocksByteCount)); + } + + if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); } uint64_t BytesToWrite = 0; @@ -5633,7 +5636,7 @@ namespace { ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); if (!ExistsResult.ExistingBlobs.empty()) { - ZEN_CONSOLE("Found {} out of {} needed blobs in remote cache in {}", + ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}", ExistsResult.ExistingBlobs.size(), BlobHashes.size(), NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); @@ -8737,7 +8740,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("Remote: {}.{}", StorageDescription, - CacheDescription.empty() ? std::string("") : fmt::format(" Cache: {}", CacheDescription)); + CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription)); return Result; }; -- cgit v1.2.3 From bbb7e0c77b9ed114baf428256f347631b3e1092c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 14:26:53 +0200 Subject: verify that we can read input files that are only hashed (#333) * output build and part details by default * output executable and version at start of builds command * verify that we can read files we do not chunk --- src/zen/cmds/builds_cmd.cpp | 121 +++++++++++++++++++++++++++++++++++++++-- src/zenutil/chunkedcontent.cpp | 8 ++- 2 files changed, 122 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index eebb6d82b..689b29db5 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -7061,6 +7063,107 @@ namespace { } } + std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) + { + ExtendableStringBuilder<512> SB; + std::vector> NameStringValuePairs; + for (CbFieldView Field : Object) + { + std::string_view Name = Field.GetName(); + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::String: + NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); + break; + case CbFieldType::IntegerPositive: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); + break; + case CbFieldType::IntegerNegative: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::BoolFalse: + NameStringValuePairs.push_back({std::string(Name), "false"}); + break; + case CbFieldType::BoolTrue: + NameStringValuePairs.push_back({std::string(Name), "true"}); + break; + case CbFieldType::Hash: + { + NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); + } + break; + case CbFieldType::Uuid: + { + StringBuilder Builder; + Accessor.AsUuid().ToString(Builder); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::DateTime: + { + ExtendableStringBuilder<64> Builder; + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::TimeSpan: + { + ExtendableStringBuilder<64> Builder; + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + break; + } + case CbFieldType::ObjectId: + NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); + break; + } + } + std::string::size_type LongestKey = 0; + for (const std::pair& KeyValue : NameStringValuePairs) + { + LongestKey = Max(KeyValue.first.length(), LongestKey); + } + for (const std::pair& KeyValue : NameStringValuePairs) + { + SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); + } + return SB.ToString(); + } + std::vector> ResolveBuildPartNames(BuildStorage& Storage, const Oid& BuildId, const std::vector& BuildPartIds, @@ -7071,14 +7174,12 @@ namespace { { Stopwatch GetBuildTimer; CbObject BuildObject = Storage.GetBuild(BuildId); - ZEN_CONSOLE("GetBuild took {}. Name: '{}' ({}, {}), Payload size: {}", + ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), BuildObject["name"sv].AsString(), - BuildObject["type"sv].AsString(), - BuildObject["Configuration"sv].AsString(), NiceBytes(BuildObject.GetSize())); - ZEN_CONSOLE_VERBOSE("Build object: {}", BuildObject); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -7170,6 +7271,7 @@ namespace { BuildPartName, NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), NiceBytes(BuildPartManifest.GetSize())); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); { CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); @@ -8750,6 +8852,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { if (SubOption == &m_ListOptions) { + if (!m_ListResultPath.empty()) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + } CbObject QueryObject; if (m_ListQueryPath.empty()) { @@ -8824,6 +8930,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_UploadOptions.help())); @@ -8979,6 +9087,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); @@ -9472,6 +9582,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -9519,7 +9630,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - // HttpClient Http(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BuildsUrl.empty() && m_StoragePath.empty()) { diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 1e8447a57..32ae2d94a 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -140,8 +140,12 @@ namespace { { ZEN_TRACE_CPU("HashOnly"); - IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); - const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); + if (Buffer.GetSize() != RawSize) + { + throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + } + const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); Lock.WithExclusiveLock([&]() { if (!RawHashToSequenceRawHashIndex.contains(Hash)) -- cgit v1.2.3 From 3ecdceceee909b57faa70ac7efa4ff58245b35d0 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 17:29:56 +0200 Subject: reduce disk io during gc (#335) * do cache bucket flush/write snapshot as part of compact to reduce disk I/O --- src/zenstore/cache/cachedisklayer.cpp | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index b2d2416be..e4d962b56 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -2731,9 +2731,10 @@ class DiskBucketStoreCompactor : public GcStoreCompactor using CacheBucket = ZenCacheDiskLayer::CacheBucket; public: - DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector>&& ExpiredStandaloneKeys) + DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector>&& ExpiredStandaloneKeys, bool FlushBucket) : m_Bucket(Bucket) , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys)) + , m_FlushBucket(FlushBucket) { m_ExpiredStandaloneKeys.shrink_to_fit(); } @@ -2960,6 +2961,10 @@ public: } } } + if (m_FlushBucket) + { + m_Bucket.Flush(); + } } virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); } @@ -2967,6 +2972,7 @@ public: private: ZenCacheDiskLayer::CacheBucket& m_Bucket; std::vector> m_ExpiredStandaloneKeys; + bool m_FlushBucket = false; }; GcStoreCompactor* @@ -2990,24 +2996,6 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) NiceBytes(Stats.FreedMemory), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } - if (Stats.DeletedCount > 0) - { - bool Expected = false; - if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true)) - { - return; - } - auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); }); - - try - { - SaveSnapshot([]() { return 0; }); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what()); - } - } }); const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); @@ -3094,7 +3082,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) return nullptr; } - return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys)); + return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys), /*FlushBucket*/ Stats.DeletedCount > 0); } bool -- cgit v1.2.3 From 571aad8c5d2ed9d975ba2fed2890116e3c8bd6ea Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 18:27:05 +0200 Subject: builds url discovery (#334) - Feature: Added `--host` option to use Jupiters list of cloud host and zen servers to resolve best hosts - Feature: Use local zenserver as builds cache if it has the `builds` service enabled and `--cloud-discovery-host` is provided and no remote zenserver cache hosts can be found - Improvement: Added `--override-host` option as a replacement for `--url` (`--url` still works, but `--override-host` is preferred) --- src/zen/cmds/builds_cmd.cpp | 247 +++++++++++++++++++----- src/zen/cmds/builds_cmd.h | 3 +- src/zenserver/buildstore/httpbuildstore.cpp | 19 +- src/zenserver/buildstore/httpbuildstore.h | 9 +- src/zenserver/frontend/frontend.cpp | 15 +- src/zenserver/frontend/frontend.h | 7 +- src/zenserver/objectstore/objectstore.cpp | 14 +- src/zenserver/objectstore/objectstore.h | 7 +- src/zenserver/projectstore/httpprojectstore.cpp | 22 ++- src/zenserver/projectstore/httpprojectstore.h | 11 +- src/zenserver/vfs/vfsservice.cpp | 15 +- src/zenserver/vfs/vfsservice.h | 9 +- src/zenserver/workspaces/httpworkspaces.cpp | 21 +- src/zenserver/workspaces/httpworkspaces.h | 10 +- src/zenserver/zenserver.cpp | 13 +- 15 files changed, 345 insertions(+), 77 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 689b29db5..1c24a3c5b 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -883,6 +883,7 @@ namespace { std::string StorageName; std::unique_ptr CacheHttp; std::unique_ptr BuildCacheStorage; + std::string CacheName; }; std::vector CalculateAbsoluteChunkOrders(const std::span LocalChunkHashes, @@ -8348,7 +8349,14 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); - Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), ""); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), ""); + Ops.add_option("cloud build", + "", + "url", + "Cloud Builds host url (legacy - use --override-host)", + cxxopts::value(m_OverrideHost), + ""); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), ""); Ops.add_option("cloud build", "", "assume-http2", @@ -8657,7 +8665,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } auto ParseStorageOptions = [&]() { - if (!m_BuildsUrl.empty()) + if (!m_OverrideHost.empty() || !m_Host.empty()) { if (!m_StoragePath.empty()) { @@ -8676,7 +8684,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; std::unique_ptr Auth; - HttpClientSettings ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2}; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; auto CreateAuthMgr = [&]() { if (!Auth) @@ -8762,7 +8773,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); } - if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider) + if (!ClientSettings.AccessTokenProvider) { ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials."); } @@ -8781,21 +8792,176 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) StorageInstance Result; + std::string BuildStorageName = ZEN_CLOUD_STORAGE; + std::string BuildCacheName; + bool CacheAssumeHttp2 = false; std::string StorageDescription; std::string CacheDescription; - if (!m_BuildsUrl.empty()) + if (!m_Host.empty() || !m_OverrideHost.empty()) { ParseAuthOptions(); - Result.BuildStorageHttp = std::make_unique(m_BuildsUrl, ClientSettings); - StorageDescription = fmt::format("cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, + } + + std::string CloudHost; + + if (!m_Host.empty()) + { + if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) + { + HttpClient DiscoveryHttpClient(m_Host, ClientSettings); + HttpClient::Response ServerInfoResponse = + DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); + if (!ServerInfoResponse.IsSuccess()) + { + throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'", + m_Host, + ServerInfoResponse.ErrorMessage(""))); + } + + std::string_view JsonResponse = ServerInfoResponse.AsText(); + CbObject ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject(); + + if (m_OverrideHost.empty()) + { + CbArrayView ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView(); + std::uint64_t ServerCount = ServerEndpointsArray.Num(); + if (ServerCount == 0) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); + } + for (CbFieldView ServerEndpointView : ServerEndpointsArray) + { + CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView(); + + std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = ServerEndpointObject["name"sv].AsString(); + + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + CloudHost = BaseUrl; + m_AssumeHttp2 = AssumeHttp2; + BuildStorageName = Name; + break; + } + } + } + if (CloudHost.empty()) + { + throw std::runtime_error( + fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host)); + } + } + + auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool { + HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); + if (TestResponse.IsSuccess()) + { + return true; + } + return false; + }; + + if (m_ZenCacheHost.empty()) + { + CbArrayView CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView(); + std::uint64_t CacheCount = CacheEndpointsArray.Num(); + for (CbFieldView CacheEndpointView : CacheEndpointsArray) + { + CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView(); + + std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = CacheEndpointObject["name"sv].AsString(); + + if (TestCacheEndpoint(BaseUrl, AssumeHttp2)) + { + m_ZenCacheHost = BaseUrl; + CacheAssumeHttp2 = AssumeHttp2; + BuildCacheName = Name; + break; + } + } + } + if (m_ZenCacheHost.empty()) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (m_ZenCacheHost.empty()) + { + std::string ZenServerLocalHostUrl = + fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (TestCacheEndpoint(ZenServerLocalHostUrl, false)) + { + m_ZenCacheHost = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + BuildCacheName = "localhost"; + } + } + }); + } + if (m_ZenCacheHost.empty()) + { + ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); + } + } + } + else if (TestCacheEndpoint(m_ZenCacheHost, false)) + { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength); + } + } + } + else + { + CloudHost = m_OverrideHost; + } + + if (!CloudHost.empty()) + { + Result.BuildStorageHttp = std::make_unique(CloudHost, ClientSettings); + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), + CloudHost, Result.BuildStorageHttp->GetSessionId(), m_Namespace, m_Bucket); Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); - Result.StorageName = ZEN_CLOUD_STORAGE; + Result.StorageName = BuildStorageName; } else if (!m_StoragePath.empty()) { @@ -8810,39 +8976,39 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } if (!m_ZenCacheHost.empty()) { - Result.CacheHttp = std::make_unique(m_ZenCacheHost, + Result.CacheHttp = std::make_unique(m_ZenCacheHost, HttpClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = false, - .AllowResume = true, - .RetryCount = 0}); - if (Result.CacheHttp->Get("/health").IsSuccess()) + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = CacheAssumeHttp2, + .AllowResume = true, + .RetryCount = 0}); + Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + m_PrimeCacheOnly); + CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", + BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName), + m_ZenCacheHost, + Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) { - Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - m_PrimeCacheOnly); - CacheDescription = fmt::format("zen cache {}. SessionId: '{}'", m_ZenCacheHost, Result.CacheHttp->GetSessionId()); - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(" {}.", m_Namespace); - } - if (!m_Bucket.empty()) - { - CacheDescription += fmt::format(" {}.", m_Bucket); - } + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); } - else + if (!m_Bucket.empty()) { - Result.CacheHttp.reset(); + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); } + Result.CacheName = BuildCacheName; + } + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheName.empty()) + { + ZEN_CONSOLE("Cache : {}", CacheDescription); } - ZEN_CONSOLE("Remote: {}.{}", - StorageDescription, - CacheDescription.empty() ? std::string("") : fmt::format(" Remote Cache: {}", CacheDescription)); return Result; }; @@ -9272,7 +9438,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); @@ -9280,7 +9446,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) m_StoragePath = StoragePath.generic_string(); } auto _ = MakeGuard([&]() { - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { DeleteDirectories(StoragePath); } @@ -9632,11 +9798,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } - if (m_BuildId.empty()) { throw zen::OptionParseException(fmt::format("build-id is required\n{}", m_DownloadOptions.help())); diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 4a77f8bd7..bc5cd235f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -34,7 +34,8 @@ private: std::string m_ZenFolderPath; // cloud builds - std::string m_BuildsUrl; + std::string m_OverrideHost; + std::string m_Host; bool m_AssumeHttp2 = false; std::string m_Namespace; std::string m_Bucket; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp index 06bfea423..c918f5683 100644 --- a/src/zenserver/buildstore/httpbuildstore.cpp +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -19,16 +19,22 @@ using namespace std::literals; ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); -HttpBuildStoreService::HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store) +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) : m_Log(logging::Get("builds")) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_BuildStore(Store) { Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); } HttpBuildStoreService::~HttpBuildStoreService() { + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); } const char* @@ -42,8 +48,6 @@ HttpBuildStoreService::Initialize() { ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); - m_StatsService.RegisterHandler("builds", *this); - m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)"); m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)"); m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); @@ -523,4 +527,13 @@ HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + } // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h index a59aa882a..50cb5db12 100644 --- a/src/zenserver/buildstore/httpbuildstore.h +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -12,16 +13,17 @@ namespace zen { class BuildStore; -class HttpBuildStoreService final : public zen::HttpService, public IHttpStatsProvider +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpBuildStoreService(HttpStatsService& StatsService, BuildStore& Store); + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); virtual ~HttpBuildStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct BuildStoreStats @@ -55,7 +57,8 @@ private: HttpRequestRouter m_Router; - HttpStatsService& m_StatsService; + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; BuildStore& m_BuildStore; BuildStoreStats m_BuildStoreStats; diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 104b26954..dfa710ae0 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -2,6 +2,7 @@ #include "frontend.h" +#include #include #include #include @@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = { namespace zen { //////////////////////////////////////////////////////////////////////////////// -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService) +: m_Directory(Directory) +, m_StatusService(StatusService) { std::filesystem::path SelfPath = GetRunningExecutablePath(); @@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { ZEN_INFO("front-end is NOT AVAILABLE"); } + m_StatusService.RegisterHandler("dashboard", *this); } HttpFrontendService::~HttpFrontendService() { + m_StatusService.UnregisterHandler("dashboard", *this); } const char* @@ -94,6 +99,14 @@ HttpFrontendService::BaseUri() const } //////////////////////////////////////////////////////////////////////////////// +void +HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 6eac20620..84ffaac42 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -3,23 +3,26 @@ #pragma once #include +#include #include "zipfs.h" #include namespace zen { -class HttpFrontendService final : public zen::HttpService +class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpFrontendService(std::filesystem::path Directory); + HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService); virtual ~HttpFrontendService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: ZipFs m_ZipFs; std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index 5af803617..8faf12165 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -219,13 +219,17 @@ private: StringBuilderBase& Builder; }; -HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg)) +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) { Inititalize(); + m_StatusService.RegisterHandler("obj", *this); } HttpObjectStoreService::~HttpObjectStoreService() { + m_StatusService.UnregisterHandler("obj", *this); } const char* @@ -244,6 +248,14 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) } } +void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpObjectStoreService::Inititalize() { diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index dae979c4c..44e50e208 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -23,14 +24,15 @@ struct ObjectStoreConfig std::vector Buckets; }; -class HttpObjectStoreService final : public zen::HttpService +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpObjectStoreService(ObjectStoreConfig Cfg); + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); virtual ~HttpObjectStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: void Inititalize(); @@ -41,6 +43,7 @@ private: void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); void PutObject(zen::HttpRouterRequest& Request); + HttpStatusService& m_StatusService; ObjectStoreConfig m_Cfg; std::mutex BucketsMutex; HttpRequestRouter m_Router; diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 6313fd69e..317a419eb 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -235,10 +235,15 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr) +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_AuthMgr(AuthMgr) { @@ -246,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, using namespace std::literals; - m_StatsService.RegisterHandler("prj", *this); - m_Router.AddPattern("project", "([[:alnum:]_.]+)"); m_Router.AddPattern("log", "([[:alnum:]_.]+)"); m_Router.AddPattern("op", "([[:digit:]]+?)"); @@ -366,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, "details\\$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); } HttpProjectService::~HttpProjectService() { m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); } const char* @@ -465,6 +472,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) { diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 8e74c57a5..295defa5c 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace zen { @@ -31,16 +32,21 @@ class ProjectStore; // refs: // -class HttpProjectService : public HttpService, public IHttpStatsProvider +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct ProjectStats @@ -89,6 +95,7 @@ private: CidStore& m_CidStore; HttpRequestRouter m_Router; Ref m_ProjectStore; + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; AuthMgr& m_AuthMgr; ProjectStats m_ProjectStats; diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp index d302a10ec..bf761f8d1 100644 --- a/src/zenserver/vfs/vfsservice.cpp +++ b/src/zenserver/vfs/vfsservice.cpp @@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { m_Impl = new Impl; @@ -136,10 +136,12 @@ VfsService::VfsService() } }, HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); } VfsService::~VfsService() { + m_StatusService.UnregisterHandler("vfs", *this); delete m_Impl; } @@ -169,8 +171,9 @@ VfsService::AddService(Ref&& Z$) #else -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { + ZEN_UNUSED(StatusService); } VfsService::~VfsService() @@ -208,6 +211,14 @@ VfsService::BaseUri() const return "/vfs/"; } +void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) { diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h index dcdc71e81..0d0168e23 100644 --- a/src/zenserver/vfs/vfsservice.h +++ b/src/zenserver/vfs/vfsservice.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -24,10 +25,10 @@ class ZenCacheStore; */ -class VfsService : public HttpService +class VfsService : public HttpService, public IHttpStatusProvider { public: - VfsService(); + explicit VfsService(HttpStatusService& StatusService); ~VfsService(); void Mount(std::string_view MountPoint); @@ -39,12 +40,14 @@ public: protected: virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct Impl; Impl* m_Impl = nullptr; - HttpRequestRouter m_Router; + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; friend struct VfsServiceDataSource; }; diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index ac0aaef8e..7ef84743e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -73,8 +73,12 @@ namespace { } // namespace -HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces) +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) : m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_Config(Cfg) , m_Workspaces(Workspaces) @@ -85,6 +89,7 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con HttpWorkspacesService::~HttpWorkspacesService() { m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); } const char* @@ -148,6 +153,15 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + void HttpWorkspacesService::Initialize() { @@ -155,8 +169,6 @@ HttpWorkspacesService::Initialize() ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); - m_StatsService.RegisterHandler("ws", *this); - m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); @@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize() HttpVerb::kGet); RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); } std::filesystem::path diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h index f01f58b86..89a8e8bdc 100644 --- a/src/zenserver/workspaces/httpworkspaces.h +++ b/src/zenserver/workspaces/httpworkspaces.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace zen { @@ -16,16 +17,20 @@ struct WorkspacesServeConfig bool AllowConfigurationChanges = false; }; -class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces); + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); virtual ~HttpWorkspacesService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct WorkspacesStats @@ -80,6 +85,7 @@ private: void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; const WorkspacesServeConfig m_Config; HttpRequestRouter m_Router; diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 45c91d691..c7cb2ba6e 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -251,13 +251,14 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); if (ServerOptions.WorksSpacesConfig.Enabled) { m_Workspaces.reset(new Workspaces()); m_HttpWorkspacesService.reset( - new HttpWorkspacesService(m_StatsService, + new HttpWorkspacesService(m_StatusService, + m_StatsService, {.SystemRootDir = ServerOptions.SystemRootDir, .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, *m_Workspaces)); @@ -295,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_HttpWorkspacesService); } - m_FrontendService = std::make_unique(m_ContentRoot); + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); if (m_FrontendService) { @@ -314,18 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique(std::move(ObjCfg)); + m_ObjStoreService = std::make_unique(m_StatusService, std::move(ObjCfg)); m_Http->RegisterService(*m_ObjStoreService); } if (ServerOptions.BuildStoreConfig.Enabled) { - m_BuildStoreService = std::make_unique(m_StatsService, *m_BuildStore); + m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); m_Http->RegisterService(*m_BuildStoreService); } #if ZEN_WITH_VFS - m_VfsService = std::make_unique(); + m_VfsService = std::make_unique(m_StatusService); m_VfsService->AddService(Ref(m_ProjectStore)); m_VfsService->AddService(Ref(m_CacheStore)); m_Http->RegisterService(*m_VfsService); -- cgit v1.2.3 From e244e2a07a0f3bf3b11d6d875d6c2c8bb169efc5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 15:46:05 +0200 Subject: added --find-max-block-count option to builds upload (#337) --- src/zen/cmds/builds_cmd.cpp | 16 +++++++++++++--- src/zen/cmds/builds_cmd.h | 3 ++- src/zenserver/projectstore/buildsremoteprojectstore.cpp | 2 +- src/zenutil/filebuildstorage.cpp | 9 ++++++++- src/zenutil/include/zenutil/buildstorage.h | 2 +- src/zenutil/include/zenutil/jupiter/jupitersession.h | 2 +- src/zenutil/jupiter/jupiterbuildstorage.cpp | 4 ++-- src/zenutil/jupiter/jupitersession.cpp | 8 +++++--- 8 files changed, 33 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 1c24a3c5b..8e8fd480a 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2900,6 +2900,7 @@ namespace { const std::filesystem::path& Path, const std::filesystem::path& ZenFolderPath, const std::filesystem::path& ManifestPath, + const uint64_t FindBlockMaxCount, const uint8_t BlockReuseMinPercentLimit, bool AllowMultiparts, const CbObject& MetaData, @@ -2939,7 +2940,7 @@ namespace { FindBlocksStatistics FindBlocksStats; std::future PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task{ - [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + [&Storage, BuildId, FindBlockMaxCount, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { ZEN_TRACE_CPU("PrepareBuild"); PrepareBuildResult Result; @@ -2976,7 +2977,7 @@ namespace { { ZEN_TRACE_CPU("FindBlocks"); Stopwatch KnownBlocksTimer; - Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount)); FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); @@ -7417,7 +7418,7 @@ namespace { std::vector AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); std::vector FoundBlocks = - ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId)); + ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, (uint64_t)-1)); for (const IoHash& BlockHash : BlockRawHashes) { @@ -8496,6 +8497,12 @@ BuildsCommand::BuildsCommand() ""); m_UploadOptions .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), ""); + m_UploadOptions.add_option("", + "", + "find-max-block-count", + "The maximum number of blocks we search for in the build context", + cxxopts::value(m_FindBlockMaxCount), + ""); m_UploadOptions.parse_positional({"local-path", "build-id"}); m_UploadOptions.positional_help("local-path build-id"); @@ -9221,6 +9228,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Path, ZenFolderPath, MakeSafeAbsolutePath(m_ManifestPath), + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -9489,6 +9497,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Path, ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -9677,6 +9686,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) DownloadPath, ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData2, diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index bc5cd235f..b40c3e08f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -89,7 +89,8 @@ private: std::string m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - bool m_PostUploadVerify = false; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; std::vector m_BuildPartNames; diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index fbb9bc344..a6583b722 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -308,7 +308,7 @@ public: { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId); + JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; if (Result.ErrorCode) diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 0fa05194f..7aa252e44 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,7 +442,7 @@ public: SimulateLatency(0, 0); } - virtual CbObject FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); @@ -451,6 +451,8 @@ public: auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; + uint64_t FoundCount = 0; + DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); CbObjectWriter Writer; @@ -469,6 +471,11 @@ public: CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); Writer.AddObject(BlockObject); + FoundCount++; + if (FoundCount == MaxBlockCount) + { + break; + } } } } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index f8c7c012c..b0665dbf8 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -55,7 +55,7 @@ public: std::function&& Receiver) = 0; virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual CbObject FindBlocks(const Oid& BuildId) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map& FloatStats) = 0; diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index fda4a7bfe..417ed7384 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -152,7 +152,7 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); JupiterResult PutBuildPartStats(std::string_view Namespace, diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index b6d9e3990..f2d190408 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -289,13 +289,13 @@ public: } } - virtual CbObject FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId); + JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount); AddStatistic(FindResult); if (!FindResult.Success) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 2e4fe5258..fde86a478 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -758,10 +758,12 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace, } JupiterResult -JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId) +JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount) { - HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId), - HttpClient::Accept(ZenContentType::kCbObject)); + const std::string Parameters = MaxBlockCount == (uint64_t)-1 ? "" : fmt::format("?count={}", MaxBlockCount); + HttpClient::Response Response = + m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks{}", Namespace, BucketId, BuildId, Parameters), + HttpClient::Accept(ZenContentType::kCbObject)); return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv); } -- cgit v1.2.3