From 8f192ab154ff9de41d4c063138478400fe2aef24 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 28 Mar 2025 14:12:42 +0100 Subject: temp path options and reduced scanning of target folder (#328) - Feature: zen: `--zen-folder-path` added to `builds` command, `list`, `upload`, `download`, `fetch-blob`, `validate-part` to control where `.zen` folder is placed and named - Improvement: Only check known files from remote state when downloading to a target folder with no local state file - Improvement: Don't move existing local to cache and back if they are untouched --- src/zenutil/chunkedcontent.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'src/zenutil/chunkedcontent.cpp') diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index bb1ee5183..1e8447a57 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -304,14 +304,22 @@ FolderContent GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector& OutDeletedPathIndexes) { ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); - FolderContent Result = {.Platform = Old.Platform}; + + const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); + + FolderContent Result = {.Platform = Old.Platform}; + Result.Paths.reserve(NewPathCount); + Result.RawSizes.reserve(NewPathCount); + Result.Attributes.reserve(NewPathCount); + Result.ModificationTicks.reserve(NewPathCount); + tsl::robin_map NewPathToIndex; - const uint32_t NewPathCount = gsl::narrow(New.Paths.size()); NewPathToIndex.reserve(NewPathCount); for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++) { NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex}); } + uint32_t OldPathCount = gsl::narrow(Old.Paths.size()); for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++) { @@ -667,6 +675,12 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); tsl::robin_map ChunkHashToChunkIndex; + const size_t ExpectedCount = BaseContent.Paths.size() - DeletedPaths.size(); + Result.Paths.reserve(ExpectedCount); + Result.RawSizes.reserve(ExpectedCount); + Result.Attributes.reserve(ExpectedCount); + Result.RawHashes.reserve(ExpectedCount); + tsl::robin_map RawHashToSequenceRawHashIndex; for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) { -- cgit v1.2.3 From bbb7e0c77b9ed114baf428256f347631b3e1092c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 1 Apr 2025 14:26:53 +0200 Subject: verify that we can read input files that are only hashed (#333) * output build and part details by default * output executable and version at start of builds command * verify that we can read files we do not chunk --- src/zenutil/chunkedcontent.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/zenutil/chunkedcontent.cpp') diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index 1e8447a57..32ae2d94a 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -140,8 +140,12 @@ namespace { { ZEN_TRACE_CPU("HashOnly"); - IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); - const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); + if (Buffer.GetSize() != RawSize) + { + throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + } + const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); Lock.WithExclusiveLock([&]() { if (!RawHashToSequenceRawHashIndex.contains(Hash)) -- cgit v1.2.3