diff options
| -rw-r--r-- | CHANGELOG.md | 3 | ||||
| -rw-r--r-- | VERSION.txt | 2 | ||||
| -rw-r--r-- | src/zenremotestore/builds/buildstorageoperations.cpp | 30 |
3 files changed, 21 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index eb2df18a3..26a3c27f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,11 @@ ## +- Improvement: Hide compress/download rates when uploading build when complete +- Improvement: Compress partial file chunks inside blocks if appropriate - Bugfix: Fixed ASSERT when using `zen workspace create` command - Bugfix: Fixed race condition in Latch. Sometimes the Wait() could early out before the completion event had been set - Bugfix: Fixed issue on MacOS where trace options would not stick due to `IterateCommandlineArgs` bug - Bugfix: If we fail to finalize a build part during `zen builds upload` due to request of upload of pre-existing blobs, fail the upload after retry attempts +- Bugfix: Don't display "Validating" progress title during download if validation is disabled ## 5.7.7 - Feature: Added `zen builds prime-cache` command to download all blocks and build blobs for a build and upload to a zen cache host diff --git a/VERSION.txt b/VERSION.txt index 1e028f6ba..67512c145 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.7.8-pre5
\ No newline at end of file +5.7.8-pre9
\ No newline at end of file diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 833a25839..2eb6132be 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -1938,7 +1938,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) { Task = "Downloading "; } - else if (m_WrittenChunkByteCount < BytesToWrite) + else if ((m_WrittenChunkByteCount < BytesToWrite) || (BytesToValidate == 0)) { Task = "Writing chunks "; } @@ -6205,10 +6205,11 @@ BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content, { ZEN_ASSERT(false); } - uint64_t RawSize = Chunk.GetSize(); - const bool ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) && - (RawSize >= m_Options.MinimumSizeForCompressInBlock) && + uint64_t RawSize = Chunk.GetSize(); + + const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock && IsChunkCompressable(m_NonCompressableExtensionHashes, Content, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, CompressionLevel)}; @@ -6241,16 +6242,15 @@ BuildsOperationUploadFolder::RebuildBlock(const ChunkedFolderContent& Content, { std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); ZEN_ASSERT(!ChunkLocations.empty()); - const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; - CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); const uint64_t RawSize = Chunk.GetSize(); - const bool ShouldCompressChunk = Lookup.RawHashToSequenceIndex.contains(ChunkHash) && - (RawSize >= m_Options.MinimumSizeForCompressInBlock) && + const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock && IsChunkCompressable(m_NonCompressableExtensionHashes, Content, Lookup, ChunkIndex); + const OodleCompressionLevel CompressionLevel = ShouldCompressChunk ? OodleCompressionLevel::VeryFast : OodleCompressionLevel::None; CompositeBuffer CompressedChunk = @@ -6691,14 +6691,16 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Co uint64_t UploadedCompressedSize = UploadedCompressedChunkSize.load() + UploadedBlockSize.load(); std::string Details = fmt::format( - "Compressed {}/{} ({}/{} {}B/s) chunks. " + "Compressed {}/{} ({}/{}{}) chunks. " "Uploaded {}/{} ({}/{}) blobs " - "({} {}bits/s)", + "({}{})", TempLooseChunksStats.CompressedChunkCount.load(), LooseChunkOrderIndexes.size(), NiceBytes(TempLooseChunksStats.CompressedChunkRawBytes), NiceBytes(TotalLooseChunksSize), - NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()), + (TempLooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) + ? "" + : fmt::format(" {}B/s", NiceNum(FilteredCompressedBytesPerSecond.GetCurrent())), UploadedBlockCount.load() + UploadedChunkCount.load(), UploadBlockCount + UploadChunkCount, @@ -6706,7 +6708,9 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Co NiceBytes(TotalRawSize), NiceBytes(UploadedCompressedSize), - NiceNum(FilteredUploadedBytesPerSecond.GetCurrent())); + (UploadedBlockCount == UploadBlockCount && UploadedChunkCount == UploadChunkCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredUploadedBytesPerSecond.GetCurrent()))); Progress.UpdateState({.Task = "Uploading blobs ", .Details = Details, |