From b625980599880e1e7d1149a51b9562e79c0f2994 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 19 Mar 2025 16:12:38 +0100 Subject: jupiter builds stats upload (#315) - Improvement: At end of build upload we post statistics to the Jupiter build stats endpoint: - `totalSize` - `reusedRatio` - `reusedBlockCount` - `reusedBlockByteCount` - `newBlockCount` - `newBlockByteCount` - `uploadedCount` - `uploadedByteCount` - `elapsedTimeSec` - `uploadedBytesPerSec` --- src/zenutil/filebuildstorage.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 47a4e1cc4..130fec355 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -502,6 +502,30 @@ public: return Result; } + virtual void PutBuildPartStats(const Oid& BuildId, + const Oid& BuildPartId, + const tsl::robin_map& FloatStats) override + { + CbObjectWriter Request; + Request.BeginObject("floatStats"sv); + for (auto It : FloatStats) + { + Request.AddFloat(It.first, It.second); + } + Request.EndObject(); + CbObject Payload = Request.Save(); + + SimulateLatency(Payload.GetSize(), 0); + + const std::filesystem::path BuildPartStatsDataPath = GetBuildPartStatsPath(BuildId, BuildPartId); + CreateDirectories(BuildPartStatsDataPath.parent_path()); + + TemporaryFile::SafeWriteFile(BuildPartStatsDataPath, Payload.GetView()); + WriteAsJson(BuildPartStatsDataPath, Payload); + + SimulateLatency(0, 0); + } + protected: std::filesystem::path GetBuildsFolder() const { return m_StoragePath / "builds"; } std::filesystem::path GetBlobsFolder() const { return m_StoragePath / "blobs"; } @@ -520,6 +544,11 @@ protected: return GetBuildPartFolder(BuildId, BuildPartId) / "metadata.cb"; } + std::filesystem::path GetBuildPartStatsPath(const Oid& BuildId, const Oid& BuildPartId) const + { + return GetBuildPartFolder(BuildId, BuildPartId) / fmt::format("stats_{}.cb", Oid::NewOid()); + } + std::filesystem::path GetBlobPayloadPath(const IoHash& RawHash) const { return GetBlobsFolder() / fmt::format("{}.cbz", RawHash); } std::filesystem::path GetBlobMetadataPath(const IoHash& RawHash) const -- cgit v1.2.3 From 28bc5ebf05984385cc0567c89b1d8e7a541ebef8 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 26 Mar 2025 17:06:23 +0100 Subject: zen build cache service (#318) - **EXPERIMENTAL** `zen builds` - Feature: `--zen-cache-host` option for `upload` and `download` operations to use a zenserver host `/builds` endpoint for storing build blob and blob metadata - Feature: New `/builds` endpoint for caching build blobs and blob metadata - `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` `GET` and `PUT` method for storing and fetching blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata` `POST` method for storing metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata` `POST` method for fetching metadata about blobs - `/builds/{namespace}/{bucket}/{buildid}/blobs/exists` `POST` method for checking existance of blobs --- src/zenutil/filebuildstorage.cpp | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 130fec355..f040e9ece 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,18 +442,19 @@ public: SimulateLatency(0, 0); } - virtual std::vector FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(BuildId), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); for (const std::filesystem::path& MetaDataFile : Content.Files) { IoHash ChunkHash; @@ -467,24 +468,28 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } } - SimulateLatency(0, sizeof(IoHash) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } - virtual std::vector GetBlockMetadata(const Oid& BuildId, std::span BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span BlockHashes) override { ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; - std::vector Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); + for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); @@ -495,10 +500,12 @@ public: m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } - SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } -- cgit v1.2.3 From fd2efb5af872a357dbc0f729f4101a330dcb4fda Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 31 Mar 2025 10:24:39 +0200 Subject: long filename support (#330) - Bugfix: Long file paths now works correctly on Windows --- src/zenutil/filebuildstorage.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f040e9ece..0fa05194f 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -235,7 +235,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); @@ -260,7 +260,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); @@ -346,7 +346,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); IoBuffer Payload; @@ -383,7 +383,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { struct WorkloadData { @@ -461,7 +461,7 @@ public: if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) { std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -493,7 +493,7 @@ public: for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); - if (std::filesystem::is_regular_file(MetaDataFile)) + if (IsFile(MetaDataFile)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); @@ -623,7 +623,7 @@ protected: BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { NeededAttachments.push_back(AttachmentHash); } -- cgit v1.2.3 From e244e2a07a0f3bf3b11d6d875d6c2c8bb169efc5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 2 Apr 2025 15:46:05 +0200 Subject: added --find-max-block-count option to builds upload (#337) --- src/zenutil/filebuildstorage.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 0fa05194f..7aa252e44 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -442,7 +442,7 @@ public: SimulateLatency(0, 0); } - virtual CbObject FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); @@ -451,6 +451,8 @@ public: auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; + uint64_t FoundCount = 0; + DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); CbObjectWriter Writer; @@ -469,6 +471,11 @@ public: CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); Writer.AddObject(BlockObject); + FoundCount++; + if (FoundCount == MaxBlockCount) + { + break; + } } } } -- cgit v1.2.3 From dec3f27c488a1dda8a2f1133361e2fda9315e0d2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 11 Apr 2025 19:22:21 +0200 Subject: fix race condition in multipart download (#358) --- src/zenutil/filebuildstorage.cpp | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 7aa252e44..f335a03a3 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -369,11 +369,11 @@ public: return IoBuffer{}; } - virtual std::vector> GetLargeBuildBlob( - const Oid& BuildId, - const IoHash& RawHash, - uint64_t ChunkSize, - std::function&& Receiver) override + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t ChunkSize, + std::function&& OnReceive, + std::function&& OnComplete) override { ZEN_TRACE_CPU("FileBuildStorage::GetLargeBuildBlob"); ZEN_UNUSED(BuildId); @@ -387,16 +387,18 @@ public: { struct WorkloadData { - std::atomic BytesRemaining; - BasicFile BlobFile; - std::function Receiver; + std::atomic BytesRemaining; + BasicFile BlobFile; + std::function OnReceive; + std::function OnComplete; }; std::shared_ptr Workload(std::make_shared()); Workload->BlobFile.Open(BlockPath, BasicFile::Mode::kRead); const uint64_t BlobSize = Workload->BlobFile.FileSize(); - Workload->Receiver = std::move(Receiver); + Workload->OnReceive = std::move(OnReceive); + Workload->OnComplete = std::move(OnComplete); Workload->BytesRemaining = BlobSize; std::vector> WorkItems; @@ -410,8 +412,12 @@ public: IoBuffer PartPayload(Size); Workload->BlobFile.Read(PartPayload.GetMutableView().GetData(), Size, Offset); m_Stats.TotalBytesRead += PartPayload.GetSize(); + Workload->OnReceive(Offset, PartPayload); uint64_t ByteRemaning = Workload->BytesRemaining.fetch_sub(Size); - Workload->Receiver(Offset, PartPayload, ByteRemaning); + if (ByteRemaning == Size) + { + Workload->OnComplete(); + } SimulateLatency(Size, PartPayload.GetSize()); }); -- cgit v1.2.3 From 9cdfac225f0dc986ce449cbe8ef72faa39025971 Mon Sep 17 00:00:00 2001 From: zousar Date: Mon, 14 Apr 2025 21:51:39 -0600 Subject: Add a list-container subcommand to zen builds command --- src/zenutil/filebuildstorage.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index f335a03a3..b77eb1d11 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -35,6 +35,26 @@ public: virtual ~FileBuildStorage() {} + virtual CbObject ListContainers() override + { + ZEN_TRACE_CPU("FileBuildStorage::ListContainers"); + + SimulateLatency(0, 0); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + m_Stats.TotalRequestCount++; + + CbObjectWriter Writer; + Writer.BeginArray("results"); + { + } + Writer.EndArray(); // results + Writer.Save(); + SimulateLatency(Writer.GetSaveSize(), 0); + return Writer.Save(); + } + virtual CbObject ListBuilds(CbObject Query) override { ZEN_TRACE_CPU("FileBuildStorage::ListBuilds"); @@ -66,7 +86,7 @@ public: } } } - Writer.EndArray(); // builds + Writer.EndArray(); // results Writer.Save(); SimulateLatency(Writer.GetSaveSize(), 0); return Writer.Save(); -- cgit v1.2.3 From 6646a12bbffd349ad234a4265e339797e815493b Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 22 Apr 2025 15:57:02 -0600 Subject: Replace container with namespaces --- src/zenutil/filebuildstorage.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index b77eb1d11..badfb4840 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -35,9 +35,10 @@ public: virtual ~FileBuildStorage() {} - virtual CbObject ListContainers() override + virtual CbObject ListNamespaces(bool bRecursive) override { - ZEN_TRACE_CPU("FileBuildStorage::ListContainers"); + ZEN_TRACE_CPU("FileBuildStorage::ListNamespaces"); + ZEN_UNUSED(bRecursive); SimulateLatency(0, 0); -- cgit v1.2.3 From a0b10b046095d57ffbdb46c83084601a832f4562 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Jun 2025 16:21:01 +0200 Subject: fixed size chunking for encrypted files (#410) - Improvement: Use fixed size block chunking for know encrypted/compressed file types - Improvement: Skip trying to compress chunks that are sourced from files that are known to be encrypted/compressed - Improvement: Add global open file cache for written files increasing throughput during download by reducing overhead of open/close of file by 80% --- src/zenutil/filebuildstorage.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index badfb4840..c389d16c5 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -678,13 +678,24 @@ protected: { return false; } - CompositeBuffer Decompressed = ValidateBuffer.DecompressToComposite(); - if (!Decompressed) + + IoHashStream Hash; + bool CouldDecompress = ValidateBuffer.DecompressToStream( + 0, + (uint64_t)-1, + [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { + ZEN_UNUSED(SourceOffset, SourceSize, Offset); + for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) + { + Hash.Append(Segment.GetView()); + } + return true; + }); + if (!CouldDecompress) { return false; } - IoHash Hash = IoHash::HashBuffer(Decompressed); - if (Hash != RawHash) + if (Hash.GetHash() != VerifyHash) { return false; } -- cgit v1.2.3 From fe4cac6a733706e6b523e9ec3e9d014571723f72 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Jun 2025 19:20:19 +0200 Subject: add retry for failed block metadata upload (#445) * add retry for failed block metadata upload --- src/zenutil/filebuildstorage.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/zenutil/filebuildstorage.cpp') diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index c389d16c5..c2cc5ab3c 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -450,7 +450,7 @@ public: return {}; } - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override + virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) override { ZEN_TRACE_CPU("FileBuildStorage::PutBlockMetadata"); ZEN_UNUSED(BuildId); @@ -467,6 +467,7 @@ public: m_Stats.TotalBytesWritten += MetaData.GetSize(); WriteAsJson(BlockMetaDataPath, MetaData); SimulateLatency(0, 0); + return true; } virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override -- cgit v1.2.3