diff options
| author | Florent Devillechabrol <[email protected]> | 2025-04-02 10:38:02 -0700 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2025-04-02 10:38:02 -0700 |
| commit | 486a22ad2c61bc1616d8745e0077eb320089bfec (patch) | |
| tree | 665d5c9002cd97c04ddffeaf211fcf8e55d01dce /src | |
| parent | Fixed missing trailing quote when converting binary data from compact binary ... (diff) | |
| parent | added --find-max-block-count option to builds upload (#337) (diff) | |
| download | zen-486a22ad2c61bc1616d8745e0077eb320089bfec.tar.xz zen-486a22ad2c61bc1616d8745e0077eb320089bfec.zip | |
Merge branch 'main' into fd-fix-binary-json
Diffstat (limited to 'src')
81 files changed, 7345 insertions, 1991 deletions
diff --git a/src/zen/cmds/admin_cmd.cpp b/src/zen/cmds/admin_cmd.cpp index 835e01151..573639c2d 100644 --- a/src/zen/cmds/admin_cmd.cpp +++ b/src/zen/cmds/admin_cmd.cpp @@ -688,7 +688,7 @@ Copy(const std::filesystem::path& Source, const std::filesystem::path& Target) static bool TryCopy(const std::filesystem::path& Source, const std::filesystem::path& Target) { - if (!std::filesystem::is_regular_file(Source)) + if (!IsFile(Source)) { return false; } @@ -717,7 +717,7 @@ CopyStateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::filesystem::path DataPath = StringToPath(m_DataPath); std::filesystem::path TargetPath = StringToPath(m_TargetPath); - if (!std::filesystem::is_directory(DataPath)) + if (!IsDir(DataPath)) { throw OptionParseException("data path must exist"); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 889ccef0b..8e8fd480a 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -6,7 +6,9 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinaryfile.h> #include <zencore/compactbinaryfmt.h> +#include <zencore/compactbinaryvalue.h> #include <zencore/compress.h> +#include <zencore/config.h> #include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -20,6 +22,7 @@ #include <zenhttp/httpclient.h> #include <zenhttp/httpclientauth.h> #include <zenhttp/httpcommon.h> +#include <zenutil/buildstoragecache.h> #include <zenutil/chunkblock.h> #include <zenutil/chunkedcontent.h> #include <zenutil/chunkedfile.h> @@ -51,6 +54,8 @@ ZEN_THIRD_PARTY_INCLUDES_END #define EXTRA_VERIFY 0 +#define ZEN_CLOUD_STORAGE "Cloud Storage" + namespace zen { namespace { static std::atomic<bool> AbortFlag = false; @@ -79,30 +84,56 @@ namespace { size_t MaxChunkEmbedSize = DefaultMaxChunkEmbedSize; }; - const ChunksBlockParameters DefaultChunksBlockParams{.MaxBlockSize = 32u * 1024u * 1024u, - .MaxChunkEmbedSize = DefaultChunkedParams.MaxSize}; - + const ChunksBlockParameters DefaultChunksBlockParams{ + .MaxBlockSize = 32u * 1024u * 1024u, + .MaxChunkEmbedSize = 2u * 1024u * 1024u // DefaultChunkedParams.MaxSize + }; const uint64_t DefaultPreferredMultipartChunkSize = 32u * 1024u * 1024u; const double DefaultLatency = 0; // .0010; const double DefaultDelayPerKBSec = 0; // 0.00005; - const std::string ZenFolderName = ".zen"; - const std::string ZenStateFilePath = fmt::format("{}/current_state.cbo", ZenFolderName); - const std::string ZenStateFileJsonPath = fmt::format("{}/current_state.json", ZenFolderName); - const std::string ZenTempFolderName = fmt::format("{}/tmp", ZenFolderName); + const bool SingleThreaded = false; + bool BoostWorkerThreads = false; - const std::string ZenTempCacheFolderName = - fmt::format("{}/cache", ZenTempFolderName); // Decompressed and verified data - chunks & sequences - const std::string ZenTempBlockFolderName = fmt::format("{}/blocks", ZenTempFolderName); // Temp storage for whole and partial blocks - const std::string ZenTempChunkFolderName = - fmt::format("{}/chunks", ZenTempFolderName); // Temp storage for decompressed and validated chunks + WorkerThreadPool& GetIOWorkerPool() + { + return SingleThreaded ? GetSyncWorkerPool() + : BoostWorkerThreads ? GetLargeWorkerPool(EWorkloadType::Burst) + : GetMediumWorkerPool(EWorkloadType::Burst); + } + WorkerThreadPool& GetNetworkPool() { return SingleThreaded ? GetSyncWorkerPool() : GetSmallWorkerPool(EWorkloadType::Burst); } - const std::string ZenTempDownloadFolderName = - fmt::format("{}/download", ZenTempFolderName); // Temp storage for unverfied downloaded blobs + const uint64_t MinimumSizeForCompressInBlock = 2u * 1024u; - const std::string ZenTempStorageFolderName = - fmt::format("{}/storage", ZenTempFolderName); // Temp storage folder for BuildStorage implementations + const std::string ZenFolderName = ".zen"; + std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.cbo"; } + // std::filesystem::path ZenStateFileJsonPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "current_state.json"; + // } + std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath) { return ZenFolderPath / "tmp"; } + + std::filesystem::path ZenTempCacheFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "cache"; // Decompressed and verified data - chunks & sequences + } + std::filesystem::path ZenTempBlockFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "blocks"; // Temp storage for whole and partial blocks + } + std::filesystem::path ZenTempChunkFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "chunks"; // Temp storage for decompressed and validated chunks + } + + std::filesystem::path ZenTempDownloadFolderPath(const std::filesystem::path& ZenFolderPath) + { + return ZenTempFolderPath(ZenFolderPath) / "download"; // Temp storage for decompressed and validated chunks + } + + // std::filesystem::path ZenTempStorageFolderPath(const std::filesystem::path& ZenFolderPath) + // { + // return ZenTempFolderPath(ZenFolderPath) / "storage"; // Temp storage folder for BuildStorage implementations + // } const std::string ZenExcludeManifestName = ".zen_exclude_manifest.txt"; @@ -133,6 +164,98 @@ namespace { ); + std::filesystem::path MakeSafeAbsolutePath(const std::string Path) + { + std::filesystem::path AbsolutePath = std::filesystem::absolute(StringToPath(Path)).make_preferred(); +#if ZEN_PLATFORM_WINDOWS && 1 + const std::string_view Prefix = "\\\\?\\"; + const std::u8string PrefixU8(Prefix.begin(), Prefix.end()); + std::u8string PathString = AbsolutePath.u8string(); + if (!PathString.empty() && !PathString.starts_with(PrefixU8)) + { + PathString.insert(0, PrefixU8); + return std::filesystem::path(PathString); + } +#endif + return AbsolutePath; + } + + void RenameFileWithRetry(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) + { + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + RenameFile(SourcePath, TargetPath, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + bool SetFileReadOnlyWithRetry(const std::filesystem::path& Path, bool ReadOnly) + { + std::error_code Ec; + bool Result = SetFileReadOnly(Path, ReadOnly, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return false; + } + Ec.clear(); + Result = SetFileReadOnly(Path, ReadOnly, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + return Result; + } + + void RemoveFileWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveFile(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsFile(Path)) + { + return; + } + Ec.clear(); + RemoveFile(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + + void RemoveDirWithRetry(const std::filesystem::path& Path) + { + std::error_code Ec; + RemoveDir(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + if (!IsDir(Path)) + { + return; + } + Ec.clear(); + RemoveDir(Path, Ec); + } + if (Ec) + { + zen::ThrowSystemError(Ec.value(), Ec.message()); + } + } + uint32_t SetNativeFileAttributes(const std::filesystem::path FilePath, SourcePlatform SourcePlatform, uint32_t Attributes) { #if ZEN_PLATFORM_WINDOWS @@ -195,34 +318,109 @@ namespace { bool CleanDirectory(const std::filesystem::path& Path, std::span<const std::string_view> ExcludeDirectories) { ZEN_TRACE_CPU("CleanDirectory"); + Stopwatch Timer; - bool CleanWipe = true; + ProgressBar Progress(UsePlainProgress); - DirectoryContent LocalDirectoryContent; - GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); - for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + std::atomic<bool> CleanWipe = true; + std::atomic<uint64_t> DiscoveredItemCount = 0; + std::atomic<uint64_t> DeletedItemCount = 0; + std::atomic<uint64_t> DeletedByteCount = 0; + ParallellWork Work(AbortFlag); + + struct AsyncVisitor : public GetDirectoryContentVisitor { - try + AsyncVisitor(const std::filesystem::path& InPath, + std::atomic<bool>& InCleanWipe, + std::atomic<uint64_t>& InDiscoveredItemCount, + std::atomic<uint64_t>& InDeletedItemCount, + std::atomic<uint64_t>& InDeletedByteCount, + std::span<const std::string_view> InExcludeDirectories) + : Path(InPath) + , CleanWipe(InCleanWipe) + , DiscoveredItemCount(InDiscoveredItemCount) + , DeletedItemCount(InDeletedItemCount) + , DeletedByteCount(InDeletedByteCount) + , ExcludeDirectories(InExcludeDirectories) { - std::filesystem::remove(LocalFilePath); } - catch (const std::exception&) + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override { - // DeleteOnClose files may be a bit slow in getting cleaned up, so pause amd retry one time - Sleep(200); - try - { - std::filesystem::remove(LocalFilePath); - } - catch (const std::exception& Ex) + ZEN_TRACE_CPU("CleanDirectory_AsyncVisitDirectory"); + if (!AbortFlag) { - ZEN_WARN("Failed removing file {}. Reason: {}", LocalFilePath, Ex.what()); - CleanWipe = false; + if (!Content.FileNames.empty()) + { + DiscoveredItemCount += Content.FileNames.size(); + + const std::string RelativeRootString = RelativeRoot.generic_string(); + bool RemoveContent = true; + for (const std::string_view ExcludeDirectory : ExcludeDirectories) + { + if (RelativeRootString.starts_with(ExcludeDirectory)) + { + if (RelativeRootString.length() > ExcludeDirectory.length()) + { + const char MaybePathDelimiter = RelativeRootString[ExcludeDirectory.length()]; + if (MaybePathDelimiter == '/' || MaybePathDelimiter == '\\' || + MaybePathDelimiter == std::filesystem::path::preferred_separator) + { + RemoveContent = false; + break; + } + } + else + { + RemoveContent = false; + break; + } + } + } + if (RemoveContent) + { + ZEN_TRACE_CPU("DeleteFiles"); + for (size_t FileIndex = 0; FileIndex < Content.FileNames.size(); FileIndex++) + { + const std::filesystem::path& FileName = Content.FileNames[FileIndex]; + const std::filesystem::path FilePath = (Path / RelativeRoot / FileName).make_preferred(); + try + { + SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + DeletedItemCount++; + DeletedByteCount += Content.FileSizes[FileIndex]; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing file {}. Reason: {}", FilePath, Ex.what()); + CleanWipe = false; + } + } + } + } } } - } + const std::filesystem::path& Path; + std::atomic<bool>& CleanWipe; + std::atomic<uint64_t>& DiscoveredItemCount; + std::atomic<uint64_t>& DeletedItemCount; + std::atomic<uint64_t>& DeletedByteCount; + std::span<const std::string_view> ExcludeDirectories; + } Visitor(Path, CleanWipe, DiscoveredItemCount, DeletedItemCount, DeletedByteCount, ExcludeDirectories); - for (const std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + GetDirectoryContent( + Path, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeFileSizes, + Visitor, + GetIOWorkerPool(), + Work.PendingWork()); + + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + DiscoveredItemCount += LocalDirectoryContent.Directories.size(); + std::vector<std::filesystem::path> DirectoriesToDelete; + DirectoriesToDelete.reserve(LocalDirectoryContent.Directories.size()); + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) { bool Leave = false; for (const std::string_view ExcludeDirectory : ExcludeDirectories) @@ -235,33 +433,84 @@ namespace { } if (!Leave) { - try - { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); - } - catch (const std::exception&) + DirectoriesToDelete.emplace_back(std::move(LocalDirPath)); + DiscoveredItemCount++; + } + } + + uint64_t LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + LastUpdateTimeMs = Timer.GetElapsedTimeMs(); + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); + }); + + for (const std::filesystem::path& DirectoryToDelete : DirectoriesToDelete) + { + ZEN_TRACE_CPU("DeleteDirs"); + try + { + std::error_code Ec; + zen::CleanDirectory(DirectoryToDelete, true, Ec); + if (Ec) { Sleep(200); - try - { - zen::CleanDirectory(LocalDirPath); - std::filesystem::remove(LocalDirPath); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed removing directory {}. Reason: {}", LocalDirPath, Ex.what()); - CleanWipe = false; - } + zen::CleanDirectory(DirectoryToDelete, true); + Ec.clear(); } + + RemoveDirWithRetry(DirectoryToDelete); + + DeletedItemCount++; + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed removing directory {}. Reason: {}", DirectoryToDelete, Ex.what()); + CleanWipe = false; + } + + uint64_t NowMs = Timer.GetElapsedTimeMs(); + if ((NowMs - LastUpdateTimeMs) >= (UsePlainProgress ? 5000 : 200)) + { + LastUpdateTimeMs = NowMs; + + uint64_t Deleted = DeletedItemCount.load(); + uint64_t DeletedBytes = DeletedByteCount.load(); + uint64_t Discovered = DiscoveredItemCount.load(); + Progress.UpdateState({.Task = "Cleaning folder ", + .Details = fmt::format("Found {}, Deleted {} ({})", Discovered, Deleted, NiceBytes(DeletedBytes)), + .TotalCount = Discovered, + .RemainingCount = Discovered - Deleted}, + false); } } + + Progress.Finish(); + + uint64_t ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (ElapsedTimeMs >= 200) + { + ZEN_CONSOLE("Wiped folder '{}' {} ({}) in {}", + Path, + DiscoveredItemCount.load(), + NiceBytes(DeletedByteCount.load()), + NiceTimeSpanMs(ElapsedTimeMs)); + } return CleanWipe; } std::string ReadAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -304,7 +553,7 @@ namespace { const IoHash& Hash, const std::string& Suffix = {}) { - std::filesystem::path TempFilePath = (TempFolderPath / (Hash.ToHexString() + Suffix)).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / (Hash.ToHexString() + Suffix); return WriteToTempFile(std::move(Buffer), TempFilePath); } @@ -402,12 +651,12 @@ namespace { std::filesystem::path GetTempChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / (RawHash.ToHexString() + ".tmp")).make_preferred(); + return CacheFolderPath / (RawHash.ToHexString() + ".tmp"); } std::filesystem::path GetFinalChunkedSequenceFileName(const std::filesystem::path& CacheFolderPath, const IoHash& RawHash) { - return (CacheFolderPath / RawHash.ToHexString()).make_preferred(); + return CacheFolderPath / RawHash.ToHexString(); } ChunkedFolderContent ScanAndChunkFolder( @@ -425,7 +674,7 @@ namespace { Path, std::move(IsAcceptedFolder), std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [](bool, std::ptrdiff_t) {}, AbortFlag); @@ -439,7 +688,7 @@ namespace { FilteredBytesHashed.Start(); ChunkedFolderContent FolderContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, ChunkController, @@ -501,6 +750,7 @@ namespace { uint64_t AcceptedBlockCount = 0; uint64_t AcceptedChunkCount = 0; uint64_t AcceptedByteCount = 0; + uint64_t AcceptedRawByteCount = 0; uint64_t RejectedBlockCount = 0; uint64_t RejectedChunkCount = 0; uint64_t RejectedByteCount = 0; @@ -539,6 +789,7 @@ namespace { uint64_t ChunkCount = 0; uint64_t ChunkByteCount = 0; std::atomic<uint64_t> CompressedChunkCount = 0; + std::atomic<uint64_t> CompressedChunkRawBytes = 0; std::atomic<uint64_t> CompressedChunkBytes = 0; uint64_t CompressChunksElapsedWallTimeUS = 0; @@ -547,6 +798,7 @@ namespace { ChunkCount += Rhs.ChunkCount; ChunkByteCount += Rhs.ChunkByteCount; CompressedChunkCount += Rhs.CompressedChunkCount; + CompressedChunkRawBytes += Rhs.CompressedChunkRawBytes; CompressedChunkBytes += Rhs.CompressedChunkBytes; CompressChunksElapsedWallTimeUS += Rhs.CompressChunksElapsedWallTimeUS; return *this; @@ -603,11 +855,9 @@ namespace { struct WriteChunkStatistics { - std::atomic<uint32_t> ChunkCountWritten = 0; - std::atomic<uint64_t> ChunkBytesWritten = 0; - uint64_t DownloadTimeUs = 0; - uint64_t WriteTimeUs = 0; - uint64_t WriteChunksElapsedWallTimeUs = 0; + uint64_t DownloadTimeUs = 0; + uint64_t WriteTimeUs = 0; + uint64_t WriteChunksElapsedWallTimeUs = 0; }; struct RebuildFolderStateStatistics @@ -626,6 +876,16 @@ namespace { uint64_t VerifyElapsedWallTimeUs = 0; }; + struct StorageInstance + { + std::unique_ptr<HttpClient> BuildStorageHttp; + std::unique_ptr<BuildStorage> BuildStorage; + std::string StorageName; + std::unique_ptr<HttpClient> CacheHttp; + std::unique_ptr<BuildStorageCache> BuildCacheStorage; + std::string CacheName; + }; + std::vector<uint32_t> CalculateAbsoluteChunkOrders(const std::span<const IoHash> LocalChunkHashes, const std::span<const uint32_t> LocalChunkOrder, const tsl::robin_map<IoHash, uint32_t, IoHash::Hasher>& ChunkHashToLocalChunkIndex, @@ -772,7 +1032,7 @@ namespace { std::span<const uint32_t> ChunkCounts, std::span<const IoHash> LocalChunkHashes, std::span<const uint64_t> LocalChunkRawSizes, - std::vector<uint32_t> AbsoluteChunkOrders, + const std::vector<uint32_t>& AbsoluteChunkOrders, const std::span<const uint32_t> LooseLocalChunkIndexes, const std::span<IoHash> BlockHashes) { @@ -1352,7 +1612,15 @@ namespace { ZEN_ASSERT(false); } uint64_t RawSize = Chunk.GetSize(); - return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + // Standalone chunk, not part of a sequence + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid)}; + } + else + { + return {RawSize, CompressedBuffer::Compress(Chunk, OodleCompressor::Mermaid, OodleCompressionLevel::None)}; + } })); } @@ -1376,13 +1644,24 @@ namespace { { std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkLocations = GetChunkSequenceLocations(Lookup, ChunkIndex); ZEN_ASSERT(!ChunkLocations.empty()); - CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, + const IoHash& ChunkHash = Content.ChunkedContent.ChunkHashes[ChunkIndex]; + CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); - ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == Content.ChunkedContent.ChunkHashes[ChunkIndex]); - CompositeBuffer CompressedChunk = - CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); - ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); + + const uint64_t RawSize = Chunk.GetSize(); + if (Lookup.RawHashToSequenceIndex.contains(ChunkHash) && RawSize >= MinimumSizeForCompressInBlock) + { + CompositeBuffer CompressedChunk = CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } + else + { + CompositeBuffer CompressedChunk = + CompressedBuffer::Compress(std::move(Chunk), OodleCompressor::Mermaid, OodleCompressionLevel::None).GetCompressed(); + ResultBuffers.insert(ResultBuffers.end(), CompressedChunk.GetSegments().begin(), CompressedChunk.GetSegments().end()); + } } return CompressedBuffer::FromCompressedNoValidate(CompositeBuffer(std::move(ResultBuffers))); }; @@ -1444,7 +1723,7 @@ namespace { for (auto& WorkItem : WorkItems) { Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(),// + NetworkPool, [WorkItem = std::move(WorkItem)](std::atomic<bool>&) { ZEN_TRACE_CPU("DownloadLargeBlob_Work"); if (!AbortFlag) @@ -1513,15 +1792,16 @@ namespace { } ValidateStats.BlockAttachmentCount = BlockAttachments.size(); - std::vector<ChunkBlockDescription> VerifyBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockAttachments); + std::vector<ChunkBlockDescription> VerifyBlockDescriptions = + ParseChunkBlockDescriptionList(Storage.GetBlockMetadatas(BuildId, BlockAttachments)); if (VerifyBlockDescriptions.size() != BlockAttachments.size()) { throw std::runtime_error(fmt::format("Uploaded blocks metadata could not all be found, {} blocks metadata is missing", BlockAttachments.size() - VerifyBlockDescriptions.size())); } - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); const std::filesystem::path TempFolder = ".zen-tmp"; @@ -1530,7 +1810,8 @@ namespace { auto __ = MakeGuard([&TempFolder]() { if (CleanDirectory(TempFolder, {})) { - std::filesystem::remove(TempFolder); + std::error_code DummyEc; + RemoveDir(TempFolder, DummyEc); } }); @@ -1787,7 +2068,6 @@ namespace { const ChunkedContentLookup& Lookup, uint32_t ChunkIndex, const std::filesystem::path& TempFolderPath, - std::atomic<uint64_t>& ReadRawBytes, LooseChunksStatistics& LooseChunksStats) { ZEN_TRACE_CPU("CompressChunk"); @@ -1808,11 +2088,11 @@ namespace { } ZEN_ASSERT_SLOW(IoHash::HashBuffer(RawSource) == ChunkHash); { - std::filesystem::path TempFilePath = (TempFolderPath / ChunkHash.ToHexString()).make_preferred(); + std::filesystem::path TempFilePath = TempFolderPath / ChunkHash.ToHexString(); BasicFile CompressedFile; std::error_code Ec; - CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncate, Ec); + CompressedFile.Open(TempFilePath, BasicFile::Mode::kTruncateDelete, Ec); if (Ec) { throw std::runtime_error( @@ -1823,7 +2103,7 @@ namespace { CompositeBuffer(SharedBuffer(RawSource)), [&](uint64_t SourceOffset, uint64_t SourceSize, uint64_t Offset, const CompositeBuffer& RangeBuffer) { ZEN_UNUSED(SourceOffset); - ReadRawBytes += SourceSize; + LooseChunksStats.CompressedChunkRawBytes += SourceSize; CompressedFile.Write(RangeBuffer, Offset); LooseChunksStats.CompressedChunkBytes += RangeBuffer.GetSize(); }); @@ -1850,7 +2130,7 @@ namespace { return Compressed.GetCompressed(); } CompressedFile.Close(); - std::filesystem::remove(TempFilePath, Ec); + RemoveFile(TempFilePath, Ec); ZEN_UNUSED(Ec); } @@ -1881,7 +2161,7 @@ namespace { void GenerateBuildBlocks(const std::filesystem::path& Path, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, - BuildStorage& Storage, + StorageInstance& Storage, const Oid& BuildId, const std::vector<std::vector<uint32_t>>& NewBlockChunks, GeneratedBlocks& OutBlocks, @@ -1904,9 +2184,8 @@ namespace { RwLock Lock; - WorkerThreadPool& GenerateBlobsPool = - GetMediumWorkerPool(EWorkloadType::Burst); // GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// - WorkerThreadPool& UploadBlocksPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool();// + WorkerThreadPool& GenerateBlobsPool = GetIOWorkerPool(); + WorkerThreadPool& UploadBlocksPool = GetNetworkPool(); FilteredRate FilteredGeneratedBytesPerSecond; FilteredRate FilteredUploadedBytesPerSecond; @@ -2005,21 +2284,35 @@ namespace { const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - Storage.PutBuildBlob(BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - std::move(Payload).GetCompressed()); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); + } + + Storage.BuildStorage->PutBuildBlob(BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + std::move(Payload).GetCompressed()); UploadStats.BlocksBytes += CompressedBlockSize; + ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(CompressedBlockSize), OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); - Storage.PutBlockMetadata(BuildId, - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, - BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector<IoHash>({BlockHash}), + std::vector<CbObject>({BlockMetaData})); + } + + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - OutBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(BlockMetaData.GetSize())); OutBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2074,9 +2367,10 @@ namespace { } } - void UploadPartBlobs(BuildStorage& Storage, + void UploadPartBlobs(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& Content, const ChunkedContentLookup& Lookup, std::span<IoHash> RawHashes, @@ -2092,8 +2386,8 @@ namespace { { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& ReadChunkPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& UploadChunkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& ReadChunkPool = GetIOWorkerPool(); + WorkerThreadPool& UploadChunkPool = GetNetworkPool(); FilteredRate FilteredGenerateBlockBytesPerSecond; FilteredRate FilteredCompressedBytesPerSecond; @@ -2150,7 +2444,7 @@ namespace { if (QueuedPendingInMemoryBlocksForUpload.load() > 16) { ZEN_TRACE_CPU("AsyncUploadBlock_WriteTempBlock"); - Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), Path / ZenTempBlockFolderName, BlockHash)); + Payload = CompositeBuffer(WriteToTempFile(std::move(Payload), ZenTempBlockFolderPath(ZenFolderPath), BlockHash)); IsInMemoryBlock = false; } else @@ -2177,18 +2471,26 @@ namespace { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + } + Storage.BuildStorage->PutBuildBlob(BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded block {} ({}) containing {} chunks", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, + BlockHash, NiceBytes(PayloadSize), NewBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); UploadedBlockSize += PayloadSize; UploadStats.BlocksBytes += PayloadSize; - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); - ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", - NewBlocks.BlockDescriptions[BlockIndex].BlockHash, - NiceBytes(BlockMetaData.GetSize())); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector<IoHash>({BlockHash}), + std::vector<CbObject>({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + ZEN_CONSOLE_VERBOSE("Uploaded block {} metadata ({})", BlockHash, NiceBytes(BlockMetaData.GetSize())); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; @@ -2214,12 +2516,17 @@ namespace { ZEN_TRACE_CPU("AsyncUploadLooseChunk"); const uint64_t PayloadSize = Payload.GetSize(); - ; + + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + } + if (PayloadSize >= LargeAttachmentSize) { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Multipart"); UploadStats.MultipartAttachmentCount++; - std::vector<std::function<void()>> MultipartWork = Storage.PutLargeBuildBlob( + std::vector<std::function<void()>> MultipartWork = Storage.BuildStorage->PutLargeBuildBlob( BuildId, RawHash, ZenContentType::kCompressedBinary, @@ -2264,7 +2571,7 @@ namespace { else { ZEN_TRACE_CPU("AsyncUploadLooseChunk_Singlepart"); - Storage.PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + Storage.BuildStorage->PutBuildBlob(BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); ZEN_CONSOLE_VERBOSE("Uploaded chunk {} ({})", RawHash, NiceBytes(PayloadSize)); UploadStats.ChunksBytes += Payload.GetSize(); UploadStats.ChunkCount++; @@ -2286,16 +2593,8 @@ namespace { std::atomic<uint64_t> GeneratedBlockCount = 0; std::atomic<uint64_t> GeneratedBlockByteCount = 0; - std::vector<uint32_t> CompressLooseChunkOrderIndexes; - std::atomic<uint64_t> QueuedPendingInMemoryBlocksForUpload = 0; - // Start upload of any pre-compressed loose chunks - for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) - { - CompressLooseChunkOrderIndexes.push_back(LooseChunkOrderIndex); - } - // Start generation of any non-prebuilt blocks and schedule upload for (const size_t BlockIndex : BlockIndexes) { @@ -2303,7 +2602,7 @@ namespace { if (!AbortFlag) { Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool() + ReadChunkPool, [&, BlockIndex](std::atomic<bool>&) { if (!AbortFlag) { @@ -2355,34 +2654,27 @@ namespace { } } - std::atomic<uint64_t> RawLooseChunkByteCount = 0; - // Start compression of any non-precompressed loose chunks and schedule upload - for (const uint32_t CompressLooseChunkOrderIndex : CompressLooseChunkOrderIndexes) + for (const uint32_t LooseChunkOrderIndex : LooseChunkOrderIndexes) { - const uint32_t ChunkIndex = LooseChunkIndexes[CompressLooseChunkOrderIndex]; + const uint32_t ChunkIndex = LooseChunkIndexes[LooseChunkOrderIndex]; Work.ScheduleWork( - ReadChunkPool, // GetSyncWorkerPool(),// ReadChunkPool, + ReadChunkPool, [&, ChunkIndex](std::atomic<bool>&) { if (!AbortFlag) { ZEN_TRACE_CPU("UploadPartBlobs_CompressChunk"); FilteredCompressedBytesPerSecond.Start(); - CompositeBuffer Payload = CompressChunk(Path, - Content, - Lookup, - ChunkIndex, - Path / ZenTempChunkFolderName, - RawLooseChunkByteCount, - LooseChunksStats); + CompositeBuffer Payload = + CompressChunk(Path, Content, Lookup, ChunkIndex, ZenTempChunkFolderPath(ZenFolderPath), LooseChunksStats); ZEN_CONSOLE_VERBOSE("Compressed chunk {} ({} -> {})", Content.ChunkedContent.ChunkHashes[ChunkIndex], NiceBytes(Content.ChunkedContent.ChunkRawSizes[ChunkIndex]), NiceBytes(Payload.GetSize())); const uint64_t ChunkRawSize = Content.ChunkedContent.ChunkRawSizes[ChunkIndex]; UploadStats.ReadFromDiskBytes += ChunkRawSize; - if (LooseChunksStats.CompressedChunkCount == CompressLooseChunkOrderIndexes.size()) + if (LooseChunksStats.CompressedChunkCount == LooseChunkOrderIndexes.size()) { FilteredCompressedBytesPerSecond.Stop(); } @@ -2397,7 +2689,7 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkBytes.load()); + FilteredCompressedBytesPerSecond.Update(LooseChunksStats.CompressedChunkRawBytes.load()); FilteredGenerateBlockBytesPerSecond.Update(GeneratedBlockByteCount.load()); FilteredUploadedBytesPerSecond.Update(UploadedCompressedChunkSize.load() + UploadedBlockSize.load()); uint64_t UploadedRawSize = UploadedRawChunkSize.load() + UploadedBlockSize.load(); @@ -2408,8 +2700,8 @@ namespace { "Uploaded {}/{} ({}/{}) blobs " "({} {}bits/s)", LooseChunksStats.CompressedChunkCount.load(), - CompressLooseChunkOrderIndexes.size(), - NiceBytes(RawLooseChunkByteCount), + LooseChunkOrderIndexes.size(), + NiceBytes(LooseChunksStats.CompressedChunkRawBytes), NiceBytes(TotalLooseChunksSize), NiceNum(FilteredCompressedBytesPerSecond.GetCurrent()), @@ -2538,9 +2830,10 @@ namespace { for (size_t KnownBlockIndex : ReuseBlockIndexes) { std::vector<uint32_t> FoundChunkIndexes; - size_t BlockSize = 0; - size_t AdjustedReuseSize = 0; - const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; + size_t BlockSize = 0; + size_t AdjustedReuseSize = 0; + size_t AdjustedRawReuseSize = 0; + const ChunkBlockDescription& KnownBlock = KnownBlocks[KnownBlockIndex]; for (size_t BlockChunkIndex = 0; BlockChunkIndex < KnownBlock.ChunkRawHashes.size(); BlockChunkIndex++) { const IoHash& BlockChunkHash = KnownBlock.ChunkRawHashes[BlockChunkIndex]; @@ -2553,6 +2846,7 @@ namespace { { FoundChunkIndexes.push_back(ChunkIndex); AdjustedReuseSize += KnownBlock.ChunkCompressedLengths[BlockChunkIndex]; + AdjustedRawReuseSize += KnownBlock.ChunkRawLengths[BlockChunkIndex]; } } } @@ -2573,6 +2867,7 @@ namespace { } FindBlocksStats.AcceptedChunkCount += FoundChunkIndexes.size(); FindBlocksStats.AcceptedByteCount += AdjustedReuseSize; + FindBlocksStats.AcceptedRawByteCount += AdjustedRawReuseSize; FindBlocksStats.AcceptedReduntantChunkCount += KnownBlock.ChunkRawHashes.size() - FoundChunkIndexes.size(); FindBlocksStats.AcceptedReduntantByteCount += BlockSize - AdjustedReuseSize; } @@ -2598,12 +2893,14 @@ namespace { return FilteredReuseBlockIndexes; }; - void UploadFolder(BuildStorage& Storage, + void UploadFolder(StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, const std::string_view BuildPartName, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::filesystem::path& ManifestPath, + const uint64_t FindBlockMaxCount, const uint8_t BlockReuseMinPercentLimit, bool AllowMultiparts, const CbObject& MetaData, @@ -2613,17 +2910,18 @@ namespace { { Stopwatch ProcessTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); CleanDirectory(ZenTempFolder, {}); auto _ = MakeGuard([&]() { if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + std::error_code DummyEc; + RemoveDir(ZenTempFolder, DummyEc); } }); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempChunkFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempChunkFolderPath(ZenFolderPath)); std::uint64_t TotalRawSize = 0; @@ -2641,54 +2939,52 @@ namespace { FindBlocksStatistics FindBlocksStats; - std::future<PrepareBuildResult> PrepBuildResultFuture = - GetSmallWorkerPool(EWorkloadType::Burst) - .EnqueueTask(std::packaged_task<PrepareBuildResult()>{ - [&Storage, BuildId, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { - ZEN_TRACE_CPU("PrepareBuild"); + std::future<PrepareBuildResult> PrepBuildResultFuture = GetNetworkPool().EnqueueTask(std::packaged_task<PrepareBuildResult()>{ + [&Storage, BuildId, FindBlockMaxCount, &MetaData, CreateBuild, AllowMultiparts, IgnoreExistingBlocks, &FindBlocksStats] { + ZEN_TRACE_CPU("PrepareBuild"); - PrepareBuildResult Result; - Stopwatch Timer; - if (CreateBuild) - { - ZEN_TRACE_CPU("CreateBuild"); + PrepareBuildResult Result; + Stopwatch Timer; + if (CreateBuild) + { + ZEN_TRACE_CPU("CreateBuild"); - Stopwatch PutBuildTimer; - CbObject PutBuildResult = Storage.PutBuild(BuildId, MetaData); - Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); - Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); - Result.PayloadSize = MetaData.GetSize(); - } - else - { - ZEN_TRACE_CPU("PutBuild"); - Stopwatch GetBuildTimer; - CbObject Build = Storage.GetBuild(BuildId); - Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); - Result.PayloadSize = Build.GetSize(); - if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) - { - Result.PreferredMultipartChunkSize = ChunkSize; - } - else if (AllowMultiparts) - { - ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", - NiceBytes(Result.PreferredMultipartChunkSize)); - } - } + Stopwatch PutBuildTimer; + CbObject PutBuildResult = Storage.BuildStorage->PutBuild(BuildId, MetaData); + Result.PrepareBuildTimeMs = PutBuildTimer.GetElapsedTimeMs(); + Result.PreferredMultipartChunkSize = PutBuildResult["chunkSize"sv].AsUInt64(Result.PreferredMultipartChunkSize); + Result.PayloadSize = MetaData.GetSize(); + } + else + { + ZEN_TRACE_CPU("PutBuild"); + Stopwatch GetBuildTimer; + CbObject Build = Storage.BuildStorage->GetBuild(BuildId); + Result.PrepareBuildTimeMs = GetBuildTimer.GetElapsedTimeMs(); + Result.PayloadSize = Build.GetSize(); + if (auto ChunkSize = Build["chunkSize"sv].AsUInt64(); ChunkSize != 0) + { + Result.PreferredMultipartChunkSize = ChunkSize; + } + else if (AllowMultiparts) + { + ZEN_WARN("PreferredMultipartChunkSize is unknown. Defaulting to '{}'", + NiceBytes(Result.PreferredMultipartChunkSize)); + } + } - if (!IgnoreExistingBlocks) - { - ZEN_TRACE_CPU("FindBlocks"); - Stopwatch KnownBlocksTimer; - Result.KnownBlocks = Storage.FindBlocks(BuildId); - FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); - FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); - Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); - } - Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - return Result; - }}); + if (!IgnoreExistingBlocks) + { + ZEN_TRACE_CPU("FindBlocks"); + Stopwatch KnownBlocksTimer; + Result.KnownBlocks = ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, FindBlockMaxCount)); + FindBlocksStats.FindBlockTimeMS = KnownBlocksTimer.GetElapsedTimeMs(); + FindBlocksStats.FoundBlockCount = Result.KnownBlocks.size(); + Result.FindBlocksTimeMs = KnownBlocksTimer.GetElapsedTimeMs(); + } + Result.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + return Result; + }}); ChunkedFolderContent LocalContent; @@ -2767,7 +3063,7 @@ namespace { { std::filesystem::path ExcludeManifestPath = Path / ZenExcludeManifestName; tsl::robin_set<std::string> ExcludeAssetPaths; - if (std::filesystem::is_regular_file(ExcludeManifestPath)) + if (IsFile(ExcludeManifestPath)) { std::vector<std::filesystem::path> AssetPaths = ParseManifest(Path, ExcludeManifestPath); ExcludeAssetPaths.reserve(AssetPaths.size()); @@ -2796,10 +3092,10 @@ namespace { } return true; }, - GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { - ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); + ZEN_CONSOLE_VERBOSE("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, AbortFlag); } @@ -2810,12 +3106,13 @@ namespace { for (const std::filesystem::path& AssetPath : AssetPaths) { Content.Paths.push_back(AssetPath); - Content.RawSizes.push_back(std::filesystem::file_size(Path / AssetPath)); + const std::filesystem::path AssetFilePath = (Path / AssetPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(AssetFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(Path / AssetPath)); + Content.Attributes.push_back(GetFileAttributes(AssetFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(Path / AssetPath)); + Content.Attributes.push_back(GetFileMode(AssetFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); LocalFolderScanStats.AcceptedFileCount++; @@ -2823,12 +3120,13 @@ namespace { if (ManifestPath.is_relative()) { Content.Paths.push_back(ManifestPath); - Content.RawSizes.push_back(std::filesystem::file_size(ManifestPath)); + const std::filesystem::path ManifestFilePath = (Path / ManifestPath).make_preferred(); + Content.RawSizes.push_back(FileSizeFromPath(ManifestFilePath)); #if ZEN_PLATFORM_WINDOWS - Content.Attributes.push_back(GetFileAttributes(ManifestPath)); + Content.Attributes.push_back(GetFileAttributes(ManifestFilePath)); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - Content.Attributes.push_back(GetFileMode(ManifestPath)); + Content.Attributes.push_back(GetFileMode(ManifestFilePath)); #endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX LocalFolderScanStats.AcceptedFileByteCount += Content.RawSizes.back(); @@ -2855,7 +3153,7 @@ namespace { FilteredBytesHashed.Start(); LocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, Content, *ChunkController, @@ -2976,16 +3274,17 @@ namespace { } FindBlocksStats.NewBlocksChunkCount = NewBlockChunkIndexes.size(); - const double AcceptedByteCountPercent = FindBlocksStats.PotentialChunkByteCount > 0 - ? (100.0 * FindBlocksStats.AcceptedByteCount / FindBlocksStats.PotentialChunkByteCount) - : 0.0; + const double AcceptedByteCountPercent = + FindBlocksStats.PotentialChunkByteCount > 0 + ? (100.0 * FindBlocksStats.AcceptedRawByteCount / FindBlocksStats.PotentialChunkByteCount) + : 0.0; const double AcceptedReduntantByteCountPercent = FindBlocksStats.AcceptedByteCount > 0 ? (100.0 * FindBlocksStats.AcceptedReduntantByteCount) / (FindBlocksStats.AcceptedByteCount + FindBlocksStats.AcceptedReduntantByteCount) : 0.0; ZEN_CONSOLE( - "Found {} chunks in {} ({}) blocks eligeble for reuse in {}\n" + "Found {} chunks in {} ({}) blocks eligible for reuse in {}\n" " Reusing {} ({}) matching chunks in {} blocks ({:.1f}%)\n" " Accepting {} ({}) redundant chunks ({:.1f}%)\n" " Rejected {} ({}) chunks in {} blocks\n" @@ -2998,7 +3297,7 @@ namespace { NiceTimeSpanMs(FindBlocksStats.FindBlockTimeMS), FindBlocksStats.AcceptedChunkCount, - NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.AcceptedBlockCount, AcceptedByteCountPercent, @@ -3204,7 +3503,8 @@ namespace { } Stopwatch PutBuildPartResultTimer; - std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = Storage.PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); + std::pair<IoHash, std::vector<IoHash>> PutBuildPartResult = + Storage.BuildStorage->PutBuildPart(BuildId, BuildPartId, BuildPartName, PartManifest); ZEN_CONSOLE("PutBuildPart took {}, payload size {}. {} attachments are needed.", NiceTimeSpanMs(PutBuildPartResultTimer.GetElapsedTimeMs()), NiceBytes(PartManifest.GetSize()), @@ -3231,8 +3531,8 @@ namespace { TempLooseChunksStats.CompressedChunkCount.load(), NiceBytes(TempLooseChunksStats.CompressedChunkBytes.load()), - NiceNum(GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, - TempLooseChunksStats.CompressedChunkBytes)), + NiceNum( + GetBytesPerSecond(TempLooseChunksStats.CompressChunksElapsedWallTimeUS, TempLooseChunksStats.ChunkByteCount)), TempUploadStats.ChunkCount.load(), NiceBytes(TempUploadStats.ChunksBytes), @@ -3243,6 +3543,7 @@ namespace { UploadPartBlobs(Storage, BuildId, Path, + ZenFolderPath, LocalContent, LocalLookup, RawHashes, @@ -3289,7 +3590,7 @@ namespace { while (!AbortFlag) { Stopwatch FinalizeBuildPartTimer; - std::vector<IoHash> Needs = Storage.FinalizeBuildPart(BuildId, BuildPartId, PartHash); + std::vector<IoHash> Needs = Storage.BuildStorage->FinalizeBuildPart(BuildId, BuildPartId, PartHash); ZEN_CONSOLE("FinalizeBuildPart took {}. {} attachments are missing.", NiceTimeSpanMs(FinalizeBuildPartTimer.GetElapsedTimeMs()), Needs.size()); @@ -3304,7 +3605,7 @@ namespace { if (CreateBuild && !AbortFlag) { Stopwatch FinalizeBuildTimer; - Storage.FinalizeBuild(BuildId); + Storage.BuildStorage->FinalizeBuild(BuildId); ZEN_CONSOLE("FinalizeBuild took {}", NiceTimeSpanMs(FinalizeBuildTimer.GetElapsedTimeMs())); } @@ -3321,7 +3622,13 @@ namespace { { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - Storage.PutBlockMetadata(BuildId, BlockHash, BlockMetaData); + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, + std::vector<IoHash>({BlockHash}), + std::vector<CbObject>({BlockMetaData})); + } + Storage.BuildStorage->PutBlockMetadata(BuildId, BlockHash, BlockMetaData); UploadStats.BlocksBytes += BlockMetaData.GetSize(); NewBlocks.MetaDataHasBeenUploaded[BlockIndex] = true; UploadBlockMetadataCount++; @@ -3340,7 +3647,7 @@ namespace { DownloadStatistics ValidateDownloadStats; if (PostUploadVerify && !AbortFlag) { - ValidateBuildPart(Storage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, BuildPartName, ValidateStats, ValidateDownloadStats); } ZEN_CONSOLE_VERBOSE( @@ -3384,6 +3691,7 @@ namespace { "\n AcceptedBlockCount: {}" "\n AcceptedChunkCount: {}" "\n AcceptedByteCount: {}" + "\n AcceptedRawByteCount: {}" "\n RejectedBlockCount: {}" "\n RejectedChunkCount: {}" "\n RejectedByteCount: {}" @@ -3401,6 +3709,7 @@ namespace { FindBlocksStats.AcceptedBlockCount, FindBlocksStats.AcceptedChunkCount, NiceBytes(FindBlocksStats.AcceptedByteCount), + NiceBytes(FindBlocksStats.AcceptedRawByteCount), FindBlocksStats.RejectedBlockCount, FindBlocksStats.RejectedChunkCount, NiceBytes(FindBlocksStats.RejectedByteCount), @@ -3570,13 +3879,13 @@ namespace { ValidateInfo); - Storage.PutBuildPartStats( + Storage.BuildStorage->PutBuildPartStats( BuildId, BuildPartId, {{"totalSize", double(LocalFolderScanStats.FoundFileByteCount.load())}, {"reusedRatio", AcceptedByteCountPercent / 100.0}, {"reusedBlockCount", double(FindBlocksStats.AcceptedBlockCount)}, - {"reusedBlockByteCount", double(FindBlocksStats.AcceptedByteCount)}, + {"reusedBlockByteCount", double(FindBlocksStats.AcceptedRawByteCount)}, {"newBlockCount", double(FindBlocksStats.NewBlocksCount)}, {"newBlockByteCount", double(FindBlocksStats.NewBlocksChunkByteCount)}, {"uploadedCount", double(UploadStats.BlockCount.load() + UploadStats.ChunkCount.load())}, @@ -3597,7 +3906,7 @@ namespace { ProgressBar ProgressBar(UsePlainProgress); - WorkerThreadPool& VerifyPool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& VerifyPool = GetIOWorkerPool(); ParallellWork Work(AbortFlag); @@ -3646,7 +3955,7 @@ namespace { if (IsAcceptedFolder(TargetPath.parent_path().generic_string())) { const uint64_t ExpectedSize = Content.RawSizes[PathIndex]; - if (!std::filesystem::exists(TargetPath)) + if (!IsFile(TargetPath)) { ErrorLock.WithExclusiveLock([&]() { Errors.push_back(fmt::format("File {} with expected size {} does not exist", TargetPath, ExpectedSize)); @@ -3656,7 +3965,7 @@ namespace { else { std::error_code Ec; - uint64_t SizeOnDisk = gsl::narrow<uint64_t>(std::filesystem::file_size(TargetPath, Ec)); + uint64_t SizeOnDisk = gsl::narrow<uint64_t>(FileSizeFromPath(TargetPath, Ec)); if (Ec) { ErrorLock.WithExclusiveLock([&]() { @@ -3873,12 +4182,34 @@ namespace { return ChunkTargetPtrs; }; + uint64_t GetChunkWriteCount(std::span<const std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters, + const ChunkedContentLookup& Lookup, + uint32_t ChunkIndex) + { + uint64_t WriteCount = 0; + std::span<const ChunkedContentLookup::ChunkSequenceLocation> ChunkSources = GetChunkSequenceLocations(Lookup, ChunkIndex); + for (const ChunkedContentLookup::ChunkSequenceLocation& Source : ChunkSources) + { + if (SequenceIndexChunksLeftToWriteCounters[Source.SequenceIndex].load() > 0) + { + WriteCount++; + } + } + return WriteCount; + }; + void FinalizeChunkSequence(const std::filesystem::path& TargetFolder, const IoHash& SequenceRawHash) { ZEN_TRACE_CPU("FinalizeChunkSequence"); - ZEN_ASSERT_SLOW(!std::filesystem::exists(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - std::filesystem::rename(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), - GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + ZEN_ASSERT_SLOW(!IsFile(GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash))); + std::error_code Ec; + RenameFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash), + GetFinalChunkedSequenceFileName(TargetFolder, SequenceRawHash), + Ec); + if (Ec) + { + throw std::system_error(Ec); + } } void FinalizeChunkSequences(const std::filesystem::path& TargetFolder, @@ -3892,8 +4223,39 @@ namespace { } } + void VerifySequence(const std::filesystem::path& TargetFolder, + const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, + uint32_t RemoteSequenceIndex) + { + ZEN_TRACE_CPU("VerifySequence"); + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + { + ZEN_TRACE_CPU("HashSequence"); + const std::uint32_t RemotePathIndex = Lookup.SequenceIndexFirstPathIndex[RemoteSequenceIndex]; + const uint64_t ExpectedSize = RemoteContent.RawSizes[RemotePathIndex]; + IoBuffer VerifyBuffer = IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash)); + const uint64_t VerifySize = VerifyBuffer.GetSize(); + if (VerifySize != ExpectedSize) + { + throw std::runtime_error(fmt::format("Written chunk sequence {} size {} does not match expected size {}", + SequenceRawHash, + VerifySize, + ExpectedSize)); + } + ZEN_TRACE_CPU("HashSequence"); + const IoHash VerifyChunkHash = IoHash::HashBuffer(std::move(VerifyBuffer)); + if (VerifyChunkHash != SequenceRawHash) + { + throw std::runtime_error( + fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); + } + } + } + void VerifyAndCompleteChunkSequencesAsync(const std::filesystem::path& TargetFolder, const ChunkedFolderContent& RemoteContent, + const ChunkedContentLookup& Lookup, std::span<const uint32_t> RemoteSequenceIndexes, ParallellWork& Work, WorkerThreadPool& VerifyPool) @@ -3908,46 +4270,33 @@ namespace { const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[RemoteSequenceIndexOffset]; Work.ScheduleWork( VerifyPool, - [&RemoteContent, TargetFolder, RemoteSequenceIndex](std::atomic<bool>&) { + [&RemoteContent, &Lookup, TargetFolder, RemoteSequenceIndex](std::atomic<bool>&) { if (!AbortFlag) { ZEN_TRACE_CPU("VerifyAndCompleteChunkSequenceAsync"); - const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); + if (!AbortFlag) { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = IoHash::HashBuffer( - IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error(fmt::format("Written chunk sequence {} hash does not match expected hash {}", - VerifyChunkHash, - SequenceRawHash)); - } + const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; + FinalizeChunkSequence(TargetFolder, SequenceRawHash); } - FinalizeChunkSequence(TargetFolder, SequenceRawHash); } }, Work.DefaultErrorFunction()); } const uint32_t RemoteSequenceIndex = RemoteSequenceIndexes[0]; + VerifySequence(TargetFolder, RemoteContent, Lookup, RemoteSequenceIndex); const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[RemoteSequenceIndex]; - { - ZEN_TRACE_CPU("HashSequence"); - const IoHash VerifyChunkHash = - IoHash::HashBuffer(IoBufferBuilder::MakeFromFile(GetTempChunkedSequenceFileName(TargetFolder, SequenceRawHash))); - if (VerifyChunkHash != SequenceRawHash) - { - throw std::runtime_error( - fmt::format("Written chunk sequence {} hash does not match expected hash {}", VerifyChunkHash, SequenceRawHash)); - } - } FinalizeChunkSequence(TargetFolder, SequenceRawHash); } bool CompleteSequenceChunk(uint32_t RemoteSequenceIndex, std::span<std::atomic<uint32_t>> SequenceIndexChunksLeftToWriteCounters) { - return SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1) == 1; + uint32_t PreviousValue = SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex].fetch_sub(1); + ZEN_ASSERT(PreviousValue >= 1); + ZEN_ASSERT(PreviousValue != (uint32_t)-1); + return PreviousValue == 1; } std::vector<uint32_t> CompleteChunkTargets(const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs, @@ -3985,8 +4334,7 @@ namespace { const BlockWriteOps& Ops, ParallellWork& Work, WorkerThreadPool& VerifyPool, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockChunkOps"); { @@ -4017,12 +4365,6 @@ namespace { FileOffset, RemoteContent.RawSizes[PathIndex]); } - WriteChunkStats.ChunkCountWritten += gsl::narrow<uint32_t>(Ops.ChunkBuffers.size()); - WriteChunkStats.ChunkBytesWritten += - std::accumulate(Ops.ChunkBuffers.begin(), - Ops.ChunkBuffers.end(), - uint64_t(0), - [](uint64_t Current, const CompositeBuffer& Buffer) -> uint64_t { return Current + Buffer.GetSize(); }); } if (!AbortFlag) { @@ -4036,7 +4378,7 @@ namespace { CompletedChunkSequences.push_back(RemoteSequenceIndex); } } - VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, CompletedChunkSequences, Work, VerifyPool); + VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, Lookup, CompletedChunkSequences, Work, VerifyPool); } } @@ -4117,11 +4459,36 @@ namespace { bool NeedsWrite = true; if (RemoteChunkIndexNeedsCopyFromSourceFlags[ChunkIndex].compare_exchange_strong(NeedsWrite, false)) { - MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock + CompressedBuffer::GetHeaderSizeForNoneEncoder(), - ChunkCompressedSize - CompressedBuffer::GetHeaderSizeForNoneEncoder()); - IoBuffer Decompressed(IoBuffer::Wrap, ChunkMemoryView.GetData(), ChunkMemoryView.GetSize()); - ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); + MemoryView ChunkMemoryView = BlockView.Mid(OffsetInBlock, ChunkCompressedSize); + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = + CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); + ZEN_ASSERT(CompressedChunk); + ZEN_ASSERT(VerifyChunkHash == ChunkHash); + ZEN_ASSERT(VerifyChunkSize == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + OodleCompressor ChunkCompressor; + OodleCompressionLevel ChunkCompressionLevel; + uint64_t ChunkBlockSize; + + bool GetCompressParametersSuccess = + CompressedChunk.TryGetCompressParameters(ChunkCompressor, ChunkCompressionLevel, ChunkBlockSize); + ZEN_ASSERT(GetCompressParametersSuccess); + + IoBuffer Decompressed; + if (ChunkCompressionLevel == OodleCompressionLevel::None) + { + MemoryView ChunkDecompressedMemoryView = ChunkMemoryView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()); + Decompressed = + IoBuffer(IoBuffer::Wrap, ChunkDecompressedMemoryView.GetData(), ChunkDecompressedMemoryView.GetSize()); + } + else + { + Decompressed = CompressedChunk.Decompress().AsIoBuffer(); + } ZEN_ASSERT(Decompressed.GetSize() == RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { OutOps.WriteOps.push_back( @@ -4162,8 +4529,7 @@ namespace { CompositeBuffer&& BlockBuffer, const ChunkedContentLookup& Lookup, std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WriteBlockToDisk"); @@ -4197,8 +4563,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4222,8 +4587,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } return false; @@ -4240,8 +4604,7 @@ namespace { uint32_t LastIncludedBlockChunkIndex, const ChunkedContentLookup& Lookup, std::span<std::atomic<bool>> RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("WritePartialBlockToDisk"); @@ -4267,8 +4630,7 @@ namespace { Ops, Work, VerifyPool, - DiskStats, - WriteChunkStats); + DiskStats); return true; } else @@ -4355,8 +4717,7 @@ namespace { void StreamDecompress(const std::filesystem::path& CacheFolderPath, const IoHash& SequenceRawHash, CompositeBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("StreamDecompress"); const std::filesystem::path TempChunkSequenceFileName = GetTempChunkedSequenceFileName(CacheFolderPath, SequenceRawHash); @@ -4390,7 +4751,6 @@ namespace { DiskStats.ReadByteCount += SourceSize; if (!AbortFlag) { - WriteChunkStats.ChunkBytesWritten += RangeBuffer.GetSize(); DecompressedTemp.Write(RangeBuffer, Offset); for (const SharedBuffer& Segment : RangeBuffer.GetSegments()) { @@ -4424,7 +4784,7 @@ namespace { throw std::runtime_error( fmt::format("Failed moving temporary file for decompressing large blob {}. Reason: {}", SequenceRawHash, Ec.message())); } - WriteChunkStats.ChunkCountWritten++; + // WriteChunkStats.ChunkCountWritten++; } bool WriteCompressedChunk(const std::filesystem::path& TargetFolder, @@ -4433,8 +4793,7 @@ namespace { const IoHash& ChunkHash, const std::vector<const ChunkedContentLookup::ChunkSequenceLocation*>& ChunkTargetPtrs, IoBuffer&& CompressedPart, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { auto ChunkHashToChunkIndexIt = RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); ZEN_ASSERT(ChunkHashToChunkIndexIt != RemoteLookup.ChunkHashToChunkIndex.end()); @@ -4444,7 +4803,7 @@ namespace { { const std::uint32_t SequenceIndex = ChunkTargetPtrs.front()->SequenceIndex; const IoHash& SequenceRawHash = RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]; - StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats, WriteChunkStats); + StreamDecompress(TargetFolder, SequenceRawHash, CompositeBuffer(std::move(CompressedPart)), DiskStats); } else { @@ -4459,15 +4818,13 @@ namespace { ChunkTargetPtrs, CompositeBuffer(std::move(Chunk)), OpenFileCache); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += ChunkRawSize; return true; } } return false; } - void AsyncWriteDownloadedChunk(const std::filesystem::path& Path, + void AsyncWriteDownloadedChunk(const std::filesystem::path& ZenFolderPath, const ChunkedFolderContent& RemoteContent, const ChunkedContentLookup& RemoteLookup, uint32_t RemoteChunkIndex, @@ -4479,8 +4836,7 @@ namespace { std::atomic<uint64_t>& WritePartsComplete, const uint64_t TotalPartWriteCount, FilteredRate& FilteredWrittenBytesPerSecond, - DiskStatistics& DiskStats, - WriteChunkStatistics& WriteChunkStats) + DiskStatistics& DiskStats) { ZEN_TRACE_CPU("AsyncWriteDownloadedChunk"); @@ -4502,8 +4858,8 @@ namespace { { Payload.SetDeleteOnClose(false); Payload = {}; - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); - std::filesystem::rename(TempBlobPath, CompressedChunkPath, Ec); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + RenameFile(TempBlobPath, CompressedChunkPath, Ec); if (Ec) { CompressedChunkPath = std::filesystem::path{}; @@ -4521,13 +4877,13 @@ namespace { { ZEN_TRACE_CPU("WriteTempChunk"); // Could not be moved and rather large, lets store it on disk - CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); + CompressedChunkPath = ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); TemporaryFile::SafeWriteFile(CompressedChunkPath, Payload); Payload = {}; } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, SequenceIndexChunksLeftToWriteCounters, CompressedChunkPath, @@ -4557,7 +4913,7 @@ namespace { } } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); bool NeedHashVerify = WriteCompressedChunk(TargetFolder, RemoteContent, @@ -4565,8 +4921,7 @@ namespace { ChunkHash, ChunkTargetPtrs, std::move(CompressedPart), - DiskStats, - WriteChunkStats); + DiskStats); if (!AbortFlag) { WritePartsComplete++; @@ -4575,13 +4930,18 @@ namespace { FilteredWrittenBytesPerSecond.Stop(); } - std::filesystem::remove(CompressedChunkPath); + RemoveFile(CompressedChunkPath); std::vector<uint32_t> CompletedSequences = CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); if (NeedHashVerify) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, RemoteContent, CompletedSequences, Work, WritePool); + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); } else { @@ -4593,9 +4953,10 @@ namespace { Work.DefaultErrorFunction()); }; - void UpdateFolder(BuildStorage& Storage, + void UpdateFolder(StorageInstance& Storage, const Oid& BuildId, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, const std::uint64_t LargeAttachmentSize, const std::uint64_t PreferredMultipartChunkSize, const ChunkedFolderContent& LocalContent, @@ -4604,6 +4965,7 @@ namespace { const std::vector<IoHash>& LooseChunkHashes, bool AllowPartialBlockRequests, bool WipeTargetFolder, + bool PrimeCacheOnly, FolderContent& OutLocalFolderState, DiskStatistics& DiskStats, CacheMappingStatistics& CacheMappingStats, @@ -4613,7 +4975,7 @@ namespace { { ZEN_TRACE_CPU("UpdateFolder"); - ZEN_UNUSED(WipeTargetFolder); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); Stopwatch IndexTimer; @@ -4623,7 +4985,7 @@ namespace { ZEN_CONSOLE("Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - const std::filesystem::path CacheFolderPath = Path / ZenTempCacheFolderName; + const std::filesystem::path CacheFolderPath = ZenTempCacheFolderPath(ZenFolderPath); Stopwatch CacheMappingTimer; @@ -4633,6 +4995,7 @@ namespace { tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedChunkHashesFound; tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedSequenceHashesFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckChunkCache"); @@ -4667,17 +5030,24 @@ namespace { if (SequenceSize == CacheDirContent.FileSizes[Index]) { CachedSequenceHashesFound.insert({FileHash, SequenceIndex}); - CacheMappingStats.CacheSequenceHashesCount += SequenceSize; - CacheMappingStats.CacheSequenceHashesByteCount++; + CacheMappingStats.CacheSequenceHashesCount++; + CacheMappingStats.CacheSequenceHashesByteCount += SequenceSize; + + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, + RemoteContent.ChunkedContent.SequenceRawHashes[SequenceIndex]); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); + continue; } } } - std::filesystem::remove(CacheDirContent.Files[Index]); + RemoveFileWithRetry(CacheDirContent.Files[Index]); } } tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> CachedBlocksFound; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_CheckBlockCache"); @@ -4690,7 +5060,7 @@ namespace { } DirectoryContent BlockDirContent; - GetDirectoryContent(Path / ZenTempBlockFolderName, + GetDirectoryContent(ZenTempBlockFolderPath(ZenFolderPath), DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::IncludeFileSizes, BlockDirContent); CachedBlocksFound.reserve(BlockDirContent.Files.size()); @@ -4718,13 +5088,15 @@ namespace { } } } - std::filesystem::remove(BlockDirContent.Files[Index]); + RemoveFileWithRetry(BlockDirContent.Files[Index]); } } std::vector<uint32_t> LocalPathIndexesMatchingSequenceIndexes; - // Pick up all whole files we can use from current local state + + if (!PrimeCacheOnly) { + // Pick up all whole files we can use from current local state ZEN_TRACE_CPU("UpdateFolder_CheckLocalChunks"); for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); RemoteSequenceIndex++) @@ -4736,6 +5108,8 @@ namespace { // const uint32_t RemoteSequenceIndex = CacheSequenceIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto CacheChunkIt = CachedChunkHashesFound.find(RemoteSequenceRawHash); CacheChunkIt != CachedChunkHashesFound.end()) @@ -4743,13 +5117,16 @@ namespace { // const uint32_t RemoteChunkIndex = CacheChunkIt->second; // const uint32_t RemotePathIndex = GetFirstPathIndexForSeqeuenceIndex(RemoteLookup, RemoteSequenceIndex); // RemoteSequenceByteCountFoundInCache += RemoteContent.RawSizes[RemotePathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RemoteSequenceRawHash); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); } else if (auto It = LocalLookup.RawHashToSequenceIndex.find(RemoteSequenceRawHash); It != LocalLookup.RawHashToSequenceIndex.end()) { const uint32_t LocalSequenceIndex = It->second; const uint32_t LocalPathIndex = GetFirstPathIndexForSeqeuenceIndex(LocalLookup, LocalSequenceIndex); - uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); + uint64_t RawSize = LocalContent.RawSizes[LocalPathIndex]; LocalPathIndexesMatchingSequenceIndexes.push_back(LocalPathIndex); CacheMappingStats.LocalPathsMatchingSequencesCount++; CacheMappingStats.LocalPathsMatchingSequencesByteCount += RawSize; @@ -4762,6 +5139,15 @@ namespace { } } } + else + { + for (uint32_t RemoteSequenceIndex = 0; RemoteSequenceIndex < RemoteContent.ChunkedContent.SequenceRawHashes.size(); + RemoteSequenceIndex++) + { + const uint32_t ChunkCount = RemoteContent.ChunkedContent.ChunkCounts[RemoteSequenceIndex]; + SequenceIndexChunksLeftToWriteCounters[RemoteSequenceIndex] = ChunkCount; + } + } // Pick up all chunks in current local state struct CacheCopyData { @@ -4779,6 +5165,7 @@ namespace { tsl::robin_map<IoHash, size_t, IoHash::Hasher> RawHashToCacheCopyDataIndex; std::vector<CacheCopyData> CacheCopyDatas; + if (!PrimeCacheOnly) { ZEN_TRACE_CPU("UpdateFolder_GetLocalChunks"); @@ -4852,38 +5239,37 @@ namespace { } } - if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty() || - !LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + if (!CachedSequenceHashesFound.empty() || !CachedChunkHashesFound.empty() || !CachedBlocksFound.empty()) { - ZEN_CONSOLE( - "Cache: {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks. Local state: {} ({}) chunk sequences, {} ({}) chunks", - CachedSequenceHashesFound.size(), - NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), - CachedChunkHashesFound.size(), - NiceBytes(CacheMappingStats.CacheChunkByteCount), - CachedBlocksFound.size(), - NiceBytes(CacheMappingStats.CacheBlocksByteCount), - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), - CacheMappingStats.LocalChunkMatchingRemoteCount, - NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); + ZEN_CONSOLE("Download cache: Found {} ({}) chunk sequences, {} ({}) chunks, {} ({}) blocks.", + CachedSequenceHashesFound.size(), + NiceBytes(CacheMappingStats.CacheSequenceHashesByteCount), + CachedChunkHashesFound.size(), + NiceBytes(CacheMappingStats.CacheChunkByteCount), + CachedBlocksFound.size(), + NiceBytes(CacheMappingStats.CacheBlocksByteCount)); } - uint32_t ChunkCountToWrite = 0; + if (!LocalPathIndexesMatchingSequenceIndexes.empty() || CacheMappingStats.LocalChunkMatchingRemoteCount > 0) + { + ZEN_CONSOLE("Local state : Found {} ({}) chunk sequences, {} ({}) chunks", + LocalPathIndexesMatchingSequenceIndexes.size(), + NiceBytes(CacheMappingStats.LocalPathsMatchingSequencesByteCount), + CacheMappingStats.LocalChunkMatchingRemoteCount, + NiceBytes(CacheMappingStats.LocalChunkMatchingRemoteByteCount)); + } + + uint64_t BytesToWrite = 0; + for (uint32_t RemoteChunkIndex = 0; RemoteChunkIndex < RemoteContent.ChunkedContent.ChunkHashes.size(); RemoteChunkIndex++) { - if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) - { - ChunkCountToWrite++; - } - else + uint64_t ChunkWriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); + if (ChunkWriteCount > 0) { - std::vector<const ChunkedContentLookup::ChunkSequenceLocation*> ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteLookup, RemoteChunkIndex); - if (!ChunkTargetPtrs.empty()) + BytesToWrite += RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] * ChunkWriteCount; + if (!RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex] = true; - ChunkCountToWrite++; } } } @@ -4900,8 +5286,8 @@ namespace { FilteredRate FilteredDownloadedBytesPerSecond; FilteredRate FilteredWrittenBytesPerSecond; - WorkerThreadPool& NetworkPool = GetSmallWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& NetworkPool = GetNetworkPool(); + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar WriteProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -4922,7 +5308,7 @@ namespace { const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); continue; } bool NeedsCopy = true; @@ -4933,7 +5319,7 @@ namespace { if (ChunkTargetPtrs.empty()) { - ZEN_DEBUG("Skipping chunk {} due to cache reuse", ChunkHash); + ZEN_CONSOLE_VERBOSE("Skipping chunk {} due to cache reuse", ChunkHash); } else { @@ -4993,118 +5379,274 @@ namespace { const std::vector<uint32_t> BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); if (!BlockChunkIndexNeeded.empty()) { - bool UsingCachedBlock = false; - if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + if (PrimeCacheOnly) { - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); - + TotalRequestCount++; TotalPartWriteCount++; - std::filesystem::path BlockPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - if (std::filesystem::exists(BlockPath)) - { - CachedChunkBlockIndexes.push_back(BlockIndex); - UsingCachedBlock = true; - } + FullBlockWorks.push_back(BlockIndex); } - - if (!UsingCachedBlock) + else { - bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); - bool CanDoPartialBlockDownload = - (BlockDescription.HeaderSize > 0) && - (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) { - std::vector<BlockRangeDescriptor> BlockRanges; + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_CacheGet"); - ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); + TotalPartWriteCount++; - uint32_t NeedBlockChunkIndexOffset = 0; - uint32_t ChunkBlockIndex = 0; - uint32_t CurrentOffset = - gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + std::filesystem::path BlockPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + if (IsFile(BlockPath)) + { + CachedChunkBlockIndexes.push_back(BlockIndex); + UsingCachedBlock = true; + } + } - BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; - while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && - ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) + if (!UsingCachedBlock) + { + bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = + (BlockDescription.HeaderSize > 0) && + (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); + if (AllowPartialBlockRequests && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) { - const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + std::vector<BlockRangeDescriptor> BlockRanges; + + ZEN_TRACE_CPU("UpdateFolder_HandleBlocks_PartialAnalysis"); + + uint32_t NeedBlockChunkIndexOffset = 0; + uint32_t ChunkBlockIndex = 0; + uint32_t CurrentOffset = + gsl::narrow<uint32_t>(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); + + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && + ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) { - if (NextRange.RangeLength > 0) + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) { - BlockRanges.push_back(NextRange); - NextRange = {.BlockIndex = BlockIndex}; + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + NextRange = {.BlockIndex = BlockIndex}; + } + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + if (NextRange.RangeLength == 0) + { + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; + } + else + { + ZEN_ASSERT(false); } - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; } - else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + if (NextRange.RangeLength > 0) { - if (NextRange.RangeLength == 0) + BlockRanges.push_back(NextRange); + } + + ZEN_ASSERT(!BlockRanges.empty()); + + std::vector<BlockRangeDescriptor> CollapsedBlockRanges; + auto It = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*It++); + while (It != BlockRanges.end()) + { + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; + if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + { + LastRange.ChunkBlockIndexCount = + (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + } + else { - NextRange.RangeStart = CurrentOffset; - NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + CollapsedBlockRanges.push_back(*It); } - NextRange.RangeLength += ChunkCompressedLength; - NextRange.ChunkBlockIndexCount++; - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - NeedBlockChunkIndexOffset++; + ++It; } - else + + const std::uint64_t WantedSize = std::accumulate( + CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + ZEN_ASSERT(WantedSize <= TotalBlockSize); + if (WantedSize > ((TotalBlockSize * 95) / 100)) { - ZEN_ASSERT(false); + ZEN_CONSOLE_VERBOSE("Using more than 95% ({}) of block {} ({}), requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); } - } - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - } - - ZEN_ASSERT(!BlockRanges.empty()); - std::vector<BlockRangeDescriptor> CollapsedBlockRanges; - auto It = BlockRanges.begin(); - CollapsedBlockRanges.push_back(*It++); - while (It != BlockRanges.end()) - { - BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); - uint64_t Slack = It->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); - uint64_t BothRangeSize = It->RangeLength + LastRange.RangeLength; - if (Slack <= Max(BothRangeSize / 8, 64u * 1024u)) // Made up heuristic - we'll see how it pans out + else if ((WantedSize > ((TotalBlockSize * 9) / 10)) && CollapsedBlockRanges.size() > 1) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 90% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 8) / 10)) && (CollapsedBlockRanges.size() > 16)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 80% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 7) / 10)) && (CollapsedBlockRanges.size() > 48)) + { + ZEN_CONSOLE_VERBOSE( + "Using more than 70% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); + } + else if ((WantedSize > ((TotalBlockSize * 6) / 10)) && (CollapsedBlockRanges.size() > 64)) { - LastRange.ChunkBlockIndexCount = - (It->ChunkBlockIndexStart + It->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; - LastRange.RangeLength = (It->RangeStart + It->RangeLength) - LastRange.RangeStart; + ZEN_CONSOLE_VERBOSE( + "Using more than 60% ({}) of block {} ({}) using {} requests, requesting full block", + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + TotalRequestCount++; + TotalPartWriteCount++; + + FullBlockWorks.push_back(BlockIndex); } else { - CollapsedBlockRanges.push_back(*It); + if (WantedSize > ((TotalBlockSize * 5) / 10)) + { + ZEN_CONSOLE_VERBOSE("Using {}% ({}) of block {} ({}) using {} requests, requesting partial block", + (WantedSize * 100) / TotalBlockSize, + NiceBytes(WantedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + CollapsedBlockRanges.size()); + } + TotalRequestCount += CollapsedBlockRanges.size(); + TotalPartWriteCount += CollapsedBlockRanges.size(); + + BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); } - ++It; } + else + { + TotalRequestCount++; + TotalPartWriteCount++; - TotalRequestCount += CollapsedBlockRanges.size(); - TotalPartWriteCount += CollapsedBlockRanges.size(); - - BlockRangeWorks.insert(BlockRangeWorks.end(), CollapsedBlockRanges.begin(), CollapsedBlockRanges.end()); - } - else - { - TotalRequestCount++; - TotalPartWriteCount++; - - FullBlockWorks.push_back(BlockIndex); + FullBlockWorks.push_back(BlockIndex); + } } } } else { - ZEN_DEBUG("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); + ZEN_CONSOLE_VERBOSE("Skipping block {} due to cache reuse", BlockDescriptions[BlockIndex].BlockHash); } } + struct BlobsExistsResult + { + tsl::robin_set<IoHash> ExistingBlobs; + uint64_t ElapsedTimeMs = 0; + }; + + BlobsExistsResult ExistsResult; + + if (Storage.BuildCacheStorage) + { + ZEN_TRACE_CPU("BlobCacheExistCheck"); + Stopwatch Timer; + + tsl::robin_set<IoHash> BlobHashesSet; + + BlobHashesSet.reserve(LooseChunkHashWorks.size() + FullBlockWorks.size()); + for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks) + { + BlobHashesSet.insert(RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); + } + for (const BlockRangeDescriptor& BlockRange : BlockRangeWorks) + { + const uint32_t BlockIndex = BlockRange.BlockIndex; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + for (uint32_t BlockIndex : FullBlockWorks) + { + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + BlobHashesSet.insert(BlockDescription.BlockHash); + } + + if (!BlobHashesSet.empty()) + { + const std::vector<IoHash> BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end()); + const std::vector<BuildStorageCache::BlobExistsResult> CacheExistsResult = + Storage.BuildCacheStorage->BlobsExists(BuildId, BlobHashes); + + if (CacheExistsResult.size() == BlobHashes.size()) + { + ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + { + if (CacheExistsResult[BlobIndex].HasBody) + { + ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]); + } + } + } + ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!ExistsResult.ExistingBlobs.empty()) + { + ZEN_CONSOLE("Remote cache : Found {} out of {} needed blobs in {}", + ExistsResult.ExistingBlobs.size(), + BlobHashes.size(), + NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); + } + } + } for (uint32_t LooseChunkHashWorkIndex = 0; LooseChunkHashWorkIndex < LooseChunkHashWorks.size(); LooseChunkHashWorkIndex++) { if (AbortFlag) @@ -5118,17 +5660,25 @@ namespace { std::move(LooseChunkHashWork.ChunkTargetPtrs); const uint32_t RemoteChunkIndex = LooseChunkHashWork.RemoteChunkIndex; + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex])) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( - WritePool, // NetworkPool, // GetSyncWorkerPool(),// + WritePool, [&, RemoteChunkIndex, ChunkTargetPtrs](std::atomic<bool>&) mutable { if (!AbortFlag) { ZEN_TRACE_CPU("UpdateFolder_ReadPreDownloaded"); std::filesystem::path ExistingCompressedChunkPath; + if (!PrimeCacheOnly) { - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - std::filesystem::path CompressedChunkPath = Path / ZenTempDownloadFolderName / ChunkHash.ToHexString(); - if (std::filesystem::exists(CompressedChunkPath)) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + std::filesystem::path CompressedChunkPath = + ZenTempDownloadFolderPath(ZenFolderPath) / ChunkHash.ToHexString(); + if (IsFile(CompressedChunkPath)) { IoBuffer ExistingCompressedPart = IoBufferBuilder::MakeFromFile(ExistingCompressedChunkPath); if (ExistingCompressedPart) @@ -5147,156 +5697,250 @@ namespace { else { std::error_code DummyEc; - std::filesystem::remove(CompressedChunkPath, DummyEc); + RemoveFile(CompressedChunkPath, DummyEc); } } } } - if (!ExistingCompressedChunkPath.empty()) + if (!AbortFlag) + { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Path, - &RemoteContent, - &RemoteLookup, - &CacheFolderPath, - &SequenceIndexChunksLeftToWriteCounters, - &Work, - &WritePool, - &DiskStats, - &WriteChunkStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - RemoteChunkIndex, - ChunkTargetPtrs, - CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); + if (!ExistingCompressedChunkPath.empty()) + { + Work.ScheduleWork( + WritePool, + [&Path, + &ZenFolderPath, + &RemoteContent, + &RemoteLookup, + &CacheFolderPath, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + RemoteChunkIndex, + ChunkTargetPtrs, + CompressedChunkPath = std::move(ExistingCompressedChunkPath)](std::atomic<bool>&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePreDownloaded"); - FilteredWrittenBytesPerSecond.Start(); + FilteredWrittenBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); - if (!CompressedPart) - { - throw std::runtime_error(fmt::format("Could not open dowloaded compressed chunk {} from {}", - ChunkHash, - CompressedChunkPath)); - } + IoBuffer CompressedPart = IoBufferBuilder::MakeFromFile(CompressedChunkPath); + if (!CompressedPart) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded compressed chunk {} from {}", + ChunkHash, + CompressedChunkPath)); + } + + std::filesystem::path TargetFolder = ZenTempCacheFolderPath(ZenFolderPath); + bool NeedHashVerify = WriteCompressedChunk(TargetFolder, + RemoteContent, + RemoteLookup, + ChunkHash, + ChunkTargetPtrs, + std::move(CompressedPart), + DiskStats); + WritePartsComplete++; + + if (!AbortFlag) + { + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } - std::filesystem::path TargetFolder = Path / ZenTempCacheFolderName; - bool NeedHashVerify = WriteCompressedChunk(TargetFolder, - RemoteContent, - RemoteLookup, - ChunkHash, - ChunkTargetPtrs, - std::move(CompressedPart), - DiskStats, - WriteChunkStats); - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex]; - WritePartsComplete++; + RemoveFile(CompressedChunkPath); + std::vector<uint32_t> CompletedSequences = + CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); + if (NeedHashVerify) + { + VerifyAndCompleteChunkSequencesAsync(TargetFolder, + RemoteContent, + RemoteLookup, + CompletedSequences, + Work, + WritePool); + } + else + { + FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + } + } + } + }, + Work.DefaultErrorFunction()); + } + else + { + Work.ScheduleWork( + NetworkPool, + [&Path, + &ZenFolderPath, + &Storage, + BuildId, + &PrimeCacheOnly, + &RemoteContent, + &RemoteLookup, + &ExistsResult, + &SequenceIndexChunksLeftToWriteCounters, + &Work, + &WritePool, + &NetworkPool, + &DiskStats, + &WriteChunkStats, + &WritePartsComplete, + TotalPartWriteCount, + TotalRequestCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + LargeAttachmentSize, + PreferredMultipartChunkSize, + RemoteChunkIndex, + ChunkTargetPtrs, + &DownloadStats](std::atomic<bool>&) mutable { if (!AbortFlag) { - if (WritePartsComplete == TotalPartWriteCount) + const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; + FilteredDownloadedBytesPerSecond.Start(); + IoBuffer BuildBlob; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + if (ExistsInCache) { - FilteredWrittenBytesPerSecond.Stop(); + BuildBlob = Storage.BuildCacheStorage->GetBuildBlob(BuildId, ChunkHash); } - - std::filesystem::remove(CompressedChunkPath); - - std::vector<uint32_t> CompletedSequences = - CompleteChunkTargets(ChunkTargetPtrs, SequenceIndexChunksLeftToWriteCounters); - if (NeedHashVerify) + if (BuildBlob) { - VerifyAndCompleteChunkSequencesAsync(TargetFolder, - RemoteContent, - CompletedSequences, - Work, - WritePool); + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); } else { - FinalizeChunkSequences(TargetFolder, RemoteContent, CompletedSequences); + if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) + { + ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); + DownloadLargeBlob( + *Storage.BuildStorage, + ZenTempDownloadFolderPath(ZenFolderPath), + BuildId, + ChunkHash, + PreferredMultipartChunkSize, + Work, + NetworkPool, + DownloadStats, + [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + if (Payload && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(Payload), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + }); + } + else + { + ZEN_TRACE_CPU("UpdateFolder_GetChunk"); + BuildBlob = Storage.BuildStorage->GetBuildBlob(BuildId, ChunkHash); + if (BuildBlob && Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->PutBuildBlob( + BuildId, + ChunkHash, + BuildBlob.GetContentType(), + CompositeBuffer(SharedBuffer(BuildBlob))); + } + if (!BuildBlob) + { + throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); + } + if (!PrimeCacheOnly) + { + if (!AbortFlag) + { + uint64_t BlobSize = BuildBlob.GetSize(); + DownloadStats.DownloadedChunkCount++; + DownloadStats.DownloadedChunkByteCount += BlobSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + AsyncWriteDownloadedChunk(ZenFolderPath, + RemoteContent, + RemoteLookup, + RemoteChunkIndex, + std::move(ChunkTargetPtrs), + Work, + WritePool, + std::move(BuildBlob), + SequenceIndexChunksLeftToWriteCounters, + WritePartsComplete, + TotalPartWriteCount, + FilteredWrittenBytesPerSecond, + DiskStats); + } + } + } } } - } - }, - Work.DefaultErrorFunction()); - } - else - { - FilteredDownloadedBytesPerSecond.Start(); - const IoHash& ChunkHash = RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; - if (RemoteContent.ChunkedContent.ChunkRawSizes[RemoteChunkIndex] >= LargeAttachmentSize) - { - ZEN_TRACE_CPU("UpdateFolder_GetLargeChunk"); - DownloadLargeBlob(Storage, - Path / ZenTempDownloadFolderName, - BuildId, - ChunkHash, - PreferredMultipartChunkSize, - Work, - NetworkPool, - DownloadStats, - [&, RemoteChunkIndex, ChunkTargetPtrs](IoBuffer&& Payload) mutable { - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(Payload), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); - }); - } - else - { - ZEN_TRACE_CPU("UpdateFolder_GetChunk"); - - IoBuffer BuildBlob = Storage.GetBuildBlob(BuildId, ChunkHash); - if (!BuildBlob) - { - throw std::runtime_error(fmt::format("Chunk {} is missing", ChunkHash)); - } - uint64_t BlobSize = BuildBlob.GetSize(); - DownloadStats.DownloadedChunkCount++; - DownloadStats.DownloadedChunkByteCount += BlobSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - AsyncWriteDownloadedChunk(Path, - RemoteContent, - RemoteLookup, - RemoteChunkIndex, - std::move(ChunkTargetPtrs), - Work, - WritePool, - std::move(BuildBlob), - SequenceIndexChunksLeftToWriteCounters, - WritePartsComplete, - TotalPartWriteCount, - FilteredWrittenBytesPerSecond, - DiskStats, - WriteChunkStats); + }, + Work.DefaultErrorFunction()); } } } @@ -5306,13 +5950,14 @@ namespace { for (size_t CopyDataIndex = 0; CopyDataIndex < CacheCopyDatas.size(); CopyDataIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, CopyDataIndex](std::atomic<bool>&) { if (!AbortFlag) { @@ -5439,16 +6084,6 @@ namespace { ChunkSource, Op.Target->Offset, RemoteContent.RawSizes[RemotePathIndex]); - for (size_t WrittenOpIndex = WriteOpIndex; WrittenOpIndex < WriteOpIndex + WriteCount; WrittenOpIndex++) - { - const WriteOp& WrittenOp = WriteOps[WrittenOpIndex]; - if (ChunkIndexesWritten.insert(WrittenOp.ChunkIndex).second) - { - WriteChunkStats.ChunkCountWritten++; - WriteChunkStats.ChunkBytesWritten += - RemoteContent.ChunkedContent.ChunkRawSizes[WrittenOp.ChunkIndex]; - } - } CacheLocalFileBytesRead += ReadLength; // TODO: This should be the sum of unique chunk sizes? @@ -5469,10 +6104,13 @@ namespace { } VerifyAndCompleteChunkSequencesAsync(CacheFolderPath, RemoteContent, + RemoteLookup, CompletedChunkSequences, Work, WritePool); - ZEN_DEBUG("Copied {} from {}", NiceBytes(CacheLocalFileBytesRead), LocalContent.Paths[LocalPathIndex]); + ZEN_CONSOLE_VERBOSE("Copied {} from {}", + NiceBytes(CacheLocalFileBytesRead), + LocalContent.Paths[LocalPathIndex]); } WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) @@ -5486,13 +6124,14 @@ namespace { for (uint32_t BlockIndex : CachedChunkBlockIndexes) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(), // WritePool, + WritePool, [&, BlockIndex](std::atomic<bool>&) mutable { if (!AbortFlag) { @@ -5501,35 +6140,38 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredWrittenBytesPerSecond.Start(); - std::filesystem::path BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + std::filesystem::path BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + IoBuffer BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); if (!BlockBuffer) { throw std::runtime_error( fmt::format("Can not read block {} at {}", BlockDescription.BlockHash, BlockChunkPath)); } - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } - WritePartsComplete++; - std::filesystem::remove(BlockChunkPath); - if (WritePartsComplete == TotalPartWriteCount) + if (!AbortFlag) { - FilteredWrittenBytesPerSecond.Stop(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + WritePartsComplete++; + RemoveFile(BlockChunkPath); + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } } }, @@ -5538,6 +6180,7 @@ namespace { for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) { + ZEN_ASSERT(!PrimeCacheOnly); if (AbortFlag) { break; @@ -5546,7 +6189,7 @@ namespace { ZEN_ASSERT(BlockRange.BlockIndex != (uint32_t)-1); const uint32_t BlockIndex = BlockRange.BlockIndex; Work.ScheduleWork( - NetworkPool, // NetworkPool, // GetSyncWorkerPool() + NetworkPool, [&, BlockIndex, BlockRange](std::atomic<bool>&) { if (!AbortFlag) { @@ -5555,131 +6198,146 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = - Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + IoBuffer BlockBuffer; + if (Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + } if (!BlockBuffer) { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + if (!BlockBuffer) { - FilteredDownloadedBytesPerSecond.Stop(); + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } - - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!AbortFlag) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + FilteredDownloadedBytesPerSecond.Stop(); + } - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) + std::filesystem::path BlockChunkPath; + + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - BlockChunkPath = std::filesystem::path{}; + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + RenameFile(TempBlobPath, BlockChunkPath, Ec); + if (Ec) + { + BlockChunkPath = std::filesystem::path{}; - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); + } } } } - } - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) - { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = - Path / ZenTempBlockFolderName / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / fmt::format("{}_{:x}_{:x}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!AbortFlag) - { - Work.ScheduleWork( - WritePool, // WritePool, // GetSyncWorkerPool(), - [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( - std::atomic<bool>&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&, BlockIndex, BlockRange, BlockChunkPath, BlockPartialBuffer = std::move(BlockBuffer)]( + std::atomic<bool>&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WritePartialBlock"); - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else - { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) + if (BlockChunkPath.empty()) { - throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error(fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockToDisk( - CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockToDisk( + CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!BlockChunkPath.empty()) + { + RemoveFile(BlockChunkPath); + } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - Work.DefaultErrorFunction()); + }, + Work.DefaultErrorFunction()); + } } } }, @@ -5692,8 +6350,15 @@ namespace { { break; } + + if (PrimeCacheOnly && ExistsResult.ExistingBlobs.contains(BlockDescriptions[BlockIndex].BlockHash)) + { + DownloadStats.RequestsCompleteCount++; + continue; + } + Work.ScheduleWork( - NetworkPool, // GetSyncWorkerPool(), // NetworkPool, + NetworkPool, [&, BlockIndex](std::atomic<bool>&) { if (!AbortFlag) { @@ -5702,133 +6367,159 @@ namespace { const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; FilteredDownloadedBytesPerSecond.Start(); - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockDescription.BlockHash); - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); - } - uint64_t BlockSize = BlockBuffer.GetSize(); - DownloadStats.DownloadedBlockCount++; - DownloadStats.DownloadedBlockByteCount += BlockSize; - DownloadStats.RequestsCompleteCount++; - if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + + IoBuffer BlockBuffer; + const bool ExistsInCache = + Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + if (ExistsInCache) { - FilteredDownloadedBytesPerSecond.Stop(); + BlockBuffer = Storage.BuildCacheStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); } - - std::filesystem::path BlockChunkPath; - - // Check if the dowloaded block is file based and we can move it directly without rewriting it + if (!BlockBuffer) { - IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && - (FileRef.FileChunkSize == BlockSize)) + BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockDescription.BlockHash); + if (BlockBuffer && Storage.BuildCacheStorage) { - ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); - std::error_code Ec; - std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); - if (!Ec) - { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - std::filesystem::rename(TempBlobPath, BlockChunkPath, Ec); - if (Ec) - { - BlockChunkPath = std::filesystem::path{}; - - // Re-open the temp file again - BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); - } - } + Storage.BuildCacheStorage->PutBuildBlob(BuildId, + BlockDescription.BlockHash, + BlockBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlockBuffer))); } } - - if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + if (!BlockBuffer) { - ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); - // Could not be moved and rather large, lets store it on disk - BlockChunkPath = Path / ZenTempBlockFolderName / BlockDescription.BlockHash.ToHexString(); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; + throw std::runtime_error(fmt::format("Block {} is missing", BlockDescription.BlockHash)); } - if (!AbortFlag) { - Work.ScheduleWork( - WritePool, // WritePool, GetSyncWorkerPool() - [&Work, - &WritePool, - &RemoteContent, - &RemoteLookup, - CacheFolderPath, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - BlockIndex, - &BlockDescriptions, - &WriteChunkStats, - &DiskStats, - &WritePartsComplete, - &TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - BlockChunkPath, - BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable { - if (!AbortFlag) - { - ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); + uint64_t BlockSize = BlockBuffer.GetSize(); + DownloadStats.DownloadedBlockCount++; + DownloadStats.DownloadedBlockByteCount += BlockSize; + DownloadStats.RequestsCompleteCount++; + if (DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; + if (!PrimeCacheOnly) + { + std::filesystem::path BlockChunkPath; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockBuffer); - } - else + // Check if the dowloaded block is file based and we can move it directly without rewriting it + { + IoBufferFileReference FileRef; + if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockSize)) + { + ZEN_TRACE_CPU("UpdateFolder_MoveTempBlock"); + std::error_code Ec; + std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); + if (!Ec) { - ZEN_ASSERT(!BlockBuffer); - BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockBuffer) + BlockBuffer.SetDeleteOnClose(false); + BlockBuffer = {}; + BlockChunkPath = + ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + RenameFile(TempBlobPath, BlockChunkPath, Ec); + if (Ec) { - throw std::runtime_error(fmt::format("Could not open dowloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + BlockChunkPath = std::filesystem::path{}; + + // Re-open the temp file again + BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); + BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); + BlockBuffer.SetDeleteOnClose(true); } } + } + } - FilteredWrittenBytesPerSecond.Start(); - if (!WriteBlockToDisk(CacheFolderPath, - RemoteContent, - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - WritePool, - CompositeBuffer(std::move(BlockBuffer)), - RemoteLookup, - RemoteChunkIndexNeedsCopyFromSourceFlags, - DiskStats, - WriteChunkStats)) - { - std::error_code DummyEc; - std::filesystem::remove(BlockChunkPath, DummyEc); - throw std::runtime_error(fmt::format("Block {} is malformed", BlockDescription.BlockHash)); - } + if (BlockChunkPath.empty() && (BlockSize > 512u * 1024u)) + { + ZEN_TRACE_CPU("UpdateFolder_WriteTempBlock"); + // Could not be moved and rather large, lets store it on disk + BlockChunkPath = ZenTempBlockFolderPath(ZenFolderPath) / BlockDescription.BlockHash.ToHexString(); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); + BlockBuffer = {}; + } - if (!BlockChunkPath.empty()) - { - std::filesystem::remove(BlockChunkPath); - } + if (!AbortFlag) + { + Work.ScheduleWork( + WritePool, + [&Work, + &WritePool, + &RemoteContent, + &RemoteLookup, + CacheFolderPath, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + BlockIndex, + &BlockDescriptions, + &WriteChunkStats, + &DiskStats, + &WritePartsComplete, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + BlockChunkPath, + BlockBuffer = std::move(BlockBuffer)](std::atomic<bool>&) mutable { + if (!AbortFlag) + { + ZEN_TRACE_CPU("UpdateFolder_WriteFullBlock"); - WritePartsComplete++; + const ChunkBlockDescription& BlockDescription = BlockDescriptions[BlockIndex]; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); - } - } - }, - Work.DefaultErrorFunction()); + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockBuffer); + } + else + { + ZEN_ASSERT(!BlockBuffer); + BlockBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockBuffer) + { + throw std::runtime_error( + fmt::format("Could not open dowloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } + } + + FilteredWrittenBytesPerSecond.Start(); + if (!WriteBlockToDisk(CacheFolderPath, + RemoteContent, + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + WritePool, + CompositeBuffer(std::move(BlockBuffer)), + RemoteLookup, + RemoteChunkIndexNeedsCopyFromSourceFlags, + DiskStats)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Block {} is malformed", BlockDescription.BlockHash)); + } + + if (!BlockChunkPath.empty()) + { + RemoveFile(BlockChunkPath); + } + + WritePartsComplete++; + + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } + } + }, + Work.DefaultErrorFunction()); + } + } } } }, @@ -5840,26 +6531,32 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - ZEN_ASSERT(ChunkCountToWrite >= WriteChunkStats.ChunkCountWritten.load()); uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); FilteredWrittenBytesPerSecond.Update(DiskStats.WriteByteCount.load()); FilteredDownloadedBytesPerSecond.Update(DownloadedBytes); - std::string Details = fmt::format("{}/{} ({} {}bits/s) downloaded. {}/{} ({} {}B/s) written.", - DownloadStats.RequestsCompleteCount.load(), - TotalRequestCount, - NiceBytes(DownloadedBytes), - NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8), - WriteChunkStats.ChunkCountWritten.load(), - ChunkCountToWrite, - NiceBytes(DiskStats.WriteByteCount.load()), - NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string DownloadRateString = + (DownloadStats.RequestsCompleteCount == TotalRequestCount) + ? "" + : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); + std::string WriteDetails = PrimeCacheOnly ? "" + : fmt::format(" {}/{} ({}B/s) written.", + NiceBytes(DiskStats.WriteByteCount.load()), + NiceBytes(BytesToWrite), + NiceNum(FilteredWrittenBytesPerSecond.GetCurrent())); + std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", + DownloadStats.RequestsCompleteCount.load(), + TotalRequestCount, + NiceBytes(DownloadedBytes), + DownloadRateString, + WriteDetails); WriteProgressBar.UpdateState( - {.Task = "Writing chunks ", + {.Task = PrimeCacheOnly ? "Downloading " : "Writing chunks ", .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(ChunkCountToWrite), - .RemainingCount = gsl::narrow<uint64_t>(ChunkCountToWrite - WriteChunkStats.ChunkCountWritten.load())}, + .TotalCount = PrimeCacheOnly ? TotalRequestCount : BytesToWrite, + .RemainingCount = PrimeCacheOnly ? (TotalRequestCount - DownloadStats.RequestsCompleteCount.load()) + : (BytesToWrite - DiskStats.WriteByteCount.load())}, false); }); } @@ -5874,21 +6571,28 @@ namespace { WriteProgressBar.Finish(); - uint32_t RawSequencesMissingWriteCount = 0; - for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) + if (!PrimeCacheOnly) { - const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; - if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + uint32_t RawSequencesMissingWriteCount = 0; + for (uint32_t SequenceIndex = 0; SequenceIndex < SequenceIndexChunksLeftToWriteCounters.size(); SequenceIndex++) { - RawSequencesMissingWriteCount++; - const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; - const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; - ZEN_ASSERT(!IncompletePath.empty()); - const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; - ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + const auto& SequenceIndexChunksLeftToWriteCounter = SequenceIndexChunksLeftToWriteCounters[SequenceIndex]; + if (SequenceIndexChunksLeftToWriteCounter.load() != 0) + { + RawSequencesMissingWriteCount++; + const uint32_t PathIndex = RemoteLookup.SequenceIndexFirstPathIndex[SequenceIndex]; + const std::filesystem::path& IncompletePath = RemoteContent.Paths[PathIndex]; + ZEN_ASSERT(!IncompletePath.empty()); + const uint32_t ExpectedSequenceCount = RemoteContent.ChunkedContent.ChunkCounts[SequenceIndex]; + ZEN_CONSOLE("{}: Max count {}, Current count {}", + IncompletePath, + ExpectedSequenceCount, + SequenceIndexChunksLeftToWriteCounter.load()); + ZEN_ASSERT(SequenceIndexChunksLeftToWriteCounter.load() <= ExpectedSequenceCount); + } } + ZEN_ASSERT(RawSequencesMissingWriteCount == 0); } - ZEN_ASSERT(RawSequencesMissingWriteCount == 0); const uint64_t DownloadedBytes = DownloadStats.DownloadedChunkByteCount.load() + DownloadStats.DownloadedBlockByteCount.load() + +DownloadStats.DownloadedPartialBlockByteCount.load(); @@ -5906,82 +6610,199 @@ namespace { WriteChunkStats.WriteTimeUs = FilteredWrittenBytesPerSecond.GetElapsedTimeUS(); } - // Move all files we will reuse to cache folder - // TODO: If WipeTargetFolder is false we could check which files are already correct and leave them in place - if (!LocalPathIndexesMatchingSequenceIndexes.empty()) + if (PrimeCacheOnly) { - ZEN_TRACE_CPU("UpdateFolder_CacheReused"); - uint64_t TotalFullFileSizeCached = 0; - for (uint32_t LocalPathIndex : LocalPathIndexesMatchingSequenceIndexes) - { - const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(LocalFilePath)); - SetFileReadOnly(LocalFilePath, false); - ZEN_ASSERT_SLOW(!std::filesystem::exists(CacheFilePath)); - std::filesystem::rename(LocalFilePath, CacheFilePath); - TotalFullFileSizeCached += std::filesystem::file_size(CacheFilePath); - } - ZEN_CONSOLE("Saved {} ({}) unchanged files in cache", - LocalPathIndexesMatchingSequenceIndexes.size(), - NiceBytes(TotalFullFileSizeCached)); + return; } - if (WipeTargetFolder) - { - ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); - Stopwatch Timer; + tsl::robin_map<uint32_t, uint32_t> RemotePathIndexToLocalPathIndex; + RemotePathIndexToLocalPathIndex.reserve(RemoteContent.Paths.size()); - // Clean target folder - ZEN_CONSOLE("Wiping {}", Path); - if (!CleanDirectory(Path, DefaultExcludeFolders)) - { - ZEN_WARN("Some files in {} could not be removed", Path); - } - RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); + tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> SequenceHashToLocalPathIndex; + std::vector<uint32_t> RemoveLocalPathIndexes; + + if (AbortFlag) + { + return; } - else + { - ZEN_TRACE_CPU("UpdateFolder_RemoveUnused"); - Stopwatch Timer; + ZEN_TRACE_CPU("UpdateFolder_PrepareTarget"); - // Remove unused tracked files - tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex; + tsl::robin_set<IoHash, IoHash::Hasher> CachedRemoteSequences; + tsl::robin_map<std::string, uint32_t> RemotePathToRemoteIndex; RemotePathToRemoteIndex.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { RemotePathToRemoteIndex.insert({RemoteContent.Paths[RemotePathIndex].generic_string(), RemotePathIndex}); } - std::vector<std::filesystem::path> LocalFilesToRemove; + + std::vector<uint32_t> FilesToCache; + + uint64_t MatchCount = 0; + uint64_t PathMismatchCount = 0; + uint64_t HashMismatchCount = 0; + std::atomic<uint64_t> CachedCount = 0; + std::atomic<uint64_t> CachedByteCount = 0; + uint64_t SkippedCount = 0; + uint64_t DeleteCount = 0; for (uint32_t LocalPathIndex = 0; LocalPathIndex < LocalContent.Paths.size(); LocalPathIndex++) { - if (!RemotePathToRemoteIndex.contains(LocalContent.Paths[LocalPathIndex].generic_string())) + if (AbortFlag) + { + break; + } + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + + ZEN_ASSERT_SLOW(IsFile((Path / LocalContent.Paths[LocalPathIndex]).make_preferred())); + + if (!WipeTargetFolder) { - const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); - if (std::filesystem::exists(LocalFilePath)) + if (auto RemotePathIt = RemotePathToRemoteIndex.find(LocalPath.generic_string()); + RemotePathIt != RemotePathToRemoteIndex.end()) { - LocalFilesToRemove.emplace_back(std::move(LocalFilePath)); + const uint32_t RemotePathIndex = RemotePathIt->second; + if (RemoteContent.RawHashes[RemotePathIndex] == RawHash) + { + // It is already in it's desired place + RemotePathIndexToLocalPathIndex[RemotePathIndex] = LocalPathIndex; + SequenceHashToLocalPathIndex.insert({RawHash, LocalPathIndex}); + MatchCount++; + continue; + } + else + { + HashMismatchCount++; + } + } + else + { + PathMismatchCount++; } } + if (RemoteLookup.RawHashToSequenceIndex.contains(RawHash)) + { + if (!CachedRemoteSequences.contains(RawHash)) + { + ZEN_TRACE_CPU("MoveToCache"); + // We need it + FilesToCache.push_back(LocalPathIndex); + CachedRemoteSequences.insert(RawHash); + } + else + { + // We already have it + SkippedCount++; + } + } + else if (!WipeTargetFolder) + { + // We don't need it + RemoveLocalPathIndexes.push_back(LocalPathIndex); + DeleteCount++; + } + } + + if (AbortFlag) + { + return; } - if (!LocalFilesToRemove.empty()) + { - ZEN_CONSOLE("Cleaning {} removed files from {}", LocalFilesToRemove.size(), Path); - for (const std::filesystem::path& LocalFilePath : LocalFilesToRemove) + ZEN_TRACE_CPU("UpdateFolder_CopyToCache"); + + Stopwatch Timer; + + WorkerThreadPool& WritePool = GetIOWorkerPool(); + + ProgressBar CacheLocalProgressBar(UsePlainProgress); + ParallellWork Work(AbortFlag); + + for (uint32_t LocalPathIndex : FilesToCache) { - SetFileReadOnly(LocalFilePath, false); - std::filesystem::remove(LocalFilePath); + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic<bool>&) { + ZEN_TRACE_CPU("UpdateFolder_AsyncCopyToCache"); + if (!AbortFlag) + { + const IoHash& RawHash = LocalContent.RawHashes[LocalPathIndex]; + const std::filesystem::path& LocalPath = LocalContent.Paths[LocalPathIndex]; + const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(!IsFile(CacheFilePath)); + const std::filesystem::path LocalFilePath = (Path / LocalPath).make_preferred(); + RenameFileWithRetry(LocalFilePath, CacheFilePath); + CachedCount++; + CachedByteCount += LocalContent.RawSizes[LocalPathIndex]; + } + }, + Work.DefaultErrorFunction()); } + + { + ZEN_TRACE_CPU("CacheLocal_Wait"); + + Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, PendingWork); + const uint64_t WorkTotal = FilesToCache.size(); + const uint64_t WorkComplete = CachedCount.load(); + std::string Details = fmt::format("{}/{} ({}) files", WorkComplete, WorkTotal, NiceBytes(CachedByteCount)); + CacheLocalProgressBar.UpdateState({.Task = "Caching local ", + .Details = Details, + .TotalCount = gsl::narrow<uint64_t>(WorkTotal), + .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete)}, + false); + }); + } + + if (AbortFlag) + { + return; + } + + CacheLocalProgressBar.Finish(); + + ZEN_DEBUG( + "Local state prep: Match: {}, PathMismatch: {}, HashMismatch: {}, Cached: {} ({}), Skipped: {}, " + "Delete: {}", + MatchCount, + PathMismatchCount, + HashMismatchCount, + CachedCount.load(), + NiceBytes(CachedByteCount.load()), + SkippedCount, + DeleteCount); + } + } + + if (WipeTargetFolder) + { + ZEN_TRACE_CPU("UpdateFolder_WipeTarget"); + Stopwatch Timer; + + // Clean target folder + if (!CleanDirectory(Path, DefaultExcludeFolders)) + { + ZEN_WARN("Some files in {} could not be removed", Path); } RebuildFolderStateStats.CleanFolderElapsedWallTimeUs = Timer.GetElapsedTimeUs(); } + if (AbortFlag) + { + return; + } + { ZEN_TRACE_CPU("UpdateFolder_FinalizeTree"); Stopwatch Timer; - WorkerThreadPool& WritePool = GetMediumWorkerPool(EWorkloadType::Burst); // GetSyncWorkerPool(); // + WorkerThreadPool& WritePool = GetIOWorkerPool(); ProgressBar RebuildProgressBar(UsePlainProgress); ParallellWork Work(AbortFlag); @@ -5991,16 +6812,52 @@ namespace { OutLocalFolderState.Attributes.resize(RemoteContent.Paths.size()); OutLocalFolderState.ModificationTicks.resize(RemoteContent.Paths.size()); + std::atomic<uint64_t> DeletedCount = 0; + + for (uint32_t LocalPathIndex : RemoveLocalPathIndexes) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork( + WritePool, + [&, LocalPathIndex](std::atomic<bool>&) { + if (!AbortFlag) + { + const std::filesystem::path LocalFilePath = (Path / LocalContent.Paths[LocalPathIndex]).make_preferred(); + SetFileReadOnlyWithRetry(LocalFilePath, false); + RemoveFileWithRetry(LocalFilePath); + DeletedCount++; + } + }, + Work.DefaultErrorFunction()); + } + std::atomic<uint64_t> TargetsComplete = 0; - std::vector<std::pair<IoHash, uint32_t>> Targets; + struct FinalizeTarget + { + IoHash RawHash; + uint32_t RemotePathIndex; + }; + + std::vector<FinalizeTarget> Targets; Targets.reserve(RemoteContent.Paths.size()); for (uint32_t RemotePathIndex = 0; RemotePathIndex < RemoteContent.Paths.size(); RemotePathIndex++) { - Targets.push_back(std::make_pair(RemoteContent.RawHashes[RemotePathIndex], RemotePathIndex)); + Targets.push_back(FinalizeTarget{.RawHash = RemoteContent.RawHashes[RemotePathIndex], .RemotePathIndex = RemotePathIndex}); } - std::sort(Targets.begin(), Targets.end(), [](const std::pair<IoHash, uint32_t>& Lhs, const std::pair<IoHash, uint32_t>& Rhs) { - return Lhs.first < Rhs.first; + std::sort(Targets.begin(), Targets.end(), [](const FinalizeTarget& Lhs, const FinalizeTarget& Rhs) { + if (Lhs.RawHash < Rhs.RawHash) + { + return true; + } + else if (Lhs.RawHash > Rhs.RawHash) + { + return false; + } + return Lhs.RemotePathIndex < Rhs.RemotePathIndex; }); size_t TargetOffset = 0; @@ -6011,93 +6868,168 @@ namespace { break; } - size_t TargetCount = 1; - const IoHash& RawHash = Targets[TargetOffset].first; - while (Targets[TargetOffset + TargetCount].first == RawHash) + size_t TargetCount = 1; + while (Targets[TargetOffset + TargetCount].RawHash == Targets[TargetOffset].RawHash) { TargetCount++; } Work.ScheduleWork( - WritePool, // GetSyncWorkerPool(),// + WritePool, [&, BaseTargetOffset = TargetOffset, TargetCount](std::atomic<bool>&) { if (!AbortFlag) { ZEN_TRACE_CPU("FinalizeTree_Work"); - size_t TargetOffset = BaseTargetOffset; - const IoHash& RawHash = Targets[TargetOffset].first; - const uint32_t FirstTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstTargetPathIndex]; - OutLocalFolderState.Paths[FirstTargetPathIndex] = FirstTargetPath; - OutLocalFolderState.RawSizes[FirstTargetPathIndex] = RemoteContent.RawSizes[FirstTargetPathIndex]; - const std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + size_t TargetOffset = BaseTargetOffset; + const IoHash& RawHash = Targets[TargetOffset].RawHash; + if (RawHash == IoHash::Zero) { - if (std::filesystem::exists(FirstTargetFilePath)) + ZEN_TRACE_CPU("ZeroSize"); + while (TargetOffset < (BaseTargetOffset + TargetCount)) { - SetFileReadOnly(FirstTargetFilePath, false); - } - CreateDirectories(FirstTargetFilePath.parent_path()); - { - BasicFile OutputFile; - OutputFile.Open(FirstTargetFilePath, BasicFile::Mode::kTruncate); + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + if (!RemotePathIndexToLocalPathIndex[RemotePathIndex]) + { + if (IsFile(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + BasicFile OutputFile; + OutputFile.Open(TargetFilePath, BasicFile::Mode::kTruncate); + } + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; } } else { - ZEN_TRACE_CPU("FinalizeTree_MoveIntoPlace"); - - const std::filesystem::path CacheFilePath = GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(CacheFilePath)); - CreateDirectories(FirstTargetFilePath.parent_path()); - if (std::filesystem::exists(FirstTargetFilePath)) + ZEN_TRACE_CPU("Files"); + ZEN_ASSERT(RemoteLookup.RawHashToSequenceIndex.contains(RawHash)); + const uint32_t FirstRemotePathIndex = Targets[TargetOffset].RemotePathIndex; + const std::filesystem::path& FirstTargetPath = RemoteContent.Paths[FirstRemotePathIndex]; + std::filesystem::path FirstTargetFilePath = (Path / FirstTargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(FirstRemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) { - SetFileReadOnly(FirstTargetFilePath, false); + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); } - std::filesystem::rename(CacheFilePath, FirstTargetFilePath); - RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; - } + else + { + if (IsFile(FirstTargetFilePath)) + { + SetFileReadOnlyWithRetry(FirstTargetFilePath, false); + } + else + { + CreateDirectories(FirstTargetFilePath.parent_path()); + } - OutLocalFolderState.Attributes[FirstTargetPathIndex] = - RemoteContent.Attributes.empty() ? GetNativeFileAttributes(FirstTargetFilePath) - : SetNativeFileAttributes(FirstTargetFilePath, - RemoteContent.Platform, - RemoteContent.Attributes[FirstTargetPathIndex]); - OutLocalFolderState.ModificationTicks[FirstTargetPathIndex] = GetModificationTickFromPath(FirstTargetFilePath); + if (auto InplaceIt = SequenceHashToLocalPathIndex.find(RawHash); + InplaceIt != SequenceHashToLocalPathIndex.end()) + { + ZEN_TRACE_CPU("Copy"); + const uint32_t LocalPathIndex = InplaceIt->second; + const std::filesystem::path& SourcePath = LocalContent.Paths[LocalPathIndex]; + std::filesystem::path SourceFilePath = (Path / SourcePath).make_preferred(); + ZEN_ASSERT_SLOW(IsFile(SourceFilePath)); + + ZEN_DEBUG("Copying from '{}' -> '{}'", SourceFilePath, FirstTargetFilePath); + CopyFile(SourceFilePath, FirstTargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + else + { + ZEN_TRACE_CPU("Rename"); + const std::filesystem::path CacheFilePath = + GetFinalChunkedSequenceFileName(CacheFolderPath, RawHash); + ZEN_ASSERT_SLOW(IsFile(CacheFilePath)); - TargetOffset++; - TargetsComplete++; - while (TargetOffset < (BaseTargetOffset + TargetCount)) - { - ZEN_TRACE_CPU("FinalizeTree_Copy"); - - ZEN_ASSERT(Targets[TargetOffset].first == RawHash); - ZEN_ASSERT_SLOW(std::filesystem::exists(FirstTargetFilePath)); - const uint32_t ExtraTargetPathIndex = Targets[TargetOffset].second; - const std::filesystem::path& ExtraTargetPath = RemoteContent.Paths[ExtraTargetPathIndex]; - const std::filesystem::path ExtraTargetFilePath = (Path / ExtraTargetPath).make_preferred(); - OutLocalFolderState.Paths[ExtraTargetPathIndex] = ExtraTargetPath; - OutLocalFolderState.RawSizes[ExtraTargetPathIndex] = RemoteContent.RawSizes[ExtraTargetPathIndex]; - CreateDirectories(ExtraTargetFilePath.parent_path()); - if (std::filesystem::exists(ExtraTargetFilePath)) - { - SetFileReadOnly(ExtraTargetFilePath, false); + RenameFileWithRetry(CacheFilePath, FirstTargetFilePath); + + RebuildFolderStateStats.FinalizeTreeFilesMovedCount++; + } } - CopyFile(FirstTargetFilePath, ExtraTargetFilePath, {.EnableClone = false}); - RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; - OutLocalFolderState.Attributes[ExtraTargetPathIndex] = + OutLocalFolderState.Paths[FirstRemotePathIndex] = FirstTargetPath; + OutLocalFolderState.RawSizes[FirstRemotePathIndex] = RemoteContent.RawSizes[FirstRemotePathIndex]; + + OutLocalFolderState.Attributes[FirstRemotePathIndex] = RemoteContent.Attributes.empty() - ? GetNativeFileAttributes(ExtraTargetFilePath) - : SetNativeFileAttributes(ExtraTargetFilePath, + ? GetNativeFileAttributes(FirstTargetFilePath) + : SetNativeFileAttributes(FirstTargetFilePath, RemoteContent.Platform, - RemoteContent.Attributes[ExtraTargetPathIndex]); - OutLocalFolderState.ModificationTicks[ExtraTargetPathIndex] = - GetModificationTickFromPath(ExtraTargetFilePath); + RemoteContent.Attributes[FirstRemotePathIndex]); + OutLocalFolderState.ModificationTicks[FirstRemotePathIndex] = + GetModificationTickFromPath(FirstTargetFilePath); TargetOffset++; TargetsComplete++; + + while (TargetOffset < (BaseTargetOffset + TargetCount)) + { + const uint32_t RemotePathIndex = Targets[TargetOffset].RemotePathIndex; + ZEN_ASSERT(Targets[TargetOffset].RawHash == RawHash); + const std::filesystem::path& TargetPath = RemoteContent.Paths[RemotePathIndex]; + std::filesystem::path TargetFilePath = (Path / TargetPath).make_preferred(); + + if (auto InPlaceIt = RemotePathIndexToLocalPathIndex.find(RemotePathIndex); + InPlaceIt != RemotePathIndexToLocalPathIndex.end()) + { + ZEN_ASSERT_SLOW(IsFile(TargetFilePath)); + } + else + { + ZEN_TRACE_CPU("Copy"); + if (IsFile(TargetFilePath)) + { + SetFileReadOnlyWithRetry(TargetFilePath, false); + } + else + { + CreateDirectories(TargetFilePath.parent_path()); + } + + ZEN_ASSERT_SLOW(IsFile(FirstTargetFilePath)); + ZEN_DEBUG("Copying from '{}' -> '{}'", FirstTargetFilePath, TargetFilePath); + CopyFile(FirstTargetFilePath, TargetFilePath, {.EnableClone = false}); + RebuildFolderStateStats.FinalizeTreeFilesCopiedCount++; + } + + OutLocalFolderState.Paths[RemotePathIndex] = TargetPath; + OutLocalFolderState.RawSizes[RemotePathIndex] = RemoteContent.RawSizes[RemotePathIndex]; + + OutLocalFolderState.Attributes[RemotePathIndex] = + RemoteContent.Attributes.empty() + ? GetNativeFileAttributes(TargetFilePath) + : SetNativeFileAttributes(TargetFilePath, + RemoteContent.Platform, + RemoteContent.Attributes[RemotePathIndex]); + OutLocalFolderState.ModificationTicks[RemotePathIndex] = GetModificationTickFromPath(TargetFilePath); + + TargetOffset++; + TargetsComplete++; + } } } }, @@ -6111,11 +7043,13 @@ namespace { Work.Wait(UsePlainProgress ? 5000 : 200, [&](bool IsAborted, std::ptrdiff_t PendingWork) { ZEN_UNUSED(IsAborted, PendingWork); - std::string Details = fmt::format("{}/{} files", TargetsComplete.load(), Targets.size()); + const uint64_t WorkTotal = Targets.size() + RemoveLocalPathIndexes.size(); + const uint64_t WorkComplete = TargetsComplete.load() + DeletedCount.load(); + std::string Details = fmt::format("{}/{} files", WorkComplete, WorkTotal); RebuildProgressBar.UpdateState({.Task = "Rebuilding state ", .Details = Details, - .TotalCount = gsl::narrow<uint64_t>(Targets.size()), - .RemainingCount = gsl::narrow<uint64_t>(Targets.size() - TargetsComplete.load())}, + .TotalCount = gsl::narrow<uint64_t>(WorkTotal), + .RemainingCount = gsl::narrow<uint64_t>(WorkTotal - WorkComplete)}, false); }); } @@ -6131,6 +7065,107 @@ namespace { } } + std::string GetCbObjectAsNiceString(CbObjectView Object, std::string_view Prefix, std::string_view Suffix) + { + ExtendableStringBuilder<512> SB; + std::vector<std::pair<std::string, std::string>> NameStringValuePairs; + for (CbFieldView Field : Object) + { + std::string_view Name = Field.GetName(); + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::String: + NameStringValuePairs.push_back({std::string(Name), std::string(Accessor.AsString())}); + break; + case CbFieldType::IntegerPositive: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerPositive())}); + break; + case CbFieldType::IntegerNegative: + NameStringValuePairs.push_back({std::string(Name), fmt::format("{}", Accessor.AsIntegerNegative())}); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.9g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + NameStringValuePairs.push_back({std::string(Name), fmt::format("{:.17g}", Value)}); + } + else + { + NameStringValuePairs.push_back({std::string(Name), "null"}); + } + } + break; + case CbFieldType::BoolFalse: + NameStringValuePairs.push_back({std::string(Name), "false"}); + break; + case CbFieldType::BoolTrue: + NameStringValuePairs.push_back({std::string(Name), "true"}); + break; + case CbFieldType::Hash: + { + NameStringValuePairs.push_back({std::string(Name), Accessor.AsHash().ToHexString()}); + } + break; + case CbFieldType::Uuid: + { + StringBuilder<Oid::StringLength + 1> Builder; + Accessor.AsUuid().ToString(Builder); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::DateTime: + { + ExtendableStringBuilder<64> Builder; + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + } + break; + case CbFieldType::TimeSpan: + { + ExtendableStringBuilder<64> Builder; + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + NameStringValuePairs.push_back({std::string(Name), Builder.ToString()}); + break; + } + case CbFieldType::ObjectId: + NameStringValuePairs.push_back({std::string(Name), Accessor.AsObjectId().ToString()}); + break; + } + } + std::string::size_type LongestKey = 0; + for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) + { + LongestKey = Max(KeyValue.first.length(), LongestKey); + } + for (const std::pair<std::string, std::string>& KeyValue : NameStringValuePairs) + { + SB.Append(fmt::format("{}{:<{}}: {}{}", Prefix, KeyValue.first, LongestKey, KeyValue.second, Suffix)); + } + return SB.ToString(); + } + std::vector<std::pair<Oid, std::string>> ResolveBuildPartNames(BuildStorage& Storage, const Oid& BuildId, const std::vector<Oid>& BuildPartIds, @@ -6140,17 +7175,13 @@ namespace { std::vector<std::pair<Oid, std::string>> Result; { Stopwatch GetBuildTimer; - - std::vector<std::pair<Oid, std::string>> AvailableParts; - - CbObject BuildObject = Storage.GetBuild(BuildId); - + CbObject BuildObject = Storage.GetBuild(BuildId); ZEN_CONSOLE("GetBuild took {}. Name: '{}', Payload size: {}", NiceTimeSpanMs(GetBuildTimer.GetElapsedTimeMs()), - BuildObject["BuildName"sv].AsString(), + BuildObject["name"sv].AsString(), NiceBytes(BuildObject.GetSize())); - ZEN_DEBUG("Build object: {}", BuildObject); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildObject, " "sv, "\n"sv)); CbObjectView PartsObject = BuildObject["parts"sv].AsObjectView(); if (!PartsObject) @@ -6160,6 +7191,8 @@ namespace { OutPreferredMultipartChunkSize = BuildObject["chunkSize"sv].AsUInt64(OutPreferredMultipartChunkSize); + std::vector<std::pair<Oid, std::string>> AvailableParts; + for (CbFieldView PartView : PartsObject) { const std::string BuildPartName = std::string(PartView.GetName()); @@ -6221,7 +7254,7 @@ namespace { return Result; } - ChunkedFolderContent GetRemoteContent(BuildStorage& Storage, + ChunkedFolderContent GetRemoteContent(StorageInstance& Storage, const Oid& BuildId, const std::vector<std::pair<Oid, std::string>>& BuildParts, std::unique_ptr<ChunkingController>& OutChunkController, @@ -6234,12 +7267,13 @@ namespace { Stopwatch GetBuildPartTimer; const Oid BuildPartId = BuildParts[0].first; const std::string_view BuildPartName = BuildParts[0].second; - CbObject BuildPartManifest = Storage.GetBuildPart(BuildId, BuildPartId); + CbObject BuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, BuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", BuildPartId, BuildPartName, NiceTimeSpanMs(GetBuildPartTimer.GetElapsedTimeMs()), NiceBytes(BuildPartManifest.GetSize())); + ZEN_CONSOLE("{}", GetCbObjectAsNiceString(BuildPartManifest, " "sv, "\n"sv)); { CbObjectView Chunker = BuildPartManifest["chunker"sv].AsObjectView(); @@ -6248,7 +7282,7 @@ namespace { OutChunkController = CreateChunkingController(ChunkerName, Parameters); } - auto ParseBuildPartManifest = [](BuildStorage& Storage, + auto ParseBuildPartManifest = [](StorageInstance& Storage, const Oid& BuildId, const Oid& BuildPartId, CbObject BuildPartManifest, @@ -6274,12 +7308,103 @@ namespace { // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them - Stopwatch GetBlockMetadataTimer; - OutBlockDescriptions = Storage.GetBlockMetadata(BuildId, BlockRawHashes); - ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - OutBlockDescriptions.size()); + { + Stopwatch GetBlockMetadataTimer; + + std::vector<ChunkBlockDescription> UnorderedList; + tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup; + if (Storage.BuildCacheStorage) + { + std::vector<CbObject> CacheBlockMetadatas = Storage.BuildCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes); + UnorderedList.reserve(CacheBlockMetadatas.size()); + for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size(); + CacheBlockMetadataIndex++) + { + const CbObject& CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex]; + ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote cache, skipping block"); + } + else + { + UnorderedList.emplace_back(std::move(Description)); + } + } + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); + } + } + + if (UnorderedList.size() < BlockRawHashes.size()) + { + std::vector<IoHash> RemainingBlockHashes; + RemainingBlockHashes.reserve(BlockRawHashes.size() - UnorderedList.size()); + for (const IoHash& BlockRawHash : BlockRawHashes) + { + if (!BlockDescriptionLookup.contains(BlockRawHash)) + { + RemainingBlockHashes.push_back(BlockRawHash); + } + } + CbObject BlockMetadatas = Storage.BuildStorage->GetBlockMetadatas(BuildId, RemainingBlockHashes); + std::vector<ChunkBlockDescription> RemainingList; + { + CbArrayView BlocksArray = BlockMetadatas["blocks"sv].AsArrayView(); + std::vector<IoHash> FoundBlockHashes; + std::vector<CbObject> FoundBlockMetadatas; + for (CbFieldView Block : BlocksArray) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); + + if (Description.BlockHash == IoHash::Zero) + { + ZEN_WARN("Unexpected/invalid block metadata received from remote store, skipping block"); + } + else + { + if (Storage.BuildCacheStorage) + { + UniqueBuffer MetaBuffer = UniqueBuffer::Alloc(Block.GetSize()); + Block.CopyTo(MetaBuffer.GetMutableView()); + CbObject BlockMetadata(MetaBuffer.MoveToShared()); + + FoundBlockHashes.push_back(Description.BlockHash); + FoundBlockMetadatas.push_back(BlockMetadata); + } + RemainingList.emplace_back(std::move(Description)); + } + } + if (Storage.BuildCacheStorage && !FoundBlockHashes.empty()) + { + Storage.BuildCacheStorage->PutBlobMetadatas(BuildId, FoundBlockHashes, FoundBlockMetadatas); + } + } + + for (size_t DescriptionIndex = 0; DescriptionIndex < RemainingList.size(); DescriptionIndex++) + { + const ChunkBlockDescription& Description = RemainingList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, UnorderedList.size() + DescriptionIndex); + } + UnorderedList.insert(UnorderedList.end(), RemainingList.begin(), RemainingList.end()); + } + + OutBlockDescriptions.reserve(BlockDescriptionLookup.size()); + for (const IoHash& BlockHash : BlockRawHashes) + { + if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) + { + OutBlockDescriptions.push_back(std::move(UnorderedList[It->second])); + } + } + + ZEN_CONSOLE("GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } if (OutBlockDescriptions.size() != BlockRawHashes.size()) { @@ -6292,7 +7417,8 @@ namespace { ZEN_CONSOLE("{} Attemping fallback options.", ErrorDescription); std::vector<ChunkBlockDescription> AugmentedBlockDescriptions; AugmentedBlockDescriptions.reserve(BlockRawHashes.size()); - std::vector<ChunkBlockDescription> FoundBlocks = Storage.FindBlocks(BuildId); + std::vector<ChunkBlockDescription> FoundBlocks = + ParseChunkBlockDescriptionList(Storage.BuildStorage->FindBlocks(BuildId, (uint64_t)-1)); for (const IoHash& BlockHash : BlockRawHashes) { @@ -6315,7 +7441,7 @@ namespace { } else { - IoBuffer BlockBuffer = Storage.GetBuildBlob(BuildId, BlockHash); + IoBuffer BlockBuffer = Storage.BuildStorage->GetBuildBlob(BuildId, BlockHash); if (!BlockBuffer) { throw std::runtime_error(fmt::format("Block {} could not be found", BlockHash)); @@ -6381,7 +7507,7 @@ namespace { const Oid& OverlayBuildPartId = BuildParts[PartIndex].first; const std::string& OverlayBuildPartName = BuildParts[PartIndex].second; Stopwatch GetOverlayBuildPartTimer; - CbObject OverlayBuildPartManifest = Storage.GetBuildPart(BuildId, OverlayBuildPartId); + CbObject OverlayBuildPartManifest = Storage.BuildStorage->GetBuildPart(BuildId, OverlayBuildPartId); ZEN_CONSOLE("GetBuildPart {} ('{}') took {}. Payload size: {}", OverlayBuildPartId, OverlayBuildPartName, @@ -6447,64 +7573,21 @@ namespace { ChunkedFolderContent GetLocalContent(GetFolderContentStatistics& LocalFolderScanStats, ChunkingStatistics& ChunkingStats, const std::filesystem::path& Path, - ChunkingController& ChunkController) + const std::filesystem::path& ZenFolderPath, + ChunkingController& ChunkController, + const ChunkedFolderContent& ReferenceContent, + FolderContent& OutLocalFolderContent) { + FolderContent LocalFolderState; ChunkedFolderContent LocalContent; - auto IsAcceptedFolder = [ExcludeFolders = DefaultExcludeFolders](const std::string_view& RelativePath) -> bool { - for (const std::string_view& ExcludeFolder : ExcludeFolders) - { - if (RelativePath.starts_with(ExcludeFolder)) - { - if (RelativePath.length() == ExcludeFolder.length()) - { - return false; - } - else if (RelativePath[ExcludeFolder.length()] == '/') - { - return false; - } - } - } - return true; - }; - - auto IsAcceptedFile = [ExcludeExtensions = - DefaultExcludeExtensions](const std::string_view& RelativePath, uint64_t, uint32_t) -> bool { - for (const std::string_view& ExcludeExtension : ExcludeExtensions) - { - if (RelativePath.ends_with(ExcludeExtension)) - { - return false; - } - } - return true; - }; - - FolderContent CurrentLocalFolderContent = GetFolderContent( - LocalFolderScanStats, - Path, - std::move(IsAcceptedFolder), - std::move(IsAcceptedFile), - GetMediumWorkerPool(EWorkloadType::Burst), - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { ZEN_DEBUG("Found {} files in '{}'...", LocalFolderScanStats.AcceptedFileCount.load(), Path); }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - - FolderContent LocalFolderState; - - bool ScanContent = true; - std::vector<uint32_t> PathIndexesOufOfDate; - if (std::filesystem::is_regular_file(Path / ZenStateFilePath)) + bool HasLocalState = false; + if (IsFile(ZenStateFilePath(ZenFolderPath))) { try { Stopwatch ReadStateTimer; - CbObject CurrentStateObject = LoadCompactBinaryObject(Path / ZenStateFilePath).Object; + CbObject CurrentStateObject = LoadCompactBinaryObject(ZenStateFilePath(ZenFolderPath)).Object; if (CurrentStateObject) { Oid CurrentBuildId; @@ -6530,124 +7613,240 @@ namespace { MergeChunkedFolderContents(SavedPartContents[0], std::span<const ChunkedFolderContent>(SavedPartContents).subspan(1)); } + HasLocalState = true; + } + } + } + ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); + } + catch (const std::exception& Ex) + { + ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + } + } - if (!LocalFolderState.AreKnownFilesEqual(CurrentLocalFolderContent)) - { - const size_t LocaStatePathCount = LocalFolderState.Paths.size(); - std::vector<std::filesystem::path> DeletedPaths; - FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, CurrentLocalFolderContent, DeletedPaths); - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } + { + const uint32_t LocalPathCount = gsl::narrow<uint32_t>(ReferenceContent.Paths.size()); + const uint32_t RemotePathCount = gsl::narrow<uint32_t>(LocalFolderState.Paths.size()); - ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", - DeletedPaths.size(), - UpdatedContent.Paths.size(), - LocaStatePathCount); - if (UpdatedContent.Paths.size() > 0) - { - uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : UpdatedContent.RawSizes) - { - ByteCountToScan += RawSize; - } - ProgressBar ProgressBar(false); - FilteredRate FilteredBytesHashed; - FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( - ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), - Path, - UpdatedContent, - ChunkController, - UsePlainProgress ? 5000 : 200, - [&](bool, std::ptrdiff_t) { - FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); - std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", - ChunkingStats.FilesProcessed.load(), - UpdatedContent.Paths.size(), - NiceBytes(ChunkingStats.BytesHashed.load()), - NiceBytes(ByteCountToScan), - NiceNum(FilteredBytesHashed.GetCurrent()), - ChunkingStats.UniqueChunksFound.load(), - NiceBytes(ChunkingStats.UniqueBytesFound.load())); - ProgressBar.UpdateState({.Task = "Scanning files ", - .Details = Details, - .TotalCount = ByteCountToScan, - .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, - false); - }, - AbortFlag); - if (AbortFlag) - { - return {}; - } - FilteredBytesHashed.Stop(); - ProgressBar.Finish(); - LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); - } - } - else + std::vector<std::filesystem::path> PathsToCheck; + PathsToCheck.reserve(LocalPathCount + RemotePathCount); + + tsl::robin_set<std::string> FileSet; + FileSet.reserve(LocalPathCount + RemotePathCount); + + for (const std::filesystem::path& LocalPath : LocalFolderState.Paths) + { + FileSet.insert(LocalPath.generic_string()); + PathsToCheck.push_back(LocalPath); + } + + for (const std::filesystem::path& RemotePath : ReferenceContent.Paths) + { + if (FileSet.insert(RemotePath.generic_string()).second) + { + PathsToCheck.push_back(RemotePath); + } + } + + const uint32_t PathCount = gsl::narrow<uint32_t>(PathsToCheck.size()); + + OutLocalFolderContent.Paths.resize(PathCount); + OutLocalFolderContent.RawSizes.resize(PathCount); + OutLocalFolderContent.Attributes.resize(PathCount); + OutLocalFolderContent.ModificationTicks.resize(PathCount); + + { + Stopwatch Timer; + auto _ = + MakeGuard([&LocalFolderScanStats, &Timer]() { LocalFolderScanStats.ElapsedWallTimeUS = Timer.GetElapsedTimeUs(); }); + + ProgressBar ProgressBar(UsePlainProgress); + + ParallellWork Work(AbortFlag); + std::atomic<uint64_t> CompletedPathCount = 0; + uint32_t PathIndex = 0; + + while (PathIndex < PathCount) + { + uint32_t PathRangeCount = Min(128u, PathCount - PathIndex); + Work.ScheduleWork( + GetIOWorkerPool(), + [PathIndex, + PathRangeCount, + &PathsToCheck, + &Path, + &OutLocalFolderContent, + &CompletedPathCount, + &LocalFolderScanStats](std::atomic<bool>&) { + for (uint32_t PathRangeIndex = PathIndex; PathRangeIndex < PathIndex + PathRangeCount; PathRangeIndex++) { - // Remove files from LocalContent no longer in LocalFolderState - tsl::robin_set<std::string> LocalFolderPaths; - LocalFolderPaths.reserve(LocalFolderState.Paths.size()); - for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) - { - LocalFolderPaths.insert(LocalFolderPath.generic_string()); - } - std::vector<std::filesystem::path> DeletedPaths; - for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) + const std::filesystem::path& FilePath = PathsToCheck[PathRangeIndex]; + std::filesystem::path LocalFilePath = (Path / FilePath).make_preferred(); + if (TryGetFileProperties(LocalFilePath, + OutLocalFolderContent.RawSizes[PathRangeIndex], + OutLocalFolderContent.ModificationTicks[PathRangeIndex], + OutLocalFolderContent.Attributes[PathRangeIndex])) { - if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) - { - DeletedPaths.push_back(LocalContentPath); - } + OutLocalFolderContent.Paths[PathRangeIndex] = std::move(FilePath); + LocalFolderScanStats.FoundFileCount++; + LocalFolderScanStats.FoundFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; + LocalFolderScanStats.AcceptedFileCount++; + LocalFolderScanStats.AcceptedFileByteCount += OutLocalFolderContent.RawSizes[PathRangeIndex]; } - if (!DeletedPaths.empty()) - { - LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); - } - - ZEN_CONSOLE("Using cached local state"); + CompletedPathCount++; } - ZEN_CONSOLE("Read local state in {}", NiceTimeSpanMs(ReadStateTimer.GetElapsedTimeMs())); - ScanContent = false; - } + }, + Work.DefaultErrorFunction()); + PathIndex += PathRangeCount; + } + Work.Wait(200, [&](bool, ptrdiff_t) { + // FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} checked, {} found", + CompletedPathCount.load(), + PathCount, + LocalFolderScanStats.FoundFileCount.load()); + ProgressBar.UpdateState({.Task = "Checking files ", + .Details = Details, + .TotalCount = PathCount, + .RemainingCount = PathCount - CompletedPathCount.load()}, + false); + }); + ProgressBar.Finish(); + } + + uint32_t WritePathIndex = 0; + for (uint32_t ReadPathIndex = 0; ReadPathIndex < PathCount; ReadPathIndex++) + { + if (!OutLocalFolderContent.Paths[ReadPathIndex].empty()) + { + if (WritePathIndex < ReadPathIndex) + { + OutLocalFolderContent.Paths[WritePathIndex] = std::move(OutLocalFolderContent.Paths[ReadPathIndex]); + OutLocalFolderContent.RawSizes[WritePathIndex] = OutLocalFolderContent.RawSizes[ReadPathIndex]; + OutLocalFolderContent.Attributes[WritePathIndex] = OutLocalFolderContent.Attributes[ReadPathIndex]; + OutLocalFolderContent.ModificationTicks[WritePathIndex] = OutLocalFolderContent.ModificationTicks[ReadPathIndex]; } + WritePathIndex++; } } - catch (const std::exception& Ex) + + OutLocalFolderContent.Paths.resize(WritePathIndex); + OutLocalFolderContent.RawSizes.resize(WritePathIndex); + OutLocalFolderContent.Attributes.resize(WritePathIndex); + OutLocalFolderContent.ModificationTicks.resize(WritePathIndex); + } + + bool ScanContent = true; + std::vector<uint32_t> PathIndexesOufOfDate; + if (HasLocalState) + { + if (!LocalFolderState.AreKnownFilesEqual(OutLocalFolderContent)) { - ZEN_CONSOLE("Failed reading state file, falling back to scannning. Reason: {}", Ex.what()); + const size_t LocaStatePathCount = LocalFolderState.Paths.size(); + std::vector<std::filesystem::path> DeletedPaths; + FolderContent UpdatedContent = GetUpdatedContent(LocalFolderState, OutLocalFolderContent, DeletedPaths); + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } + + ZEN_CONSOLE("Updating state, {} local files deleted and {} local files updated out of {}", + DeletedPaths.size(), + UpdatedContent.Paths.size(), + LocaStatePathCount); + if (UpdatedContent.Paths.size() > 0) + { + uint64_t ByteCountToScan = 0; + for (const uint64_t RawSize : UpdatedContent.RawSizes) + { + ByteCountToScan += RawSize; + } + ProgressBar ProgressBar(false); + FilteredRate FilteredBytesHashed; + FilteredBytesHashed.Start(); + ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + ChunkingStats, + GetIOWorkerPool(), + Path, + UpdatedContent, + ChunkController, + UsePlainProgress ? 5000 : 200, + [&](bool, std::ptrdiff_t) { + FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); + std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", + ChunkingStats.FilesProcessed.load(), + UpdatedContent.Paths.size(), + NiceBytes(ChunkingStats.BytesHashed.load()), + NiceBytes(ByteCountToScan), + NiceNum(FilteredBytesHashed.GetCurrent()), + ChunkingStats.UniqueChunksFound.load(), + NiceBytes(ChunkingStats.UniqueBytesFound.load())); + ProgressBar.UpdateState({.Task = "Scanning files ", + .Details = Details, + .TotalCount = ByteCountToScan, + .RemainingCount = ByteCountToScan - ChunkingStats.BytesHashed.load()}, + false); + }, + AbortFlag); + if (AbortFlag) + { + return {}; + } + FilteredBytesHashed.Stop(); + ProgressBar.Finish(); + LocalContent = MergeChunkedFolderContents(LocalContent, {{UpdatedLocalContent}}); + } + } + else + { + // Remove files from LocalContent no longer in LocalFolderState + tsl::robin_set<std::string> LocalFolderPaths; + LocalFolderPaths.reserve(LocalFolderState.Paths.size()); + for (const std::filesystem::path& LocalFolderPath : LocalFolderState.Paths) + { + LocalFolderPaths.insert(LocalFolderPath.generic_string()); + } + std::vector<std::filesystem::path> DeletedPaths; + for (const std::filesystem::path& LocalContentPath : LocalContent.Paths) + { + if (!LocalFolderPaths.contains(LocalContentPath.generic_string())) + { + DeletedPaths.push_back(LocalContentPath); + } + } + if (!DeletedPaths.empty()) + { + LocalContent = DeletePathsFromChunkedContent(LocalContent, DeletedPaths); + } } + ScanContent = false; } if (ScanContent) { uint64_t ByteCountToScan = 0; - for (const uint64_t RawSize : CurrentLocalFolderContent.RawSizes) + for (const uint64_t RawSize : OutLocalFolderContent.RawSizes) { ByteCountToScan += RawSize; } ProgressBar ProgressBar(false); FilteredRate FilteredBytesHashed; FilteredBytesHashed.Start(); - ChunkedFolderContent UpdatedLocalContent = ChunkFolderContent( + LocalContent = ChunkFolderContent( ChunkingStats, - GetMediumWorkerPool(EWorkloadType::Burst), + GetIOWorkerPool(), Path, - CurrentLocalFolderContent, + OutLocalFolderContent, ChunkController, UsePlainProgress ? 5000 : 200, [&](bool, std::ptrdiff_t) { FilteredBytesHashed.Update(ChunkingStats.BytesHashed.load()); std::string Details = fmt::format("{}/{} ({}/{}, {}B/s) scanned, {} ({}) chunks found", ChunkingStats.FilesProcessed.load(), - CurrentLocalFolderContent.Paths.size(), + OutLocalFolderContent.Paths.size(), NiceBytes(ChunkingStats.BytesHashed.load()), - ByteCountToScan, + NiceBytes(ByteCountToScan), NiceNum(FilteredBytesHashed.GetCurrent()), ChunkingStats.UniqueChunksFound.load(), NiceBytes(ChunkingStats.UniqueBytesFound.load())); @@ -6670,31 +7869,35 @@ namespace { return LocalContent; } - void DownloadFolder(BuildStorage& Storage, + void DownloadFolder(StorageInstance& Storage, const Oid& BuildId, const std::vector<Oid>& BuildPartIds, std::span<const std::string> BuildPartNames, const std::filesystem::path& Path, + const std::filesystem::path& ZenFolderPath, bool AllowMultiparts, bool AllowPartialBlockRequests, bool WipeTargetFolder, - bool PostDownloadVerify) + bool PostDownloadVerify, + bool PrimeCacheOnly) { ZEN_TRACE_CPU("DownloadFolder"); + ZEN_ASSERT((!PrimeCacheOnly) || (PrimeCacheOnly && (!AllowPartialBlockRequests))); + Stopwatch DownloadTimer; - const std::filesystem::path ZenTempFolder = Path / ZenTempFolderName; + const std::filesystem::path ZenTempFolder = ZenTempFolderPath(ZenFolderPath); CreateDirectories(ZenTempFolder); - CreateDirectories(Path / ZenTempBlockFolderName); - CreateDirectories(Path / ZenTempCacheFolderName); - CreateDirectories(Path / ZenTempDownloadFolderName); + CreateDirectories(ZenTempBlockFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempCacheFolderPath(ZenFolderPath)); + CreateDirectories(ZenTempDownloadFolderPath(ZenFolderPath)); std::uint64_t PreferredMultipartChunkSize = 32u * 1024u * 1024u; std::vector<std::pair<Oid, std::string>> AllBuildParts = - ResolveBuildPartNames(Storage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); + ResolveBuildPartNames(*Storage.BuildStorage, BuildId, BuildPartIds, BuildPartNames, PreferredMultipartChunkSize); std::vector<ChunkedFolderContent> PartContents; @@ -6706,26 +7909,36 @@ namespace { ChunkedFolderContent RemoteContent = GetRemoteContent(Storage, BuildId, AllBuildParts, ChunkController, PartContents, BlockDescriptions, LooseChunkHashes); - const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; - if (!ChunkController) - { - ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); - ChunkController = CreateBasicChunkingController(); - } - + const std::uint64_t LargeAttachmentSize = AllowMultiparts ? PreferredMultipartChunkSize * 4u : (std::uint64_t)-1; GetFolderContentStatistics LocalFolderScanStats; ChunkingStatistics ChunkingStats; ChunkedFolderContent LocalContent; - if (std::filesystem::is_directory(Path)) + FolderContent LocalFolderContent; + if (!PrimeCacheOnly) { - if (!WipeTargetFolder) + if (IsDir(Path)) { - LocalContent = GetLocalContent(LocalFolderScanStats, ChunkingStats, Path, *ChunkController); + if (!WipeTargetFolder) + { + if (!ChunkController) + { + ZEN_CONSOLE("Warning: Unspecified chunking algorith, using default"); + ChunkController = CreateBasicChunkingController(); + } + + LocalContent = GetLocalContent(LocalFolderScanStats, + ChunkingStats, + Path, + ZenFolderPath, + *ChunkController, + RemoteContent, + LocalFolderContent); + } + } + else + { + CreateDirectories(Path); } - } - else - { - CreateDirectories(Path); } if (AbortFlag) { @@ -6778,6 +7991,12 @@ namespace { { ZEN_CONSOLE("Local state is identical to build to download. All done. Completed in {}.", NiceTimeSpanMs(DownloadTimer.GetElapsedTimeMs())); + + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderContent); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); } else { @@ -6786,7 +8005,7 @@ namespace { { BuildPartString.Append(fmt::format(" {} ({})", BuildPart.second, BuildPart.first)); } - ZEN_CONSOLE("Downloading build {}, parts:{}", BuildId, BuildPartString.ToView()); + ZEN_CONSOLE("Downloading build {}, parts:{} to '{}'", BuildId, BuildPartString.ToView(), Path); FolderContent LocalFolderState; DiskStatistics DiskStats; @@ -6799,6 +8018,7 @@ namespace { UpdateFolder(Storage, BuildId, Path, + ZenFolderPath, LargeAttachmentSize, PreferredMultipartChunkSize, LocalContent, @@ -6807,6 +8027,7 @@ namespace { LooseChunkHashes, AllowPartialBlockRequests, WipeTargetFolder, + PrimeCacheOnly, LocalFolderState, DiskStats, CacheMappingStats, @@ -6816,20 +8037,23 @@ namespace { if (!AbortFlag) { - VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); + if (!PrimeCacheOnly) + { + VerifyFolder(RemoteContent, Path, PostDownloadVerify, VerifyFolderStats); - Stopwatch WriteStateTimer; - CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); + Stopwatch WriteStateTimer; + CbObject StateObject = CreateStateObject(BuildId, AllBuildParts, PartContents, LocalFolderState); - CreateDirectories((Path / ZenStateFilePath).parent_path()); - TemporaryFile::SafeWriteFile(Path / ZenStateFilePath, StateObject.GetView()); - ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); + CreateDirectories(ZenStateFilePath(ZenFolderPath).parent_path()); + TemporaryFile::SafeWriteFile(ZenStateFilePath(ZenFolderPath), StateObject.GetView()); + ZEN_CONSOLE("Wrote local state in {}", NiceTimeSpanMs(WriteStateTimer.GetElapsedTimeMs())); #if 0 - ExtendableStringBuilder<1024> SB; - CompactBinaryToJson(StateObject, SB); - WriteFile(Path / ZenStateFileJsonPath, IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); + ExtendableStringBuilder<1024> SB; + CompactBinaryToJson(StateObject, SB); + WriteFile(ZenStateFileJsonPath(ZenFolderPath), IoBuffer(IoBuffer::Wrap, SB.Data(), SB.Size())); #endif // 0 + } const uint64_t DownloadCount = DownloadStats.DownloadedChunkCount.load() + DownloadStats.DownloadedBlockCount.load() + DownloadStats.DownloadedPartialBlockCount.load(); const uint64_t DownloadByteCount = DownloadStats.DownloadedChunkByteCount.load() + @@ -6863,9 +8087,26 @@ namespace { NiceTimeSpanMs(VerifyFolderStats.VerifyElapsedWallTimeUs / 1000)); } } + if (PrimeCacheOnly) + { + if (Storage.BuildCacheStorage) + { + Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + if (Remaining == 0) + { + ZEN_CONSOLE("Build cache upload complete"); + } + else + { + ZEN_CONSOLE("Waiting for build cache to complete uploading. {} blobs remaining", Remaining); + } + return !AbortFlag; + }); + } + } if (CleanDirectory(ZenTempFolder, {})) { - std::filesystem::remove(ZenTempFolder); + RemoveDirWithRetry(ZenTempFolder); } } @@ -7109,7 +8350,14 @@ BuildsCommand::BuildsCommand() auto AddCloudOptions = [this, &AddAuthOptions](cxxopts::Options& Ops) { AddAuthOptions(Ops); - Ops.add_option("cloud build", "", "url", "Cloud Builds URL", cxxopts::value(m_BuildsUrl), "<url>"); + Ops.add_option("cloud build", "", "override-host", "Cloud Builds URL", cxxopts::value(m_OverrideHost), "<override-host>"); + Ops.add_option("cloud build", + "", + "url", + "Cloud Builds host url (legacy - use --override-host)", + cxxopts::value(m_OverrideHost), + "<url>"); + Ops.add_option("cloud build", "", "host", "Cloud Builds host", cxxopts::value(m_Host), "<host>"); Ops.add_option("cloud build", "", "assume-http2", @@ -7131,11 +8379,32 @@ BuildsCommand::BuildsCommand() "<jsonmetadata>"); }; + auto AddCacheOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("cache", "", "zen-cache-host", "Host ip and port for zen builds cache", cxxopts::value(m_ZenCacheHost), "<zenhost>"); + }; + auto AddOutputOptions = [this](cxxopts::Options& Ops) { Ops.add_option("output", "", "plain-progress", "Show progress using plain output", cxxopts::value(m_PlainProgress), "<progress>"); Ops.add_option("output", "", "verbose", "Enable verbose console output", cxxopts::value(m_Verbose), "<verbose>"); }; + auto AddWorkerOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "boost-workers", + "Increase the number of worker threads - may cause computer to less responsive", + cxxopts::value(m_BoostWorkerThreads), + "<boostworkers>"); + }; + + auto AddZenFolderOptions = [this](cxxopts::Options& Ops) { + Ops.add_option("", + "", + "zen-folder-path", + fmt::format("Path to zen state and temp folders. Defaults to [--local-path/]{}", ZenFolderName), + cxxopts::value(m_ZenFolderPath), + "<boostworkers>"); + }; m_Options.add_option("", "v", "verb", @@ -7149,6 +8418,7 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ListOptions); AddFileOptions(m_ListOptions); AddOutputOptions(m_ListOptions); + AddZenFolderOptions(m_ListOptions); m_ListOptions.add_options()("h,help", "Print help"); m_ListOptions.add_option("", "", @@ -7169,6 +8439,9 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_UploadOptions); AddFileOptions(m_UploadOptions); AddOutputOptions(m_UploadOptions); + AddCacheOptions(m_UploadOptions); + AddWorkerOptions(m_UploadOptions); + AddZenFolderOptions(m_UploadOptions); m_UploadOptions.add_options()("h,help", "Print help"); m_UploadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); m_UploadOptions.add_option("", @@ -7224,6 +8497,12 @@ BuildsCommand::BuildsCommand() "<manifestpath>"); m_UploadOptions .add_option("", "", "verify", "Enable post upload verify of all uploaded data", cxxopts::value(m_PostUploadVerify), "<verify>"); + m_UploadOptions.add_option("", + "", + "find-max-block-count", + "The maximum number of blocks we search for in the build context", + cxxopts::value(m_FindBlockMaxCount), + "<maxblockcount>"); m_UploadOptions.parse_positional({"local-path", "build-id"}); m_UploadOptions.positional_help("local-path build-id"); @@ -7232,6 +8511,16 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_DownloadOptions); AddFileOptions(m_DownloadOptions); AddOutputOptions(m_DownloadOptions); + AddCacheOptions(m_DownloadOptions); + AddZenFolderOptions(m_DownloadOptions); + AddWorkerOptions(m_DownloadOptions); + m_DownloadOptions.add_option("cache", + "", + "cache-prime-only", + "Only download blobs missing in cache and upload to cache", + cxxopts::value(m_PrimeCacheOnly), + "<cacheprimeonly>"); + m_DownloadOptions.add_options()("h,help", "Print help"); m_DownloadOptions.add_option("", "l", "local-path", "Root file system folder for build", cxxopts::value(m_Path), "<local-path>"); m_DownloadOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); @@ -7263,12 +8552,14 @@ BuildsCommand::BuildsCommand() "Allow request for partial chunk blocks. Defaults to true.", cxxopts::value(m_AllowPartialBlockRequests), "<allowpartialblockrequests>"); + m_DownloadOptions .add_option("", "", "verify", "Enable post download verify of all tracked files", cxxopts::value(m_PostDownloadVerify), "<verify>"); m_DownloadOptions.parse_positional({"local-path", "build-id", "build-part-name"}); m_DownloadOptions.positional_help("local-path build-id build-part-name"); AddOutputOptions(m_DiffOptions); + AddWorkerOptions(m_DiffOptions); m_DiffOptions.add_options()("h,help", "Print help"); m_DiffOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); m_DiffOptions.add_option("", "c", "compare-path", "Root file system folder used as diff", cxxopts::value(m_DiffPath), "<diff-path>"); @@ -7284,6 +8575,8 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_TestOptions); AddFileOptions(m_TestOptions); AddOutputOptions(m_TestOptions); + AddCacheOptions(m_TestOptions); + AddWorkerOptions(m_TestOptions); m_TestOptions.add_options()("h,help", "Print help"); m_TestOptions.add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); m_TestOptions.add_option("", @@ -7304,6 +8597,8 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_FetchBlobOptions); AddFileOptions(m_FetchBlobOptions); AddOutputOptions(m_FetchBlobOptions); + AddCacheOptions(m_FetchBlobOptions); + AddZenFolderOptions(m_FetchBlobOptions); m_FetchBlobOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); m_FetchBlobOptions .add_option("", "", "blob-hash", "IoHash in hex form identifying the blob to download", cxxopts::value(m_BlobHash), "<blob-hash>"); @@ -7313,6 +8608,8 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_ValidateBuildPartOptions); AddFileOptions(m_ValidateBuildPartOptions); AddOutputOptions(m_ValidateBuildPartOptions); + AddWorkerOptions(m_ValidateBuildPartOptions); + AddZenFolderOptions(m_ValidateBuildPartOptions); m_ValidateBuildPartOptions.add_option("", "", "build-id", "Build Id", cxxopts::value(m_BuildId), "<id>"); m_ValidateBuildPartOptions.add_option("", "", @@ -7333,6 +8630,8 @@ BuildsCommand::BuildsCommand() AddCloudOptions(m_MultiTestDownloadOptions); AddFileOptions(m_MultiTestDownloadOptions); AddOutputOptions(m_MultiTestDownloadOptions); + AddCacheOptions(m_MultiTestDownloadOptions); + AddWorkerOptions(m_MultiTestDownloadOptions); m_MultiTestDownloadOptions .add_option("", "l", "local-path", "Root file system folder used as base", cxxopts::value(m_Path), "<local-path>"); m_MultiTestDownloadOptions.add_option("", "", "build-ids", "Build Ids list separated by ','", cxxopts::value(m_BuildIds), "<ids>"); @@ -7373,7 +8672,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } auto ParseStorageOptions = [&]() { - if (!m_BuildsUrl.empty()) + if (!m_OverrideHost.empty() || !m_Host.empty()) { if (!m_StoragePath.empty()) { @@ -7385,16 +8684,23 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) fmt::format("namespace and bucket options are required for url option\n{}", m_Options.help())); } } + else if (m_StoragePath.empty()) + { + throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); + } }; std::unique_ptr<AuthMgr> Auth; - HttpClientSettings ClientSettings{.AssumeHttp2 = m_AssumeHttp2, .AllowResume = true, .RetryCount = 2}; + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; auto CreateAuthMgr = [&]() { if (!Auth) { - std::filesystem::path DataRoot = m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : StringToPath(m_SystemRootDir); - + std::filesystem::path DataRoot = + m_SystemRootDir.empty() ? PickDefaultSystemRootDirectory() : MakeSafeAbsolutePath(m_SystemRootDir); if (m_EncryptionKey.empty()) { m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; @@ -7448,7 +8754,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else if (!m_AccessTokenPath.empty()) { - std::string ResolvedAccessToken = ReadAccessTokenFromFile(StringToPath(m_AccessTokenPath)); + std::string ResolvedAccessToken = ReadAccessTokenFromFile(MakeSafeAbsolutePath(m_AccessTokenPath)); if (!ResolvedAccessToken.empty()) { ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); @@ -7474,7 +8780,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); } - if (!m_BuildsUrl.empty() && !ClientSettings.AccessTokenProvider) + if (!ClientSettings.AccessTokenProvider) { ZEN_CONSOLE("Warning: No auth provider given, attempting operation without credentials."); } @@ -7486,15 +8792,243 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) }; ParseOutputOptions(); - try - { - if (SubOption == &m_ListOptions) + auto CreateBuildStorage = [&](BuildStorage::Statistics& StorageStats, + BuildStorageCache::Statistics& StorageCacheStats, + const std::filesystem::path& TempPath) -> StorageInstance { + ParseStorageOptions(); + + StorageInstance Result; + + std::string BuildStorageName = ZEN_CLOUD_STORAGE; + std::string BuildCacheName; + bool CacheAssumeHttp2 = false; + std::string StorageDescription; + std::string CacheDescription; + + if (!m_Host.empty() || !m_OverrideHost.empty()) { - ParseStorageOptions(); ParseAuthOptions(); + } + + std::string CloudHost; + + if (!m_Host.empty()) + { + if (m_OverrideHost.empty() || m_ZenCacheHost.empty()) + { + HttpClient DiscoveryHttpClient(m_Host, ClientSettings); + HttpClient::Response ServerInfoResponse = + DiscoveryHttpClient.Get("/api/v1/status/servers", HttpClient::Accept(HttpContentType::kJSON)); + if (!ServerInfoResponse.IsSuccess()) + { + throw std::runtime_error(fmt::format("Failed to get list of servers from discovery url '{}'. Reason: '{}'", + m_Host, + ServerInfoResponse.ErrorMessage(""))); + } + + std::string_view JsonResponse = ServerInfoResponse.AsText(); + CbObject ResponseObjectView = LoadCompactBinaryFromJson(JsonResponse).AsObject(); - HttpClient Http(m_BuildsUrl, ClientSettings); + if (m_OverrideHost.empty()) + { + CbArrayView ServerEndpointsArray = ResponseObjectView["serverEndpoints"sv].AsArrayView(); + std::uint64_t ServerCount = ServerEndpointsArray.Num(); + if (ServerCount == 0) + { + throw std::runtime_error(fmt::format("Failed to find any builds hosts at {}", m_Host)); + } + for (CbFieldView ServerEndpointView : ServerEndpointsArray) + { + CbObjectView ServerEndpointObject = ServerEndpointView.AsObjectView(); + std::string_view BaseUrl = ServerEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = ServerEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = ServerEndpointObject["name"sv].AsString(); + + HttpClientSettings TestClientSettings{.LogCategory = "httpbuildsclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); + if (TestResponse.IsSuccess()) + { + CloudHost = BaseUrl; + m_AssumeHttp2 = AssumeHttp2; + BuildStorageName = Name; + break; + } + } + } + if (CloudHost.empty()) + { + throw std::runtime_error( + fmt::format("Failed to find any usable builds hosts out of {} using {}", ServerCount, m_Host)); + } + } + + auto TestCacheEndpoint = [](std::string_view BaseUrl, const bool AssumeHttp2) -> bool { + HttpClientSettings TestClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{1000}, + .Timeout = std::chrono::milliseconds{2000}, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 0}; + HttpClient TestHttpClient(BaseUrl, TestClientSettings); + HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); + if (TestResponse.IsSuccess()) + { + return true; + } + return false; + }; + + if (m_ZenCacheHost.empty()) + { + CbArrayView CacheEndpointsArray = ResponseObjectView["cacheEndpoints"sv].AsArrayView(); + std::uint64_t CacheCount = CacheEndpointsArray.Num(); + for (CbFieldView CacheEndpointView : CacheEndpointsArray) + { + CbObjectView CacheEndpointObject = CacheEndpointView.AsObjectView(); + + std::string_view BaseUrl = CacheEndpointObject["baseUrl"sv].AsString(); + if (!BaseUrl.empty()) + { + const bool AssumeHttp2 = CacheEndpointObject["assumeHttp2"sv].AsBool(false); + std::string_view Name = CacheEndpointObject["name"sv].AsString(); + + if (TestCacheEndpoint(BaseUrl, AssumeHttp2)) + { + m_ZenCacheHost = BaseUrl; + CacheAssumeHttp2 = AssumeHttp2; + BuildCacheName = Name; + break; + } + } + } + if (m_ZenCacheHost.empty()) + { + ZenServerState State; + if (State.InitializeReadOnly()) + { + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + if (m_ZenCacheHost.empty()) + { + std::string ZenServerLocalHostUrl = + fmt::format("http://127.0.0.1:{}", Entry.EffectiveListenPort.load()); + if (TestCacheEndpoint(ZenServerLocalHostUrl, false)) + { + m_ZenCacheHost = ZenServerLocalHostUrl; + CacheAssumeHttp2 = false; + BuildCacheName = "localhost"; + } + } + }); + } + if (m_ZenCacheHost.empty()) + { + ZEN_CONSOLE("Warning: Failed to find any usable cache hosts out of {} using {}", CacheCount, m_Host); + } + } + } + else if (TestCacheEndpoint(m_ZenCacheHost, false)) + { + std::string::size_type HostnameStart = 0; + std::string::size_type HostnameLength = std::string::npos; + if (auto StartPos = m_ZenCacheHost.find("//"); StartPos != std::string::npos) + { + HostnameStart = StartPos + 2; + } + if (auto EndPos = m_ZenCacheHost.find("/", HostnameStart); EndPos != std::string::npos) + { + HostnameLength = EndPos - HostnameStart; + } + BuildCacheName = m_ZenCacheHost.substr(HostnameStart, HostnameLength); + } + } + } + else + { + CloudHost = m_OverrideHost; + } + + if (!CloudHost.empty()) + { + Result.BuildStorageHttp = std::make_unique<HttpClient>(CloudHost, ClientSettings); + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", + BuildStorageName.empty() ? "" : fmt::format("{}, ", BuildStorageName), + CloudHost, + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket); + Result.BuildStorage = + CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, TempPath / "storage"); + Result.StorageName = BuildStorageName; + } + else if (!m_StoragePath.empty()) + { + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); + StorageDescription = fmt::format("folder {}", StoragePath); + Result.BuildStorage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); + Result.StorageName = fmt::format("Disk {}", StoragePath.stem()); + } + else + { + throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); + } + if (!m_ZenCacheHost.empty()) + { + Result.CacheHttp = std::make_unique<HttpClient>(m_ZenCacheHost, + HttpClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = CacheAssumeHttp2, + .AllowResume = true, + .RetryCount = 0}); + Result.BuildCacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + m_PrimeCacheOnly); + CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", + BuildCacheName.empty() ? "" : fmt::format("{}, ", BuildCacheName), + m_ZenCacheHost, + Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + } + Result.CacheName = BuildCacheName; + } + ZEN_CONSOLE("Remote: {}", StorageDescription); + if (!Result.CacheName.empty()) + { + ZEN_CONSOLE("Cache : {}", CacheDescription); + } + return Result; + }; + + BoostWorkerThreads = m_BoostWorkerThreads; + + try + { + if (SubOption == &m_ListOptions) + { + if (!m_ListResultPath.empty()) + { + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); + } CbObject QueryObject; if (m_ListQueryPath.empty()) { @@ -7505,7 +9039,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListQueryPath = StringToPath(m_ListQueryPath); + std::filesystem::path ListQueryPath = MakeSafeAbsolutePath(m_ListQueryPath); if (ToLower(ListQueryPath.extension().string()) == ".cbo") { QueryObject = LoadCompactBinaryObject(IoBufferBuilder::MakeFromFile(ListQueryPath)); @@ -7525,28 +9059,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } BuildStorage::Statistics StorageStats; - std::unique_ptr<BuildStorage> Storage; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE_VERBOSE("Querying builds in cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, std::filesystem::path{}); - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE_VERBOSE("Querying builds in folder '{}'.", StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; - CbObject Response = Storage->ListBuilds(QueryObject); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + + CbObject Response = Storage.BuildStorage->ListBuilds(QueryObject); ZEN_ASSERT(ValidateCompactBinary(Response.GetView(), CbValidateMode::All) == CbValidateError::None); if (m_ListResultPath.empty()) { @@ -7556,7 +9084,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - std::filesystem::path ListResultPath = StringToPath(m_ListResultPath); + std::filesystem::path ListResultPath = MakeSafeAbsolutePath(m_ListResultPath); if (ToLower(ListResultPath.extension().string()) == ".cbo") { MemoryView ResponseView = Response.GetView(); @@ -7575,10 +9103,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_UploadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_Path.empty()) { @@ -7610,7 +9135,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); if (m_BuildPartName.empty()) { @@ -7645,48 +9170,31 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Invalid build part id\n{}", m_UploadOptions.help())); } + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); + BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Uploading '{}' from '{}' to cloud endpoint '{}'. SessionId: '{}'. Namespace '{}', Bucket '{}', {}BuildId '{}'", - m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Uploading '{}' from '{}' to folder '{}'. {}BuildId '{}'", - m_BuildPartName, - Path, - StoragePath, - GeneratedBuildId ? "Generated " : "", - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, m_WriteMetadataAsJson, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); CbObject MetaData; if (m_CreateBuild) { if (!m_BuildMetadataPath.empty()) { - std::filesystem::path MetadataPath = StringToPath(m_BuildMetadataPath); + std::filesystem::path MetadataPath = MakeSafeAbsolutePath(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(MetadataPath).Flatten(); std::string_view Json(reinterpret_cast<const char*>(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -7713,12 +9221,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, Path, - StringToPath(m_ManifestPath), + ZenFolderPath, + MakeSafeAbsolutePath(m_ManifestPath), + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -7735,7 +9245,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7751,10 +9261,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_DownloadOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_Path.empty()) { @@ -7775,6 +9282,22 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + if (m_PostDownloadVerify && m_PrimeCacheOnly) + { + throw zen::OptionParseException( + fmt::format("'cache-prime-only' option is not compatible with 'verify' option\n{}", m_DownloadOptions.help())); + } + + if (m_Clean && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'clean' option when 'cache-prime-only' is enabled"); + } + + if (m_AllowPartialBlockRequests && m_PrimeCacheOnly) + { + ZEN_WARN("ignoring 'allow-partial-block-requests' option when 'cache-prime-only' is enabled"); + } + std::vector<Oid> BuildPartIds; for (const std::string& BuildPartId : m_BuildPartIds) { @@ -7786,45 +9309,27 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - BuildId, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading '{}' to '{}' from folder {}. BuildId '{}'", BuildId, Path, StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); - DownloadFolder(*Storage, + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); + + DownloadFolder(Storage, BuildId, BuildPartIds, m_BuildPartNames, Path, + ZenFolderPath, m_AllowMultiparts, - m_AllowPartialBlockRequests, + m_AllowPartialBlockRequests && !m_PrimeCacheOnly, m_Clean, - m_PostDownloadVerify); + m_PostDownloadVerify, + m_PrimeCacheOnly); if (false) { @@ -7835,7 +9340,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - StorageName, + Storage.StorageName, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -7859,8 +9364,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("compare-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); - std::filesystem::path DiffPath = StringToPath(m_DiffPath); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + std::filesystem::path DiffPath = MakeSafeAbsolutePath(m_DiffPath); DiffFolders(Path, DiffPath, m_OnlyChunked); return AbortFlag ? 11 : 0; } @@ -7872,10 +9377,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); // m_StoragePath = "D:\\buildstorage"; // m_Path = "F:\\Saved\\DownloadedBuilds\\++Fortnite+Main-CL-XXXXXXXX\\WindowsClient"; // std::vector<std::string> BuildIdStrings{"07d3942f0e7f4ca1b13b0587", @@ -7886,34 +9387,15 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) // "07d3964f919d577a321a1fdd", // "07d396a6ce875004e16b9528"}; - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Downloading {} to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - FormatArray<std::string>(m_BuildIds, " "sv), - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Downloading {}'to '{}' from folder {}", FormatArray<std::string>(m_BuildIds, " "sv), Path, StoragePath); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? Path / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); Stopwatch Timer; for (const std::string& BuildIdString : m_BuildIds) @@ -7923,15 +9405,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw zen::OptionParseException(fmt::format("invalid build id {}\n{}", BuildIdString, m_DownloadOptions.help())); } - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId, {}, {}, Path, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, BuildIdString == m_BuildIds.front(), - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download cancelled"); @@ -7945,71 +9429,45 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_TestOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - if (m_Path.empty()) { throw zen::OptionParseException(fmt::format("local-path is required\n{}", m_DownloadOptions.help())); } - std::filesystem::path Path = StringToPath(m_Path); + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); m_BuildId = Oid::NewOid().ToString(); m_BuildPartName = Path.filename().string(); m_BuildPartId = Oid::NewOid().ToString(); m_CreateBuild = true; - BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; + const Oid BuildId = Oid::FromHexString(m_BuildId); + const Oid BuildPartId = Oid::FromHexString(m_BuildPartId); - std::filesystem::path StoragePath = StringToPath(m_StoragePath); + std::filesystem::path StoragePath = MakeSafeAbsolutePath(m_StoragePath); - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { StoragePath = (GetRunningExecutablePath().parent_path() / ".tmpstore").make_preferred(); CreateDirectories(StoragePath); CleanDirectory(StoragePath, {}); + m_StoragePath = StoragePath.generic_string(); } auto _ = MakeGuard([&]() { - if (m_BuildsUrl.empty() && StoragePath.empty()) + if (m_OverrideHost.empty() && StoragePath.empty()) { DeleteDirectories(StoragePath); } }); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!StoragePath.empty()) - { - ZEN_CONSOLE("Using '{}' to '{}' from folder {}. BuildId '{}'", - m_BuildPartName.empty() ? m_BuildPartId : m_BuildPartName, - Path, - StoragePath, - BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + BuildStorage::Statistics StorageStats; + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_test"); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? DownloadPath / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); auto MakeMetaData = [](const Oid& BuildId) -> CbObject { CbObjectWriter BuildMetaDataWriter; @@ -8032,12 +9490,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("Upload Build {}, Part {} ({})\n{}", m_BuildId, BuildPartId, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId, BuildPartId, m_BuildPartName, Path, + ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData, @@ -8050,9 +9510,18 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) return 11; } - const std::filesystem::path DownloadPath = Path.parent_path() / (m_BuildPartName + "_download"); ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}'", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, true, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + ZenFolderPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + true, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Download failed."); @@ -8064,7 +9533,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + ZenFolderPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (identical target)"); @@ -8115,11 +9594,11 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SourceSize > 256) { Work.ScheduleWork( - GetMediumWorkerPool(EWorkloadType::Burst), - [SourceSize, FilePath](std::atomic<bool>&) { + GetIOWorkerPool(), + [SourceSize, FilePath = std::filesystem::path(FilePath)](std::atomic<bool>&) { if (!AbortFlag) { - bool IsReadOnly = SetFileReadOnly(FilePath, false); + bool IsReadOnly = SetFileReadOnlyWithRetry(FilePath, false); { BasicFile Source(FilePath, BasicFile::Mode::kWrite); uint64_t RangeSize = Min(SourceSize / 3, 512u * 1024u); @@ -8146,7 +9625,10 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } break; case 1: - std::filesystem::remove(FilePath); + { + (void)SetFileReadOnlyWithRetry(FilePath, false); + RemoveFileWithRetry(FilePath); + } break; default: break; @@ -8168,7 +9650,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + ZenFolderPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed. (scrambled target)"); @@ -8187,12 +9679,14 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ZEN_CONSOLE("\nUpload scrambled Build {}, Part {} ({})\n{}\n", BuildId2, BuildPartId2, m_BuildPartName, SB.ToView()); } - UploadFolder(*Storage, + UploadFolder(Storage, BuildId2, BuildPartId2, m_BuildPartName, DownloadPath, + ZenFolderPath, {}, + m_FindBlockMaxCount, m_BlockReuseMinPercentLimit, m_AllowMultiparts, MetaData2, @@ -8206,7 +9700,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (original)", BuildId, BuildPartId, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, BuildId, {BuildPartId}, {}, DownloadPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, true); + DownloadFolder(Storage, + BuildId, + {BuildPartId}, + {}, + DownloadPath, + ZenFolderPath, + m_AllowMultiparts, + m_AllowPartialBlockRequests, + false, + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8214,15 +9718,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nDownload Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8230,15 +9736,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } ZEN_CONSOLE("\nRe-download Build {}, Part {} ({}) to '{}' (scrambled)", BuildId2, BuildPartId2, m_BuildPartName, DownloadPath); - DownloadFolder(*Storage, + DownloadFolder(Storage, BuildId2, {BuildPartId2}, {}, DownloadPath, + ZenFolderPath, m_AllowMultiparts, m_AllowPartialBlockRequests, false, - true); + true, + false); if (AbortFlag) { ZEN_CONSOLE("Re-download failed."); @@ -8250,11 +9758,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_FetchBlobOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BlobHash.empty()) { throw zen::OptionParseException(fmt::format("Blob hash string is missing\n{}", m_UploadOptions.help())); @@ -8266,44 +9770,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("Blob hash string is invalid\n{}", m_UploadOptions.help())); } - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } + const Oid BuildId = Oid::FromHexString(m_BuildId); + + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); BuildStorage::Statistics StorageStats; - const Oid BuildId = Oid::FromHexString(m_BuildId); - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; + BuildStorageCache::Statistics StorageCacheStats; - std::filesystem::path Path = StringToPath(m_Path); + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); uint64_t CompressedSize; uint64_t DecompressedSize; - ValidateBlob(*Storage, BuildId, BlobHash, CompressedSize, DecompressedSize); + ValidateBlob(*Storage.BuildStorage, BuildId, BlobHash, CompressedSize, DecompressedSize); if (AbortFlag) { return 11; @@ -8317,15 +9806,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ValidateBuildPartOptions) { - ParseStorageOptions(); - ParseAuthOptions(); - - HttpClient Http(m_BuildsUrl, ClientSettings); - - if (m_BuildsUrl.empty() && m_StoragePath.empty()) - { - throw zen::OptionParseException(fmt::format("At least one storage option is required\n{}", m_UploadOptions.help())); - } + ZEN_CONSOLE("Running {}: {}", GetRunningExecutablePath(), ZEN_CFG_VERSION_BUILD_STRING_FULL); if (m_BuildId.empty()) { @@ -8342,39 +9823,29 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException(fmt::format("build-part-id conflicts with build-part-name\n{}", m_DownloadOptions.help())); } + std::filesystem::path Path = MakeSafeAbsolutePath(m_Path); + BuildStorage::Statistics StorageStats; - std::unique_ptr<BuildStorage> Storage; - std::string StorageName; + BuildStorageCache::Statistics StorageCacheStats; + + const std::filesystem::path ZenFolderPath = + m_ZenFolderPath.empty() ? MakeSafeAbsolutePath(".") / ZenFolderName : MakeSafeAbsolutePath(m_ZenFolderPath); + CreateDirectories(ZenFolderPath); + auto _ = MakeGuard([ZenFolderPath]() { + if (CleanDirectory(ZenFolderPath, {})) + { + std::error_code DummyEc; + RemoveDir(ZenFolderPath, DummyEc); + } + }); - std::filesystem::path Path = StringToPath(m_Path); + StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, ZenTempFolderPath(ZenFolderPath)); - if (!m_BuildsUrl.empty()) - { - ZEN_CONSOLE("Using from cloud endpoint {}. SessionId: '{}'. Namespace '{}', Bucket '{}', BuildId '{}'", - m_BuildsUrl, - Http.GetSessionId(), - m_Namespace, - m_Bucket, - BuildId); - Storage = CreateJupiterBuildStorage(Log(), Http, StorageStats, m_Namespace, m_Bucket, Path / ZenTempStorageFolderName); - StorageName = "Cloud DDC"; - } - else if (!m_StoragePath.empty()) - { - std::filesystem::path StoragePath = StringToPath(m_StoragePath); - ZEN_CONSOLE("Using folder {}. BuildId '{}'", StoragePath, BuildId); - Storage = CreateFileBuildStorage(StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - StorageName = fmt::format("Disk {}", StoragePath.stem()); - } - else - { - throw zen::OptionParseException(fmt::format("Storage option is missing\n{}", m_UploadOptions.help())); - } Oid BuildPartId = Oid::TryFromHexString(m_BuildPartId); ValidateStatistics ValidateStats; DownloadStatistics DownloadStats; - ValidateBuildPart(*Storage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); + ValidateBuildPart(*Storage.BuildStorage, BuildId, BuildPartId, m_BuildPartName, ValidateStats, DownloadStats); return AbortFlag ? 13 : 0; } diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index 1634975c1..b40c3e08f 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -27,11 +27,15 @@ private: std::string m_SystemRootDir; - bool m_PlainProgress = false; - bool m_Verbose = false; + bool m_PlainProgress = false; + bool m_Verbose = false; + bool m_BoostWorkerThreads = false; + + std::string m_ZenFolderPath; // cloud builds - std::string m_BuildsUrl; + std::string m_OverrideHost; + std::string m_Host; bool m_AssumeHttp2 = false; std::string m_Namespace; std::string m_Bucket; @@ -40,6 +44,10 @@ private: std::string m_StoragePath; bool m_WriteMetadataAsJson = false; + // cache + std::string m_ZenCacheHost; + bool m_PrimeCacheOnly = false; + std::string m_BuildId; bool m_CreateBuild = false; std::string m_BuildMetadataPath; @@ -81,7 +89,8 @@ private: std::string m_Path; cxxopts::Options m_UploadOptions{"upload", "Upload a folder"}; - bool m_PostUploadVerify = false; + uint64_t m_FindBlockMaxCount = 10000; + bool m_PostUploadVerify = false; cxxopts::Options m_DownloadOptions{"download", "Download a folder"}; std::vector<std::string> m_BuildPartNames; diff --git a/src/zen/cmds/cache_cmd.cpp b/src/zen/cmds/cache_cmd.cpp index 6ec6a80db..185edc35d 100644 --- a/src/zen/cmds/cache_cmd.cpp +++ b/src/zen/cmds/cache_cmd.cpp @@ -629,7 +629,7 @@ CacheGetCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_OutputPath.empty()) { TargetPath = std::filesystem::path(m_OutputPath); - if (std::filesystem::is_directory(TargetPath)) + if (IsDir(TargetPath)) { TargetPath = TargetPath / (m_AttachmentHash.empty() ? m_ValueKey : m_AttachmentHash); } diff --git a/src/zen/cmds/copy_cmd.cpp b/src/zen/cmds/copy_cmd.cpp index cc6ddd505..53e80c896 100644 --- a/src/zen/cmds/copy_cmd.cpp +++ b/src/zen/cmds/copy_cmd.cpp @@ -64,8 +64,8 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } } - const bool IsFileCopy = std::filesystem::is_regular_file(m_CopySource); - const bool IsDirCopy = std::filesystem::is_directory(m_CopySource); + const bool IsFileCopy = IsFile(m_CopySource); + const bool IsDirCopy = IsDir(m_CopySource); if (!IsFileCopy && !IsDirCopy) { @@ -79,20 +79,14 @@ CopyCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (IsDirCopy) { - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - const bool IsTargetDir = std::filesystem::is_directory(ToPath); - if (!IsTargetDir) - { - if (std::filesystem::is_regular_file(ToPath)) - { - throw std::runtime_error("Attempted copy of directory into file"); - } - } + throw std::runtime_error("Attempted copy of directory into file"); } - else + + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } std::filesystem::path ToCanonical = std::filesystem::canonical(ToPath, Ec); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 6bc499f03..13c7c4b23 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,7 +41,7 @@ namespace { std::string ReadJupiterAccessTokenFromFile(const std::filesystem::path& Path) { - if (!std::filesystem::is_regular_file(Path)) + if (!IsFile(Path)) { throw std::runtime_error(fmt::format("the file '{}' does not exist", Path)); } @@ -2185,7 +2185,7 @@ OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return 1; } - std::filesystem::remove_all(TmpPath); + DeleteDirectories(TmpPath); ZEN_CONSOLE("mirrored {} files from {} oplog entries successfully", FileCount.load(), OplogEntryCount); diff --git a/src/zen/cmds/rpcreplay_cmd.cpp b/src/zen/cmds/rpcreplay_cmd.cpp index 5b88a1f73..4fc38d92a 100644 --- a/src/zen/cmds/rpcreplay_cmd.cpp +++ b/src/zen/cmds/rpcreplay_cmd.cpp @@ -196,7 +196,7 @@ RpcReplayCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("Rpc replay command requires a path"); } - if (!std::filesystem::exists(m_RecordingPath) || !std::filesystem::is_directory(m_RecordingPath)) + if (!IsDir(m_RecordingPath)) { throw std::runtime_error(fmt::format("could not find recording at '{}'", m_RecordingPath)); } diff --git a/src/zen/cmds/run_cmd.cpp b/src/zen/cmds/run_cmd.cpp index a99ba9704..309b8996a 100644 --- a/src/zen/cmds/run_cmd.cpp +++ b/src/zen/cmds/run_cmd.cpp @@ -100,7 +100,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - CleanDirectory(BaseDirectory); + CleanDirectory(BaseDirectory, /*ForceRemoveReadOnlyFiles*/ false); } } diff --git a/src/zen/cmds/serve_cmd.cpp b/src/zen/cmds/serve_cmd.cpp index f87725e36..64039e4c9 100644 --- a/src/zen/cmds/serve_cmd.cpp +++ b/src/zen/cmds/serve_cmd.cpp @@ -67,7 +67,7 @@ ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) throw zen::OptionParseException("command requires a root path"); } - if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + if (!IsDir(m_RootPath)) { throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); } diff --git a/src/zen/cmds/status_cmd.cpp b/src/zen/cmds/status_cmd.cpp index 4d1534e05..88c0b22a2 100644 --- a/src/zen/cmds/status_cmd.cpp +++ b/src/zen/cmds/status_cmd.cpp @@ -33,7 +33,7 @@ StatusCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_DataDir.empty()) { std::filesystem::path DataDir = StringToPath(m_DataDir); - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zen/cmds/up_cmd.cpp b/src/zen/cmds/up_cmd.cpp index 44a41146c..aacc115a0 100644 --- a/src/zen/cmds/up_cmd.cpp +++ b/src/zen/cmds/up_cmd.cpp @@ -159,7 +159,7 @@ AttachCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; @@ -229,7 +229,7 @@ DownCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!DataDir.empty()) { - if (!std::filesystem::is_regular_file(DataDir / ".lock")) + if (!IsFile(DataDir / ".lock")) { ZEN_CONSOLE("lock file does not exist in directory '{}'", DataDir); return 1; diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index a181bbd66..ea526399c 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -590,7 +590,7 @@ TemporaryFile::MoveTemporaryIntoPlace(std::filesystem::path FinalFileName, std:: // deleting the temporary file BasicFile::Close(); - std::filesystem::rename(m_TempPath, FinalFileName, Ec); + RenameFile(m_TempPath, FinalFileName, Ec); if (Ec) { @@ -984,9 +984,9 @@ TEST_CASE("TemporaryFile") TmpFile.CreateTemporary(std::filesystem::current_path(), Ec); CHECK(!Ec); Path = TmpFile.GetPath(); - CHECK(std::filesystem::exists(Path)); + CHECK(IsFile(Path)); } - CHECK(std::filesystem::exists(Path) == false); + CHECK(IsFile(Path) == false); } SUBCASE("MoveIntoPlace") @@ -997,11 +997,11 @@ TEST_CASE("TemporaryFile") CHECK(!Ec); std::filesystem::path TempPath = TmpFile.GetPath(); std::filesystem::path FinalPath = std::filesystem::current_path() / "final"; - CHECK(std::filesystem::exists(TempPath)); + CHECK(IsFile(TempPath)); TmpFile.MoveTemporaryIntoPlace(FinalPath, Ec); CHECK(!Ec); - CHECK(std::filesystem::exists(TempPath) == false); - CHECK(std::filesystem::exists(FinalPath)); + CHECK(IsFile(TempPath) == false); + CHECK(IsFile(FinalPath)); } } diff --git a/src/zencore/except.cpp b/src/zencore/except.cpp index d5eabea9d..610b0ced5 100644 --- a/src/zencore/except.cpp +++ b/src/zencore/except.cpp @@ -47,7 +47,7 @@ ThrowSystemException([[maybe_unused]] HRESULT hRes, [[maybe_unused]] std::string { if (HRESULT_FACILITY(hRes) == FACILITY_WIN32) { - throw std::system_error(std::error_code(hRes & 0xffff, std::system_category()), std::string(Message)); + throw std::system_error(std::error_code(HRESULT_CODE(hRes), std::system_category()), std::string(Message)); } else { diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 05e2bf049..4ec563ba3 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -86,16 +86,9 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) } bool -CreateDirectories(const wchar_t* Dir) +CreateDirectories(const wchar_t* Path) { - // This may be suboptimal, in that it appears to try and create directories - // from the root on up instead of from some directory which is known to - // be present - // - // We should implement a smarter version at some point since this can be - // pretty expensive in aggregate - - return std::filesystem::create_directories(Dir); + return CreateDirectories(std::filesystem::path(Path)); } // Erase all files and directories in a given directory, leaving an empty directory @@ -212,75 +205,327 @@ DeleteDirectoriesInternal(const wchar_t* DirPath) bool CleanDirectory(const wchar_t* DirPath, bool KeepDotFiles) { - if (std::filesystem::exists(DirPath)) + if (IsDir(DirPath)) { return WipeDirectory(DirPath, KeepDotFiles); } return CreateDirectories(DirPath); } + +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_WINDOWS +const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; +#else +const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; #endif // ZEN_PLATFORM_WINDOWS +const uint32_t FileModeWriteEnableFlags = 0222; + +bool +IsFileAttributeReadOnly(uint32_t FileAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; +#else + return (FileAttributes & 0x00000001) != 0; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsFileModeReadOnly(uint32_t FileMode) +{ + return (FileMode & FileModeWriteEnableFlags) == 0; +} + +uint32_t +MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +{ + return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); +} + +uint32_t +MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) +{ + return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); +} + bool -CreateDirectories(const std::filesystem::path& Dir) +RemoveDirNative(const std::filesystem::path& Path, std::error_code& Ec) { - if (Dir.string().ends_with(":")) +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::RemoveDirectory(Path.native().c_str()); + if (!Success) { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } return false; } - while (!std::filesystem::is_directory(Dir)) + return true; +#else + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFileNative(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + BOOL Success = ::DeleteFile(NativePath); + if (!Success) { - if (Dir.has_parent_path()) + if (ForceRemoveReadOnlyFiles) { - CreateDirectories(Dir.parent_path()); + DWORD FileAttributes = ::GetFileAttributes(NativePath); + if ((FileAttributes != INVALID_FILE_ATTRIBUTES) && IsFileAttributeReadOnly(FileAttributes) != 0) + { + ::SetFileAttributes(NativePath, MakeFileAttributeReadOnly(FileAttributes, false)); + Success = ::DeleteFile(NativePath); + } } - std::error_code ErrorCode; - std::filesystem::create_directory(Dir, ErrorCode); - if (ErrorCode) + if (!Success) { - throw std::system_error(ErrorCode, fmt::format("Failed to create directories for '{}'", Dir.string())); + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + } + return true; +#else + if (!ForceRemoveReadOnlyFiles) + { + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + const uint32_t Mode = (uint32_t)Stat.st_mode; + if (IsFileModeReadOnly(Mode)) + { + Ec = MakeErrorCode(EACCES); + return false; + } + } + return std::filesystem::remove(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +static void +WipeDirectoryContentInternal(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + DirectoryContent LocalDirectoryContent; + GetDirectoryContent(Path, DirectoryContentFlags::IncludeDirs | DirectoryContentFlags::IncludeFiles, LocalDirectoryContent); + for (const std::filesystem::path& LocalFilePath : LocalDirectoryContent.Files) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsFile(LocalFilePath)) + { + RemoveFileNative(LocalFilePath, ForceRemoveReadOnlyFiles, Ec); + } + } + if (Ec) + { + return; + } + } + + for (std::filesystem::path& LocalDirPath : LocalDirectoryContent.Directories) + { + WipeDirectoryContentInternal(LocalDirPath, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + return; + } + + RemoveDirNative(LocalDirPath, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(LocalDirPath)) + { + RemoveDirNative(LocalDirPath, Ec); + } + } + if (Ec) + { + return; } - return true; } - return false; } bool -DeleteDirectories(const std::filesystem::path& Dir) +CreateDirectory(const std::filesystem::path& Path, std::error_code& Ec) { - std::error_code ErrorCode; - return std::filesystem::remove_all(Dir, ErrorCode); +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::CreateDirectory(Path.native().c_str(), nullptr); + if (!Success) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_EXISTS: + case ERROR_ALREADY_EXISTS: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return Success; +#else + return std::filesystem::create_directory(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS } bool -CleanDirectory(const std::filesystem::path& Dir) +CreateDirectories(const std::filesystem::path& Path) { - if (std::filesystem::exists(Dir)) + std::error_code Ec; + bool Success = CreateDirectories(Path, Ec); + if (Ec) { - bool Success = true; + throw std::system_error(Ec, fmt::format("Failed to create directories for '{}'", Path.string())); + } + return Success; +} - for (const auto& Item : std::filesystem::directory_iterator(Dir)) - { - std::error_code ErrorCode; - const uintmax_t RemovedCount = std::filesystem::remove_all(Item, ErrorCode); +bool +CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec) +{ + if (Path.string().ends_with(":")) + { + return false; + } + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return false; + } + if (Exists) + { + return false; + } - Success = Success && !ErrorCode && RemovedCount; + if (Path.has_parent_path()) + { + bool Result = CreateDirectories(Path.parent_path(), Ec); + if (Ec) + { + return Result; } + } + return CreateDirectory(Path, Ec); +} + +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles) +{ + std::error_code Ec; + bool Result = CleanDirectory(Path, ForceRemoveReadOnlyFiles, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to clean directory for '{}'", Path.string())); + } + return Result; +} + +bool +CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; + } + if (Exists) + { + WipeDirectoryContentInternal(Path, ForceRemoveReadOnlyFiles, Ec); + return false; + } + return CreateDirectory(Path, Ec); +} + +bool +DeleteDirectories(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = DeleteDirectories(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to delete directories for '{}'", Path.string())); + } + return Result; +} - return Success; +bool +DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec) +{ + bool Exists = IsDir(Path, Ec); + if (Ec) + { + return Exists; } - return CreateDirectories(Dir); + if (Exists) + { + WipeDirectoryContentInternal(Path, false, Ec); + if (Ec) + { + return false; + } + bool Result = RemoveDirNative(Path, Ec); + for (size_t Retries = 0; Ec && Retries < 3; Retries++) + { + Sleep(100 + int(Retries * 50)); + Ec.clear(); + if (IsDir(Path)) + { + Result = RemoveDirNative(Path, Ec); + } + } + return Result; + } + return false; } bool -CleanDirectoryExceptDotFiles(const std::filesystem::path& Dir) +CleanDirectoryExceptDotFiles(const std::filesystem::path& Path) { #if ZEN_PLATFORM_WINDOWS const bool KeepDotFiles = true; - return CleanDirectory(Dir.c_str(), KeepDotFiles); + return CleanDirectory(Path.c_str(), KeepDotFiles); #else - ZEN_UNUSED(Dir); + ZEN_UNUSED(Path); ZEN_NOT_IMPLEMENTED(); #endif @@ -637,7 +882,7 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop { // Validate arguments - if (FromPath.empty() || !std::filesystem::is_directory(FromPath)) + if (FromPath.empty() || !IsDir(FromPath)) throw std::runtime_error("invalid CopyTree source directory specified"); if (ToPath.empty()) @@ -646,16 +891,13 @@ CopyTree(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop if (Options.MustClone && !SupportsBlockRefCounting(FromPath)) throw std::runtime_error(fmt::format("cloning not possible from '{}'", FromPath)); - if (std::filesystem::exists(ToPath)) + if (IsFile(ToPath)) { - if (!std::filesystem::is_directory(ToPath)) - { - throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); - } + throw std::runtime_error(fmt::format("specified CopyTree target '{}' is not a directory", ToPath)); } - else + if (!IsDir(ToPath)) { - std::filesystem::create_directories(ToPath); + CreateDirectories(ToPath); } if (Options.MustClone && !SupportsBlockRefCounting(ToPath)) @@ -811,7 +1053,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { Outfile.Close(); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else @@ -819,7 +1061,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { close(Fd); std::error_code DummyEc; - std::filesystem::remove(Path, DummyEc); + RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); } #endif // ZEN_PLATFORM_WINDOWS @@ -1172,7 +1414,7 @@ void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { #if ZEN_PLATFORM_WINDOWS - uint64_t FileInfoBuffer[8 * 1024]; + std::vector<uint64_t> FileInfoBuffer(8 * 1024); FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; @@ -1183,7 +1425,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr if (FAILED(hRes)) { - if (hRes == ERROR_FILE_NOT_FOUND || hRes == ERROR_PATH_NOT_FOUND) + if (HRESULT_CODE(hRes) == ERROR_FILE_NOT_FOUND || HRESULT_CODE(hRes) == ERROR_PATH_NOT_FOUND) { // Directory no longer exist, treat it as empty return; @@ -1193,8 +1435,9 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr while (Continue) { - BOOL Success = GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer, sizeof FileInfoBuffer); - FibClass = FileIdBothDirectoryInfo; // Set up for next iteration + BOOL Success = + GetFileInformationByHandleEx(RootDirHandle, FibClass, FileInfoBuffer.data(), (DWORD)(FileInfoBuffer.size() * sizeof(uint64_t))); + FibClass = FileIdBothDirectoryInfo; // Set up for next iteration uint64_t EntryOffset = 0; @@ -1213,7 +1456,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr for (;;) { const FILE_ID_BOTH_DIR_INFO* DirInfo = - reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer) + EntryOffset); + reinterpret_cast<const FILE_ID_BOTH_DIR_INFO*>(reinterpret_cast<const uint8_t*>(FileInfoBuffer.data()) + EntryOffset); std::wstring_view FileName(DirInfo->FileName, DirInfo->FileNameLength / sizeof(wchar_t)); @@ -1338,6 +1581,172 @@ CanonicalPath(std::filesystem::path InPath, std::error_code& Ec) #endif } +bool +IsFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a file", Path.string())); + } + return Result; +} + +bool +IsFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == 0; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISREG(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +IsDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Result = IsDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to test if path '{}' is a directory", Path.string())); + } + return Result; +} + +bool +IsDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + DWORD Attributes = ::GetFileAttributes(Path.native().c_str()); + if (Attributes == INVALID_FILE_ATTRIBUTES) + { + DWORD LastError = GetLastError(); + switch (LastError) + { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_BAD_NETPATH: + case ERROR_INVALID_DRIVE: + break; + default: + Ec = MakeErrorCode(LastError); + break; + } + return false; + } + return (Attributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY; +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err != 0) + { + int32_t err = errno; + if (err == ENOENT) + { + Ec.clear(); + return false; + } + } + if (S_ISDIR(Stat.st_mode)) + { + return true; + } + return false; +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveFile(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveFile(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove file '{}'", Path.string())); + } + return Success; +} + +bool +RemoveFile(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveFileNative(Path, false, Ec); +#else + bool IsDirectory = std::filesystem::is_directory(Path, Ec); + if (IsDirectory) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveFileNative(Path, false, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +bool +RemoveDir(const std::filesystem::path& Path) +{ + std::error_code Ec; + bool Success = RemoveDir(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to remove directory '{}'", Path.string())); + } + return Success; +} + +bool +RemoveDir(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + return RemoveDirNative(Path, Ec); +#else + bool IsFile = std::filesystem::is_regular_file(Path, Ec); + if (IsFile) + { + Ec = MakeErrorCode(EPERM); + return false; + } + Ec.clear(); + return RemoveDirNative(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec) { @@ -1435,6 +1844,49 @@ PathFromHandle(void* NativeHandle, std::error_code& Ec) } uint64_t +FileSizeFromPath(const std::filesystem::path& Path) +{ + std::error_code Ec; + uint64_t Size = FileSizeFromPath(Path, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get file size for path '{}'", Path.string())); + } + return Size; +} + +uint64_t +FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + void* Handle = ::CreateFile(Path.native().c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + DWORD LastError = GetLastError(); + Ec = MakeErrorCode(LastError); + return 0; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + LARGE_INTEGER FileSize; + BOOL Success = GetFileSizeEx(Handle, &FileSize); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + return 0; + } + return FileSize.QuadPart; +#else + return std::filesystem::file_size(Path, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +uint64_t FileSizeFromHandle(void* NativeHandle) { uint64_t FileSize = ~0ull; @@ -1483,7 +1935,13 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) // PathFromHandle void* Handle; #if ZEN_PLATFORM_WINDOWS - Handle = CreateFileW(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + Handle = CreateFileW(Filename.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); if (Handle == INVALID_HANDLE_VALUE) { ThrowLastError(fmt::format("Failed to open file {} to check modification tick.", Filename)); @@ -1493,7 +1951,7 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) uint64_t ModificatonTick = GetModificationTickFromHandle(Handle, Ec); if (Ec) { - ThrowSystemError(Ec.value(), Ec.message()); + throw std::system_error(Ec, fmt::format("Failed to get modification tick for path '{}'", Filename.string())); } return ModificatonTick; #else @@ -1507,6 +1965,102 @@ GetModificationTickFromPath(const std::filesystem::path& Filename) #endif } +bool +TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes) +{ +#if ZEN_PLATFORM_WINDOWS + const std::filesystem::path::value_type* NativePath = Path.native().c_str(); + { + void* Handle = CreateFileW(NativePath, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr); + if (Handle == INVALID_HANDLE_VALUE) + { + return false; + } + auto _ = MakeGuard([Handle]() { CloseHandle(Handle); }); + + BY_HANDLE_FILE_INFORMATION Bhfh = {}; + if (!GetFileInformationByHandle(Handle, &Bhfh)) + { + return false; + } + OutSize = uint64_t(Bhfh.nFileSizeHigh) << 32 | Bhfh.nFileSizeLow; + OutModificationTick = ((uint64_t(Bhfh.ftLastWriteTime.dwHighDateTime) << 32) | Bhfh.ftLastWriteTime.dwLowDateTime); + OutNativeModeOrAttributes = Bhfh.dwFileAttributes; + return true; + } +#else + struct stat Stat; + int err = stat(Path.native().c_str(), &Stat); + if (err) + { + return false; + } + OutModificationTick = gsl::narrow<uint64_t>(Stat.st_mtime); + OutSize = size_t(Stat.st_size); + OutNativeModeOrAttributes = (uint32_t)Stat.st_mode; + return true; +#endif +} + +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameFile(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename path from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFileEx(SourcePath.native().c_str(), TargetPath.native().c_str(), MOVEFILE_REPLACE_EXISTING); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath) +{ + std::error_code Ec; + RenameDirectory(SourcePath, TargetPath, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to rename directory from '{}' to '{}'", SourcePath.string(), TargetPath.string())); + } +} + +void +RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec) +{ +#if ZEN_PLATFORM_WINDOWS + BOOL Success = ::MoveFile(SourcePath.native().c_str(), TargetPath.native().c_str()); + if (!Success) + { + Ec = MakeErrorCodeFromLastError(); + } +#else + return std::filesystem::rename(SourcePath, TargetPath, Ec); +#endif // ZEN_PLATFORM_WINDOWS +} + std::filesystem::path GetRunningExecutablePath() { @@ -1793,7 +2347,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) }; auto IsEmpty = [](const std::filesystem::path& Path, std::error_code& Ec) -> bool { - bool Exists = std::filesystem::exists(Path, Ec); + bool Exists = IsFile(Path, Ec); if (Ec) { return false; @@ -1802,7 +2356,7 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) { return true; } - uintmax_t Size = std::filesystem::file_size(Path, Ec); + uintmax_t Size = FileSizeFromPath(Path, Ec); if (Ec) { return false; @@ -1821,17 +2375,17 @@ RotateFiles(const std::filesystem::path& Filename, std::size_t MaxFiles) for (auto i = MaxFiles; i > 0; i--) { std::filesystem::path src = GetFileName(i - 1); - if (!std::filesystem::exists(src)) + if (!IsFile(src)) { continue; } std::error_code DummyEc; std::filesystem::path target = GetFileName(i); - if (std::filesystem::exists(target, DummyEc)) + if (IsFile(target, DummyEc)) { - std::filesystem::remove(target, DummyEc); + RemoveFile(target, DummyEc); } - std::filesystem::rename(src, target, DummyEc); + RenameFile(src, target, DummyEc); } } @@ -1868,16 +2422,16 @@ RotateDirectories(const std::filesystem::path& DirectoryName, std::size_t MaxDir { const std::filesystem::path SourcePath = GetPathForIndex(i - 1); - if (std::filesystem::exists(SourcePath)) + if (IsDir(SourcePath)) { std::filesystem::path TargetPath = GetPathForIndex(i); std::error_code DummyEc; - if (std::filesystem::exists(TargetPath, DummyEc)) + if (IsDir(TargetPath, DummyEc)) { - std::filesystem::remove_all(TargetPath, DummyEc); + DeleteDirectories(TargetPath, DummyEc); } - std::filesystem::rename(SourcePath, TargetPath, DummyEc); + RenameDirectory(SourcePath, TargetPath, DummyEc); } } @@ -1936,22 +2490,46 @@ PickDefaultSystemRootDirectory() #if ZEN_PLATFORM_WINDOWS uint32_t -GetFileAttributes(const std::filesystem::path& Filename) +GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec) { DWORD Attributes = ::GetFileAttributes(Filename.native().c_str()); if (Attributes == INVALID_FILE_ATTRIBUTES) { - ThrowLastError(fmt::format("failed to get attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Attributes; } +uint32_t +GetFileAttributes(const std::filesystem::path& Filename) +{ + std::error_code Ec; + uint32_t Result = zen::GetFileAttributes(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to get attributes of file '{}'", Filename.string())); + } + return Result; +} + void -SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec) { if (::SetFileAttributes(Filename.native().c_str(), Attributes) == 0) { - ThrowLastError(fmt::format("failed to set attributes of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + } +} + +void +SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) +{ + std::error_code Ec; + zen::SetFileAttributes(Filename, Attributes, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set attributes of file {}", Filename.string())); } } @@ -1962,98 +2540,117 @@ SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes) uint32_t GetFileMode(const std::filesystem::path& Filename) { + std::error_code Ec; + uint32_t Result = GetFileMode(Filename, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("Failed to get mode of file {}", Filename)); + } + return Result; +} + +uint32_t +GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec) +{ struct stat Stat; int err = stat(Filename.native().c_str(), &Stat); if (err) { - ThrowLastError(fmt::format("Failed to get mode of file {}", Filename)); + Ec = MakeErrorCodeFromLastError(); + return 0; } return (uint32_t)Stat.st_mode; } void -SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes) +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode) { - int err = chmod(Filename.native().c_str(), (mode_t)Attributes); - if (err) + std::error_code Ec; + SetFileMode(Filename, Mode, Ec); + if (Ec) { - ThrowLastError(fmt::format("Failed to set mode of file {}", Filename)); + throw std::system_error(Ec, fmt::format("Failed to set mode of file {}", Filename)); } } -#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - -#if ZEN_PLATFORM_WINDOWS -const uint32_t FileAttributesSystemReadOnlyFlag = FILE_ATTRIBUTE_READONLY; -#else -const uint32_t FileAttributesSystemReadOnlyFlag = 0x00000001; -#endif // ZEN_PLATFORM_WINDOWS - -const uint32_t FileModeWriteEnableFlags = 0222; - -bool -IsFileAttributeReadOnly(uint32_t FileAttributes) -{ -#if ZEN_PLATFORM_WINDOWS - return (FileAttributes & FileAttributesSystemReadOnlyFlag) != 0; -#else - return (FileAttributes & 0x00000001) != 0; -#endif // ZEN_PLATFORM_WINDOWS -} - -bool -IsFileModeReadOnly(uint32_t FileMode) -{ - return (FileMode & FileModeWriteEnableFlags) == 0; -} - -uint32_t -MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly) +void +SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec) { - return ReadOnly ? (FileAttributes | FileAttributesSystemReadOnlyFlag) : (FileAttributes & ~FileAttributesSystemReadOnlyFlag); + int err = chmod(Filename.native().c_str(), (mode_t)Mode); + if (err) + { + Ec = MakeErrorCodeFromLastError(); + } } -uint32_t -MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly) -{ - return ReadOnly ? (FileMode & ~FileModeWriteEnableFlags) : (FileMode | FileModeWriteEnableFlags); -} +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool -SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec) { #if ZEN_PLATFORM_WINDOWS - uint32_t CurrentAttributes = GetFileAttributes(Filename); - uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); + uint32_t CurrentAttributes = GetFileAttributes(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewAttributes = MakeFileAttributeReadOnly(CurrentAttributes, ReadOnly); if (CurrentAttributes != NewAttributes) { - SetFileAttributes(Filename, NewAttributes); + SetFileAttributes(Filename, NewAttributes, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - uint32_t CurrentMode = GetFileMode(Filename); - uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); + uint32_t CurrentMode = GetFileMode(Filename, Ec); + if (Ec) + { + return false; + } + uint32_t NewMode = MakeFileModeReadOnly(CurrentMode, ReadOnly); if (CurrentMode != NewMode) { - SetFileMode(Filename, NewMode); + SetFileMode(Filename, NewMode, Ec); + if (Ec) + { + return false; + } return true; } #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC return false; } +bool +SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) +{ + std::error_code Ec; + bool Result = SetFileReadOnly(Filename, ReadOnly, Ec); + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to set read only mode of file '{}'", Filename.string())); + } + return Result; +} + std::filesystem::path StringToPath(const std::string_view& Path) { + std::string_view UnquotedPath = Path; + if (Path.length() > 2 && Path.front() == '\"' && Path.back() == '\"') { - return std::filesystem::path(Path.substr(1, Path.length() - 2)).make_preferred(); + UnquotedPath = Path.substr(1, Path.length() - 2); } - else + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) { - return std::filesystem::path(Path).make_preferred(); + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); } + return std::filesystem::path(UnquotedPath).make_preferred(); } ////////////////////////////////////////////////////////////////////////// @@ -2076,7 +2673,7 @@ TEST_CASE("filesystem") path BinPath = GetRunningExecutablePath(); const bool ExpectedExe = PathToUtf8(BinPath.stem().native()).ends_with("-test"sv) || BinPath.stem() == "zenserver"; CHECK(ExpectedExe); - CHECK(is_regular_file(BinPath)); + CHECK(IsFile(BinPath)); // PathFromHandle void* Handle; @@ -2129,6 +2726,80 @@ TEST_CASE("filesystem") CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); } +TEST_CASE("Filesystem.Basics") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; + CleanDirectory(TestBaseDir, true); + DeleteDirectories(TestBaseDir); + CHECK(!IsDir(TestBaseDir)); + CHECK(CleanDirectory(TestBaseDir, false)); + CHECK(IsDir(TestBaseDir)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(!IsDir(TestBaseDir / "no_such_thing")); + CHECK(!IsDir("hgjda/cev_/q12")); + CHECK(!IsFile(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "no_such_thing")); + CHECK(!IsFile("hgjda/cev_/q12")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir) == 0); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "no_such_file")); + CHECK(!CreateDirectories(TestBaseDir)); + CHECK(CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(!CreateDirectories(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep")); + CHECK(IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit" / "deep" / "no")); + CHECK_THROWS(WriteFile(TestBaseDir / "nested" / "a", IoBuffer(20))); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "a" / "yo", IoBuffer(20))); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "nested" / "a" / "yo") == 20); + CHECK(!IsFile(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(!IsDir(TestBaseDir / "nested" / "a" / "bit")); + CHECK(!DeleteDirectories(TestBaseDir / "nested" / "a" / "bit")); + CHECK(IsDir(TestBaseDir / "nested" / "a")); + CHECK(DeleteDirectories(TestBaseDir / "nested")); + CHECK(!IsFile(TestBaseDir / "nested" / "a" / "yo")); + CHECK(CreateDirectories(TestBaseDir / "nested" / "deeper")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "deeper" / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK(IsDir(TestBaseDir / "new_place")); + CHECK(!IsFile(TestBaseDir / "new_place")); + CHECK_THROWS(RenameDirectory(TestBaseDir / "nested" / "deeper", TestBaseDir / "new_place")); + CHECK(!RemoveDir(TestBaseDir / "nested" / "deeper")); + CHECK(RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(FileSizeFromPath(TestBaseDir / "new_place" / "yo")); + CHECK(!RemoveFile(TestBaseDir / "new_place" / "yo")); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested")); + CHECK_THROWS(RemoveDir(TestBaseDir)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "yo", IoBuffer(20))); + CHECK_NOTHROW(RenameFile(TestBaseDir / "yo", TestBaseDir / "new_place" / "yo")); + CHECK(!IsFile(TestBaseDir / "yo")); + CHECK(IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(FileSizeFromPath(TestBaseDir / "new_place" / "yo") == 20); + CHECK_THROWS(RemoveDir(TestBaseDir / "new_place" / "yo")); + CHECK(DeleteDirectories(TestBaseDir)); + CHECK(!IsFile(TestBaseDir / "new_place" / "yo")); + CHECK(!IsDir(TestBaseDir)); + CHECK(!IsDir(TestBaseDir / "nested")); + CHECK(CreateDirectories(TestBaseDir / "nested")); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK_THROWS(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK_THROWS(CleanDirectory(TestBaseDir, false)); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", false)); + CHECK(RemoveFile(TestBaseDir / "nested" / "readonly")); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK_NOTHROW(WriteFile(TestBaseDir / "nested" / "readonly", IoBuffer(20))); + CHECK(SetFileReadOnly(TestBaseDir / "nested" / "readonly", true)); + CHECK(!CleanDirectory(TestBaseDir / "nested", true)); + CHECK(!CleanDirectory(TestBaseDir, false)); + CHECK(RemoveDir(TestBaseDir)); +} + TEST_CASE("WriteFile") { std::filesystem::path TempFile = GetRunningExecutablePath().parent_path(); @@ -2163,7 +2834,7 @@ TEST_CASE("WriteFile") CHECK_EQ(memcmp(MagicTest.Data, MagicsReadback.Data[0].Data(), MagicTest.Size), 0); } - std::filesystem::remove(TempFile); + RemoveFile(TempFile); } TEST_CASE("DiskSpaceInfo") @@ -2220,7 +2891,7 @@ TEST_CASE("PathBuilder") TEST_CASE("RotateDirectories") { std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".test"; - CleanDirectory(TestBaseDir); + CleanDirectory(TestBaseDir, false); std::filesystem::path RotateDir = TestBaseDir / "rotate_dir" / "dir_to_rotate"; IoBuffer DummyFileData = IoBufferBuilder::MakeCloneFromMemory("blubb", 5); @@ -2234,16 +2905,16 @@ TEST_CASE("RotateDirectories") const int RotateMax = 10; NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); NewDir(); - CHECK(std::filesystem::exists(RotateDir)); + CHECK(IsDir(RotateDir)); RotateDirectories(RotateDir, RotateMax); - CHECK(!std::filesystem::exists(RotateDir)); - CHECK(std::filesystem::exists(DirWithSuffix(1))); - CHECK(std::filesystem::exists(DirWithSuffix(2))); + CHECK(!IsDir(RotateDir)); + CHECK(IsDir(DirWithSuffix(1))); + CHECK(IsDir(DirWithSuffix(2))); for (int i = 0; i < RotateMax; ++i) { @@ -2253,16 +2924,16 @@ TEST_CASE("RotateDirectories") CHECK_EQ(IsError, false); } - CHECK(!std::filesystem::exists(RotateDir)); + CHECK(!IsDir(RotateDir)); for (int i = 0; i < RotateMax; ++i) { - CHECK(std::filesystem::exists(DirWithSuffix(i + 1))); + CHECK(IsDir(DirWithSuffix(i + 1))); } for (int i = RotateMax; i < RotateMax + 5; ++i) { - CHECK(!std::filesystem::exists(DirWithSuffix(RotateMax + i + 1))); + CHECK(!IsDir(DirWithSuffix(RotateMax + i + 1))); } } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 9a2b15d1d..66deffa6f 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -20,21 +20,35 @@ class WorkerThreadPool; /** Delete directory (after deleting any contents) */ -ZENCORE_API bool DeleteDirectories(const std::filesystem::path& dir); +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path); + +/** Delete directory (after deleting any contents) + */ +ZENCORE_API bool DeleteDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists. + + Will also create any required parent direCleanDirectoryctories + */ +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path); /** Ensure directory exists. Will also create any required parent directories */ -ZENCORE_API bool CreateDirectories(const std::filesystem::path& dir); +ZENCORE_API bool CreateDirectories(const std::filesystem::path& Path, std::error_code& Ec); + +/** Ensure directory exists and delete contents (if any) before returning + */ +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectory(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectory(const std::filesystem::path& Path, bool ForceRemoveReadOnlyFiles, std::error_code& Ec); /** Ensure directory exists and delete contents (if any) before returning */ -ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& dir); +ZENCORE_API bool CleanDirectoryExceptDotFiles(const std::filesystem::path& Path); /** Map native file handle to a path */ @@ -44,6 +58,46 @@ ZENCORE_API std::filesystem::path PathFromHandle(void* NativeHandle, std::error_ */ ZENCORE_API std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path); + +/** Query file size + */ +ZENCORE_API uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); + /** Query file size from native file handle */ ZENCORE_API uint64_t FileSizeFromHandle(void* NativeHandle); @@ -56,6 +110,27 @@ ZENCORE_API uint64_t GetModificationTickFromHandle(void* NativeHandle, std::erro */ ZENCORE_API uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +ZENCORE_API bool TryGetFileProperties(const std::filesystem::path& Path, + uint64_t& OutSize, + uint64_t& OutModificationTick, + uint32_t& OutNativeModeOrAttributes); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a file, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); + +/** Move a directory, if the files are not on the same drive the function will fail + */ +ZENCORE_API void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); + ZENCORE_API std::filesystem::path GetRunningExecutablePath(); /** Set the max open file handle count to max allowed for the current process on Linux and MacOS @@ -277,12 +352,16 @@ std::filesystem::path PickDefaultSystemRootDirectory(); #if ZEN_PLATFORM_WINDOWS uint32_t GetFileAttributes(const std::filesystem::path& Filename); +uint32_t GetFileAttributes(const std::filesystem::path& Filename, std::error_code& Ec); void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes); +void SetFileAttributes(const std::filesystem::path& Filename, uint32_t Attributes, std::error_code& Ec); #endif // ZEN_PLATFORM_WINDOWS #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC uint32_t GetFileMode(const std::filesystem::path& Filename); -void SetFileMode(const std::filesystem::path& Filename, uint32_t Attributes); +uint32_t GetFileMode(const std::filesystem::path& Filename, std::error_code& Ec); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode); +void SetFileMode(const std::filesystem::path& Filename, uint32_t Mode, std::error_code& Ec); #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC bool IsFileAttributeReadOnly(uint32_t FileAttributes); @@ -290,6 +369,7 @@ bool IsFileModeReadOnly(uint32_t FileMode); uint32_t MakeFileAttributeReadOnly(uint32_t FileAttributes, bool ReadOnly); uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); +bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); std::filesystem::path StringToPath(const std::string_view& Path); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 0761521dc..2fe5b8948 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -60,7 +60,7 @@ GetPidStatus(int Pid, std::error_code& OutEc) { std::filesystem::path EntryPath = std::filesystem::path("/proc") / fmt::format("{}", Pid); std::filesystem::path StatPath = EntryPath / "stat"; - if (std::filesystem::is_regular_file(StatPath)) + if (IsFile(StatPath)) { FILE* StatFile = fopen(StatPath.c_str(), "r"); if (StatFile) diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 641d5508a..9f50de032 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -4,6 +4,7 @@ #if ZEN_WITH_TESTS +# include <zencore/filesystem.h> # include <zencore/session.h> # include "zencore/string.h" @@ -19,8 +20,8 @@ CreateTemporaryDirectory() std::error_code Ec; std::filesystem::path DirPath = std::filesystem::temp_directory_path() / GetSessionIdString() / IntNum(++Sequence).c_str(); - std::filesystem::remove_all(DirPath, Ec); - std::filesystem::create_directories(DirPath); + DeleteDirectories(DirPath, Ec); + CreateDirectories(DirPath); return DirPath; } @@ -32,14 +33,14 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() : m_RootPath(CreateTemporar ScopedTemporaryDirectory::ScopedTemporaryDirectory(std::filesystem::path Directory) : m_RootPath(Directory) { std::error_code Ec; - std::filesystem::remove_all(Directory, Ec); - std::filesystem::create_directories(Directory); + DeleteDirectories(Directory, Ec); + CreateDirectories(Directory); } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { std::error_code Ec; - std::filesystem::remove_all(m_RootPath, Ec); + DeleteDirectories(m_RootPath, Ec); } IoBuffer diff --git a/src/zenhttp/auth/authmgr.cpp b/src/zenhttp/auth/authmgr.cpp index 1a9892d5c..8f7befc80 100644 --- a/src/zenhttp/auth/authmgr.cpp +++ b/src/zenhttp/auth/authmgr.cpp @@ -379,7 +379,7 @@ private: AuthState.EndArray(); } - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); std::optional<std::string> Reason; diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 30711a432..f3baf37ce 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -45,6 +45,7 @@ namespace detail { TempPayloadFile() : m_FileHandle(nullptr), m_WriteOffset(0) {} ~TempPayloadFile() { + ZEN_TRACE_CPU("TempPayloadFile::Close"); try { if (m_FileHandle) @@ -87,6 +88,7 @@ namespace detail { std::error_code Open(const std::filesystem::path& TempFolderPath) { + ZEN_TRACE_CPU("TempPayloadFile::Open"); ZEN_ASSERT(m_FileHandle == nullptr); std::uint64_t TmpIndex = ((std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) & 0xffffffffu) << 32) | @@ -131,6 +133,7 @@ namespace detail { std::error_code Write(std::string_view DataString) { + ZEN_TRACE_CPU("TempPayloadFile::Write"); const uint8_t* DataPtr = (const uint8_t*)DataString.data(); size_t DataSize = DataString.size(); if (DataSize >= CacheBufferSize) @@ -165,6 +168,7 @@ namespace detail { IoBuffer DetachToIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::DetachToIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -180,6 +184,7 @@ namespace detail { IoBuffer BorrowIoBuffer() { + ZEN_TRACE_CPU("TempPayloadFile::BorrowIoBuffer"); if (std::error_code Ec = Flush(); Ec) { ThrowSystemError(Ec.value(), Ec.message()); @@ -193,6 +198,7 @@ namespace detail { uint64_t GetSize() const { return m_WriteOffset; } void ResetWritePos(uint64_t WriteOffset) { + ZEN_TRACE_CPU("TempPayloadFile::ResetWritePos"); Flush(); m_WriteOffset = WriteOffset; } @@ -200,6 +206,7 @@ namespace detail { private: std::error_code Flush() { + ZEN_TRACE_CPU("TempPayloadFile::Flush"); if (m_CacheBufferOffset == 0) { return {}; @@ -211,6 +218,7 @@ namespace detail { std::error_code AppendData(const void* Data, uint64_t Size) { + ZEN_TRACE_CPU("TempPayloadFile::AppendData"); ZEN_ASSERT(m_FileHandle != nullptr); const uint64_t MaxChunkSize = 2u * 1024 * 1024 * 1024; @@ -314,7 +322,11 @@ CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffe const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) { - ZEN_WARN("HttpClient client error (session: {}): {}", SessionId, HttpResponse); + if (HttpResponse.error.code != cpr::ErrorCode::OPERATION_TIMEDOUT && + HttpResponse.error.code != cpr::ErrorCode::CONNECTION_FAILURE && HttpResponse.error.code != cpr::ErrorCode::REQUEST_CANCELLED) + { + ZEN_WARN("HttpClient client failure (session: {}): {}", SessionId, HttpResponse); + } // Client side failure code return HttpClient::Response{ @@ -376,6 +388,7 @@ ShouldRetry(const cpr::Response& Response) static bool ValidatePayload(cpr::Response& Response, std::unique_ptr<detail::TempPayloadFile>& PayloadFile) { + ZEN_TRACE_CPU("ValidatePayload"); IoBuffer ResponseBuffer = (Response.text.empty() && PayloadFile) ? PayloadFile->BorrowIoBuffer() : IoBuffer(IoBuffer::Wrap, Response.text.data(), Response.text.size()); @@ -535,12 +548,14 @@ struct HttpClient::Impl : public RefCounted inline cpr::Session* operator->() const { return CprSession; } inline cpr::Response Get() { + ZEN_TRACE_CPU("HttpClient::Impl::Get"); cpr::Response Result = CprSession->Get(); ZEN_TRACE("GET {}", Result); return Result; } inline cpr::Response Download(cpr::WriteCallback&& Write, std::optional<cpr::HeaderCallback>&& Header = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Download"); if (Header) { CprSession->SetHeaderCallback(std::move(Header.value())); @@ -553,12 +568,14 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Head() { + ZEN_TRACE_CPU("HttpClient::Impl::Head"); cpr::Response Result = CprSession->Head(); ZEN_TRACE("HEAD {}", Result); return Result; } inline cpr::Response Put(std::optional<cpr::ReadCallback>&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Put"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -570,6 +587,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Post(std::optional<cpr::ReadCallback>&& Read = {}) { + ZEN_TRACE_CPU("HttpClient::Impl::Post"); if (Read) { CprSession->SetReadCallback(std::move(Read.value())); @@ -581,6 +599,7 @@ struct HttpClient::Impl : public RefCounted } inline cpr::Response Delete() { + ZEN_TRACE_CPU("HttpClient::Impl::Delete"); cpr::Response Result = CprSession->Delete(); ZEN_TRACE("DELETE {}", Result); return Result; @@ -620,6 +639,7 @@ HttpClient::Impl::Impl(LoggerRef Log) : m_Log(Log) HttpClient::Impl::~Impl() { + ZEN_TRACE_CPU("HttpClient::Impl::~Impl"); m_SessionLock.WithExclusiveLock([&] { for (auto CprSession : m_Sessions) { @@ -638,6 +658,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, const std::string_view SessionId, std::optional<HttpClientAccessToken> AccessToken) { + ZEN_TRACE_CPU("HttpClient::Impl::AllocSession"); cpr::Session* CprSession = nullptr; m_SessionLock.WithExclusiveLock([&] { if (!m_Sessions.empty()) @@ -694,6 +715,7 @@ HttpClient::Impl::AllocSession(const std::string_view BaseUrl, void HttpClient::Impl::ReleaseSession(cpr::Session* CprSession) { + ZEN_TRACE_CPU("HttpClient::Impl::ReleaseSession"); CprSession->SetUrl({}); CprSession->SetHeader({}); CprSession->SetBody({}); @@ -718,6 +740,7 @@ HttpClient::~HttpClient() bool HttpClient::Authenticate() { + ZEN_TRACE_CPU("HttpClient::Authenticate"); std::optional<HttpClientAccessToken> Token = GetAccessToken(); if (!Token) { @@ -729,6 +752,7 @@ HttpClient::Authenticate() const std::optional<HttpClientAccessToken> HttpClient::GetAccessToken() { + ZEN_TRACE_CPU("HttpClient::GetAccessToken"); if (!m_ConnectionSettings.AccessTokenProvider.has_value()) { return {}; diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index 74da9ab05..05a23d675 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -73,9 +73,11 @@ struct fmt::formatter<cpr::Response> if (zen::IsHttpSuccessCode(Response.status_code)) { return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}", + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}", Response.url.str(), Response.status_code, + Response.error.message, + int(Response.error.code), Response.uploaded_bytes, Response.downloaded_bytes, NiceResponseTime.c_str()); @@ -92,29 +94,35 @@ struct fmt::formatter<cpr::Response> zen::ExtendableStringBuilder<256> Sb; std::string_view Json = Obj.ToJson(Sb).ToView(); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Json, - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}). Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Json, + Response.reason); } else { zen::BodyLogFormatter Body(Response.text); - return fmt::format_to(Ctx.out(), - "Url: {}, Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", - Response.url.str(), - Response.status_code, - Response.uploaded_bytes, - Response.downloaded_bytes, - NiceResponseTime.c_str(), - Body.GetText(), - Response.reason); + return fmt::format_to( + Ctx.out(), + "Url: {}, Status: {}, Error: '{}' ({}), Bytes: {}/{} (Up/Down), Elapsed: {}, Response: '{}', Reason: '{}'", + Response.url.str(), + Response.status_code, + Response.error.message, + int(Response.error.code), + Response.uploaded_bytes, + Response.downloaded_bytes, + NiceResponseTime.c_str(), + Body.GetText(), + Response.reason); } } } diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index ae80851e4..9d423ecbc 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -279,11 +279,10 @@ FormatPackageMessageInternal(const CbPackage& Data, FormatFlags Flags, void* Tar { IoBuffer ObjIoBuffer = AttachmentObject.GetBuffer().AsIoBuffer(); ZEN_ASSERT(ObjIoBuffer.GetSize() > 0); - ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); - *AttachmentInfo++ = {.PayloadSize = ObjIoBuffer.Size(), .Flags = CbAttachmentEntry::kIsObject, .AttachmentHash = Attachment.GetHash()}; + ResponseBuffers.emplace_back(std::move(ObjIoBuffer)); } else if (const CompositeBuffer& AttachmentBinary = Attachment.AsCompositeBinary()) { @@ -500,30 +499,25 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint { if (Entry.Flags & CbAttachmentEntry::kIsObject) { + CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); + if (!CompBuf) + { + // First payload is always a compact binary object + MalformedAttachments.push_back( + std::make_pair(i, + fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}", + AttachmentBuffer.GetSize(), + Entry.AttachmentHash))); + } + CbObject AttachmentObject = LoadCompactBinaryObject(std::move(CompBuf)); if (i == 0) { - CompressedBuffer CompBuf(CompressedBuffer::FromCompressedNoValidate(IoBuffer(AttachmentBuffer))); - if (CompBuf) - { - Package.SetObject(LoadCompactBinaryObject(std::move(CompBuf))); - } - else - { - // First payload is always a compact binary object - MalformedAttachments.push_back( - std::make_pair(i, - fmt::format("Invalid format, expected compressed buffer for CbObject (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); - } + // First payload is always a compact binary object + Package.SetObject(AttachmentObject); } else { - MalformedAttachments.push_back(std::make_pair( - i, - fmt::format("Invalid format, compressed object attachments are not currently supported (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); + Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash)); } } else @@ -547,17 +541,14 @@ ParsePackageMessage(IoBuffer Payload, std::function<IoBuffer(const IoHash&, uint { if (Entry.Flags & CbAttachmentEntry::kIsObject) { + CbObject AttachmentObject = LoadCompactBinaryObject(AttachmentBuffer); if (i == 0) { - Package.SetObject(LoadCompactBinaryObject(AttachmentBuffer)); + Package.SetObject(AttachmentObject); } else { - MalformedAttachments.push_back( - std::make_pair(i, - fmt::format("Invalid format, object attachments are not currently supported (size {}) for {}", - AttachmentBuffer.GetSize(), - Entry.AttachmentHash))); + Attachments.emplace_back(CbAttachment(AttachmentObject, Entry.AttachmentHash)); } } else if (AttachmentSize > 0) diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 027a35998..78a735ea0 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -3306,18 +3306,18 @@ GenerateFolderContent(const std::filesystem::path& RootPath) std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -3473,7 +3473,7 @@ TEST_CASE("workspaces.create") while (true) { std::error_code Ec; - std::filesystem::remove_all(Root2Path / Share2Path, Ec); + DeleteDirectories(Root2Path / Share2Path, Ec); if (!Ec) break; } @@ -3630,7 +3630,7 @@ TEST_CASE("workspaces.lifetimes") } // Wipe system config - std::filesystem::remove_all(SystemRootPath); + DeleteDirectories(SystemRootPath); // Restart @@ -3696,8 +3696,8 @@ TEST_CASE("workspaces.share") uint64_t Size = FileObject["size"sv].AsUInt64(); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); - CHECK(std::filesystem::file_size(AbsFilePath) == Size); + CHECK(IsFile(AbsFilePath)); + CHECK(FileSizeFromPath(AbsFilePath) == Size); Files.insert_or_assign(ChunkId, std::make_pair(AbsFilePath, Size)); } } @@ -3720,7 +3720,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } } @@ -3740,7 +3740,7 @@ TEST_CASE("workspaces.share") CHECK(ChunkId != Oid::Zero); std::u8string_view Path = FileObject["clientpath"sv].AsU8String(); std::filesystem::path AbsFilePath = SharePath / Path; - CHECK(std::filesystem::is_regular_file(AbsFilePath)); + CHECK(IsFile(AbsFilePath)); } } diff --git a/src/zenserver/admin/admin.cpp b/src/zenserver/admin/admin.cpp index 2888f5450..73166e608 100644 --- a/src/zenserver/admin/admin.cpp +++ b/src/zenserver/admin/admin.cpp @@ -20,6 +20,7 @@ #include <zenstore/cidstore.h> #include <zenstore/gc.h> +#include <zenstore/buildstore/buildstore.h> #include <zenstore/cache/structuredcachestore.h> #include <zenutil/workerpools.h> #include "config.h" @@ -39,7 +40,7 @@ struct DirStats DirStats GetStatsForDirectory(std::filesystem::path Dir) { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) return {}; struct StatsTraversal : public GetDirectoryContentVisitor @@ -105,6 +106,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) @@ -112,6 +114,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, , m_CacheStore(CacheStore) , m_CidStore(CidStore) , m_ProjectStore(ProjectStore) +, m_BuildStore(BuildStore) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { @@ -306,6 +309,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, Response << "Interval" << ToTimeSpan(State.Config.Interval); Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); @@ -401,6 +405,14 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } } + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt<uint64_t>(Param)) @@ -782,6 +794,10 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, { m_ProjectStore->Flush(); } + if (m_BuildStore) + { + m_BuildStore->Flush(); + } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); diff --git a/src/zenserver/admin/admin.h b/src/zenserver/admin/admin.h index 563c4f536..e7821dead 100644 --- a/src/zenserver/admin/admin.h +++ b/src/zenserver/admin/admin.h @@ -12,6 +12,7 @@ class JobQueue; class ZenCacheStore; class CidStore; class ProjectStore; +class BuildStore; struct ZenServerOptions; class HttpAdminService : public zen::HttpService @@ -28,6 +29,7 @@ public: ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, + BuildStore* BuildStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions); ~HttpAdminService(); @@ -42,6 +44,7 @@ private: ZenCacheStore* m_CacheStore; CidStore* m_CidStore; ProjectStore* m_ProjectStore; + BuildStore* m_BuildStore; LogPaths m_LogPaths; const ZenServerOptions& m_ServerOptions; }; diff --git a/src/zenserver/buildstore/httpbuildstore.cpp b/src/zenserver/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..c918f5683 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.cpp @@ -0,0 +1,539 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenhttp/packageformat.h> +#include <zenstore/buildstore/buildstore.h> +#include <zenutil/workerpools.h> + +#include <numeric> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_Router.AddPattern("namespace", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize >= BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector<IoHash> BlobHashes; + std::vector<IoBuffer> MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector<IoBuffer> BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector<CbAttachment> Attachments; + tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector<BuildStore::BlobExistsResult> BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/buildstore/httpbuildstore.h b/src/zenserver/buildstore/httpbuildstore.h new file mode 100644 index 000000000..50cb5db12 --- /dev/null +++ b/src/zenserver/buildstore/httpbuildstore.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> + +#include <filesystem> + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index 809092378..52f539dcd 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -377,6 +377,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + ////// network LuaOptions.AddOption("network.httpserverclass"sv, ServerOptions.HttpServerConfig.ServerClass, "http"sv); LuaOptions.AddOption("network.httpserverthreads"sv, ServerOptions.HttpServerConfig.ThreadCount, "http-threads"sv); @@ -492,6 +495,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); ////// security LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); @@ -960,6 +966,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) options.add_option("gc", "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + + options.add_option("gc", + "", "disk-reserve-size", "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), @@ -1031,6 +1044,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<std::vector<std::string>>(BucketConfigs), ""); + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("stats", "", "statsd", @@ -1096,7 +1116,7 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) if (DataDir.empty()) throw zen::OptionParseException("You must explicitly specify a data directory when specifying a base snapshot"); - if (!std::filesystem::is_directory(ServerOptions.BaseSnapshotDir)) + if (!IsDir(ServerOptions.BaseSnapshotDir)) throw OptionParseException(fmt::format("Snapshot directory must be a directory: '{}", BaseSnapshotDir)); } diff --git a/src/zenserver/config.h b/src/zenserver/config.h index c7781aada..a87b6f8b3 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -59,11 +59,17 @@ struct ZenProjectStoreEvictionPolicy int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; }; +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + struct ZenGcConfig { // ZenCasEvictionPolicy Cas; ZenCacheEvictionPolicy Cache; ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; int32_t MonitorIntervalSeconds = 30; int32_t IntervalSeconds = 0; bool CollectSmallObjects = true; @@ -130,6 +136,11 @@ struct ZenProjectStoreConfig bool StoreProjectAttachmentMetaData = false; }; +struct ZenBuildStoreConfig +{ + bool Enabled = false; +}; + struct ZenWorkspacesConfig { bool Enabled = false; @@ -145,6 +156,7 @@ struct ZenServerOptions zen::HttpServerConfig HttpServerConfig; ZenStructuredCacheConfig StructuredCacheConfig; ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; ZenStatsConfig StatsConfig; ZenWorkspacesConfig WorksSpacesConfig; std::filesystem::path SystemRootDir; // System root directory (used for machine level config) diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 31d9e1c94..dfa710ae0 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -2,6 +2,7 @@ #include "frontend.h" +#include <zencore/compactbinarybuilder.h> #include <zencore/endian.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> @@ -26,7 +27,9 @@ static unsigned char gHtmlZipData[] = { namespace zen { //////////////////////////////////////////////////////////////////////////////// -HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Directory(Directory) +HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService) +: m_Directory(Directory) +, m_StatusService(StatusService) { std::filesystem::path SelfPath = GetRunningExecutablePath(); @@ -50,7 +53,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { break; } - if (std::filesystem::is_regular_file(ParentPath / "xmake.lua", ErrorCode)) + if (IsFile(ParentPath / "xmake.lua", ErrorCode)) { if (ErrorCode) { @@ -59,7 +62,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di std::filesystem::path HtmlDir = ParentPath / "src" / "zenserver" / "frontend" / "html"; - if (std::filesystem::is_directory(HtmlDir, ErrorCode)) + if (IsDir(HtmlDir, ErrorCode)) { m_Directory = HtmlDir; } @@ -81,10 +84,12 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory) : m_Di { ZEN_INFO("front-end is NOT AVAILABLE"); } + m_StatusService.RegisterHandler("dashboard", *this); } HttpFrontendService::~HttpFrontendService() { + m_StatusService.UnregisterHandler("dashboard", *this); } const char* @@ -95,6 +100,14 @@ HttpFrontendService::BaseUri() const //////////////////////////////////////////////////////////////////////////////// void +HttpFrontendService::HandleStatusRequest(zen::HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { using namespace std::literals; diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 6eac20620..84ffaac42 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -3,23 +3,26 @@ #pragma once #include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> #include "zipfs.h" #include <filesystem> namespace zen { -class HttpFrontendService final : public zen::HttpService +class HttpFrontendService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpFrontendService(std::filesystem::path Directory); + HttpFrontendService(std::filesystem::path Directory, HttpStatusService& StatusService); virtual ~HttpFrontendService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: ZipFs m_ZipFs; std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 65a3ef39b..f127cb0a3 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -59,6 +59,62 @@ export class Page extends ZenPage } } + _find_iohash_field(container, name) + { + const found_field = container.find(name); + if (found_field != undefined) + { + var found_value = found_field.as_value(); + if (found_value instanceof Uint8Array) + { + var ret = ""; + for (var x of found_value) + ret += x.toString(16).padStart(2, "0"); + return ret; + } + } + return null; + } + + async _build_meta(section, entry) + { + var tree = {} + const cookart = this._find_iohash_field(entry, "CookPackageArtifacts"); + if (cookart != null) + { + tree["cook"] = { CookPackageArtifacts: cookart}; + } + + if (Object.keys(tree).length == 0) + return; + + const sub_section = section.add_section("meta"); + + for (const cat_name in tree) + { + const cat_section = sub_section.add_section(cat_name); + const table = cat_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + Object.entries(tree[cat_name]).forEach(([key, value]) => + { + const row = table.add_row(key); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + ); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, value); + }); + } + } + async _build_page() { var entry = await this._entry; @@ -78,8 +134,16 @@ export class Page extends ZenPage delete tree["$id"]; - const sub_section = section.add_section("deps"); - this._build_deps(sub_section, tree); + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("deps"); + this._build_deps(sub_section, tree); + } + } + + // meta + { + this._build_meta(section, entry); } // data @@ -128,7 +192,6 @@ export class Page extends ZenPage ); link.first_child().attr("download", `${io_hash}_${base_name}`); - const do_nothing = () => void(0); const action_tb = new Toolbar(row.get_cell(-1), true); action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index d5419d342..78ddd39a0 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -406,17 +406,17 @@ main(int argc, char* argv[]) if (!DeleteReason.empty()) { - if (std::filesystem::exists(ServerOptions.DataDir)) + if (IsDir(ServerOptions.DataDir)) { ZEN_CONSOLE_INFO("deleting files from '{}' ({})", ServerOptions.DataDir, DeleteReason); DeleteDirectories(ServerOptions.DataDir); } } - if (!std::filesystem::exists(ServerOptions.DataDir)) + if (!IsDir(ServerOptions.DataDir)) { ServerOptions.IsFirstRun = true; - std::filesystem::create_directories(ServerOptions.DataDir); + CreateDirectories(ServerOptions.DataDir); } if (!ServerOptions.BaseSnapshotDir.empty()) diff --git a/src/zenserver/objectstore/objectstore.cpp b/src/zenserver/objectstore/objectstore.cpp index e757ef84e..8faf12165 100644 --- a/src/zenserver/objectstore/objectstore.cpp +++ b/src/zenserver/objectstore/objectstore.cpp @@ -219,13 +219,17 @@ private: StringBuilderBase& Builder; }; -HttpObjectStoreService::HttpObjectStoreService(ObjectStoreConfig Cfg) : m_Cfg(std::move(Cfg)) +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) { Inititalize(); + m_StatusService.RegisterHandler("obj", *this); } HttpObjectStoreService::~HttpObjectStoreService() { + m_StatusService.UnregisterHandler("obj", *this); } const char* @@ -245,13 +249,21 @@ HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) } void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpObjectStoreService::Inititalize() { namespace fs = std::filesystem; ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; - if (!fs::exists(BucketsPath)) + if (!IsDir(BucketsPath)) { CreateDirectories(BucketsPath); } @@ -324,7 +336,7 @@ HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; { std::lock_guard _(BucketsMutex); - if (!fs::exists(BucketPath)) + if (!IsDir(BucketPath)) { CreateDirectories(BucketPath); ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); @@ -406,7 +418,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); FileSystemTraversal Traversal; - if (std::filesystem::exists(FullPath)) + if (IsDir(FullPath)) { std::lock_guard _(BucketsMutex); Traversal.TraverseFileSystem(FullPath, FileVisitor); @@ -475,7 +487,7 @@ HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::st } const fs::path FilePath = BucketDir / RelativeBucketPath; - if (!fs::exists(FilePath)) + if (!IsFile(FilePath)) { ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); @@ -576,7 +588,7 @@ HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) { std::lock_guard _(BucketsMutex); - if (!fs::exists(FileDirectory)) + if (!IsDir(FileDirectory)) { CreateDirectories(FileDirectory); } diff --git a/src/zenserver/objectstore/objectstore.h b/src/zenserver/objectstore/objectstore.h index dae979c4c..44e50e208 100644 --- a/src/zenserver/objectstore/objectstore.h +++ b/src/zenserver/objectstore/objectstore.h @@ -3,6 +3,7 @@ #pragma once #include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> #include <atomic> #include <filesystem> #include <mutex> @@ -23,14 +24,15 @@ struct ObjectStoreConfig std::vector<BucketConfig> Buckets; }; -class HttpObjectStoreService final : public zen::HttpService +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider { public: - HttpObjectStoreService(ObjectStoreConfig Cfg); + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); virtual ~HttpObjectStoreService(); virtual const char* BaseUri() const override; virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: void Inititalize(); @@ -41,6 +43,7 @@ private: void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); void PutObject(zen::HttpRouterRequest& Request); + HttpStatusService& m_StatusService; ObjectStoreConfig m_Cfg; std::mutex BucketsMutex; HttpRequestRouter m_Router; diff --git a/src/zenserver/projectstore/buildsremoteprojectstore.cpp b/src/zenserver/projectstore/buildsremoteprojectstore.cpp index fbb9bc344..a6583b722 100644 --- a/src/zenserver/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenserver/projectstore/buildsremoteprojectstore.cpp @@ -308,7 +308,7 @@ public: { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client()); - JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId); + JupiterResult FindResult = Session.FindBlocks(m_Namespace, m_Bucket, m_BuildId, (uint64_t)-1); AddStats(FindResult); GetKnownBlocksResult Result{ConvertResult(FindResult)}; if (Result.ErrorCode) diff --git a/src/zenserver/projectstore/fileremoteprojectstore.cpp b/src/zenserver/projectstore/fileremoteprojectstore.cpp index 98e292d91..375e44e59 100644 --- a/src/zenserver/projectstore/fileremoteprojectstore.cpp +++ b/src/zenserver/projectstore/fileremoteprojectstore.cpp @@ -73,7 +73,7 @@ public: ContainerObject.IterateAttachments([&](CbFieldView FieldView) { IoHash AttachmentHash = FieldView.AsBinaryAttachment(); std::filesystem::path AttachmentPath = GetAttachmentPath(AttachmentHash); - if (!std::filesystem::exists(AttachmentPath)) + if (!IsFile(AttachmentPath)) { Result.Needs.insert(AttachmentHash); } @@ -111,7 +111,7 @@ public: Stopwatch Timer; SaveAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::exists(ChunkPath)) + if (!IsFile(ChunkPath)) { try { @@ -182,7 +182,7 @@ public: for (const IoHash& RawHash : BlockHashes) { std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (std::filesystem::is_regular_file(ChunkPath)) + if (IsFile(ChunkPath)) { ExistingBlockHashes.push_back(RawHash); } @@ -203,7 +203,7 @@ public: Stopwatch Timer; LoadAttachmentResult Result; std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!std::filesystem::is_regular_file(ChunkPath)) + if (!IsFile(ChunkPath)) { Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); @@ -246,7 +246,7 @@ private: LoadContainerResult Result; std::filesystem::path SourcePath = m_OutputPath; SourcePath.append(Name); - if (!std::filesystem::is_regular_file(SourcePath)) + if (!IsFile(SourcePath)) { Result.ErrorCode = gsl::narrow<int>(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index 47748dd90..317a419eb 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -8,6 +8,7 @@ #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> #include <zencore/compactbinaryutil.h> +#include <zencore/compactbinaryvalidation.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> @@ -234,10 +235,15 @@ namespace { ////////////////////////////////////////////////////////////////////////// -HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, HttpStatsService& StatsService, AuthMgr& AuthMgr) +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr) : m_Log(logging::Get("project")) , m_CidStore(Store) , m_ProjectStore(Projects) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_AuthMgr(AuthMgr) { @@ -245,8 +251,6 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, using namespace std::literals; - m_StatsService.RegisterHandler("prj", *this); - m_Router.AddPattern("project", "([[:alnum:]_.]+)"); m_Router.AddPattern("log", "([[:alnum:]_.]+)"); m_Router.AddPattern("op", "([[:digit:]]+?)"); @@ -365,11 +369,15 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, "details\\$/{project}/{log}/{chunk}", [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); } HttpProjectService::~HttpProjectService() { m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); } const char* @@ -465,6 +473,15 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) } void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) { ZEN_TRACE_CPU("ProjectService::ProjectList"); @@ -885,10 +902,63 @@ HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) case HttpVerb::kGet: { IoBuffer Value; - std::pair<HttpResponseCode, std::string> Result = - m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, AcceptType, Value, nullptr); + std::pair<HttpResponseCode, std::string> Result = m_ProjectStore->GetChunk(ProjectId, OplogId, Cid, Value, nullptr); if (Result.first == HttpResponseCode::OK) { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (DecompressedBuffer) + { + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } + else + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); + } + } m_ProjectStats.ChunkHitCount++; return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); } diff --git a/src/zenserver/projectstore/httpprojectstore.h b/src/zenserver/projectstore/httpprojectstore.h index 8e74c57a5..295defa5c 100644 --- a/src/zenserver/projectstore/httpprojectstore.h +++ b/src/zenserver/projectstore/httpprojectstore.h @@ -5,6 +5,7 @@ #include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> #include <zenstore/cidstore.h> namespace zen { @@ -31,16 +32,21 @@ class ProjectStore; // refs: // -class HttpProjectService : public HttpService, public IHttpStatsProvider +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpProjectService(CidStore& Store, ProjectStore* InProjectStore, HttpStatsService& StatsService, AuthMgr& AuthMgr); + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr); ~HttpProjectService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct ProjectStats @@ -89,6 +95,7 @@ private: CidStore& m_CidStore; HttpRequestRouter m_Router; Ref<ProjectStore> m_ProjectStore; + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; AuthMgr& m_AuthMgr; ProjectStats m_ProjectStats; diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 86791e29a..1966eeef9 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -58,7 +58,7 @@ namespace { std::filesystem::path DroppedBucketPath; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return true; } @@ -68,7 +68,7 @@ namespace { std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), MovedId); DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { if (!DeleteDirectories(DroppedBucketPath)) { @@ -77,7 +77,7 @@ namespace { Dir); continue; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Drop directory '{}' for '{}' still exists after remove, attempting different name.", DroppedBucketPath, Dir); continue; @@ -88,13 +88,13 @@ namespace { do { std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { OutDeleteDir = DroppedBucketPath; return true; } - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { ZEN_INFO("Can't rename '{}' to still existing drop directory '{}'. Reason: '{}'. Attempting different name.", Dir, @@ -486,7 +486,7 @@ struct ProjectStore::OplogStorage : public RefCounted [[nodiscard]] bool Exists() const { return Exists(m_OplogStoragePath); } [[nodiscard]] static bool Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(GetLogPath(BasePath)) && std::filesystem::exists(GetBlobsPath(BasePath)); + return IsFile(GetLogPath(BasePath)) && IsFile(GetBlobsPath(BasePath)); } [[nodiscard]] bool IsValid() const { return IsValid(m_OplogStoragePath); } [[nodiscard]] static bool IsValid(const std::filesystem::path& BasePath) @@ -496,13 +496,13 @@ struct ProjectStore::OplogStorage : public RefCounted void WipeState() const { std::error_code Ec; - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); } static bool Delete(const std::filesystem::path& BasePath) { return DeleteDirectories(BasePath); } - uint64_t OpBlobsSize() const { return std::filesystem::file_size(GetBlobsPath()); } + uint64_t OpBlobsSize() const { return FileSizeFromPath(GetBlobsPath()); } uint64_t OpsSize() const { return OpsSize(m_OplogStoragePath); } static uint64_t OpsSize(const std::filesystem::path& BasePath) @@ -510,7 +510,7 @@ struct ProjectStore::OplogStorage : public RefCounted if (Exists(BasePath)) { std::error_code DummyEc; - return std::filesystem::file_size(GetLogPath(BasePath)) + std::filesystem::file_size(GetBlobsPath(BasePath)); + return FileSizeFromPath(GetLogPath(BasePath)) + FileSizeFromPath(GetBlobsPath(BasePath)); } return 0; } @@ -689,7 +689,7 @@ struct ProjectStore::OplogStorage : public RefCounted m_OpBlobs.Close(); Oplog.Close(); - std::filesystem::rename(OplogPath, GetLogPath(), Ec); + RenameFile(OplogPath, GetLogPath(), Ec); if (Ec) { throw std::system_error( @@ -702,9 +702,9 @@ struct ProjectStore::OplogStorage : public RefCounted if (Ec) { // We failed late - clean everything up as best we can - std::filesystem::remove(OpBlobs.GetPath(), Ec); - std::filesystem::remove(GetLogPath(), Ec); - std::filesystem::remove(GetBlobsPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); + RemoveFile(GetLogPath(), Ec); + RemoveFile(GetBlobsPath(), Ec); throw std::system_error(Ec, fmt::format("Oplog::Compact failed to rename temporary oplog file from '{}' to '{}'", OpBlobs.GetPath(), @@ -739,7 +739,7 @@ struct ProjectStore::OplogStorage : public RefCounted } catch (const std::exception& /*Ex*/) { - std::filesystem::remove(OpBlobs.GetPath(), Ec); + RemoveFile(OpBlobs.GetPath(), Ec); throw; } } @@ -1108,7 +1108,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, ZEN_WARN("Invalid oplog found at '{}'. Wiping state for oplog.", m_BasePath); m_Storage->WipeState(); std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } } m_Storage->Open(/* IsCreate */ !StoreExists); @@ -1116,7 +1116,7 @@ ProjectStore::Oplog::Oplog(std::string_view Id, m_MetaPath = m_BasePath / "ops.meta"sv; m_MetaValid = !IsFileOlderThan(m_MetaPath, m_Storage->GetBlobsPath()); - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); } ProjectStore::Oplog::~Oplog() @@ -1142,7 +1142,7 @@ ProjectStore::Oplog::Flush() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } uint64_t LogCount = m_Storage->LogCount(); @@ -1238,19 +1238,19 @@ ProjectStore::Oplog::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = OplogStorage::OpsSize(BasePath); std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::exists(StateFilePath)) + if (IsFile(StateFilePath)) { - Size += std::filesystem::file_size(StateFilePath); + Size += FileSizeFromPath(StateFilePath); } std::filesystem::path MetaFilePath = BasePath / "ops.meta"sv; - if (std::filesystem::exists(MetaFilePath)) + if (IsFile(MetaFilePath)) { - Size += std::filesystem::file_size(MetaFilePath); + Size += FileSizeFromPath(MetaFilePath); } std::filesystem::path IndexFilePath = BasePath / "ops.zidx"sv; - if (std::filesystem::exists(IndexFilePath)) + if (IsFile(IndexFilePath)) { - Size += std::filesystem::file_size(IndexFilePath); + Size += FileSizeFromPath(IndexFilePath); } return Size; @@ -1303,7 +1303,7 @@ ProjectStore::Oplog::ExistsAt(const std::filesystem::path& BasePath) using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - return std::filesystem::is_regular_file(StateFilePath); + return IsFile(StateFilePath); } bool @@ -1337,7 +1337,7 @@ ProjectStore::Oplog::Read() if (!m_MetaValid) { std::error_code DummyEc; - std::filesystem::remove(m_MetaPath, DummyEc); + RemoveFile(m_MetaPath, DummyEc); } ReadIndexSnapshot(); @@ -1438,7 +1438,7 @@ ProjectStore::Oplog::Reset() m_Storage = new OplogStorage(this, m_BasePath); m_Storage->Open(true); m_MetaValid = false; - CleanDirectory(m_TempPath); + CleanDirectory(m_TempPath, /*ForceRemoveReadOnlyFiles*/ false); Write(); } // Erase content on disk @@ -1457,7 +1457,7 @@ ProjectStore::Oplog::ReadStateFile(const std::filesystem::path& BasePath, std::f using namespace std::literals; std::filesystem::path StateFilePath = BasePath / "oplog.zcb"sv; - if (std::filesystem::is_regular_file(StateFilePath)) + if (IsFile(StateFilePath)) { // ZEN_INFO("oplog '{}/{}': config read from '{}'", m_OuterProject->Identifier, m_OplogId, StateFilePath); @@ -1536,7 +1536,7 @@ ProjectStore::Oplog::Validate(std::atomic_bool& IsCancelledFlag, WorkerThreadPoo if (File.Hash == IoHash::Zero) { std::filesystem::path FilePath = m_OuterProject->RootDir / File.ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { ResultLock.WithExclusiveLock([&]() { Result.MissingFiles.push_back({KeyHash, File}); }); HasMissingEntries = true; @@ -1625,10 +1625,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() fs::path TempIndexPath = m_BasePath / "ops.zidx.tmp"; // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to clean up temp snapshot at {}, reason: '{}'", GetOuterProject()->Identifier, @@ -1641,9 +1641,9 @@ ProjectStore::Oplog::WriteIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -1778,11 +1778,11 @@ ProjectStore::Oplog::WriteIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to restore old snapshot from {}, reason: '{}'", @@ -1793,10 +1793,10 @@ ProjectStore::Oplog::WriteIndexSnapshot() } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("oplog '{}/{}': snapshot failed to remove temporary file {}, reason: '{}'", m_OuterProject->Identifier, @@ -1814,7 +1814,7 @@ ProjectStore::Oplog::ReadIndexSnapshot() ZEN_TRACE_CPU("Oplog::ReadIndexSnapshot"); std::filesystem::path IndexPath = m_BasePath / "ops.zidx"; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint64_t EntryCount = 0; Stopwatch Timer; @@ -3133,7 +3133,7 @@ ProjectStore::Project::~Project() bool ProjectStore::Project::Exists(const std::filesystem::path& BasePath) { - return std::filesystem::exists(BasePath / "Project.zcb"); + return IsFile(BasePath / "Project.zcb"); } void @@ -3207,7 +3207,7 @@ ProjectStore::Project::ReadAccessTimes() using namespace std::literals; std::filesystem::path ProjectAccessTimesFilePath = m_OplogStoragePath / "AccessTimes.zcb"sv; - if (!std::filesystem::exists(ProjectAccessTimesFilePath)) + if (!IsFile(ProjectAccessTimesFilePath)) { return; } @@ -3598,14 +3598,14 @@ ProjectStore::Project::TotalSize(const std::filesystem::path& BasePath) uint64_t Size = 0; std::filesystem::path AccessTimesFilePath = BasePath / "AccessTimes.zcb"sv; - if (std::filesystem::exists(AccessTimesFilePath)) + if (IsFile(AccessTimesFilePath)) { - Size += std::filesystem::file_size(AccessTimesFilePath); + Size += FileSizeFromPath(AccessTimesFilePath); } std::filesystem::path ProjectFilePath = BasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectFilePath)) + if (IsFile(ProjectFilePath)) { - Size += std::filesystem::file_size(ProjectFilePath); + Size += FileSizeFromPath(ProjectFilePath); } return Size; @@ -3717,7 +3717,7 @@ ProjectStore::Project::IsExpired(const std::string& EntryName, if (!MarkerPath.empty()) { std::error_code Ec; - if (std::filesystem::exists(MarkerPath, Ec)) + if (IsFile(MarkerPath, Ec)) { if (Ec) { @@ -3870,7 +3870,7 @@ void ProjectStore::DiscoverProjects() { ZEN_MEMSCOPE(GetProjectstoreTag()); - if (!std::filesystem::exists(m_ProjectBasePath)) + if (!IsDir(m_ProjectBasePath)) { return; } @@ -3979,7 +3979,7 @@ ProjectStore::StorageSize() const GcStorageSize Result; { - if (std::filesystem::exists(m_ProjectBasePath)) + if (IsDir(m_ProjectBasePath)) { DirectoryContent ProjectsFolderContent; GetDirectoryContent(m_ProjectBasePath, DirectoryContentFlags::IncludeDirs, ProjectsFolderContent); @@ -3987,7 +3987,7 @@ ProjectStore::StorageSize() const for (const std::filesystem::path& ProjectBasePath : ProjectsFolderContent.Directories) { std::filesystem::path ProjectStateFilePath = ProjectBasePath / "Project.zcb"sv; - if (std::filesystem::exists(ProjectStateFilePath)) + if (IsFile(ProjectStateFilePath)) { Result.DiskSize += Project::TotalSize(ProjectBasePath); DirectoryContent DirContent; @@ -4770,7 +4770,6 @@ std::pair<HttpResponseCode, std::string> ProjectStore::GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag) { @@ -4812,16 +4811,7 @@ ProjectStore::GetChunk(const std::string_view ProjectId, } } - if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary) - { - CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(OutChunk)); - OutChunk = Compressed.Decompress().AsIoBuffer(); - OutChunk.SetContentType(ZenContentType::kBinary); - } - else - { - OutChunk.SetContentType(ZenContentType::kCompressedBinary); - } + OutChunk.SetContentType(ZenContentType::kCompressedBinary); return {HttpResponseCode::OK, {}}; } @@ -7253,7 +7243,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project1FilePath); + RemoveFile(Project1FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7282,7 +7272,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2Oplog1Path); + RemoveFile(Project2Oplog1Path); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() - std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() - std::chrono::hours(24), @@ -7311,7 +7301,7 @@ TEST_CASE("project.store.gc") CHECK(ProjectStore.OpenProject("proj2"sv)); } - std::filesystem::remove(Project2FilePath); + RemoveFile(Project2FilePath); { GcSettings Settings = {.CacheExpireTime = GcClock::Now() + std::chrono::hours(24), .ProjectStoreExpireTime = GcClock::Now() + std::chrono::hours(24), @@ -8035,7 +8025,7 @@ TEST_CASE("project.store.rpc.getchunks") CompositeBuffer Buffer = Attachment->AsCompositeBinary(); CHECK_EQ(IoHash::HashBuffer(IoBuffer(ReadFile(FilesOpIdAttachments[0].second).Flatten(), 81823, 5434)), IoHash::HashBuffer(Buffer)); - CHECK_EQ(Chunk["Size"sv].AsUInt64(), std::filesystem::file_size(FilesOpIdAttachments[0].second)); + CHECK_EQ(Chunk["Size"sv].AsUInt64(), FileSizeFromPath(FilesOpIdAttachments[0].second)); CHECK(!Chunk.FindView("RawSize")); } { @@ -8516,12 +8506,7 @@ TEST_CASE("project.store.partial.read") uint64_t ModificationTag = 0; IoBuffer Chunk; CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::OK); IoHash RawHash; uint64_t RawSize; @@ -8530,12 +8515,7 @@ TEST_CASE("project.store.partial.read") CHECK(ModificationTag != 0); CHECK(ProjectStore - .GetChunk("proj1"sv, - "oplog1"sv, - Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), - HttpContentType::kCompressedBinary, - Chunk, - &ModificationTag) + .GetChunk("proj1"sv, "oplog1"sv, Attachments[OpIds[1]][0].second.DecodeRawHash().ToHexString(), Chunk, &ModificationTag) .first == HttpResponseCode::NotModified); } diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8f2d3ce0d..368da5ea4 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -449,7 +449,6 @@ public: std::pair<HttpResponseCode, std::string> GetChunk(const std::string_view ProjectId, const std::string_view OplogId, const std::string_view Cid, - ZenContentType AcceptType, IoBuffer& OutChunk, uint64_t* OptionalInOutModificationTag); diff --git a/src/zenserver/projectstore/remoteprojectstore.cpp b/src/zenserver/projectstore/remoteprojectstore.cpp index a7263da83..f96b3e185 100644 --- a/src/zenserver/projectstore/remoteprojectstore.cpp +++ b/src/zenserver/projectstore/remoteprojectstore.cpp @@ -1212,7 +1212,7 @@ BuildContainer(CidStore& ChunkStore, { std::string_view ServerPath = View["serverpath"sv].AsString(); std::filesystem::path FilePath = Project.RootDir / ServerPath; - if (!std::filesystem::is_regular_file(FilePath)) + if (!IsFile(FilePath)) { remotestore_impl::ReportMessage( OptionalContext, @@ -3083,9 +3083,9 @@ LoadOplog(CidStore& ChunkStore, OptionalContext]() { auto _ = MakeGuard([&DechunkLatch, &TempFileName] { std::error_code Ec; - if (std::filesystem::exists(TempFileName, Ec)) + if (IsFile(TempFileName, Ec)) { - std::filesystem::remove(TempFileName, Ec); + RemoveFile(TempFileName, Ec); if (Ec) { ZEN_INFO("Failed to remove temporary file '{}'. Reason: {}", TempFileName, Ec.message()); diff --git a/src/zenserver/vfs/vfsservice.cpp b/src/zenserver/vfs/vfsservice.cpp index d302a10ec..bf761f8d1 100644 --- a/src/zenserver/vfs/vfsservice.cpp +++ b/src/zenserver/vfs/vfsservice.cpp @@ -61,7 +61,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { m_Impl = new Impl; @@ -136,10 +136,12 @@ VfsService::VfsService() } }, HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); } VfsService::~VfsService() { + m_StatusService.UnregisterHandler("vfs", *this); delete m_Impl; } @@ -169,8 +171,9 @@ VfsService::AddService(Ref<ZenCacheStore>&& Z$) #else -VfsService::VfsService() +VfsService::VfsService(HttpStatusService& StatusService) : m_StatusService(StatusService) { + ZEN_UNUSED(StatusService); } VfsService::~VfsService() @@ -209,6 +212,14 @@ VfsService::BaseUri() const } void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) { m_Router.HandleRequest(HttpServiceRequest); diff --git a/src/zenserver/vfs/vfsservice.h b/src/zenserver/vfs/vfsservice.h index dcdc71e81..0d0168e23 100644 --- a/src/zenserver/vfs/vfsservice.h +++ b/src/zenserver/vfs/vfsservice.h @@ -4,6 +4,7 @@ #include <zenbase/refcount.h> #include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> #include <zenvfs/vfs.h> #include <memory> @@ -24,10 +25,10 @@ class ZenCacheStore; */ -class VfsService : public HttpService +class VfsService : public HttpService, public IHttpStatusProvider { public: - VfsService(); + explicit VfsService(HttpStatusService& StatusService); ~VfsService(); void Mount(std::string_view MountPoint); @@ -39,12 +40,14 @@ public: protected: virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct Impl; Impl* m_Impl = nullptr; - HttpRequestRouter m_Router; + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; friend struct VfsServiceDataSource; }; diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp index 8a4b977ad..7ef84743e 100644 --- a/src/zenserver/workspaces/httpworkspaces.cpp +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -73,8 +73,12 @@ namespace { } // namespace -HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces) +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) : m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) , m_StatsService(StatsService) , m_Config(Cfg) , m_Workspaces(Workspaces) @@ -84,7 +88,8 @@ HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, con HttpWorkspacesService::~HttpWorkspacesService() { - m_StatsService.UnregisterHandler("prj", *this); + m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); } const char* @@ -149,14 +154,21 @@ HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) } void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void HttpWorkspacesService::Initialize() { using namespace std::literals; ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); - m_StatsService.RegisterHandler("ws", *this); - m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); @@ -238,6 +250,9 @@ HttpWorkspacesService::Initialize() HttpVerb::kGet); RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); } std::filesystem::path @@ -1100,7 +1115,7 @@ HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& Workspace } } - if (!std::filesystem::is_directory(Workspace.RootPath / NewConfig.SharePath)) + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) { return ServerRequest.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h index f01f58b86..89a8e8bdc 100644 --- a/src/zenserver/workspaces/httpworkspaces.h +++ b/src/zenserver/workspaces/httpworkspaces.h @@ -5,6 +5,7 @@ #include <zencore/stats.h> #include <zenhttp/httpserver.h> #include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> namespace zen { @@ -16,16 +17,20 @@ struct WorkspacesServeConfig bool AllowConfigurationChanges = false; }; -class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider { public: - HttpWorkspacesService(HttpStatsService& StatsService, const WorkspacesServeConfig& Cfg, Workspaces& Workspaces); + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); virtual ~HttpWorkspacesService(); virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: struct WorkspacesStats @@ -80,6 +85,7 @@ private: void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + HttpStatusService& m_StatusService; HttpStatsService& m_StatsService; const WorkspacesServeConfig m_Config; HttpRequestRouter m_Router; diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index f84bc0b00..c7cb2ba6e 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include <zencore/trace.h> #include <zencore/workthreadpool.h> #include <zenhttp/httpserver.h> +#include <zenstore/buildstore/buildstore.h> #include <zenstore/cidstore.h> #include <zenstore/scrubcontext.h> #include <zenstore/workspaces.h> @@ -250,18 +251,26 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen *m_JobQueue, *m_OpenProcessCache, ProjectStore::Configuration{}); - m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr}); if (ServerOptions.WorksSpacesConfig.Enabled) { m_Workspaces.reset(new Workspaces()); m_HttpWorkspacesService.reset( - new HttpWorkspacesService(m_StatsService, + new HttpWorkspacesService(m_StatusService, + m_StatsService, {.SystemRootDir = ServerOptions.SystemRootDir, .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, *m_Workspaces)); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + BuildStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "builds"; + m_BuildStore = std::make_unique<BuildStore>(std::move(ObjCfg), m_GcManager); + } + if (ServerOptions.StructuredCacheConfig.Enabled) { InitializeStructuredCache(ServerOptions); @@ -287,7 +296,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_HttpWorkspacesService); } - m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); + m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService); if (m_FrontendService) { @@ -306,12 +315,18 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen ObjCfg.Buckets.push_back(std::move(NewBucket)); } - m_ObjStoreService = std::make_unique<HttpObjectStoreService>(std::move(ObjCfg)); + m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg)); m_Http->RegisterService(*m_ObjStoreService); } + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore); + m_Http->RegisterService(*m_BuildStoreService); + } + #if ZEN_WITH_VFS - m_VfsService = std::make_unique<VfsService>(); + m_VfsService = std::make_unique<VfsService>(m_StatusService); m_VfsService->AddService(Ref<ProjectStore>(m_ProjectStore)); m_VfsService->AddService(Ref<ZenCacheStore>(m_CacheStore)); m_Http->RegisterService(*m_VfsService); @@ -327,6 +342,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, .Enabled = ServerOptions.GcConfig.Enabled, .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, @@ -347,6 +363,7 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_CacheStore.Get(), m_CidStore.get(), m_ProjectStore, + m_BuildStore.get(), HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, @@ -418,7 +435,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) { std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; - if (ManifestVersion != 0 && std::filesystem::is_regular_file(ManifestSkipSchemaChangePath)) + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) { ZEN_INFO( "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " @@ -467,7 +484,7 @@ ZenServer::InitializeState(const ZenServerOptions& ServerOptions) { ZEN_INFO("Deleting '{}'", DirEntry.path()); - std::filesystem::remove_all(DirEntry.path(), Ec); + DeleteDirectories(DirEntry.path(), Ec); if (Ec) { @@ -801,6 +818,9 @@ ZenServer::Cleanup() m_ObjStoreService.reset(); m_FrontendService.reset(); + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_StructuredCacheService.reset(); m_UpstreamService.reset(); m_UpstreamCache.reset(); @@ -895,7 +915,7 @@ ZenServer::CheckStateMarker() std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; try { - if (!std::filesystem::exists(StateMarkerPath)) + if (!IsFile(StateMarkerPath)) { ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); RequestExit(1); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 80054dc35..5cfa04ba1 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -25,6 +25,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include <zenstore/cache/structuredcachestore.h> #include <zenstore/gc.h> #include "admin/admin.h" +#include "buildstore/httpbuildstore.h" #include "cache/httpstructuredcache.h" #include "diag/diagsvcs.h" #include "frontend/frontend.h" @@ -127,6 +128,8 @@ private: Ref<ZenCacheStore> m_CacheStore; std::unique_ptr<OpenProcessCache> m_OpenProcessCache; HttpTestService m_TestService; + std::unique_ptr<BuildStore> m_BuildStore; + #if ZEN_WITH_TESTS HttpTestingService m_TestingService; #endif @@ -140,6 +143,7 @@ private: HttpHealthService m_HealthService; std::unique_ptr<HttpFrontendService> m_FrontendService; std::unique_ptr<HttpObjectStoreService> m_ObjStoreService; + std::unique_ptr<HttpBuildStoreService> m_BuildStoreService; std::unique_ptr<VfsService> m_VfsService; std::unique_ptr<JobQueue> m_JobQueue; std::unique_ptr<HttpAdminService> m_AdminService; diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index e5b312984..c56971520 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -2,6 +2,7 @@ #include <zencore/filesystem.h> #include <zencore/logging.h> +#include <zenstore/buildstore/buildstore.h> #include <zenstore/zenstore.h> #include <zencore/memory/newdelete.h> @@ -18,6 +19,7 @@ main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) zen::zenstore_forcelinktests(); zen::logging::InitializeLogging(); + zen::buildstore_forcelink(); zen::MaximizeOpenFileCount(); return ZEN_RUN_TESTS(argc, argv); diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index e976c061d..7cc09be15 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -85,7 +85,7 @@ BlockStoreFile::Create(uint64_t InitialSize) ZEN_TRACE_CPU("BlockStoreFile::Create"); auto ParentPath = m_Path.parent_path(); - if (!std::filesystem::is_directory(ParentPath)) + if (!IsDir(ParentPath)) { CreateDirectories(ParentPath); } @@ -215,7 +215,7 @@ IsMetaDataValid(const std::filesystem::path& BlockPath, const std::filesystem::p } if (MetaWriteTime < BlockWriteTime) { - std::filesystem::remove(MetaPath, Ec); + RemoveFile(MetaPath, Ec); return false; } return true; @@ -239,7 +239,7 @@ BlockStoreFile::MetaSize() const if (IsMetaDataValid(m_Path, MetaPath)) { std::error_code DummyEc; - if (uint64_t Size = std::filesystem::file_size(MetaPath, DummyEc); !DummyEc) + if (uint64_t Size = FileSizeFromPath(MetaPath, DummyEc); !DummyEc) { return Size; } @@ -252,7 +252,7 @@ BlockStoreFile::RemoveMeta() { std::filesystem::path MetaPath = GetMetaPath(); std::error_code DummyEc; - std::filesystem::remove(MetaPath, DummyEc); + RemoveFile(MetaPath, DummyEc); } std::filesystem::path @@ -291,7 +291,7 @@ BlockStore::Initialize(const std::filesystem::path& BlocksBasePath, uint64_t Max m_MaxBlockSize = MaxBlockSize; m_MaxBlockCount = MaxBlockCount; - if (std::filesystem::is_directory(m_BlocksBasePath)) + if (IsDir(m_BlocksBasePath)) { uint32_t NextBlockIndex = 0; std::vector<std::filesystem::path> FoldersToScan; @@ -500,7 +500,7 @@ BlockStore::GetFreeBlockIndex(uint32_t ProbeIndex, RwLock::ExclusiveLockScope&, { OutBlockPath = GetBlockPath(m_BlocksBasePath, ProbeIndex); std::error_code Ec; - bool Exists = std::filesystem::exists(OutBlockPath, Ec); + bool Exists = IsFile(OutBlockPath, Ec); if (Ec) { ZEN_WARN("Failed to probe existence of file '{}' when trying to allocate a new block. Reason: '{}'", @@ -578,7 +578,7 @@ BlockStore::WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, cons } void -BlockStore::WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback) +BlockStore::WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback) { ZEN_MEMSCOPE(GetBlocksTag()); ZEN_TRACE_CPU("BlockStore::WriteChunks"); @@ -1375,14 +1375,14 @@ TEST_CASE("blockstore.blockfile") BoopChunk = File1.GetChunk(5, 5); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); const char* Data = static_cast<const char*>(DataChunk.GetData()); CHECK(std::string(Data) == "data"); const char* Boop = static_cast<const char*>(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(std::filesystem::exists(RootDirectory / "1")); + CHECK(IsFile(RootDirectory / "1")); { IoBuffer DataChunk; @@ -1401,7 +1401,7 @@ TEST_CASE("blockstore.blockfile") const char* Boop = static_cast<const char*>(BoopChunk.GetData()); CHECK(std::string(Boop) == "boop"); } - CHECK(!std::filesystem::exists(RootDirectory / "1")); + CHECK(!IsFile(RootDirectory / "1")); } namespace blockstore::impl { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp new file mode 100644 index 000000000..f26901458 --- /dev/null +++ b/src/zenstore/buildstore/buildstore.cpp @@ -0,0 +1,1496 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenstore/buildstore/buildstore.h> + +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/memory/llm.h> +#include <zencore/scopeguard.h> +#include <zencore/trace.h> +#include <zencore/workthreadpool.h> + +#include <zencore/uid.h> +#include <zencore/xxhash.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <gsl/gsl-lite.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_WITH_TESTS +# include <zencore/compactbinarybuilder.h> +# include <zencore/compress.h> +# include <zencore/testing.h> +# include <zencore/testutils.h> +#endif // ZEN_WITH_TESTS + +namespace zen { +const FLLMTag& +GetBuildstoreTag() +{ + static FLLMTag _("store", FLLMTag("builds")); + + return _; +} + +using namespace std::literals; + +namespace blobstore::impl { + + const std::string BaseName = "builds"; + const char* IndexExtension = ".uidx"; + const char* LogExtension = ".slog"; + + std::filesystem::path GetBlobIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + IndexExtension); + } + + std::filesystem::path GetBlobLogPath(const std::filesystem::path& RootDirectory) { return RootDirectory / (BaseName + LogExtension); } + + std::filesystem::path GetMetaIndexPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + IndexExtension); + } + + std::filesystem::path GetMetaLogPath(const std::filesystem::path& RootDirectory) + { + return RootDirectory / (BaseName + "_meta" + LogExtension); + } +} // namespace blobstore::impl + +BuildStore::BuildStore(const BuildStoreConfig& Config, GcManager& Gc) +: m_Config(Config) +, m_Gc(Gc) +, m_LargeBlobStore(m_Gc) +, m_SmallBlobStore(Gc) +, m_MetadataBlockStore() +{ + ZEN_TRACE_CPU("BuildStore::BuildStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + try + { + std::filesystem::path BlobLogPath = blobstore::impl::GetBlobLogPath(Config.RootDirectory); + std::filesystem::path MetaLogPath = blobstore::impl::GetMetaLogPath(Config.RootDirectory); + bool IsNew = !(IsFile(BlobLogPath) && IsFile(MetaLogPath)); + + if (!IsNew) + { + m_BlobLogFlushPosition = ReadPayloadLog(RwLock::ExclusiveLockScope(m_Lock), BlobLogPath, 0); + m_MetaLogFlushPosition = ReadMetadataLog(RwLock::ExclusiveLockScope(m_Lock), MetaLogPath, 0); + } + m_LargeBlobStore.Initialize(Config.RootDirectory / "file_cas", IsNew); + m_SmallBlobStore.Initialize(Config.RootDirectory, + "blob_cas", + m_Config.SmallBlobBlockStoreMaxBlockSize, + m_Config.SmallBlobBlockStoreAlignement, + IsNew); + m_MetadataBlockStore.Initialize(Config.RootDirectory / "metadata", m_Config.MetadataBlockStoreMaxBlockSize, 1u << 20); + { + BlockStore::BlockIndexSet KnownBlocks; + for (const BlobEntry& Blob : m_BlobEntries) + { + if (const MetadataIndex MetaIndex = Blob.Metadata; MetaIndex) + { + const MetadataEntry& Metadata = m_MetadataEntries[MetaIndex]; + KnownBlocks.insert(Metadata.Location.BlockIndex); + } + } + m_MetadataBlockStore.SyncExistingBlocksOnDisk(KnownBlocks); + } + + m_PayloadlogFile.Open(BlobLogPath, CasLogFile::Mode::kWrite); + m_MetadatalogFile.Open(MetaLogPath, CasLogFile::Mode::kWrite); + + m_Gc.AddGcReferencer(*this); + m_Gc.AddGcReferenceLocker(*this); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to initialize build store. Reason: '{}'", Ex.what()); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + } +} + +BuildStore::~BuildStore() +{ + try + { + ZEN_TRACE_CPU("BuildStore::~BuildStore"); + m_Gc.RemoveGcReferenceLocker(*this); + m_Gc.RemoveGcReferencer(*this); + Flush(); + m_MetadatalogFile.Close(); + m_PayloadlogFile.Close(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~BuildStore() threw exception: {}", Ex.what()); + } +} + +void +BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) +{ + ZEN_TRACE_CPU("BuildStore::PutBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + ZEN_ASSERT(Payload.GetContentType() == ZenContentType::kCompressedBinary); + { + RwLock::SharedLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex BlobIndex = It->second; + if (m_BlobEntries[BlobIndex].Payload) + { + return; + } + } + } + + PayloadEntry Entry; + if (Payload.GetSize() > m_Config.SmallBlobBlockStoreMaxBlockEmbedSize) + { + CasStore::InsertResult Result = m_LargeBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = PayloadEntry::kStandalone}; + } + else + { + CasStore::InsertResult Result = m_SmallBlobStore.InsertChunk(Payload, BlobHash); + ZEN_UNUSED(Result); + Entry = {.Flags = 0}; + } + m_PayloadlogFile.Append(PayloadDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + RwLock::ExclusiveLockScope _(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Payload) + { + m_PayloadEntries[Blob.Payload] = Entry; + } + else + { + Blob.Payload = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + PayloadIndex NewPayloadIndex = PayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size())); + // we only remove during GC and compact this then... + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } +} + +IoBuffer +BuildStore::GetBlob(const IoHash& BlobHash) +{ + ZEN_TRACE_CPU("BuildStore::GetBlob"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + RwLock::SharedLockScope Lock(m_Lock); + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + Blob.LastAccessTime = GcClock::TickCount(); + if (Blob.Payload) + { + const PayloadEntry& Entry = m_PayloadEntries[Blob.Payload]; + const bool IsStandalone = (Entry.Flags & PayloadEntry::kStandalone) != 0; + Lock.ReleaseNow(); + + IoBuffer Chunk; + if (IsStandalone) + { + ZEN_TRACE_CPU("GetLarge"); + Chunk = m_LargeBlobStore.FindChunk(BlobHash); + } + else + { + ZEN_TRACE_CPU("GetSmall"); + Chunk = m_SmallBlobStore.FindChunk(BlobHash); + } + if (Chunk) + { + Chunk.SetContentType(ZenContentType::kCompressedBinary); + return Chunk; + } + else + { + ZEN_WARN("Inconsistencies in build store, {} is in index but not {}", BlobHash, IsStandalone ? "on disk" : "in block"); + } + } + } + return {}; +} + +std::vector<BuildStore::BlobExistsResult> +BuildStore::BlobsExists(std::span<const IoHash> BlobHashes) +{ + ZEN_TRACE_CPU("BuildStore::BlobsExists"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector<BuildStore::BlobExistsResult> Result; + Result.reserve(BlobHashes.size()); + RwLock::SharedLockScope _(m_Lock); + for (const IoHash& BlobHash : BlobHashes) + { + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + bool HasPayload = !!Blob.Payload; + bool HasMetadata = !!Blob.Metadata; + Result.push_back(BlobExistsResult{.HasBody = HasPayload, .HasMetadata = HasMetadata}); + } + else + { + Result.push_back({}); + } + } + return Result; +} + +void +BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas) +{ + ZEN_TRACE_CPU("BuildStore::PutMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + size_t WriteBlobIndex = 0; + m_MetadataBlockStore.WriteChunks(MetaDatas, m_Config.MetadataBlockStoreAlignement, [&](std::span<BlockStoreLocation> Locations) { + RwLock::ExclusiveLockScope _(m_Lock); + for (size_t LocationIndex = 0; LocationIndex < Locations.size(); LocationIndex++) + { + const IoBuffer& Data = MetaDatas[WriteBlobIndex]; + const IoHash& BlobHash = BlobHashes[WriteBlobIndex]; + const BlockStoreLocation& Location = Locations[LocationIndex]; + + MetadataEntry Entry = {.Location = Location, .ContentType = Data.GetContentType(), .Flags = 0}; + m_MetadatalogFile.Append(MetadataDiskEntry{.Entry = Entry, .BlobHash = BlobHash}); + + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& Blob = m_BlobEntries[ExistingBlobIndex]; + if (Blob.Metadata) + { + m_MetadataEntries[Blob.Metadata] = Entry; + } + else + { + Blob.Metadata = MetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + } + Blob.LastAccessTime = GcClock::TickCount(); + } + else + { + MetadataIndex NewMetadataIndex = MetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Entry); + + const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert({BlobHash, NewBlobIndex}); + } + WriteBlobIndex++; + if (m_TrackedCacheKeys) + { + m_TrackedCacheKeys->insert(BlobHash); + } + } + }); +} + +std::vector<IoBuffer> +BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool) +{ + ZEN_TRACE_CPU("BuildStore::GetMetadatas"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + std::vector<BlockStoreLocation> MetaLocations; + std::vector<size_t> MetaLocationResultIndexes; + MetaLocations.reserve(BlobHashes.size()); + MetaLocationResultIndexes.reserve(BlobHashes.size()); + tsl::robin_set<uint32_t> ReferencedBlocks; + + std::vector<IoBuffer> Result; + std::vector<ZenContentType> ResultContentTypes; + Result.resize(BlobHashes.size()); + ResultContentTypes.resize(BlobHashes.size(), ZenContentType::kUnknownContentType); + { + RwLock::SharedLockScope _(m_Lock); + for (size_t Index = 0; Index < BlobHashes.size(); Index++) + { + const IoHash& BlobHash = BlobHashes[Index]; + if (auto It = m_BlobLookup.find(BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlobEntry = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlobEntry.Metadata) + { + const MetadataEntry& ExistingMetadataEntry = m_MetadataEntries[ExistingBlobEntry.Metadata]; + MetaLocations.push_back(ExistingMetadataEntry.Location); + MetaLocationResultIndexes.push_back(Index); + ReferencedBlocks.insert(ExistingMetadataEntry.Location.BlockIndex); + ResultContentTypes[Index] = ExistingMetadataEntry.ContentType; + } + ExistingBlobEntry.LastAccessTime = AccessTime(GcClock::TickCount()); + } + } + } + + auto DoOneBlock = [&](std::span<const size_t> ChunkIndexes) { + if (ChunkIndexes.size() < 4) + { + for (size_t ChunkIndex : ChunkIndexes) + { + IoBuffer Chunk = m_MetadataBlockStore.TryGetChunk(MetaLocations[ChunkIndex]); + if (Chunk) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = std::move(Chunk); + } + } + return true; + } + return m_MetadataBlockStore.IterateBlock( + MetaLocations, + ChunkIndexes, + [&](size_t ChunkIndex, const void* Data, uint64_t Size) { + if (Data != nullptr) + { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = IoBuffer(IoBuffer::Clone, Data, Size); + } + return true; + }, + [&](size_t ChunkIndex, BlockStoreFile& File, uint64_t Offset, uint64_t Size) { + size_t ResultIndex = MetaLocationResultIndexes[ChunkIndex]; + Result[ResultIndex] = File.GetChunk(Offset, Size); + return true; + }, + 8u * 1024u); + }; + + if (!MetaLocations.empty()) + { + Latch WorkLatch(1); + + m_MetadataBlockStore.IterateChunks(MetaLocations, [&](uint32_t BlockIndex, std::span<const size_t> ChunkIndexes) -> bool { + ZEN_UNUSED(BlockIndex); + if (ChunkIndexes.size() == MetaLocations.size() || OptionalWorkerPool == nullptr || ReferencedBlocks.size() == 1) + { + return DoOneBlock(ChunkIndexes); + } + else + { + ZEN_ASSERT(OptionalWorkerPool != nullptr); + WorkLatch.AddCount(1); + try + { + OptionalWorkerPool->ScheduleWork([&, ChunkIndexes = std::vector<size_t>(ChunkIndexes.begin(), ChunkIndexes.end())]() { + auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); + try + { + DoOneBlock(ChunkIndexes); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Failed getting metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + }); + } + catch (const std::exception& Ex) + { + WorkLatch.CountDown(); + ZEN_ERROR("Failed dispatching async work to fetch metadata for {} chunks. Reason: {}", ChunkIndexes.size(), Ex.what()); + } + return true; + } + }); + + WorkLatch.CountDown(); + WorkLatch.Wait(); + } + for (size_t Index = 0; Index < Result.size(); Index++) + { + if (Result[Index]) + { + Result[Index].SetContentType(ResultContentTypes[Index]); + } + } + return Result; +} + +void +BuildStore::Flush() +{ + ZEN_TRACE_CPU("BuildStore::Flush"); + try + { + m_LargeBlobStore.Flush(); + m_SmallBlobStore.Flush(); + m_MetadataBlockStore.Flush(false); + + m_PayloadlogFile.Flush(); + m_MetadatalogFile.Flush(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("BuildStore::Flush failed. Reason: {}", Ex.what()); + } +} + +void +BuildStore::CompactState() +{ + ZEN_TRACE_CPU("BuildStore::CompactState"); + + std::vector<BlobEntry> BlobEntries; + std::vector<PayloadEntry> PayloadEntries; + std::vector<MetadataEntry> MetadataEntries; + + tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> BlobLookup; + + RwLock::ExclusiveLockScope _(m_Lock); + const size_t EntryCount = m_BlobLookup.size(); + BlobLookup.reserve(EntryCount); + const size_t PayloadCount = m_PayloadEntries.size(); + PayloadEntries.reserve(PayloadCount); + const size_t MetadataCount = m_MetadataEntries.size(); + MetadataEntries.reserve(MetadataCount); + + for (auto LookupIt : m_BlobLookup) + { + const IoHash& BlobHash = LookupIt.first; + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const BlobIndex WriteBlobIndex(gsl::narrow<uint32_t>(BlobEntries.size())); + BlobEntries.push_back(ReadBlobEntry); + BlobEntry& WriteBlobEntry = BlobEntries.back(); + + if (WriteBlobEntry.Payload) + { + const PayloadEntry& ReadPayloadEntry = m_PayloadEntries[ReadBlobEntry.Payload]; + WriteBlobEntry.Payload = PayloadIndex(gsl::narrow<uint32_t>(PayloadEntries.size())); + PayloadEntries.push_back(ReadPayloadEntry); + } + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_MetadataEntries[ReadBlobEntry.Metadata]; + WriteBlobEntry.Metadata = MetadataIndex(gsl::narrow<uint32_t>(MetadataEntries.size())); + MetadataEntries.push_back(ReadMetadataEntry); + } + + BlobLookup.insert({BlobHash, WriteBlobIndex}); + } + m_BlobEntries.swap(BlobEntries); + m_PayloadEntries.swap(PayloadEntries); + m_MetadataEntries.swap(MetadataEntries); + m_BlobLookup.swap(BlobLookup); +} + +uint64_t +BuildStore::ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadPayloadLog"); + if (!IsFile(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' payload log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile<PayloadDiskEntry> CasLog; + if (!CasLog.IsValid(LogPath)) + { + RemoveFile(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full payload log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const PayloadDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & PayloadEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Metadata) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Payload = {}; + } + } + return; + } + + if (!ValidatePayloadDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid payload entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Payload) + { + const PayloadIndex ExistingPayloadIndex = ExistingBlob.Payload; + m_PayloadEntries[ExistingPayloadIndex] = Record.Entry; + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + ExistingBlob.Payload = NewPayloadIndex; + } + } + else + { + const PayloadIndex NewPayloadIndex(gsl::narrow<uint32_t>(m_PayloadEntries.size())); + m_PayloadEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Payload = NewPayloadIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid payload entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +uint64_t +BuildStore::ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount) +{ + ZEN_TRACE_CPU("BuildStore::ReadMetadataLog"); + if (!IsFile(LogPath)) + { + return 0; + } + + uint64_t LogEntryCount = 0; + Stopwatch Timer; + const auto _ = MakeGuard([&] { + ZEN_INFO("read build store '{}' metadata log containing {} entries in {}", + LogPath, + LogEntryCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + TCasLogFile<MetadataDiskEntry> CasLog; + if (!CasLog.IsValid(LogPath)) + { + RemoveFile(LogPath); + return 0; + } + CasLog.Open(LogPath, CasLogFile::Mode::kRead); + if (!CasLog.Initialize()) + { + return 0; + } + + const uint64_t EntryCount = CasLog.GetLogCount(); + if (EntryCount < SkipEntryCount) + { + ZEN_WARN("reading full metadata log at '{}', reason: Log position from index snapshot is out of range", LogPath); + SkipEntryCount = 0; + } + + LogEntryCount = EntryCount - SkipEntryCount; + uint64_t InvalidEntryCount = 0; + + CasLog.Replay( + [&](const MetadataDiskEntry& Record) { + std::string InvalidEntryReason; + if (Record.Entry.Flags & MetadataEntry::kTombStone) + { + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + // Note: this leaves m_BlobLookup and other arrays with 'holes' in them, this will get clean up in compact gc operation + if (auto ExistingIt = m_BlobLookup.find(Record.BlobHash); ExistingIt != m_BlobLookup.end()) + { + if (!m_BlobEntries[ExistingIt->second].Payload) + { + m_BlobLookup.erase(ExistingIt); + } + else + { + m_BlobEntries[ExistingIt->second].Metadata = {}; + } + } + return; + } + + if (!ValidateMetadataDiskEntry(Record, InvalidEntryReason)) + { + ZEN_WARN("skipping invalid metadata entry in '{}', reason: '{}'", LogPath, InvalidEntryReason); + ++InvalidEntryCount; + return; + } + if (auto It = m_BlobLookup.find(Record.BlobHash); It != m_BlobLookup.end()) + { + const BlobIndex ExistingBlobIndex = It->second; + BlobEntry& ExistingBlob = m_BlobEntries[ExistingBlobIndex]; + if (ExistingBlob.Metadata) + { + const MetadataIndex ExistingMetadataIndex = ExistingBlob.Metadata; + m_MetadataEntries[ExistingMetadataIndex] = Record.Entry; + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + ExistingBlob.Metadata = NewMetadataIndex; + } + } + else + { + const MetadataIndex NewMetadataIndex(gsl::narrow<uint32_t>(m_MetadataEntries.size())); + m_MetadataEntries.push_back(Record.Entry); + + const BlobIndex NewBlobIndex(gsl::narrow<uint32_t>(m_BlobEntries.size())); + m_BlobEntries.push_back(BlobEntry{.Metadata = NewMetadataIndex, .LastAccessTime = AccessTime(GcClock::TickCount())}); + m_BlobLookup.insert_or_assign(Record.BlobHash, NewBlobIndex); + } + }, + SkipEntryCount); + + if (InvalidEntryCount) + { + ZEN_WARN("found {} invalid metadata entries in '{}'", InvalidEntryCount, LogPath); + } + + return LogEntryCount; +} + +bool +BuildStore::ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & ~(PayloadEntry::kTombStone | PayloadEntry::kStandalone)) + { + OutReason = fmt::format("Invalid flags {} for entry {}", Entry.Entry.Flags, Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & PayloadEntry::kTombStone) + { + return true; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0 || Entry.Entry.Reserved3 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +bool +BuildStore::ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason) +{ + if (Entry.BlobHash == IoHash::Zero) + { + OutReason = fmt::format("Invalid blob hash {} for meta entry", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Location.Size == 0) + { + OutReason = fmt::format("Invalid meta blob size {} for meta entry", Entry.Entry.Location.Size); + return false; + } + if (Entry.Entry.Reserved1 != 0 || Entry.Entry.Reserved2 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Entry.Flags & MetadataEntry::kTombStone) + { + return true; + } + if (Entry.Entry.ContentType == ZenContentType::kCOUNT) + { + OutReason = fmt::format("Invalid content type for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + if (Entry.Reserved1 != 0 || Entry.Reserved2 != 0 || Entry.Reserved3 != 0 || Entry.Reserved4 != 0) + { + OutReason = fmt::format("Invalid reserved fields for meta entry {}", Entry.BlobHash.ToHexString()); + return false; + } + return true; +} + +class BuildStoreGcReferenceChecker : public GcReferenceChecker +{ +public: + BuildStoreGcReferenceChecker(BuildStore& Store) : m_Store(Store) {} + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + + virtual void PreCache(GcCtx& Ctx) override { ZEN_UNUSED(Ctx); } + + virtual void UpdateLockedState(GcCtx& Ctx) override + { + ZEN_TRACE_CPU("Builds::UpdateLockedState"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + m_References.reserve(m_Store.m_BlobLookup.size()); + for (const auto& It : m_Store.m_BlobLookup) + { + const BuildStore::BlobIndex ExistingBlobIndex = It.second; + if (m_Store.m_BlobEntries[ExistingBlobIndex].Payload) + { + m_References.push_back(It.first); + } + } + FilterReferences(Ctx, fmt::format("buildstore [LOCKSTATE] '{}'", "buildstore"), m_References); + } + + virtual std::span<IoHash> GetUnusedReferences(GcCtx& Ctx, std::span<IoHash> IoCids) override + { + ZEN_UNUSED(Ctx); + ZEN_TRACE_CPU("Builds::GetUnusedReferences"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + size_t InitialCount = IoCids.size(); + size_t UsedCount = InitialCount; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [FILTER REFERENCES] '{}': filtered out {} used references out of {} in {}", + "buildstore", + UsedCount, + InitialCount, + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + std::span<IoHash> UnusedReferences = KeepUnusedReferences(m_References, IoCids); + UsedCount = IoCids.size() - UnusedReferences.size(); + return UnusedReferences; + } + +private: + BuildStore& m_Store; + std::vector<IoHash> m_References; +}; + +std::string +BuildStore::GetGcName(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Config.RootDirectory.string()); +} + +class BuildStoreGcCompator : public GcStoreCompactor +{ + using BlobEntry = BuildStore::BlobEntry; + using PayloadEntry = BuildStore::PayloadEntry; + using MetadataEntry = BuildStore::MetadataEntry; + using MetadataDiskEntry = BuildStore::MetadataDiskEntry; + using BlobIndex = BuildStore::BlobIndex; + using PayloadIndex = BuildStore::PayloadIndex; + using MetadataIndex = BuildStore::MetadataIndex; + +public: + BuildStoreGcCompator(BuildStore& Store, std::vector<IoHash>&& RemovedBlobs) : m_Store(Store), m_RemovedBlobs(std::move(RemovedBlobs)) {} + + virtual void CompactStore(GcCtx& Ctx, GcCompactStoreStats& Stats, const std::function<uint64_t()>& ClaimDiskReserveCallback) override + { + ZEN_UNUSED(ClaimDiskReserveCallback); + ZEN_TRACE_CPU("Builds::CompactStore"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (!Ctx.Settings.Verbose) + { + return; + } + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': RemovedDisk: {} in {}", + m_Store.m_Config.RootDirectory, + NiceBytes(Stats.RemovedDisk), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + }); + + if (!m_RemovedBlobs.empty()) + { + if (Ctx.Settings.CollectSmallObjects) + { + m_Store.m_Lock.WithExclusiveLock([this]() { m_Store.m_TrackedCacheKeys = std::make_unique<HashSet>(); }); + auto __ = MakeGuard([this]() { m_Store.m_Lock.WithExclusiveLock([&]() { m_Store.m_TrackedCacheKeys.reset(); }); }); + + BlockStore::BlockUsageMap BlockUsage; + { + RwLock::SharedLockScope __(m_Store.m_Lock); + + for (auto LookupIt : m_Store.m_BlobLookup) + { + const BlobIndex ReadBlobIndex = LookupIt.second; + const BlobEntry& ReadBlobEntry = m_Store.m_BlobEntries[ReadBlobIndex]; + + if (ReadBlobEntry.Metadata) + { + const MetadataEntry& ReadMetadataEntry = m_Store.m_MetadataEntries[ReadBlobEntry.Metadata]; + + uint32_t BlockIndex = ReadMetadataEntry.Location.BlockIndex; + uint64_t ChunkSize = RoundUp(ReadMetadataEntry.Location.Size, m_Store.m_Config.MetadataBlockStoreAlignement); + + if (auto BlockUsageIt = BlockUsage.find(BlockIndex); BlockUsageIt != BlockUsage.end()) + { + BlockStore::BlockUsageInfo& Info = BlockUsageIt.value(); + Info.EntryCount++; + Info.DiskUsage += ChunkSize; + } + else + { + BlockUsage.insert_or_assign(BlockIndex, + BlockStore::BlockUsageInfo{.DiskUsage = ChunkSize, .EntryCount = 1}); + } + } + } + } + + BlockStore::BlockEntryCountMap BlocksToCompact = m_Store.m_MetadataBlockStore.GetBlocksToCompact(BlockUsage, 90); + BlockStoreCompactState BlockCompactState; + std::vector<IoHash> BlockCompactStateKeys; + BlockCompactState.IncludeBlocks(BlocksToCompact); + + if (BlocksToCompact.size() > 0) + { + { + RwLock::SharedLockScope ___(m_Store.m_Lock); + for (const auto& Entry : m_Store.m_BlobLookup) + { + BlobIndex Index = Entry.second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + if (BlockCompactState.AddKeepLocation(m_Store.m_MetadataEntries[Meta].Location)) + { + BlockCompactStateKeys.push_back(Entry.first); + } + } + } + } + + if (Ctx.Settings.IsDeleteMode) + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': compacting {} blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + + m_Store.m_MetadataBlockStore.CompactBlocks( + BlockCompactState, + m_Store.m_Config.MetadataBlockStoreAlignement, + [&](const BlockStore::MovedChunksArray& MovedArray, uint64_t FreedDiskSpace) { + std::vector<MetadataDiskEntry> MovedEntries; + MovedEntries.reserve(MovedArray.size()); + RwLock::ExclusiveLockScope _(m_Store.m_Lock); + for (const std::pair<size_t, BlockStoreLocation>& Moved : MovedArray) + { + size_t ChunkIndex = Moved.first; + const IoHash& Key = BlockCompactStateKeys[ChunkIndex]; + + ZEN_ASSERT(m_Store.m_TrackedCacheKeys); + if (m_Store.m_TrackedCacheKeys->contains(Key)) + { + continue; + } + + if (auto It = m_Store.m_BlobLookup.find(Key); It != m_Store.m_BlobLookup.end()) + { + const BlobIndex Index = It->second; + + if (MetadataIndex Meta = m_Store.m_BlobEntries[Index].Metadata; Meta) + { + m_Store.m_MetadataEntries[Meta].Location = Moved.second; + MovedEntries.push_back( + MetadataDiskEntry{.Entry = m_Store.m_MetadataEntries[Meta], .BlobHash = Key}); + } + } + } + m_Store.m_MetadatalogFile.Append(MovedEntries); + Stats.RemovedDisk += FreedDiskSpace; + if (Ctx.IsCancelledFlag.load()) + { + return false; + } + return true; + }, + ClaimDiskReserveCallback, + fmt::format("GCV2: buildstore [COMPACT] '{}': ", m_Store.m_Config.RootDirectory)); + } + else + { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [COMPACT] '{}': skipped compacting of {} eligible blocks", + m_Store.m_Config.RootDirectory, + BlocksToCompact.size()); + } + } + } + } + } + } + + virtual std::string GetGcName(GcCtx& Ctx) override + { + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + return fmt::format("buildstore: '{}'", m_Store.m_Config.RootDirectory.string()); + } + +private: + BuildStore& m_Store; + const std::vector<IoHash> m_RemovedBlobs; +}; + +GcStoreCompactor* +BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) +{ + ZEN_TRACE_CPU("Builds::RemoveExpiredData"); + ZEN_MEMSCOPE(GetBuildstoreTag()); + + auto Log = [&Ctx]() { return Ctx.Logger; }; + + Stopwatch Timer; + const auto _ = MakeGuard([&] { + if (Ctx.Settings.Verbose) + { + ZEN_INFO("GCV2: buildstore [REMOVE EXPIRED] '{}': Count: {}, Expired: {}, Deleted: {}, FreedMemory: {} in {}", + m_Config.RootDirectory, + Stats.CheckedCount, + Stats.FoundCount, + Stats.DeletedCount, + NiceBytes(Stats.FreedMemory), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + } + }); + + const GcClock::Tick ExpireTicks = Ctx.Settings.BuildStoreExpireTime.time_since_epoch().count(); + + std::vector<IoHash> ExpiredBlobs; + { + RwLock::SharedLockScope __(m_Lock); + for (const auto& It : m_BlobLookup) + { + const BlobIndex ReadBlobIndex = It.second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + if (AccessTick < ExpireTicks) + { + ExpiredBlobs.push_back(It.first); + } + } + Stats.CheckedCount += m_BlobLookup.size(); + Stats.FoundCount += ExpiredBlobs.size(); + } + + std::vector<IoHash> RemovedBlobs; + if (!ExpiredBlobs.empty()) + { + if (Ctx.Settings.IsDeleteMode) + { + RemovedBlobs.reserve(ExpiredBlobs.size()); + + std::vector<PayloadDiskEntry> RemovedPayloads; + std::vector<MetadataDiskEntry> RemoveMetadatas; + + RwLock::ExclusiveLockScope __(m_Lock); + if (Ctx.IsCancelledFlag.load()) + { + return nullptr; + } + + for (const IoHash& ExpiredBlob : ExpiredBlobs) + { + if (auto It = m_BlobLookup.find(ExpiredBlob); It != m_BlobLookup.end()) + { + const BlobIndex ReadBlobIndex = It->second; + const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; + + const GcClock::Tick AccessTick = ReadBlobEntry.LastAccessTime; + + if (AccessTick < ExpireTicks) + { + if (ReadBlobEntry.Payload) + { + RemovedPayloads.push_back( + PayloadDiskEntry{.Entry = m_PayloadEntries[ReadBlobEntry.Payload], .BlobHash = ExpiredBlob}); + RemovedPayloads.back().Entry.Flags |= PayloadEntry::kTombStone; + m_PayloadEntries[ReadBlobEntry.Payload] = {}; + m_BlobEntries[ReadBlobIndex].Payload = {}; + } + if (ReadBlobEntry.Metadata) + { + RemoveMetadatas.push_back( + MetadataDiskEntry{.Entry = m_MetadataEntries[ReadBlobEntry.Metadata], .BlobHash = ExpiredBlob}); + RemoveMetadatas.back().Entry.Flags |= MetadataEntry::kTombStone; + m_MetadataEntries[ReadBlobEntry.Metadata] = {}; + m_BlobEntries[ReadBlobIndex].Metadata = {}; + } + + m_BlobLookup.erase(It); + + RemovedBlobs.push_back(ExpiredBlob); + Stats.DeletedCount++; + } + } + } + if (!RemovedPayloads.empty()) + { + m_PayloadlogFile.Append(RemovedPayloads); + } + if (!RemoveMetadatas.empty()) + { + m_MetadatalogFile.Append(RemoveMetadatas); + } + } + } + + if (!RemovedBlobs.empty()) + { + CompactState(); + } + + return new BuildStoreGcCompator(*this, std::move(RemovedBlobs)); +} + +std::vector<GcReferenceChecker*> +BuildStore::CreateReferenceCheckers(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + ZEN_MEMSCOPE(GetBuildstoreTag()); + return {new BuildStoreGcReferenceChecker(*this)}; +} + +std::vector<GcReferenceValidator*> +BuildStore::CreateReferenceValidators(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + return {}; +} + +std::vector<RwLock::SharedLockScope> +BuildStore::LockState(GcCtx& Ctx) +{ + ZEN_UNUSED(Ctx); + std::vector<RwLock::SharedLockScope> Locks; + Locks.emplace_back(RwLock::SharedLockScope(m_Lock)); + return Locks; +} + +/* + ___________ __ + \__ ___/___ _______/ |_ ______ + | |_/ __ \ / ___/\ __\/ ___/ + | |\ ___/ \___ \ | | \___ \ + |____| \___ >____ > |__| /____ > + \/ \/ \/ +*/ + +#if ZEN_WITH_TESTS + +TEST_CASE("BuildStore.Blobs") +{ + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector<IoHash> CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& RawHash : CompressedBlobsHashes) + { + IoBuffer Payload = Store.GetBlob(RawHash); + CHECK(Payload); + CHECK(Payload.GetContentType() == ZenContentType::kCompressedBinary); + IoHash VerifyRawHash; + uint64_t VerifyRawSize; + CompressedBuffer CompressedBlob = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Payload)), VerifyRawHash, VerifyRawSize); + CHECK(CompressedBlob); + CHECK(VerifyRawHash == RawHash); + IoBuffer Decompressed = CompressedBlob.Decompress().AsIoBuffer(); + CHECK(IoHash::HashBuffer(Decompressed) == RawHash); + } + } +} + +namespace blockstore::testing { + IoBuffer MakeMetaData(const IoHash& BlobHash, const std::vector<std::pair<std::string, std::string>>& KeyValues) + { + CbObjectWriter Writer; + Writer.AddHash("rawHash"sv, BlobHash); + Writer.BeginObject("values"); + { + for (const auto& V : KeyValues) + { + Writer.AddString(V.first, V.second); + } + } + Writer.EndObject(); // values + return Writer.Save().GetBuffer().AsIoBuffer(); + }; + +} // namespace blockstore::testing + +TEST_CASE("BuildStore.Metadata") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector<IoHash> BlobHashes; + std::vector<IoBuffer> MetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + + for (size_t I = 0; I < 5; I++) + { + BlobHashes.push_back(IoHash::HashBuffer(&I, sizeof(I))); + MetaPayloads.push_back(MakeMetaData(BlobHashes.back(), {{"index", fmt::format("{}", I)}})); + MetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(BlobHashes, MetaPayloads); + + std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + std::vector<IoBuffer> ValidateMetaPayloads = Store.GetMetadatas(BlobHashes, nullptr); + CHECK(ValidateMetaPayloads.size() == MetaPayloads.size()); + for (size_t I = 0; I < ValidateMetaPayloads.size(); I++) + { + const IoHash ExpectedHash = IoHash::HashBuffer(MetaPayloads[I]); + CHECK_EQ(IoHash::HashBuffer(ValidateMetaPayloads[I]), ExpectedHash); + } + for (const IoHash& BlobHash : BlobHashes) + { + CHECK(!Store.GetBlob(BlobHash)); + } + } + std::vector<IoHash> CompressedBlobsHashes; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + for (const auto& MetadataIt : MetadataPayloads) + { + CHECK(!MetadataIt); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } + + std::vector<IoBuffer> BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK_EQ(IoHash::HashBuffer(MetadataPayload), IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + + BlobMetaPayloads.clear(); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back( + MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}, {"replaced", fmt::format("{}", true)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK_EQ(IoHash::HashBuffer(DecompressedBlob), BlobHash); + } + } +} + +TEST_CASE("BuildStore.GC") +{ + using namespace blockstore::testing; + + ScopedTemporaryDirectory _; + + BuildStoreConfig Config; + Config.RootDirectory = _.Path() / "build_store"; + + std::vector<IoHash> CompressedBlobsHashes; + std::vector<IoBuffer> BlobMetaPayloads; + { + GcManager Gc; + BuildStore Store(Config, Gc); + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + Store.PutBlob(CompressedBlobsHashes.back(), Payload); + } + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + BlobMetaPayloads.push_back(MakeMetaData(BlobHash, {{"blobHash", fmt::format("{}", BlobHash)}})); + BlobMetaPayloads.back().SetContentType(ZenContentType::kCbObject); + } + Store.PutMetadatas(CompressedBlobsHashes, BlobMetaPayloads); + } + { + GcManager Gc; + BuildStore Store(Config, Gc); + + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() - std::chrono::hours(1), + .CollectSmallObjects = false, + .IsDeleteMode = false, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(Blob); + IoBuffer DecompressedBlob = CompressedBuffer::FromCompressedNoValidate(std::move(Blob)).Decompress().AsIoBuffer(); + CHECK(DecompressedBlob); + CHECK(IoHash::HashBuffer(DecompressedBlob) == BlobHash); + } + + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(IoHash::HashBuffer(MetadataPayload) == IoHash::HashBuffer(BlobMetaPayloads[I])); + } + } + { + GcResult Result = Gc.CollectGarbage(GcSettings{.BuildStoreExpireTime = GcClock::Now() + std::chrono::hours(1), + .CollectSmallObjects = true, + .IsDeleteMode = true, + .Verbose = true}); + CHECK(!Result.WasCancelled); + for (const IoHash& BlobHash : CompressedBlobsHashes) + { + IoBuffer Blob = Store.GetBlob(BlobHash); + CHECK(!Blob); + } + + std::vector<IoBuffer> MetadataPayloads = Store.GetMetadatas(CompressedBlobsHashes, nullptr); + CHECK(MetadataPayloads.size() == BlobMetaPayloads.size()); + for (size_t I = 0; I < MetadataPayloads.size(); I++) + { + const IoBuffer& MetadataPayload = MetadataPayloads[I]; + CHECK(!MetadataPayload); + } + } + } +} + +void +buildstore_forcelink() +{ +} + +#endif + +} // namespace zen diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 61552fafc..e4d962b56 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -200,21 +200,21 @@ namespace cache::impl { int DropIndex = 0; do { - if (!std::filesystem::exists(Dir)) + if (!IsDir(Dir)) { return false; } std::string DroppedName = fmt::format("[dropped]{}({})", Dir.filename().string(), DropIndex); std::filesystem::path DroppedBucketPath = Dir.parent_path() / DroppedName; - if (std::filesystem::exists(DroppedBucketPath)) + if (IsDir(DroppedBucketPath)) { DropIndex++; continue; } std::error_code Ec; - std::filesystem::rename(Dir, DroppedBucketPath, Ec); + RenameDirectory(Dir, DroppedBucketPath, Ec); if (!Ec) { DeleteDirectories(DroppedBucketPath); @@ -909,16 +909,16 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(bool FlushLockPosition, { std::filesystem::path LogPath = cache::impl::GetLogPath(m_BucketDir, m_BucketName); - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { - if (!std::filesystem::remove(LogPath, Ec) || Ec) + if (!RemoveFile(LogPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean log file '{}', removing index at '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); std::error_code RemoveIndexEc; - std::filesystem::remove(IndexPath, RemoveIndexEc); + RemoveFile(IndexPath, RemoveIndexEc); } } } @@ -939,7 +939,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const { ZEN_TRACE_CPU("Z$::Bucket::ReadIndexFile"); - if (!std::filesystem::is_regular_file(IndexPath)) + if (!IsFile(IndexPath)) { return 0; } @@ -1023,7 +1023,7 @@ ZenCacheDiskLayer::CacheBucket::ReadLog(RwLock::ExclusiveLockScope&, const std:: { ZEN_TRACE_CPU("Z$::Bucket::ReadLog"); - if (!std::filesystem::is_regular_file(LogPath)) + if (!IsFile(LogPath)) { return 0; } @@ -1103,37 +1103,37 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco if (IsNew) { - fs::remove(LogPath); - fs::remove(IndexPath); - fs::remove_all(m_BlocksBasePath); + RemoveFile(LogPath); + RemoveFile(IndexPath); + DeleteDirectories(m_BlocksBasePath); } CreateDirectories(m_BucketDir); m_BlockStore.Initialize(m_BlocksBasePath, m_Configuration.MaxBlockSize, BlockStoreDiskLocation::MaxBlockIndex + 1); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexLock, IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<DiskIndexEntry>::IsValid(LogPath)) { LogEntryCount = ReadLog(IndexLock, LogPath, m_LogFlushPosition); } - else if (fs::is_regular_file(LogPath)) + else if (IsFile(LogPath)) { ZEN_WARN("removing invalid log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -2146,7 +2146,7 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); std::error_code Ec; - uintmax_t size = std::filesystem::file_size(DataFilePath.ToPath(), Ec); + uintmax_t size = FileSizeFromPath(DataFilePath.ToPath(), Ec); if (Ec) { ReportBadKey(HashKey); @@ -2287,11 +2287,11 @@ ZenCacheDiskLayer::CacheBucket::ScrubStorage(ScrubContext& Ctx) BuildPath(Path, Entry.Key); fs::path FilePath = Path.ToPath(); RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key)); - if (fs::is_regular_file(FilePath)) + if (IsFile(FilePath)) { ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8()); std::error_code Ec; - fs::remove(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... + RemoveFile(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... } } } @@ -2424,7 +2424,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (CleanUpTempFile) { std::error_code Ec; - std::filesystem::remove(DataFile.GetPath(), Ec); + RemoveFile(DataFile.GetPath(), Ec); if (Ec) { ZEN_WARN("Failed to clean up temporary file '{}' for put in '{}', reason '{}'", @@ -2452,7 +2452,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); // We do a speculative remove of the file instead of probing with a exists call and check the error code instead - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec) { if (Ec.value() != ENOENT) @@ -2460,7 +2460,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c ZEN_WARN("Failed to remove file '{}' for put in '{}', reason: '{}', retrying.", FsPath, m_BucketDir, Ec.message()); Sleep(100); Ec.clear(); - std::filesystem::remove(FsPath, Ec); + RemoveFile(FsPath, Ec); if (Ec && Ec.value() != ENOENT) { throw std::system_error(Ec, fmt::format("Failed to remove file '{}' for put in '{}'", FsPath, m_BucketDir)); @@ -2731,9 +2731,10 @@ class DiskBucketStoreCompactor : public GcStoreCompactor using CacheBucket = ZenCacheDiskLayer::CacheBucket; public: - DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys) + DiskBucketStoreCompactor(CacheBucket& Bucket, std::vector<std::pair<IoHash, uint64_t>>&& ExpiredStandaloneKeys, bool FlushBucket) : m_Bucket(Bucket) , m_ExpiredStandaloneKeys(std::move(ExpiredStandaloneKeys)) + , m_FlushBucket(FlushBucket) { m_ExpiredStandaloneKeys.shrink_to_fit(); } @@ -2791,7 +2792,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': deleting standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - if (!fs::remove(FilePath, Ec)) + if (!RemoveFile(FilePath, Ec)) { continue; } @@ -2812,7 +2813,7 @@ public: ZEN_DEBUG("GCV2: cachebucket [COMPACT] '{}': checking standalone cache file '{}'", m_Bucket.m_BucketDir, Path.ToUtf8()); std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(FilePath, Ec); + bool Existed = IsFile(FilePath, Ec); if (Ec) { ZEN_WARN("GCV2: cachebucket [COMPACT] '{}': failed checking cache payload file '{}'. Reason '{}'", @@ -2960,6 +2961,10 @@ public: } } } + if (m_FlushBucket) + { + m_Bucket.Flush(); + } } virtual std::string GetGcName(GcCtx& Ctx) override { return m_Bucket.GetGcName(Ctx); } @@ -2967,6 +2972,7 @@ public: private: ZenCacheDiskLayer::CacheBucket& m_Bucket; std::vector<std::pair<IoHash, uint64_t>> m_ExpiredStandaloneKeys; + bool m_FlushBucket = false; }; GcStoreCompactor* @@ -2990,24 +2996,6 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) NiceBytes(Stats.FreedMemory), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } - if (Stats.DeletedCount > 0) - { - bool Expected = false; - if (m_IsFlushing || !m_IsFlushing.compare_exchange_strong(Expected, true)) - { - return; - } - auto FlushingGuard = MakeGuard([&] { m_IsFlushing.store(false); }); - - try - { - SaveSnapshot([]() { return 0; }); - } - catch (const std::exception& Ex) - { - ZEN_WARN("Failed to write index and manifest after RemoveExpiredData in '{}'. Reason: '{}'", m_BucketDir, Ex.what()); - } - } }); const GcClock::Tick ExpireTicks = Ctx.Settings.CacheExpireTime.time_since_epoch().count(); @@ -3094,7 +3082,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) return nullptr; } - return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys)); + return new DiskBucketStoreCompactor(*this, std::move(ExpiredStandaloneKeys), /*FlushBucket*/ Stats.DeletedCount > 0); } bool diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 73c10a6db..ed42f254e 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -118,7 +118,7 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) // Ensure root directory exists - create if it doesn't exist already - std::filesystem::create_directories(m_Config.RootDirectory); + CreateDirectories(m_Config.RootDirectory); // Open or create manifest diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 6c7b1b297..492ce9317 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -37,7 +37,7 @@ CasLogFile::~CasLogFile() bool CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) { - if (!std::filesystem::is_regular_file(FileName)) + if (!IsFile(FileName)) { return false; } diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 2be0542db..184251da7 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -226,7 +226,7 @@ CasContainerStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) } std::vector<CasStore::InsertResult> -CasContainerStrategy::InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes) +CasContainerStrategy::InsertChunks(std::span<const IoBuffer> Chunks, std::span<const IoHash> ChunkHashes) { ZEN_MEMSCOPE(GetCasContainerTag()); @@ -323,7 +323,7 @@ CasContainerStrategy::FilterChunks(HashKeySet& InOutChunks) } bool -CasContainerStrategy::IterateChunks(std::span<IoHash> ChunkHashes, +CasContainerStrategy::IterateChunks(std::span<const IoHash> ChunkHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit) @@ -927,10 +927,10 @@ CasContainerStrategy::MakeIndexSnapshot() fs::path TempIndexPath = cas::impl::GetTempIndexPath(m_RootDirectory, m_ContainerBaseName); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", TempIndexPath, Ec.message()); return; @@ -939,9 +939,9 @@ CasContainerStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, TempIndexPath); + RenameFile(IndexPath, TempIndexPath); } // Write the current state of the location map to a new index state @@ -992,21 +992,21 @@ CasContainerStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(TempIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(TempIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", TempIndexPath, Ec.message()); } } } - if (fs::is_regular_file(TempIndexPath)) + if (IsFile(TempIndexPath)) { std::error_code Ec; - if (!fs::remove(TempIndexPath, Ec) || Ec) + if (!RemoveFile(TempIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", TempIndexPath, Ec.message()); } @@ -1092,7 +1092,7 @@ CasContainerStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t Ski if (!TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath)) { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); return 0; } @@ -1155,7 +1155,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) if (IsNewStore) { - std::filesystem::remove_all(BasePath); + DeleteDirectories(BasePath); } CreateDirectories(BasePath); @@ -1165,19 +1165,19 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) std::filesystem::path LogPath = cas::impl::GetLogPath(m_RootDirectory, m_ContainerBaseName); std::filesystem::path IndexPath = cas::impl::GetIndexPath(m_RootDirectory, m_ContainerBaseName); - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<CasDiskIndexEntry>::IsValid(LogPath)) { @@ -1186,7 +1186,7 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } diff --git a/src/zenstore/compactcas.h b/src/zenstore/compactcas.h index 07e620086..2eb4c233a 100644 --- a/src/zenstore/compactcas.h +++ b/src/zenstore/compactcas.h @@ -52,11 +52,11 @@ struct CasContainerStrategy final : public GcStorage, public GcReferenceStore ~CasContainerStrategy(); CasStore::InsertResult InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash); - std::vector<CasStore::InsertResult> InsertChunks(std::span<IoBuffer> Chunks, std::span<IoHash> ChunkHashes); + std::vector<CasStore::InsertResult> InsertChunks(std::span<const IoBuffer> Chunks, std::span<const IoHash> ChunkHashes); IoBuffer FindChunk(const IoHash& ChunkHash); bool HaveChunk(const IoHash& ChunkHash); void FilterChunks(HashKeySet& InOutChunks); - bool IterateChunks(std::span<IoHash> ChunkHashes, + bool IterateChunks(std::span<const IoHash> ChunkHashes, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit); diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 34db51aa9..14bdc41f0 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -176,10 +176,10 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN if (IsNewStore) { - std::filesystem::remove(LogPath); - std::filesystem::remove(IndexPath); + RemoveFile(LogPath); + RemoveFile(IndexPath); - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { // We need to explicitly only delete sharded root folders as the cas manifest, tinyobject and smallobject cas folders may reside // in this folder as well @@ -211,24 +211,24 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN Traversal.TraverseFileSystem(m_RootDirectory, CasVisitor); for (const std::filesystem::path& SharededRoot : CasVisitor.ShardedRoots) { - std::filesystem::remove_all(SharededRoot); + DeleteDirectories(SharededRoot); } } } - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { uint32_t IndexVersion = 0; m_LogFlushPosition = ReadIndexFile(IndexPath, IndexVersion); if (IndexVersion == 0) { ZEN_WARN("removing invalid index file at '{}'", IndexPath); - std::filesystem::remove(IndexPath); + RemoveFile(IndexPath); } } uint64_t LogEntryCount = 0; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { if (TCasLogFile<FileCasIndexEntry>::IsValid(LogPath)) { @@ -237,7 +237,7 @@ FileCasStrategy::Initialize(const std::filesystem::path& RootDirectory, bool IsN else { ZEN_WARN("removing invalid cas log at '{}'", LogPath); - std::filesystem::remove(LogPath); + RemoveFile(LogPath); } } @@ -327,7 +327,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); std::error_code Ec; - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (Ec) { throw std::system_error(Ec, fmt::format("unable to move existing CAS file {} to {}", ChunkPath, TempPath)); @@ -452,7 +452,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: { PayloadFile.Close(); std::error_code DummyEc; - std::filesystem::remove(ChunkPath, DummyEc); + RemoveFile(ChunkPath, DummyEc); throw; } bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); @@ -503,7 +503,7 @@ FileCasStrategy::SafeOpenChunk(const IoHash& ChunkHash, uint64 ExpectedSize) { std::error_code Ec; std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { Chunk.SetDeleteOnClose(true); @@ -574,7 +574,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) ShardingHelper Name(m_RootDirectory, ChunkHash); const std::filesystem::path ChunkPath = Name.ShardedPath.ToPath(); - uint64_t FileSize = static_cast<uint64_t>(std::filesystem::file_size(ChunkPath, Ec)); + uint64_t FileSize = static_cast<uint64_t>(FileSizeFromPath(ChunkPath, Ec)); if (Ec) { ZEN_WARN("get file size FAILED, file cas '{}'", ChunkPath); @@ -582,9 +582,9 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) } ZEN_DEBUG("deleting CAS payload file '{}' {}", ChunkPath, NiceBytes(FileSize)); - std::filesystem::remove(ChunkPath, Ec); + RemoveFile(ChunkPath, Ec); - if (!Ec || !std::filesystem::exists(ChunkPath)) + if (!Ec || !IsFile(ChunkPath)) { { RwLock::ExclusiveLockScope _(m_Lock); @@ -941,10 +941,10 @@ FileCasStrategy::MakeIndexSnapshot() fs::path STmpIndexPath = GetTempIndexPath(m_RootDirectory); // Move index away, we keep it if something goes wrong - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to clean up temp snapshot at {}, reason: '{}'", STmpIndexPath, Ec.message()); return; @@ -953,9 +953,9 @@ FileCasStrategy::MakeIndexSnapshot() try { - if (fs::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { - fs::rename(IndexPath, STmpIndexPath); + RenameFile(IndexPath, STmpIndexPath); } // Write the current state of the location map to a new index state @@ -1004,21 +1004,21 @@ FileCasStrategy::MakeIndexSnapshot() // Restore any previous snapshot - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - fs::remove(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless - fs::rename(STmpIndexPath, IndexPath, Ec); + RemoveFile(IndexPath, Ec); // We don't care if this fails, we try to move the old temp file regardless + RenameFile(STmpIndexPath, IndexPath, Ec); if (Ec) { ZEN_WARN("snapshot failed to restore old snapshot from {}, reason: '{}'", STmpIndexPath, Ec.message()); } } } - if (fs::is_regular_file(STmpIndexPath)) + if (IsFile(STmpIndexPath)) { std::error_code Ec; - if (!fs::remove(STmpIndexPath, Ec) || Ec) + if (!RemoveFile(STmpIndexPath, Ec) || Ec) { ZEN_WARN("snapshot failed to remove temporary file {}, reason: '{}'", STmpIndexPath, Ec.message()); } @@ -1032,7 +1032,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& using namespace filecas::impl; std::vector<FileCasIndexEntry> Entries; - if (std::filesystem::is_regular_file(IndexPath)) + if (IsFile(IndexPath)) { Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1077,7 +1077,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& return 0; } - if (std::filesystem::is_directory(m_RootDirectory)) + if (IsDir(m_RootDirectory)) { ZEN_INFO("missing index for file cas, scanning for cas files in {}", m_RootDirectory); TCasLogFile<FileCasIndexEntry> CasLog; @@ -1116,7 +1116,7 @@ FileCasStrategy::ReadLog(const std::filesystem::path& LogPath, uint64_t SkipEntr using namespace filecas::impl; - if (std::filesystem::is_regular_file(LogPath)) + if (IsFile(LogPath)) { uint64_t LogEntryCount = 0; Stopwatch Timer; @@ -1274,12 +1274,12 @@ public: ChunkPath); } std::error_code Ec; - uint64_t SizeOnDisk = std::filesystem::file_size(ChunkPath, Ec); + uint64_t SizeOnDisk = FileSizeFromPath(ChunkPath, Ec); if (Ec) { SizeOnDisk = 0; } - bool Existed = std::filesystem::remove(ChunkPath, Ec); + bool Existed = RemoveFile(ChunkPath, Ec); if (Ec) { // Target file may be open for read, attempt to move it to a temp file and mark it delete on close @@ -1290,7 +1290,7 @@ public: if (OldChunk) { std::filesystem::path TempPath(ChunkPath.parent_path() / Oid::NewOid().ToString()); - std::filesystem::rename(ChunkPath, TempPath, Ec); + RenameFile(ChunkPath, TempPath, Ec); if (!Ec) { OldChunk.SetDeleteOnClose(true); @@ -1317,7 +1317,7 @@ public: else { std::error_code Ec; - bool Existed = std::filesystem::is_regular_file(ChunkPath, Ec); + bool Existed = IsFile(ChunkPath, Ec); if (Ec) { if (Ctx.Settings.Verbose) @@ -1516,7 +1516,7 @@ TEST_CASE("cas.chunk.moveoverwrite") Payload1.SetDeleteOnClose(true); CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, true); - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1526,9 +1526,9 @@ TEST_CASE("cas.chunk.moveoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash()); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1554,7 +1554,7 @@ TEST_CASE("cas.chunk.moveoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; @@ -1598,9 +1598,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, true); - CHECK(std::filesystem::exists(Payload1Path)); + CHECK(IsFile(Payload1Path)); Payload1 = {}; - CHECK(!std::filesystem::exists(Payload1Path)); + CHECK(!IsFile(Payload1Path)); } { std::filesystem::path Payload1BPath{TempDir.Path() / "payload_1"}; @@ -1611,9 +1611,9 @@ TEST_CASE("cas.chunk.copyoverwrite") CasStore::InsertResult Result = FileCas.InsertChunk(Payload1B, CompressedPayload1.DecodeRawHash(), CasStore::InsertMode::kCopyOnly); CHECK_EQ(Result.New, false); - CHECK(std::filesystem::exists(Payload1BPath)); + CHECK(IsFile(Payload1BPath)); Payload1B = {}; - CHECK(!std::filesystem::exists(Payload1BPath)); + CHECK(!IsFile(Payload1BPath)); } IoBuffer FetchedPayload = FileCas.FindChunk(CompressedPayload1.DecodeRawHash()); @@ -1640,7 +1640,7 @@ TEST_CASE("cas.chunk.copyoverwrite") } Payload2 = {}; - CHECK(!std::filesystem::exists(Payload2Path)); + CHECK(!IsFile(Payload2Path)); { IoHash RawHash; diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 7ac10d613..ac4dda83f 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -62,11 +62,11 @@ namespace { { if (Size == 0) { - std::filesystem::remove(Path); + RemoveFile(Path); return std::error_code{}; } CreateDirectories(Path.parent_path()); - if (std::filesystem::is_regular_file(Path) && std::filesystem::file_size(Path) == Size) + if (IsFile(Path) && FileSizeFromPath(Path) == Size) { return std::error_code(); } @@ -1081,7 +1081,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_INFO("GCV2: Locking state for {} reference checkers", ReferenceCheckers.size()); { ZEN_TRACE_CPU("GcV2::LockReferencers"); - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceLockers Latch WorkLeft(1); { @@ -1108,7 +1108,7 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::UpdateLockedState"); // Locking all references checkers so we have a steady state of which references are used - // From this point we have blocked all writes to all References (DiskBucket/ProjectStore) until + // From this point we have blocked all writes to all References (DiskBucket/ProjectStore/BuildStore) until // we delete the ReferenceCheckers Latch WorkLeft(1); @@ -1262,12 +1262,12 @@ GcManager::CollectGarbage(const GcSettings& Settings) ZEN_TRACE_CPU("GcV2::CompactStores"); auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(Settings.DiskReservePath)) + if (!IsFile(Settings.DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(Settings.DiskReservePath); - if (std::filesystem::remove(Settings.DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(Settings.DiskReservePath); + if (RemoveFile(Settings.DiskReservePath)) { return ReclaimedSize; } @@ -1557,7 +1557,7 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_Config.LightweightInterval = m_Config.MonitorInterval; } - std::filesystem::create_directories(Config.RootDirectory); + CreateDirectories(Config.RootDirectory); std::error_code Ec = CreateGCReserve(m_Config.RootDirectory / "reserve.gc", m_Config.DiskReserveSize); if (Ec) @@ -1739,6 +1739,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons { Writer << "CacheExpireTime"sv << ToDateTime(Settings.CacheExpireTime); Writer << "ProjectStoreExpireTime"sv << ToDateTime(Settings.ProjectStoreExpireTime); + Writer << "BuildStoreExpireTime"sv << ToDateTime(Settings.BuildStoreExpireTime); Writer << "CollectSmallObjects"sv << Settings.CollectSmallObjects; Writer << "IsDeleteMode"sv << Settings.IsDeleteMode; Writer << "SkipCidDelete"sv << Settings.SkipCidDelete; @@ -1849,7 +1850,7 @@ GcScheduler::GetState() const if (Result.Config.DiskReserveSize != 0) { Ec.clear(); - Result.HasDiskReserve = std::filesystem::is_regular_file(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; + Result.HasDiskReserve = IsFile(Result.Config.RootDirectory / "reserve.gc", Ec) && !Ec; } if (Result.Status != GcSchedulerStatus::kRunning) @@ -1940,6 +1941,7 @@ GcScheduler::SchedulerThread() std::chrono::seconds LightweightGcInterval = m_Config.LightweightInterval; std::chrono::seconds MaxCacheDuration = m_Config.MaxCacheDuration; std::chrono::seconds MaxProjectStoreDuration = m_Config.MaxProjectStoreDuration; + std::chrono::seconds MaxBuildStoreDuration = m_Config.MaxBuildStoreDuration; uint64_t DiskSizeSoftLimit = m_Config.DiskSizeSoftLimit; bool SkipCid = false; GcVersion UseGCVersion = m_Config.UseGCVersion; @@ -1975,6 +1977,10 @@ GcScheduler::SchedulerThread() { MaxProjectStoreDuration = TriggerParams.MaxProjectStoreDuration; } + if (TriggerParams.MaxBuildStoreDuration != std::chrono::seconds::max()) + { + MaxBuildStoreDuration = TriggerParams.MaxBuildStoreDuration; + } if (TriggerParams.DiskSizeSoftLimit != 0) { DiskSizeSoftLimit = TriggerParams.DiskSizeSoftLimit; @@ -2046,6 +2052,8 @@ GcScheduler::SchedulerThread() MaxCacheDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxCacheDuration; GcClock::TimePoint ProjectStoreExpireTime = MaxProjectStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxProjectStoreDuration; + GcClock::TimePoint BuildStoreExpireTime = + MaxBuildStoreDuration == GcClock::Duration::max() ? GcClock::TimePoint::min() : Now - MaxBuildStoreDuration; const GcStorageSize TotalSize = m_GcManager.TotalStorageSize(); @@ -2102,6 +2110,10 @@ GcScheduler::SchedulerThread() { ProjectStoreExpireTime = SizeBasedExpireTime; } + if (SizeBasedExpireTime > BuildStoreExpireTime) + { + BuildStoreExpireTime = SizeBasedExpireTime; + } } std::chrono::seconds RemainingTimeUntilGc = @@ -2227,6 +2239,7 @@ GcScheduler::SchedulerThread() bool GcSuccess = CollectGarbage(CacheExpireTime, ProjectStoreExpireTime, + BuildStoreExpireTime, DoDelete, CollectSmallObjects, SkipCid, @@ -2333,6 +2346,7 @@ GcScheduler::ScrubStorage(bool DoDelete, bool SkipCid, std::chrono::seconds Time bool GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, @@ -2375,12 +2389,12 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { // We are low on disk, check if we can release our extra storage reserve, if we can't bail from doing GC auto ClaimDiskReserve = [&]() -> uint64_t { - if (!std::filesystem::is_regular_file(DiskReservePath)) + if (!IsFile(DiskReservePath)) { return 0; } - uint64_t ReclaimedSize = std::filesystem::file_size(DiskReservePath); - if (std::filesystem::remove(DiskReservePath)) + uint64_t ReclaimedSize = FileSizeFromPath(DiskReservePath); + if (RemoveFile(DiskReservePath)) { return ReclaimedSize; } @@ -2416,6 +2430,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcSettings Settings = {.CacheExpireTime = CacheExpireTime, .ProjectStoreExpireTime = ProjectStoreExpireTime, + .BuildStoreExpireTime = BuildStoreExpireTime, .CollectSmallObjects = CollectSmallObjects, .IsDeleteMode = Delete, .SkipCidDelete = SkipCid, @@ -2447,6 +2462,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, } SB.Append(fmt::format(" Cache cutoff time: {}\n", Settings.CacheExpireTime)); SB.Append(fmt::format(" Project store cutoff time: {}\n", Settings.ProjectStoreExpireTime)); + SB.Append(fmt::format(" Build store cutoff time: {}\n", Settings.BuildStoreExpireTime)); }; { @@ -2552,6 +2568,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, if (Delete) { GcClock::TimePoint KeepRangeStart = Min(CacheExpireTime, ProjectStoreExpireTime); + KeepRangeStart = Min(KeepRangeStart, BuildStoreExpireTime); m_LastGcExpireTime = KeepRangeStart; std::unique_lock Lock(m_GcMutex); m_DiskUsageWindow.KeepRange(KeepRangeStart.time_since_epoch().count(), GcClock::Duration::max().count()); diff --git a/src/zenstore/include/zenstore/accesstime.h b/src/zenstore/include/zenstore/accesstime.h new file mode 100644 index 000000000..a28dc908b --- /dev/null +++ b/src/zenstore/include/zenstore/accesstime.h @@ -0,0 +1,47 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenstore/gc.h> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch +struct AccessTime +{ + explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} + AccessTime& operator=(GcClock::Tick Tick) noexcept + { + SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); + return *this; + } + operator GcClock::Tick() const noexcept + { + return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) + .count(); + } + + AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} + AccessTime& operator=(AccessTime&& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + AccessTime& operator=(const AccessTime& Rhs) noexcept + { + SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); + return *this; + } + +private: + static uint32_t ToSeconds(GcClock::Tick Tick) + { + return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count()); + } + std::atomic_uint32_t SecondsSinceEpoch; +}; + +} // namespace zen diff --git a/src/zenstore/include/zenstore/blockstore.h b/src/zenstore/include/zenstore/blockstore.h index 97357e5cb..0c72a13aa 100644 --- a/src/zenstore/include/zenstore/blockstore.h +++ b/src/zenstore/include/zenstore/blockstore.h @@ -156,7 +156,7 @@ public: void WriteChunk(const void* Data, uint64_t Size, uint32_t Alignment, const WriteChunkCallback& Callback); typedef std::function<void(std::span<BlockStoreLocation> Locations)> WriteChunksCallback; - void WriteChunks(std::span<IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback); + void WriteChunks(std::span<const IoBuffer> Datas, uint32_t Alignment, const WriteChunksCallback& Callback); IoBuffer TryGetChunk(const BlockStoreLocation& Location) const; void Flush(bool ForceNewBlock); diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h new file mode 100644 index 000000000..302af5f9c --- /dev/null +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -0,0 +1,186 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenstore/blockstore.h> + +#include <zencore/iohash.h> +#include <zenstore/accesstime.h> +#include <zenstore/caslog.h> +#include <zenstore/gc.h> +#include "../compactcas.h" +#include "../filecas.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include <tsl/robin_map.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +struct BuildStoreConfig +{ + std::filesystem::path RootDirectory; + uint32_t SmallBlobBlockStoreMaxBlockSize = 256 * 1024 * 1024; + uint64_t SmallBlobBlockStoreMaxBlockEmbedSize = 1 * 1024 * 1024; + uint32_t SmallBlobBlockStoreAlignement = 16; + uint32_t MetadataBlockStoreMaxBlockSize = 64 * 1024 * 1024; + uint32_t MetadataBlockStoreAlignement = 8; +}; + +class BuildStore : public GcReferencer, public GcReferenceLocker //, public GcStorage +{ +public: + explicit BuildStore(const BuildStoreConfig& Config, GcManager& Gc); + virtual ~BuildStore(); + + void PutBlob(const IoHash& BlobHashes, const IoBuffer& Payload); + IoBuffer GetBlob(const IoHash& BlobHashes); + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + std::vector<BlobExistsResult> BlobsExists(std::span<const IoHash> BlobHashes); + + void PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoBuffer> MetaDatas); + std::vector<IoBuffer> GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* OptionalWorkerPool); + + void Flush(); + +private: + void CompactState(); + + uint64_t ReadPayloadLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + uint64_t ReadMetadataLog(const RwLock::ExclusiveLockScope&, const std::filesystem::path& LogPath, uint64_t SkipEntryCount); + + //////// GcReferencer + virtual std::string GetGcName(GcCtx& Ctx) override; + virtual GcStoreCompactor* RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) override; + virtual std::vector<GcReferenceChecker*> CreateReferenceCheckers(GcCtx& Ctx) override; + virtual std::vector<GcReferenceValidator*> CreateReferenceValidators(GcCtx& Ctx) override; + + //////// GcReferenceLocker + virtual std::vector<RwLock::SharedLockScope> LockState(GcCtx& Ctx) override; + +#pragma pack(push) +#pragma pack(1) + struct PayloadEntry + { + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + static const uint8_t kStandalone = 0x20u; // This payload is stored as a standalone value + + uint8_t Flags = 0; + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + }; + static_assert(sizeof(PayloadEntry) == 4); + + struct PayloadDiskEntry + { + PayloadEntry Entry; // 4 bytes + IoHash BlobHash; // 20 bytes + }; + static_assert(sizeof(PayloadDiskEntry) == 24); + + struct MetadataEntry + { + BlockStoreLocation Location; // 12 bytes + + ZenContentType ContentType = ZenContentType::kCOUNT; // 1 byte + static const uint8_t kTombStone = 0x10u; // Represents a deleted key/value + uint8_t Flags = 0; // 1 byte + + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + }; + static_assert(sizeof(MetadataEntry) == 16); + + struct MetadataDiskEntry + { + MetadataEntry Entry; // 16 bytes + IoHash BlobHash; // 20 bytes + uint8_t Reserved1 = 0; + uint8_t Reserved2 = 0; + uint8_t Reserved3 = 0; + uint8_t Reserved4 = 0; + }; + static_assert(sizeof(MetadataDiskEntry) == 40); + +#pragma pack(pop) + + static bool ValidatePayloadDiskEntry(const PayloadDiskEntry& Entry, std::string& OutReason); + static bool ValidateMetadataDiskEntry(const MetadataDiskEntry& Entry, std::string& OutReason); + + struct PayloadIndex + { + uint32_t Index = std::numeric_limits<uint32_t>::max(); + + operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); }; + PayloadIndex() = default; + explicit PayloadIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const PayloadIndex& Other) const = default; + }; + + struct MetadataIndex + { + uint32_t Index = std::numeric_limits<uint32_t>::max(); + + operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); }; + MetadataIndex() = default; + explicit MetadataIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const MetadataIndex& Other) const = default; + }; + + struct BlobIndex + { + uint32_t Index = std::numeric_limits<uint32_t>::max(); + + operator bool() const { return Index != std::numeric_limits<uint32_t>::max(); }; + BlobIndex() = default; + explicit BlobIndex(size_t InIndex) : Index(uint32_t(InIndex)) {} + operator size_t() const { return Index; }; + inline auto operator<=>(const BlobIndex& Other) const = default; + }; + + struct BlobEntry + { + PayloadIndex Payload; + MetadataIndex Metadata; + AccessTime LastAccessTime; + }; + static_assert(sizeof(BlobEntry) == 12); + + const BuildStoreConfig m_Config; + GcManager& m_Gc; + + RwLock m_Lock; + + std::vector<PayloadEntry> m_PayloadEntries; + std::vector<MetadataEntry> m_MetadataEntries; + + std::vector<BlobEntry> m_BlobEntries; + tsl::robin_map<IoHash, BlobIndex, IoHash::Hasher> m_BlobLookup; + + FileCasStrategy m_LargeBlobStore; + CasContainerStrategy m_SmallBlobStore; + BlockStore m_MetadataBlockStore; + + TCasLogFile<PayloadDiskEntry> m_PayloadlogFile; + TCasLogFile<MetadataDiskEntry> m_MetadatalogFile; + uint64_t m_BlobLogFlushPosition = 0; + uint64_t m_MetaLogFlushPosition = 0; + + std::unique_ptr<HashSet> m_TrackedCacheKeys; + + friend class BuildStoreGcReferenceChecker; + friend class BuildStoreGcReferencePruner; + friend class BuildStoreGcCompator; +}; + +void buildstore_forcelink(); + +} // namespace zen diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 05400c784..5a51718d3 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -5,6 +5,7 @@ #include "cacheshared.h" #include <zencore/stats.h> +#include <zenstore/accesstime.h> #include <zenstore/blockstore.h> #include <zenstore/caslog.h> diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 521c78bb1..ef1b803de 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -72,42 +72,4 @@ struct CacheContentStats bool IsKnownBadBucketName(std::string_view BucketName); bool ValidateIoBuffer(ZenContentType ContentType, IoBuffer Buffer); -////////////////////////////////////////////////////////////////////////// - -// This store the access time as seconds since epoch internally in a 32-bit value giving is a range of 136 years since epoch -struct AccessTime -{ - explicit AccessTime(GcClock::Tick Tick) noexcept : SecondsSinceEpoch(ToSeconds(Tick)) {} - AccessTime& operator=(GcClock::Tick Tick) noexcept - { - SecondsSinceEpoch.store(ToSeconds(Tick), std::memory_order_relaxed); - return *this; - } - operator GcClock::Tick() const noexcept - { - return std::chrono::duration_cast<GcClock::Duration>(std::chrono::seconds(SecondsSinceEpoch.load(std::memory_order_relaxed))) - .count(); - } - - AccessTime(AccessTime&& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime(const AccessTime& Rhs) noexcept : SecondsSinceEpoch(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed)) {} - AccessTime& operator=(AccessTime&& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - AccessTime& operator=(const AccessTime& Rhs) noexcept - { - SecondsSinceEpoch.store(Rhs.SecondsSinceEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); - return *this; - } - -private: - static uint32_t ToSeconds(GcClock::Tick Tick) - { - return gsl::narrow<uint32_t>(std::chrono::duration_cast<std::chrono::seconds>(GcClock::Duration(Tick)).count()); - } - std::atomic_uint32_t SecondsSinceEpoch; -}; - } // namespace zen diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 3daae0a93..67aadef71 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -55,6 +55,7 @@ struct GcSettings { GcClock::TimePoint CacheExpireTime = GcClock::Now(); GcClock::TimePoint ProjectStoreExpireTime = GcClock::Now(); + GcClock::TimePoint BuildStoreExpireTime = GcClock::Now(); bool CollectSmallObjects = false; bool IsDeleteMode = false; bool SkipCidDelete = false; @@ -412,6 +413,7 @@ struct GcSchedulerConfig std::chrono::seconds Interval{}; std::chrono::seconds MaxCacheDuration{86400}; std::chrono::seconds MaxProjectStoreDuration{604800}; + std::chrono::seconds MaxBuildStoreDuration{604800}; bool CollectSmallObjects = true; bool Enabled = true; uint64_t DiskReserveSize = 1ul << 28; @@ -496,6 +498,7 @@ public: bool CollectSmallObjects = false; std::chrono::seconds MaxCacheDuration = std::chrono::seconds::max(); std::chrono::seconds MaxProjectStoreDuration = std::chrono::seconds::max(); + std::chrono::seconds MaxBuildStoreDuration = std::chrono::seconds::max(); uint64_t DiskSizeSoftLimit = 0; bool SkipCid = false; bool SkipDelete = false; @@ -528,6 +531,7 @@ private: void SchedulerThread(); bool CollectGarbage(const GcClock::TimePoint& CacheExpireTime, const GcClock::TimePoint& ProjectStoreExpireTime, + const GcClock::TimePoint& BuildStoreExpireTime, bool Delete, bool CollectSmallObjects, bool SkipCid, diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index 02a83d2a6..0ca2adab2 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -444,7 +444,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& RootPath = Workspace->GetConfig().RootPath; std::filesystem::path ConfigPath = RootPath / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::string Error; std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = ReadWorkspaceConfig(m_Log, RootPath, Error); @@ -458,7 +458,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) { const std::filesystem::path& SharePath = Configuration.SharePath; - if (std::filesystem::is_directory(RootPath / SharePath)) + if (IsDir(RootPath / SharePath)) { DeletedShares.erase(Configuration.Id); @@ -808,7 +808,7 @@ Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& Work ZEN_DEBUG("Reading workspaces state from {}", WorkspaceStatePath); const std::filesystem::path ConfigPath = WorkspaceStatePath / WorkspacesConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector<Workspaces::WorkspaceConfiguration> Workspaces = WorkspacesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -847,7 +847,7 @@ Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::p ZEN_DEBUG("Reading workspace state from {}", WorkspaceRoot); std::filesystem::path ConfigPath = WorkspaceRoot / WorkspaceConfigName; - if (std::filesystem::exists(ConfigPath)) + if (IsFile(ConfigPath)) { std::vector<Workspaces::WorkspaceShareConfiguration> WorkspaceShares = WorkspaceSharesFromJson(IoBufferBuilder::MakeFromFile(ConfigPath), OutError); @@ -886,7 +886,7 @@ Workspaces::AddWorkspace(const LoggerRef& Log, const std::filesystem::path& Work { throw std::invalid_argument(fmt::format("invalid root path '{}' for workspace {}", Configuration.RootPath, Configuration.Id)); } - if (!std::filesystem::is_directory(Configuration.RootPath)) + if (!IsDir(Configuration.RootPath)) { throw std::invalid_argument( fmt::format("workspace root path '{}' does not exist for workspace '{}'", Configuration.RootPath, Configuration.Id)); @@ -965,7 +965,7 @@ Workspaces::AddWorkspaceShare(const LoggerRef& Log, throw std::invalid_argument( fmt::format("workspace share path '{}' is not a sub-path of workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); } - if (!std::filesystem::is_directory(WorkspaceRoot / Configuration.SharePath)) + if (!IsDir(WorkspaceRoot / Configuration.SharePath)) { throw std::invalid_argument( fmt::format("workspace share path '{}' does not exist in workspace path '{}'", Configuration.SharePath, WorkspaceRoot)); @@ -1244,7 +1244,7 @@ Workspaces::FindWorkspaceShare(const Oid& WorkspaceId, const Oid& ShareId, bool const Workspaces::WorkspaceConfiguration& WorkspaceConfig = Workspace->GetConfig(); const Workspaces::WorkspaceShareConfiguration& ShareConfig = Share->GetConfig(); std::filesystem::path FullSharePath = WorkspaceConfig.RootPath / ShareConfig.SharePath; - if (std::filesystem::is_directory(FullSharePath)) + if (IsDir(FullSharePath)) { if (ForceRefresh || !Share->IsInitialized()) { @@ -1306,18 +1306,18 @@ namespace { std::filesystem::path EmptyFolder(RootPath / "empty_folder"); std::filesystem::path FirstFolder(RootPath / "first_folder"); - std::filesystem::create_directory(FirstFolder); + CreateDirectories(FirstFolder); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob1.bin", CreateRandomBlob(22))); Result.push_back(std::make_pair(FirstFolder / "first_folder_blob2.bin", CreateRandomBlob(122))); std::filesystem::path SecondFolder(RootPath / "second_folder"); - std::filesystem::create_directory(SecondFolder); + CreateDirectories(SecondFolder); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob1.bin", CreateRandomBlob(522))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob2.bin", CreateRandomBlob(122))); Result.push_back(std::make_pair(SecondFolder / "second_folder_blob3.bin", CreateRandomBlob(225))); std::filesystem::path SecondFolderChild(SecondFolder / "child_in_second"); - std::filesystem::create_directory(SecondFolderChild); + CreateDirectories(SecondFolderChild); Result.push_back(std::make_pair(SecondFolderChild / "second_child_folder_blob1.bin", CreateRandomBlob(622))); for (const auto& It : Result) @@ -1365,13 +1365,13 @@ TEST_CASE("workspaces.scanfolder") Structure->IterateEntries([&](const Oid& Id, const FolderStructure::FileEntry& Entry) { std::filesystem::path AbsPath = RootPath / Entry.RelativePath; - CHECK(std::filesystem::is_regular_file(AbsPath)); - CHECK(std::filesystem::file_size(AbsPath) == Entry.Size); + CHECK(IsFile(AbsPath)); + CHECK(FileSizeFromPath(AbsPath) == Entry.Size); const FolderStructure::FileEntry* FindEntry = Structure->FindEntry(Id); CHECK(FindEntry); std::filesystem::path Path = RootPath / FindEntry->RelativePath; CHECK(AbsPath == Path); - CHECK(std::filesystem::file_size(AbsPath) == FindEntry->Size); + CHECK(FileSizeFromPath(AbsPath) == FindEntry->Size); }); } diff --git a/src/zenutil/buildstoragecache.cpp b/src/zenutil/buildstoragecache.cpp new file mode 100644 index 000000000..f273ac699 --- /dev/null +++ b/src/zenutil/buildstoragecache.cpp @@ -0,0 +1,407 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenutil/buildstoragecache.h> + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/fmtutils.h> +#include <zencore/scopeguard.h> +#include <zencore/timer.h> +#include <zencore/trace.h> +#include <zencore/workthreadpool.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/packageformat.h> +#include <zenutil/workerpools.h> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <tsl/robin_set.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +class ZenBuildStorageCache : public BuildStorageCache +{ +public: + explicit ZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) + : m_HttpClient(HttpClient) + , m_Stats(Stats) + , m_Namespace(Namespace.empty() ? "none" : Namespace) + , m_Bucket(Bucket.empty() ? "none" : Bucket) + , m_TempFolderPath(std::filesystem::path(TempFolderPath).make_preferred()) + , m_BoostBackgroundThreadCount(BoostBackgroundThreadCount) + , m_BackgroundWorkPool(m_BoostBackgroundThreadCount ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)) + , m_PendingBackgroundWorkCount(1) + , m_CancelBackgroundWork(false) + { + } + + virtual ~ZenBuildStorageCache() + { + try + { + m_CancelBackgroundWork.store(true); + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + m_PendingBackgroundWorkCount.Wait(); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("~ZenBuildStorageCache() failed with: {}", Ex.what()); + } + } + + void ScheduleBackgroundWork(std::function<void()>&& Work) + { + m_PendingBackgroundWorkCount.AddCount(1); + try + { + m_BackgroundWorkPool.ScheduleWork([this, Work = std::move(Work)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::BackgroundWork"); + auto _ = MakeGuard([this]() { m_PendingBackgroundWorkCount.CountDown(); }); + if (!m_CancelBackgroundWork) + { + try + { + Work(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed executing background upload to build cache. Reason: {}", Ex.what()); + } + } + }); + } + catch (const std::exception& Ex) + { + m_PendingBackgroundWorkCount.CountDown(); + ZEN_ERROR("Failed scheduling background upload to build cache. Reason: {}", Ex.what()); + } + } + + virtual void PutBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + ZenContentType ContentType, + const CompositeBuffer& Payload) override + { + ZEN_ASSERT(!IsFlushed); + ZEN_ASSERT(ContentType == ZenContentType::kCompressedBinary); + ScheduleBackgroundWork( + [this, BuildId = Oid(BuildId), RawHash = IoHash(RawHash), ContentType, Payload = CompositeBuffer(Payload)]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBuildBlob"); + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::Response CacheResponse = + m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + Payload, + ContentType); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual IoBuffer GetBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t RangeOffset, uint64_t RangeBytes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlob"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + HttpClient::KeyValueMap Headers; + if (RangeOffset != 0 || RangeBytes != (uint64_t)-1) + { + Headers.Entries.insert({"Range", fmt::format("bytes={}-{}", RangeOffset, RangeOffset + RangeBytes - 1)}); + } + CreateDirectories(m_TempFolderPath); + HttpClient::Response CacheResponse = + m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + m_TempFolderPath, + Headers); + AddStatistic(CacheResponse); + if (CacheResponse.IsSuccess()) + { + return CacheResponse.ResponsePayload; + } + return {}; + } + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) override + { + ZEN_ASSERT(!IsFlushed); + ScheduleBackgroundWork([this, + BuildId = Oid(BuildId), + BlobRawHashes = std::vector<IoHash>(BlobHashes.begin(), BlobHashes.end()), + MetaDatas = std::vector<CbObject>(MetaDatas.begin(), MetaDatas.end())]() { + ZEN_TRACE_CPU("ZenBuildStorageCache::PutBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + const uint64_t BlobCount = BlobRawHashes.size(); + + CbPackage RequestPackage; + std::vector<CbAttachment> Attachments; + tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter RequestWriter; + RequestWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + RequestWriter.AddHash(BlobRawHashes[BlockHashIndex]); + } + RequestWriter.EndArray(); // blobHashes + + RequestWriter.BeginArray("metadatas"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + const IoHash ObjectHash = MetaDatas[BlockHashIndex].GetHash(); + RequestWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(MetaDatas[BlockHashIndex], ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + + RequestWriter.EndArray(); // metadatas + + RequestPackage.SetObject(RequestWriter.Save()); + } + RequestPackage.AddAttachments(Attachments); + + CompositeBuffer RpcRequestBuffer = FormatPackageMessageBuffer(RequestPackage); + + HttpClient::Response CacheResponse = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/putBlobMetadata", m_Namespace, m_Bucket, BuildId), + RpcRequestBuffer, + ZenContentType::kCbPackage); + AddStatistic(CacheResponse); + if (!CacheResponse.IsSuccess()) + { + ZEN_DEBUG("Failed posting blob metadata to cache: {}", CacheResponse.ErrorMessage(""sv)); + } + }); + } + + virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBlobMetadatas"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/getBlobMetadata", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + std::vector<CbObject> Result; + + CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload); + CbObject ResponseObject = ResponsePackage.GetObject(); + + CbArrayView BlobHashArray = ResponseObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadatasArray = ResponseObject["metadatas"sv].AsArrayView(); + Result.reserve(MetadatasArray.Num()); + auto BlobHashesIt = BlobHashes.begin(); + auto BlobHashArrayIt = begin(BlobHashArray); + auto MetadataArrayIt = begin(MetadatasArray); + while (MetadataArrayIt != end(MetadatasArray)) + { + const IoHash BlobHash = (*BlobHashArrayIt).AsHash(); + while (BlobHash != *BlobHashesIt) + { + ZEN_ASSERT(BlobHashesIt != BlobHashes.end()); + BlobHashesIt++; + } + + ZEN_ASSERT(BlobHash == *BlobHashesIt); + + const IoHash MetaHash = (*MetadataArrayIt).AsAttachment(); + const CbAttachment* MetaAttachment = ResponsePackage.FindAttachment(MetaHash); + ZEN_ASSERT(MetaAttachment); + + CbObject Metadata = MetaAttachment->AsObject(); + Result.emplace_back(std::move(Metadata)); + + BlobHashArrayIt++; + MetadataArrayIt++; + BlobHashesIt++; + } + return Result; + } + return {}; + } + + virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::BlobsExists"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Request; + + Request.BeginArray("blobHashes"sv); + for (const IoHash& BlobHash : BlobHashes) + { + Request.AddHash(BlobHash); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response Response = m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/exists", m_Namespace, m_Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + AddStatistic(Response); + if (Response.IsSuccess()) + { + CbObject ResponseObject = LoadCompactBinaryObject(Response.ResponsePayload); + if (!ResponseObject) + { + throw std::runtime_error("BlobExists reponse is invalid, failed to load payload as compact binary object"); + } + CbArrayView BlobsExistsArray = ResponseObject["blobExists"sv].AsArrayView(); + if (!BlobsExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'blobExists' array is missing"); + } + if (BlobsExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error(fmt::format("BlobExists reponse is invalid, 'blobExists' array contains {} entries, expected {}", + BlobsExistsArray.Num(), + BlobHashes.size())); + } + + CbArrayView MetadatasExistsArray = ResponseObject["metadataExists"sv].AsArrayView(); + if (!MetadatasExistsArray) + { + throw std::runtime_error("BlobExists reponse is invalid, 'metadataExists' array is missing"); + } + if (MetadatasExistsArray.Num() != BlobHashes.size()) + { + throw std::runtime_error( + fmt::format("BlobExists reponse is invalid, 'metadataExists' array contains {} entries, expected {}", + MetadatasExistsArray.Num(), + BlobHashes.size())); + } + + std::vector<BlobExistsResult> Result; + Result.reserve(BlobHashes.size()); + auto BlobExistsIt = begin(BlobsExistsArray); + auto MetadataExistsIt = begin(MetadatasExistsArray); + while (BlobExistsIt != end(BlobsExistsArray)) + { + ZEN_ASSERT(MetadataExistsIt != end(MetadatasExistsArray)); + + const bool HasBody = (*BlobExistsIt).AsBool(); + const bool HasMetadata = (*MetadataExistsIt).AsBool(); + + Result.push_back({.HasBody = HasBody, .HasMetadata = HasMetadata}); + + BlobExistsIt++; + MetadataExistsIt++; + } + return Result; + } + return {}; + } + + virtual void Flush(int32_t UpdateInteralMS, std::function<bool(intptr_t Remaining)>&& UpdateCallback) override + { + if (IsFlushed) + { + return; + } + if (!IsFlushed) + { + m_PendingBackgroundWorkCount.CountDown(); + IsFlushed = true; + } + if (m_PendingBackgroundWorkCount.Wait(100)) + { + return; + } + while (true) + { + intptr_t Remaining = m_PendingBackgroundWorkCount.Remaining(); + if (UpdateCallback(Remaining)) + { + if (m_PendingBackgroundWorkCount.Wait(UpdateInteralMS)) + { + UpdateCallback(0); + return; + } + } + else + { + m_CancelBackgroundWork.store(true); + } + } + } + +private: + void AddStatistic(const HttpClient::Response& Result) + { + m_Stats.TotalBytesWritten += Result.UploadedBytes; + m_Stats.TotalBytesRead += Result.DownloadedBytes; + m_Stats.TotalRequestTimeUs += uint64_t(Result.ElapsedSeconds * 1000000.0); + m_Stats.TotalRequestCount++; + } + + HttpClient& m_HttpClient; + BuildStorageCache::Statistics& m_Stats; + const std::string m_Namespace; + const std::string m_Bucket; + const std::filesystem::path m_TempFolderPath; + const bool m_BoostBackgroundThreadCount; + bool IsFlushed = false; + + WorkerThreadPool& m_BackgroundWorkPool; + Latch m_PendingBackgroundWorkCount; + std::atomic<bool> m_CancelBackgroundWork; +}; + +std::unique_ptr<BuildStorageCache> +CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount) +{ + return std::make_unique<ZenBuildStorageCache>(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BoostBackgroundThreadCount); +} + +} // namespace zen diff --git a/src/zenutil/cache/rpcrecording.cpp b/src/zenutil/cache/rpcrecording.cpp index 1f951167d..380c182b2 100644 --- a/src/zenutil/cache/rpcrecording.cpp +++ b/src/zenutil/cache/rpcrecording.cpp @@ -46,7 +46,7 @@ struct RecordedRequestsWriter void BeginWrite(const std::filesystem::path& BasePath) { m_BasePath = BasePath; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void EndWrite() @@ -426,7 +426,7 @@ RecordedRequestsSegmentWriter::BeginWrite(const std::filesystem::path& BasePath, m_BasePath = BasePath; m_SegmentIndex = SegmentIndex; m_RequestBaseIndex = RequestBaseIndex; - std::filesystem::create_directories(m_BasePath); + CreateDirectories(m_BasePath); } void @@ -1051,14 +1051,14 @@ public: static bool IsCompatible(const std::filesystem::path& BasePath) { - if (std::filesystem::exists(BasePath / "rpc_recording_info.zcb")) + if (IsFile(BasePath / "rpc_recording_info.zcb")) { return true; } const std::filesystem::path SegmentZero = BasePath / MakeSegmentPath(0); - if (std::filesystem::exists(SegmentZero / "rpc_segment_info.zcb") && std::filesystem::exists(SegmentZero / "index.bin")) + if (IsFile(SegmentZero / "rpc_segment_info.zcb") && IsFile(SegmentZero / "index.bin")) { // top-level metadata is missing, possibly because of premature exit // on the recording side diff --git a/src/zenutil/chunkblock.cpp b/src/zenutil/chunkblock.cpp index f3c14edc4..abfc0fb63 100644 --- a/src/zenutil/chunkblock.cpp +++ b/src/zenutil/chunkblock.cpp @@ -52,7 +52,7 @@ ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject) return {}; } std::vector<ChunkBlockDescription> Result; - CbArrayView Blocks = BlocksObject["blocks"].AsArrayView(); + CbArrayView Blocks = BlocksObject["blocks"sv].AsArrayView(); Result.reserve(Blocks.Num()); for (CbFieldView BlockView : Blocks) { diff --git a/src/zenutil/chunkedcontent.cpp b/src/zenutil/chunkedcontent.cpp index bb1ee5183..32ae2d94a 100644 --- a/src/zenutil/chunkedcontent.cpp +++ b/src/zenutil/chunkedcontent.cpp @@ -140,8 +140,12 @@ namespace { { ZEN_TRACE_CPU("HashOnly"); - IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); - const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); + IoBuffer Buffer = IoBufferBuilder::MakeFromFile((FolderPath / Path).make_preferred()); + if (Buffer.GetSize() != RawSize) + { + throw std::runtime_error(fmt::format("Failed opening file '{}' for hashing", FolderPath / Path)); + } + const IoHash Hash = IoHash::HashBuffer(Buffer, &Stats.BytesHashed); Lock.WithExclusiveLock([&]() { if (!RawHashToSequenceRawHashIndex.contains(Hash)) @@ -304,14 +308,22 @@ FolderContent GetUpdatedContent(const FolderContent& Old, const FolderContent& New, std::vector<std::filesystem::path>& OutDeletedPathIndexes) { ZEN_TRACE_CPU("FolderContent::GetUpdatedContent"); - FolderContent Result = {.Platform = Old.Platform}; + + const uint32_t NewPathCount = gsl::narrow<uint32_t>(New.Paths.size()); + + FolderContent Result = {.Platform = Old.Platform}; + Result.Paths.reserve(NewPathCount); + Result.RawSizes.reserve(NewPathCount); + Result.Attributes.reserve(NewPathCount); + Result.ModificationTicks.reserve(NewPathCount); + tsl::robin_map<std::string, uint32_t> NewPathToIndex; - const uint32_t NewPathCount = gsl::narrow<uint32_t>(New.Paths.size()); NewPathToIndex.reserve(NewPathCount); for (uint32_t NewPathIndex = 0; NewPathIndex < NewPathCount; NewPathIndex++) { NewPathToIndex.insert({New.Paths[NewPathIndex].generic_string(), NewPathIndex}); } + uint32_t OldPathCount = gsl::narrow<uint32_t>(Old.Paths.size()); for (uint32_t OldPathIndex = 0; OldPathIndex < OldPathCount; OldPathIndex++) { @@ -667,6 +679,12 @@ DeletePathsFromChunkedContent(const ChunkedFolderContent& BaseContent, std::span const ChunkedContentLookup BaseLookup = BuildChunkedContentLookup(BaseContent); tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> ChunkHashToChunkIndex; + const size_t ExpectedCount = BaseContent.Paths.size() - DeletedPaths.size(); + Result.Paths.reserve(ExpectedCount); + Result.RawSizes.reserve(ExpectedCount); + Result.Attributes.reserve(ExpectedCount); + Result.RawHashes.reserve(ExpectedCount); + tsl::robin_map<IoHash, uint32_t, IoHash::Hasher> RawHashToSequenceRawHashIndex; for (uint32_t PathIndex = 0; PathIndex < BaseContent.Paths.size(); PathIndex++) { diff --git a/src/zenutil/chunkingcontroller.cpp b/src/zenutil/chunkingcontroller.cpp index 2a7057a46..a5ebce193 100644 --- a/src/zenutil/chunkingcontroller.cpp +++ b/src/zenutil/chunkingcontroller.cpp @@ -41,9 +41,13 @@ class BasicChunkingController : public ChunkingController { public: BasicChunkingController(std::span<const std::string_view> ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) : m_ChunkExcludeExtensions(ExcludeExtensions.begin(), ExcludeExtensions.end()) + , m_ExcludeElfFiles(ExcludeElfFiles) + , m_ExcludeMachOFiles(ExcludeMachOFiles) , m_ChunkFileSizeLimit(ChunkFileSizeLimit) , m_ChunkingParams(ChunkingParams) { @@ -51,6 +55,8 @@ public: BasicChunkingController(CbObjectView Parameters) : m_ChunkExcludeExtensions(ReadStringArray(Parameters["ChunkExcludeExtensions"sv].AsArrayView())) + , m_ExcludeElfFiles(Parameters["ExcludeElfFiles"sv].AsBool(DefaultChunkingExcludeElfFiles)) + , m_ExcludeMachOFiles(Parameters["ExcludeMachOFiles"sv].AsBool(DefaultChunkingExcludeMachOFiles)) , m_ChunkFileSizeLimit(Parameters["ChunkFileSizeLimit"sv].AsUInt64(DefaultChunkingFileSizeLimit)) , m_ChunkingParams(ReadChunkParams(Parameters["ChunkingParams"sv].AsObjectView())) { @@ -73,6 +79,25 @@ public: } BasicFile Buffer(InputPath, BasicFile::Mode::kRead); + if (m_ExcludeElfFiles && Buffer.FileSize() > 4) + { + uint32_t ElfCheck = 0; + Buffer.Read(&ElfCheck, 4, 0); + if (ElfCheck == 0x464c457f) + { + return false; + } + } + if (m_ExcludeMachOFiles && Buffer.FileSize() > 4) + { + uint32_t MachOCheck = 0; + Buffer.Read(&MachOCheck, 4, 0); + if ((MachOCheck == 0xfeedface) || (MachOCheck == 0xcefaedfe)) + { + return false; + } + } + OutChunked = ChunkData(Buffer, 0, RawSize, m_ChunkingParams, &BytesProcessed, &AbortFlag); return true; } @@ -90,6 +115,10 @@ public: } } Writer.EndArray(); // ChunkExcludeExtensions + + Writer.AddBool("ExcludeElfFiles"sv, m_ExcludeElfFiles); + Writer.AddBool("ExcludeMachOFiles"sv, m_ExcludeMachOFiles); + Writer.AddInteger("ChunkFileSizeLimit"sv, m_ChunkFileSizeLimit); Writer.BeginObject("ChunkingParams"sv); { @@ -106,6 +135,8 @@ public: protected: const std::vector<std::string> m_ChunkExcludeExtensions; + const bool m_ExcludeElfFiles = false; + const bool m_ExcludeMachOFiles = false; const uint64_t m_ChunkFileSizeLimit; const ChunkedParams m_ChunkingParams; }; @@ -230,10 +261,16 @@ protected: std::unique_ptr<ChunkingController> CreateBasicChunkingController(std::span<const std::string_view> ExcludeExtensions, + bool ExcludeElfFiles, + bool ExcludeMachOFiles, uint64_t ChunkFileSizeLimit, const ChunkedParams& ChunkingParams) { - return std::make_unique<BasicChunkingController>(ExcludeExtensions, ChunkFileSizeLimit, ChunkingParams); + return std::make_unique<BasicChunkingController>(ExcludeExtensions, + ExcludeElfFiles, + ExcludeMachOFiles, + ChunkFileSizeLimit, + ChunkingParams); } std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters) diff --git a/src/zenutil/filebuildstorage.cpp b/src/zenutil/filebuildstorage.cpp index 130fec355..7aa252e44 100644 --- a/src/zenutil/filebuildstorage.cpp +++ b/src/zenutil/filebuildstorage.cpp @@ -235,7 +235,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); TemporaryFile::SafeWriteFile(BlockPath, Payload.Flatten().GetView()); @@ -260,7 +260,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { CreateDirectories(BlockPath.parent_path()); @@ -346,7 +346,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { BasicFile File(BlockPath, BasicFile::Mode::kRead); IoBuffer Payload; @@ -383,7 +383,7 @@ public: m_Stats.TotalRequestCount++; const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { struct WorkloadData { @@ -442,63 +442,77 @@ public: SimulateLatency(0, 0); } - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("FileBuildStorage::FindBlocks"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(BuildId), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; + uint64_t FoundCount = 0; + DirectoryContent Content; GetDirectoryContent(GetBlobsMetadataFolder(), DirectoryContentFlags::IncludeFiles, Content); - std::vector<ChunkBlockDescription> Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); for (const std::filesystem::path& MetaDataFile : Content.Files) { IoHash ChunkHash; if (IoHash::TryParse(MetaDataFile.stem().string(), ChunkHash)) { std::filesystem::path BlockPath = GetBlobPayloadPath(ChunkHash); - if (std::filesystem::is_regular_file(BlockPath)) + if (IsFile(BlockPath)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); + FoundCount++; + if (FoundCount == MaxBlockCount) + { + break; + } } } } - SimulateLatency(0, sizeof(IoHash) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override { ZEN_TRACE_CPU("FileBuildStorage::GetBlockMetadata"); ZEN_UNUSED(BuildId); - SimulateLatency(0, 0); + SimulateLatency(sizeof(Oid) + sizeof(IoHash) * BlockHashes.size(), 0); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); m_Stats.TotalRequestCount++; - std::vector<ChunkBlockDescription> Result; + CbObjectWriter Writer; + Writer.BeginArray("blocks"); + for (const IoHash& BlockHash : BlockHashes) { std::filesystem::path MetaDataFile = GetBlobMetadataPath(BlockHash); - if (std::filesystem::is_regular_file(MetaDataFile)) + if (IsFile(MetaDataFile)) { IoBuffer BlockMetaDataPayload = ReadFile(MetaDataFile).Flatten(); m_Stats.TotalBytesRead += BlockMetaDataPayload.GetSize(); CbObject BlockObject = CbObject(SharedBuffer(BlockMetaDataPayload)); - Result.emplace_back(ParseChunkBlockDescription(BlockObject)); + Writer.AddObject(BlockObject); } } - SimulateLatency(sizeof(BlockHashes) * BlockHashes.size(), sizeof(ChunkBlockDescription) * Result.size()); + Writer.EndArray(); // blocks + CbObject Result = Writer.Save(); + SimulateLatency(0, Result.GetSize()); return Result; } @@ -616,7 +630,7 @@ protected: BuildPartObject.IterateAttachments([&](CbFieldView FieldView) { const IoHash AttachmentHash = FieldView.AsBinaryAttachment(); const std::filesystem::path BlockPath = GetBlobPayloadPath(AttachmentHash); - if (!std::filesystem::is_regular_file(BlockPath)) + if (!IsFile(BlockPath)) { NeededAttachments.push_back(AttachmentHash); } diff --git a/src/zenutil/include/zenutil/buildstorage.h b/src/zenutil/include/zenutil/buildstorage.h index 2ebd65a00..b0665dbf8 100644 --- a/src/zenutil/include/zenutil/buildstorage.h +++ b/src/zenutil/include/zenutil/buildstorage.h @@ -54,9 +54,9 @@ public: uint64_t ChunkSize, std::function<void(uint64_t Offset, const IoBuffer& Chunk, uint64_t BytesRemaining)>&& Receiver) = 0; - virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) = 0; - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; + virtual void PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) = 0; virtual void PutBuildPartStats(const Oid& BuildId, const Oid& BuildPartId, const tsl::robin_map<std::string, double>& FloatStats) = 0; }; diff --git a/src/zenutil/include/zenutil/buildstoragecache.h b/src/zenutil/include/zenutil/buildstoragecache.h new file mode 100644 index 000000000..cab35328d --- /dev/null +++ b/src/zenutil/include/zenutil/buildstoragecache.h @@ -0,0 +1,57 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/logging.h> + +#include <zencore/compactbinary.h> +#include <zencore/compositebuffer.h> +#include <zenutil/chunkblock.h> + +namespace zen { + +class HttpClient; + +class BuildStorageCache +{ +public: + struct Statistics + { + std::atomic<uint64_t> TotalBytesRead = 0; + std::atomic<uint64_t> TotalBytesWritten = 0; + std::atomic<uint64_t> TotalRequestCount = 0; + std::atomic<uint64_t> TotalRequestTimeUs = 0; + std::atomic<uint64_t> TotalExecutionTimeUs = 0; + }; + + virtual ~BuildStorageCache() {} + + virtual void PutBuildBlob(const Oid& BuildId, const IoHash& RawHash, ZenContentType ContentType, const CompositeBuffer& Payload) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; + + virtual void PutBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes, std::span<const CbObject> MetaDatas) = 0; + virtual std::vector<CbObject> GetBlobMetadatas(const Oid& BuildId, std::span<const IoHash> BlobHashes) = 0; + + struct BlobExistsResult + { + bool HasBody = 0; + bool HasMetadata = 0; + }; + + virtual std::vector<BlobExistsResult> BlobsExists(const Oid& BuildId, std::span<const IoHash> BlobHashes) = 0; + + virtual void Flush( + int32_t UpdateInteralMS, + std::function<bool(intptr_t Remaining)>&& UpdateCallback = [](intptr_t) { return true; }) = 0; +}; + +std::unique_ptr<BuildStorageCache> CreateZenBuildStorageCache(HttpClient& HttpClient, + BuildStorageCache::Statistics& Stats, + std::string_view Namespace, + std::string_view Bucket, + const std::filesystem::path& TempFolderPath, + bool BoostBackgroundThreadCount); +} // namespace zen diff --git a/src/zenutil/include/zenutil/chunkingcontroller.h b/src/zenutil/include/zenutil/chunkingcontroller.h index 246f4498a..970917fb0 100644 --- a/src/zenutil/include/zenutil/chunkingcontroller.h +++ b/src/zenutil/include/zenutil/chunkingcontroller.h @@ -12,6 +12,8 @@ namespace zen { const std::vector<std::string_view> DefaultChunkingExcludeExtensions = {".exe", ".dll", ".pdb", ".self", ".mp4"}; +const bool DefaultChunkingExcludeElfFiles = true; +const bool DefaultChunkingExcludeMachOFiles = true; const ChunkedParams DefaultChunkedParams = {.MinSize = ((8u * 1u) * 1024u) - 128u, .MaxSize = 128u * 1024u, @@ -40,6 +42,8 @@ public: std::unique_ptr<ChunkingController> CreateBasicChunkingController( std::span<const std::string_view> ExcludeExtensions = DefaultChunkingExcludeExtensions, + bool ExcludeElfFiles = DefaultChunkingExcludeElfFiles, + bool ExcludeMachOFiles = DefaultChunkingExcludeMachOFiles, uint64_t ChunkFileSizeLimit = DefaultChunkingFileSizeLimit, const ChunkedParams& ChunkingParams = DefaultChunkedParams); std::unique_ptr<ChunkingController> CreateBasicChunkingController(CbObjectView Parameters); diff --git a/src/zenutil/include/zenutil/jupiter/jupitersession.h b/src/zenutil/include/zenutil/jupiter/jupitersession.h index fda4a7bfe..417ed7384 100644 --- a/src/zenutil/include/zenutil/jupiter/jupitersession.h +++ b/src/zenutil/include/zenutil/jupiter/jupitersession.h @@ -152,7 +152,7 @@ public: const Oid& BuildId, const Oid& PartId, const IoHash& RawHash); - JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId); + JupiterResult FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount); JupiterResult GetBlockMetadata(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, IoBuffer Payload); JupiterResult PutBuildPartStats(std::string_view Namespace, diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 758722156..cd28bdcb2 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -27,7 +27,6 @@ public: { ZEN_MEMSCOPE(ELLMTag::Logging); - ZEN_MEMSCOPE(ELLMTag::Logging); std::error_code Ec; if (RotateOnOpen) { diff --git a/src/zenutil/include/zenutil/workerpools.h b/src/zenutil/include/zenutil/workerpools.h index 9683ad720..df2033bca 100644 --- a/src/zenutil/include/zenutil/workerpools.h +++ b/src/zenutil/include/zenutil/workerpools.h @@ -21,6 +21,9 @@ WorkerThreadPool& GetMediumWorkerPool(EWorkloadType WorkloadType); // Worker pool with std::thread::hardware_concurrency() / 8 worker threads, but at least one thread WorkerThreadPool& GetSmallWorkerPool(EWorkloadType WorkloadType); +// Worker pool with minimum number of worker threads, but at least one thread +WorkerThreadPool& GetTinyWorkerPool(EWorkloadType WorkloadType); + // Special worker pool that does not use worker thread but issues all scheduled work on the calling thread // This is useful for debugging when multiple async thread can make stepping in debugger complicated WorkerThreadPool& GetSyncWorkerPool(); diff --git a/src/zenutil/jupiter/jupiterbuildstorage.cpp b/src/zenutil/jupiter/jupiterbuildstorage.cpp index d70fd8c00..f2d190408 100644 --- a/src/zenutil/jupiter/jupiterbuildstorage.cpp +++ b/src/zenutil/jupiter/jupiterbuildstorage.cpp @@ -49,7 +49,7 @@ public: { throw std::runtime_error(fmt::format("Failed listing builds: {} ({})", ListResult.Reason, ListResult.ErrorCode)); } - return PayloadToJson("Failed listing builds"sv, ListResult.Response); + return PayloadToCbObject("Failed listing builds"sv, ListResult.Response); } virtual CbObject PutBuild(const Oid& BuildId, const CbObject& MetaData) override @@ -66,7 +66,7 @@ public: { throw std::runtime_error(fmt::format("Failed creating build: {} ({})", PutResult.Reason, PutResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); + return PayloadToCbObject(fmt::format("Failed creating build: {}", BuildId), PutResult.Response); } virtual CbObject GetBuild(const Oid& BuildId) override @@ -81,7 +81,7 @@ public: { throw std::runtime_error(fmt::format("Failed fetching build: {} ({})", GetBuildResult.Reason, GetBuildResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build {}:", BuildId), GetBuildResult.Response); } virtual void FinalizeBuild(const Oid& BuildId) override @@ -134,7 +134,7 @@ public: GetBuildPartResult.Reason, GetBuildPartResult.ErrorCode)); } - return PayloadToJson(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); + return PayloadToCbObject(fmt::format("Failed fetching build part {}:", BuildPartId), GetBuildPartResult.Response); } virtual std::vector<IoHash> FinalizeBuildPart(const Oid& BuildId, const Oid& BuildPartId, const IoHash& PartHash) override @@ -289,22 +289,22 @@ public: } } - virtual std::vector<ChunkBlockDescription> FindBlocks(const Oid& BuildId) override + virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) override { ZEN_TRACE_CPU("Jupiter::FindBlocks"); Stopwatch ExecutionTimer; auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); - JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId); + JupiterResult FindResult = m_Session.FindBlocks(m_Namespace, m_Bucket, BuildId, MaxBlockCount); AddStatistic(FindResult); if (!FindResult.Success) { throw std::runtime_error(fmt::format("Failed fetching known blocks: {} ({})", FindResult.Reason, FindResult.ErrorCode)); } - return ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching known blocks"sv, FindResult.Response)); + return PayloadToCbObject("Failed fetching known blocks"sv, FindResult.Response); } - virtual std::vector<ChunkBlockDescription> GetBlockMetadata(const Oid& BuildId, std::span<const IoHash> BlockHashes) override + virtual CbObject GetBlockMetadatas(const Oid& BuildId, std::span<const IoHash> BlockHashes) override { ZEN_TRACE_CPU("Jupiter::GetBlockMetadata"); @@ -328,24 +328,7 @@ public: throw std::runtime_error( fmt::format("Failed fetching block metadatas: {} ({})", GetBlockMetadataResult.Reason, GetBlockMetadataResult.ErrorCode)); } - std::vector<ChunkBlockDescription> UnorderedList = - ParseChunkBlockDescriptionList(PayloadToJson("Failed fetching block metadatas", GetBlockMetadataResult.Response)); - tsl::robin_map<IoHash, size_t, IoHash::Hasher> BlockDescriptionLookup; - for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) - { - const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; - BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); - } - std::vector<ChunkBlockDescription> SortedBlockDescriptions; - SortedBlockDescriptions.reserve(BlockDescriptionLookup.size()); - for (const IoHash& BlockHash : BlockHashes) - { - if (auto It = BlockDescriptionLookup.find(BlockHash); It != BlockDescriptionLookup.end()) - { - SortedBlockDescriptions.push_back(std::move(UnorderedList[It->second])); - } - } - return SortedBlockDescriptions; + return PayloadToCbObject("Failed fetching block metadatas", GetBlockMetadataResult.Response); } virtual void PutBuildPartStats(const Oid& BuildId, @@ -373,7 +356,7 @@ public: } private: - static CbObject PayloadToJson(std::string_view Context, const IoBuffer& Payload) + static CbObject PayloadToCbObject(std::string_view Context, const IoBuffer& Payload) { if (Payload.GetContentType() == ZenContentType::kJSON) { diff --git a/src/zenutil/jupiter/jupitersession.cpp b/src/zenutil/jupiter/jupitersession.cpp index 2e4fe5258..fde86a478 100644 --- a/src/zenutil/jupiter/jupitersession.cpp +++ b/src/zenutil/jupiter/jupitersession.cpp @@ -758,10 +758,12 @@ JupiterSession::FinalizeBuildPart(std::string_view Namespace, } JupiterResult -JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId) +JupiterSession::FindBlocks(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, uint64_t MaxBlockCount) { - HttpClient::Response Response = m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks", Namespace, BucketId, BuildId), - HttpClient::Accept(ZenContentType::kCbObject)); + const std::string Parameters = MaxBlockCount == (uint64_t)-1 ? "" : fmt::format("?count={}", MaxBlockCount); + HttpClient::Response Response = + m_HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}/blocks/listBlocks{}", Namespace, BucketId, BuildId, Parameters), + HttpClient::Accept(ZenContentType::kCbObject)); return detail::ConvertResponse(Response, "JupiterSession::FindBlocks"sv); } diff --git a/src/zenutil/workerpools.cpp b/src/zenutil/workerpools.cpp index e3165e838..797034978 100644 --- a/src/zenutil/workerpools.cpp +++ b/src/zenutil/workerpools.cpp @@ -11,9 +11,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { namespace { - const int LargeWorkerThreadPoolTreadCount = gsl::narrow<int>(std::thread::hardware_concurrency()); + const int LargeWorkerThreadPoolTreadCount = gsl::narrow<int>(Max(std::thread::hardware_concurrency() - 1u, 2u)); const int MediumWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 4u), 2u)); const int SmallWorkerThreadPoolTreadCount = gsl::narrow<int>(Max((std::thread::hardware_concurrency() / 8u), 1u)); + const int TinyWorkerThreadPoolTreadCount = 1; static bool IsShutDown = false; @@ -35,6 +36,9 @@ namespace { WorkerPool BurstSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(burst)"}; WorkerPool BackgroundSmallWorkerPool = {.TreadCount = SmallWorkerThreadPoolTreadCount, .Name = "SmallThreadPool(bkg)"}; + WorkerPool BurstTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(burst)"}; + WorkerPool BackgroundTinyWorkerPool = {.TreadCount = TinyWorkerThreadPoolTreadCount, .Name = "TinyThreadPool(bkg)"}; + WorkerPool SyncWorkerPool = {.TreadCount = 0, .Name = "SyncThreadPool"}; WorkerThreadPool& EnsurePoolPtr(WorkerPool& Pool) @@ -75,6 +79,12 @@ GetSmallWorkerPool(EWorkloadType WorkloadType) } WorkerThreadPool& +GetTinyWorkerPool(EWorkloadType WorkloadType) +{ + return EnsurePoolPtr(WorkloadType == EWorkloadType::Burst ? BurstTinyWorkerPool : BackgroundTinyWorkerPool); +} + +WorkerThreadPool& GetSyncWorkerPool() { return EnsurePoolPtr(SyncWorkerPool); @@ -91,6 +101,8 @@ ShutdownWorkerPools() BackgroundMediumWorkerPool.Pool.reset(); BurstSmallWorkerPool.Pool.reset(); BackgroundSmallWorkerPool.Pool.reset(); + BurstTinyWorkerPool.Pool.reset(); + BackgroundTinyWorkerPool.Pool.reset(); SyncWorkerPool.Pool.reset(); } } // namespace zen diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index b36f11741..c0c2a754a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -534,7 +534,7 @@ ZenServerEnvironment::CreateNewTestDir() TestDir << "test"sv << int64_t(ZenServerTestCounter.fetch_add(1)); std::filesystem::path TestPath = m_TestBaseDir / TestDir.c_str(); - ZEN_ASSERT(!std::filesystem::exists(TestPath)); + ZEN_ASSERT(!IsDir(TestPath)); ZEN_INFO("Creating new test dir @ '{}'", TestPath); @@ -568,7 +568,7 @@ ZenServerInstance::~ZenServerInstance() { Shutdown(); std::error_code DummyEc; - std::filesystem::remove(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); + RemoveFile(std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"), DummyEc); } catch (const std::exception& Err) { @@ -1033,7 +1033,7 @@ std::string ZenServerInstance::GetLogOutput() const { std::filesystem::path OutputPath = std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - if (std::filesystem::is_regular_file(OutputPath)) + if (IsFile(OutputPath)) { FileContents Contents = ReadFile(OutputPath); if (!Contents.ErrorCode) @@ -1123,7 +1123,7 @@ ValidateLockFileInfo(const LockFileInfo& Info, std::string& OutReason) OutReason = fmt::format("listen port ({}) is not valid", Info.EffectiveListenPort); return false; } - if (!std::filesystem::is_directory(Info.DataDir)) + if (!IsDir(Info.DataDir)) { OutReason = fmt::format("data directory ('{}') does not exist", Info.DataDir); return false; diff --git a/src/zenvfs/vfsprovider.cpp b/src/zenvfs/vfsprovider.cpp index a3cfe9d15..9cec5372a 100644 --- a/src/zenvfs/vfsprovider.cpp +++ b/src/zenvfs/vfsprovider.cpp @@ -373,13 +373,12 @@ VfsProvider::Initialize() std::filesystem::path ManifestPath = Root / ".zen_vfs"; bool HaveManifest = false; - if (std::filesystem::exists(Root)) + if (IsFile(Root)) + { + throw std::runtime_error("specified VFS root exists but is not a directory"); + } + if (IsDir(Root)) { - if (!std::filesystem::is_directory(Root)) - { - throw std::runtime_error("specified VFS root exists but is not a directory"); - } - std::error_code Ec; m_RootPath = WideToUtf8(CanonicalPath(Root, Ec).c_str()); @@ -388,7 +387,7 @@ VfsProvider::Initialize() throw std::system_error(Ec); } - if (std::filesystem::exists(ManifestPath)) + if (IsFile(ManifestPath)) { FileContents ManifestData = zen::ReadFile(ManifestPath); CbObject Manifest = LoadCompactBinaryObject(ManifestData.Flatten()); |