diff options
| author | Stefan Boberg <[email protected]> | 2026-04-23 18:16:57 +0200 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2026-04-23 18:16:57 +0200 |
| commit | 0232b991cd7d8e3a2114ea30e4591dd3e7b65c36 (patch) | |
| tree | 94730e7594fd09ae1fa820391ce311f6daf13905 /src/zenstore | |
| parent | Fix forward declaration order for s_GotSigWinch and SigWinchHandler (diff) | |
| parent | trace: declare Region event name fields as AnsiString (#1012) (diff) | |
| download | archived-zen-sb/zen-help.tar.xz archived-zen-sb/zen-help.zip | |
Merge branch 'main' into sb/zen-helpsb/zen-help
- Combine HelpCommand (this branch) with HistoryCommand (main) in zen CLI dispatcher
- Keep filter-aware TuiPickOne rewrite; adopt main's ASCII arrow glyphs in doc comment
Diffstat (limited to 'src/zenstore')
| -rw-r--r-- | src/zenstore/blockstore.cpp | 79 | ||||
| -rw-r--r-- | src/zenstore/buildstore/buildstore.cpp | 6 | ||||
| -rw-r--r-- | src/zenstore/cache/cachedisklayer.cpp | 34 | ||||
| -rw-r--r-- | src/zenstore/cache/structuredcachestore.cpp | 34 | ||||
| -rw-r--r-- | src/zenstore/cidstore.cpp | 12 | ||||
| -rw-r--r-- | src/zenstore/compactcas.cpp | 19 | ||||
| -rw-r--r-- | src/zenstore/filecas.cpp | 10 | ||||
| -rw-r--r-- | src/zenstore/gc.cpp | 11 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/cache/cachepolicy.h | 10 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/cache/structuredcachestore.h | 3 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/cidstore.h | 22 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/memorycidstore.h | 68 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/projectstore.h | 13 | ||||
| -rw-r--r-- | src/zenstore/include/zenstore/zenstore.h | 37 | ||||
| -rw-r--r-- | src/zenstore/memorycidstore.cpp | 143 | ||||
| -rw-r--r-- | src/zenstore/projectstore.cpp | 86 | ||||
| -rw-r--r-- | src/zenstore/workspaces.cpp | 23 | ||||
| -rw-r--r-- | src/zenstore/zenstore.cpp | 20 |
18 files changed, 465 insertions, 165 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 6528fcb2f..f25a0404a 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -106,31 +106,30 @@ BlockStoreFile::Create(uint64_t InitialSize) void* FileHandle = m_File.Handle(); + // File was truncated to 0 bytes; make the size cache reflect that explicitly. + m_CachedFileSize.store(0, std::memory_order::relaxed); + // We map our m_IoBuffer beyond the file size as we will grow it over time and want // to be able to create sub-buffers of all the written range later m_IoBuffer = IoBuffer(IoBuffer::File, FileHandle, 0, InitialSize, false); } +// `m_CachedFileSize` is the single source of truth for the file size. It is +// seeded by `Open` (from the filesystem) or `Create` (to 0 after truncate), +// and grown monotonically by `Write` via CAS. The BlockStore owns its block +// files exclusively, so nothing else can change the on-disk size behind our +// back - no filesystem round-trip is needed here. uint64_t BlockStoreFile::FileSize() const { - uint64_t CachedSize = m_CachedFileSize; - if (CachedSize == 0) - { - std::error_code Ec; - uint64_t Size = m_File.FileSize(Ec); - if (Ec) - { - ZEN_WARN("Failed to get file size of block {}. Reason: {}", m_Path, Ec.message()); - return 0; - } - uint64_t Expected = 0; - m_CachedFileSize.compare_exchange_strong(Expected, Size); - return Size; - } - return CachedSize; + return m_CachedFileSize.load(std::memory_order::relaxed); } +// Safe to call while other threads hold a Ref<BlockStoreFile> and read: the +// actual unlink is deferred until the last open handle closes (Windows pending +// delete / POSIX unlinked-but-open), and the Ref keeps this object alive until +// all readers drop it. Callers may therefore release the BlockStore's insert +// lock, hand out a Ref, and concurrently mark the file here. void BlockStoreFile::MarkAsDeleteOnClose() { @@ -141,7 +140,13 @@ BlockStoreFile::MarkAsDeleteOnClose() IoBuffer BlockStoreFile::GetChunk(uint64_t Offset, uint64_t Size) { - if (Offset + Size > m_IoBuffer.GetSize()) + // Bound the request against the actual written file size rather than the + // mapped buffer size. `m_IoBuffer` is created at `InitialSize` (= block + // capacity) on `Create`, so a bare `m_IoBuffer.GetSize()` check would + // accept offsets inside the mapped region but past the real end of the + // file, returning a view over uninitialised / zero-filled bytes. + const uint64_t WrittenSize = FileSize(); + if (Offset > WrittenSize || Size > WrittenSize - Offset) { return {}; } @@ -503,13 +508,17 @@ BlockStore::SyncExistingBlocksOnDisk(const BlockIndexSet& KnownBlocks) } for (std::uint32_t BlockIndex : DeleteBlocks) { - std::filesystem::path BlockPath = GetBlockPath(m_BlocksBasePath, BlockIndex); - if (m_ChunkBlocks[BlockIndex]) + auto It = m_ChunkBlocks.find(BlockIndex); + if (It == m_ChunkBlocks.end()) { - m_TotalSize.fetch_sub(m_ChunkBlocks[BlockIndex]->TotalSize(), std::memory_order::relaxed); - m_ChunkBlocks[BlockIndex]->MarkAsDeleteOnClose(); + continue; + } + if (It->second) + { + m_TotalSize.fetch_sub(It->second->TotalSize(), std::memory_order::relaxed); + It->second->MarkAsDeleteOnClose(); } - m_ChunkBlocks.erase(BlockIndex); + m_ChunkBlocks.erase(It); } return MissingBlocks; } @@ -657,14 +666,19 @@ BlockStore::RemoveActiveWriteBlock(uint32_t BlockIndex) eastl::fixed_vector<Ref<BlockStoreFile>, 2> FlushBlocks; { RwLock::ExclusiveLockScope _(m_InsertLock); - m_ActiveWriteBlocks.erase(std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), BlockIndex)); + auto ActiveIt = std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), BlockIndex); + ZEN_ASSERT(ActiveIt != m_ActiveWriteBlocks.end()); + m_ActiveWriteBlocks.erase(ActiveIt); for (auto It = m_BlocksToFlush.begin(); It != m_BlocksToFlush.end();) { const uint32_t FlushBlockIndex = *It; if (std::find(m_ActiveWriteBlocks.begin(), m_ActiveWriteBlocks.end(), FlushBlockIndex) == m_ActiveWriteBlocks.end()) { - FlushBlocks.push_back(m_ChunkBlocks[FlushBlockIndex]); - ZEN_DEBUG("Flushing block {} at '{}'", FlushBlockIndex, GetBlockPath(m_BlocksBasePath, FlushBlockIndex)); + if (auto BlockIt = m_ChunkBlocks.find(FlushBlockIndex); BlockIt != m_ChunkBlocks.end() && BlockIt->second) + { + FlushBlocks.push_back(BlockIt->second); + ZEN_DEBUG("Flushing block {} at '{}'", FlushBlockIndex, GetBlockPath(m_BlocksBasePath, FlushBlockIndex)); + } It = m_BlocksToFlush.erase(It); } else @@ -1195,9 +1209,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, { { RwLock::ExclusiveLockScope _l(m_InsertLock); - if (m_ChunkBlocks[NewBlockIndex] == NewBlockFile) + if (auto It = m_ChunkBlocks.find(NewBlockIndex); It != m_ChunkBlocks.end() && It->second == NewBlockFile) { - m_ChunkBlocks.erase(NewBlockIndex); + m_ChunkBlocks.erase(It); } } if (NewBlockFile->IsOpen()) @@ -1370,8 +1384,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, ZEN_ERROR("{}get disk space in '{}' FAILED, reason: '{}'", LogPrefix, m_BlocksBasePath, Error.message()); { RwLock::ExclusiveLockScope _l(m_InsertLock); - ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile); - m_ChunkBlocks.erase(NextBlockIndex); + auto It = m_ChunkBlocks.find(NextBlockIndex); + ZEN_ASSERT(It != m_ChunkBlocks.end() && It->second == NewBlockFile); + m_ChunkBlocks.erase(It); } ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen()); NewBlockFile = nullptr; @@ -1390,8 +1405,9 @@ BlockStore::CompactBlocks(const BlockStoreCompactState& CompactState, NiceBytes(Space.Free + ReclaimedSpace)); { RwLock::ExclusiveLockScope _l(m_InsertLock); - ZEN_ASSERT(m_ChunkBlocks[NextBlockIndex] == NewBlockFile); - m_ChunkBlocks.erase(NextBlockIndex); + auto It = m_ChunkBlocks.find(NextBlockIndex); + ZEN_ASSERT(It != m_ChunkBlocks.end() && It->second == NewBlockFile); + m_ChunkBlocks.erase(It); } ZEN_ASSERT_SLOW(!NewBlockFile->IsOpen()); NewBlockFile = nullptr; @@ -1537,8 +1553,7 @@ BlockStore::SetMetaData(uint32_t BlockIndex, const IoBuffer& Payload) if (It->second->SetMetaData(Payload)) { uint64_t NewMetaSize = It->second->MetaSize(); - m_TotalSize += NewMetaSize; - m_TotalSize -= OldMetaSize; + m_TotalSize.fetch_add(NewMetaSize - OldMetaSize, std::memory_order::relaxed); } } } diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index dff1c3c61..a08741d31 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -1052,7 +1052,7 @@ public: { ZEN_TRACE_CPU("Builds::PreCache"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1107,7 +1107,7 @@ public: ZEN_TRACE_CPU("Builds::GetUnusedReferences"); ZEN_MEMSCOPE(GetBuildstoreTag()); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); size_t InitialCount = IoCids.size(); size_t UsedCount = InitialCount; @@ -1152,7 +1152,7 @@ BuildStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) ZEN_TRACE_CPU("Builds::RemoveExpiredData"); ZEN_MEMSCOPE(GetBuildstoreTag()); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index 4640309d9..22b9d0be5 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -140,25 +140,37 @@ namespace cache::impl { const char* LogExtension = ".slog"; const char* MetaExtension = ".meta"; + std::filesystem::path GetBucketFilePath(const std::filesystem::path& BucketDir, const std::string& BucketName, const char* Extension) + { + ExtendablePathBuilder<256> Path; + Path.Append(BucketDir); + Path /= BucketName.c_str(); + Path << Extension; + return Path.ToPath(); + } + std::filesystem::path GetIndexPath(const std::filesystem::path& BucketDir, const std::string& BucketName) { - return BucketDir / (BucketName + IndexExtension); + return GetBucketFilePath(BucketDir, BucketName, IndexExtension); } std::filesystem::path GetMetaPath(const std::filesystem::path& BucketDir, const std::string& BucketName) { - return BucketDir / (BucketName + MetaExtension); + return GetBucketFilePath(BucketDir, BucketName, MetaExtension); } std::filesystem::path GetLogPath(const std::filesystem::path& BucketDir, const std::string& BucketName) { - return BucketDir / (BucketName + LogExtension); + return GetBucketFilePath(BucketDir, BucketName, LogExtension); } std::filesystem::path GetManifestPath(const std::filesystem::path& BucketDir, const std::string& BucketName) { ZEN_UNUSED(BucketName); - return BucketDir / "zen_manifest"; + ExtendablePathBuilder<256> Path; + Path.Append(BucketDir); + Path /= "zen_manifest"; + return Path.ToPath(); } bool ValidateCacheBucketIndexEntry(const DiskIndexEntry& Entry, std::string& OutReason) @@ -3083,7 +3095,7 @@ public: { ZEN_TRACE_CPU("Z$::Bucket::CompactStore"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -3338,7 +3350,7 @@ ZenCacheDiskLayer::CacheBucket::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) { ZEN_TRACE_CPU("Z$::Bucket::RemoveExpiredData"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); size_t TotalEntries = 0; @@ -3502,7 +3514,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, { ZEN_TRACE_CPU("Z$::Bucket::GetReferencesLocked"); - auto Log = [&Logger]() { return Logger; }; + ZEN_SCOPED_LOG(Logger); auto GetAttachments = [&](const IoHash& RawHash, MemoryView Data) -> bool { if (CbValidateError Error = ValidateCompactBinary(Data, CbValidateMode::Default); Error == CbValidateError::None) @@ -3718,7 +3730,7 @@ public: { ZEN_TRACE_CPU("Z$::Bucket::PreCache"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -3753,7 +3765,7 @@ public: { ZEN_TRACE_CPU("Z$::Bucket::UpdateLockedState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -3784,7 +3796,7 @@ public: { ZEN_TRACE_CPU("Z$::Bucket::GetUnusedReferences"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); const size_t InitialCount = IoCids.size(); size_t UsedCount = InitialCount; @@ -3818,7 +3830,7 @@ ZenCacheDiskLayer::CacheBucket::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("Z$::Bucket::CreateReferenceCheckers"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index cff0e9a35..990238e1e 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -468,7 +468,7 @@ ZenCacheStore::LogWorker() LoggerRef ZCacheLog(logging::Get("z$")); - auto Log = [&ZCacheLog]() -> LoggerRef { return ZCacheLog; }; + ZEN_SCOPED_LOG(ZCacheLog); std::vector<AccessLogItem> Items; while (true) @@ -817,7 +817,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) std::function<void()> PostDropOp; { RwLock::ExclusiveLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(InNamespace); It != m_Namespaces.end()) { ZenCacheNamespace& Namespace = *It->second; m_DroppedNamespaces.push_back(std::move(It->second)); @@ -888,13 +888,13 @@ ZenCacheNamespace* ZenCacheStore::GetNamespace(std::string_view Namespace) { RwLock::SharedLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) { return It->second.get(); } if (Namespace == DefaultNamespace) { - if (auto It = m_Namespaces.find(std::string(UE4DDCNamespaceName)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(UE4DDCNamespaceName); It != m_Namespaces.end()) { return It->second.get(); } @@ -907,17 +907,17 @@ ZenCacheStore::GetNamespace(std::string_view Namespace) } RwLock::ExclusiveLockScope __(m_NamespacesLock); - if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) { return It->second.get(); } auto NewNamespace = - m_Namespaces.insert_or_assign(std::string(Namespace), - std::make_unique<ZenCacheNamespace>(m_Gc, - m_JobQueue, - m_BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace), - m_Configuration.NamespaceConfig)); + m_Namespaces.try_emplace(std::string(Namespace), + std::make_unique<ZenCacheNamespace>(m_Gc, + m_JobQueue, + m_BasePath / fmt::format("{}{}", NamespaceDiskPrefix, Namespace), + m_Configuration.NamespaceConfig)); if (m_CapturedNamespaces) { @@ -931,13 +931,13 @@ const ZenCacheNamespace* ZenCacheStore::FindNamespace(std::string_view Namespace) const { RwLock::SharedLockScope _(m_NamespacesLock); - if (auto It = m_Namespaces.find(std::string(Namespace)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(Namespace); It != m_Namespaces.end()) { return It->second.get(); } if (Namespace == DefaultNamespace) { - if (auto It = m_Namespaces.find(std::string(UE4DDCNamespaceName)); It != m_Namespaces.end()) + if (auto It = m_Namespaces.find(UE4DDCNamespaceName); It != m_Namespaces.end()) { return It->second.get(); } @@ -1086,11 +1086,9 @@ ZenCacheStore::GetBucketInfo(std::string_view NamespaceName, std::string_view Bu std::vector<RwLock::SharedLockScope> ZenCacheStore::LockState(GcCtx& Ctx) { + ZEN_UNUSED(Ctx); ZEN_TRACE_CPU("CacheStore::LockState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; - ZEN_UNUSED(Log); - std::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_NamespacesLock)); for (auto& NamespaceIt : m_Namespaces) @@ -1211,7 +1209,7 @@ public: { ZEN_TRACE_CPU("Z$::UpdateLockedState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; @@ -1276,7 +1274,7 @@ public: { ZEN_TRACE_CPU("Z$::GetUnusedReferences"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); const size_t InitialCount = IoCids.size(); size_t UsedCount = InitialCount; @@ -1309,7 +1307,7 @@ ZenCacheStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("CacheStore::CreateReferenceCheckers"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index b20d8f565..ac8a75a58 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -188,12 +188,24 @@ CidStore::Initialize(const CidStoreConfiguration& Config) } CidStore::InsertResult +CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash) +{ + return m_Impl->AddChunk(ChunkData, RawHash, InsertMode::kMayBeMovedInPlace); +} + +CidStore::InsertResult CidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode) { return m_Impl->AddChunk(ChunkData, RawHash, Mode); } std::vector<CidStore::InsertResult> +CidStore::AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes) +{ + return m_Impl->AddChunks(ChunkDatas, RawHashes, InsertMode::kMayBeMovedInPlace); +} + +std::vector<CidStore::InsertResult> CidStore::AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, InsertMode Mode) { return m_Impl->AddChunks(ChunkDatas, RawHashes, Mode); diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 43dc389e2..6f1e1d701 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -698,7 +698,7 @@ public: ZEN_TRACE_CPU("CasContainer::CompactStore"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -875,7 +875,7 @@ public: ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::RemoveUnreferencedData"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -958,7 +958,7 @@ CasContainerStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) ZEN_MEMSCOPE(GetCasContainerTag()); ZEN_TRACE_CPU("CasContainer::CreateReferencePruner"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1391,7 +1391,7 @@ TEST_CASE("compactcas.compact.gc") { ScopedTemporaryDirectory TempDir; - const int kIterationCount = 1000; + const int kIterationCount = 200; std::vector<IoHash> Keys(kIterationCount); @@ -1504,7 +1504,7 @@ TEST_CASE("compactcas.threadedinsert") ScopedTemporaryDirectory TempDir; const uint64_t kChunkSize = 1048; - const int32_t kChunkCount = 2048; + const int32_t kChunkCount = 512; uint64_t ExpectedSize = 0; tsl::robin_map<IoHash, IoBuffer, IoHash::Hasher> Chunks; @@ -1803,7 +1803,7 @@ TEST_CASE("compactcas.restart") } const uint64_t kChunkSize = 1048 + 395; - const size_t kChunkCount = 7167; + const size_t kChunkCount = 2000; std::vector<IoHash> Hashes; Hashes.reserve(kChunkCount); @@ -1984,9 +1984,8 @@ TEST_CASE("compactcas.iteratechunks") WorkerThreadPool ThreadPool(Max(GetHardwareConcurrency() - 1u, 2u), "put"); const uint64_t kChunkSize = 1048 + 395; - const size_t kChunkCount = 63840; + const size_t kChunkCount = 10000; - for (uint32_t N = 0; N < 2; N++) { GcManager Gc; CasContainerStrategy Cas(Gc); @@ -2017,7 +2016,7 @@ TEST_CASE("compactcas.iteratechunks") size_t BatchCount = Min<size_t>(kChunkCount - Offset, 512u); WorkLatch.AddCount(1); ThreadPool.ScheduleWork( - [N, &WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount]() { + [&WorkLatch, &InsertLock, &ChunkHashesLookup, &ExpectedSize, &Hashes, &Cas, Offset, BatchCount]() { auto _ = MakeGuard([&WorkLatch]() { WorkLatch.CountDown(); }); std::vector<IoBuffer> BatchBlobs; @@ -2028,7 +2027,7 @@ TEST_CASE("compactcas.iteratechunks") while (BatchBlobs.size() < BatchCount) { IoBuffer Chunk = CreateRandomBlob( - N + kChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377)); + kChunkSize + ((BatchHashes.size() % 100) + (BatchHashes.size() % 7) * 315u + Offset % 377)); IoHash Hash = IoHash::HashBuffer(Chunk); { RwLock::ExclusiveLockScope __(InsertLock); diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 0088afe6e..b254d06ab 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -315,7 +315,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: // File-based chunks have special case handling whereby we move the file into // place in the file store directory, thus avoiding unnecessary copying - if (MoveToFile(ChunkPath, Chunk)) + if (std::error_code MoveEc = MoveToFile(ChunkPath, Chunk); !MoveEc) { bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); return CasStore::InsertResult{.New = IsNew}; @@ -337,7 +337,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: if (Mode == CasStore::InsertMode::kMayBeMovedInPlace) { - if (MoveToFile(ChunkPath, Chunk)) + if (std::error_code MoveEc = MoveToFile(ChunkPath, Chunk); !MoveEc) { bool IsNew = UpdateIndex(ChunkHash, Chunk.Size()); return CasStore::InsertResult{.New = IsNew}; @@ -1231,7 +1231,7 @@ public: ZEN_MEMSCOPE(GetFileCasTag()); ZEN_TRACE_CPU("FileCas::CompactStore"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1375,7 +1375,7 @@ public: ZEN_MEMSCOPE(GetFileCasTag()); ZEN_TRACE_CPU("FileCas::RemoveUnreferencedData"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -1458,7 +1458,7 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) ZEN_TRACE_CPU("FileCas::CreateReferencePruner"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index f3edf804d..2bcd086fc 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -546,7 +546,7 @@ FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHash>& InOu return false; } - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); const bool Filter = Ctx.Settings.AttachmentRangeMax != IoHash::Max || Ctx.Settings.AttachmentRangeMin != IoHash::Zero; @@ -2063,6 +2063,14 @@ GcScheduler::GetState() const { { std::unique_lock Lock(m_GcMutex); + + if (m_TriggerGcParams || m_TriggerScrubParams) + { + // If a trigger is pending, treat it as running + Result.Status = GcSchedulerStatus::kRunning; + return Result; + } + Result.LastFullGcTime = m_LastGcTime; Result.LastFullGCDiff = m_LastFullGCDiff; Result.LastFullGcDuration = m_LastFullGcDuration; @@ -2766,6 +2774,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, ZEN_MEMSCOPE(GetGcTag()); ZEN_TRACE_CPU("GcScheduler::CollectGarbage"); + ZEN_TRACE_REGION_CAT("GcScheduler::CollectGarbage", "gc"); try { diff --git a/src/zenstore/include/zenstore/cache/cachepolicy.h b/src/zenstore/include/zenstore/cache/cachepolicy.h index 7773cd3d1..4a062a0c2 100644 --- a/src/zenstore/include/zenstore/cache/cachepolicy.h +++ b/src/zenstore/include/zenstore/cache/cachepolicy.h @@ -163,9 +163,9 @@ private: friend class CacheRecordPolicyBuilder; friend class OptionalCacheRecordPolicy; - CachePolicy RecordPolicy = CachePolicy::Default; - CachePolicy DefaultValuePolicy = CachePolicy::Default; - RefPtr<const Private::ICacheRecordPolicyShared> Shared; + CachePolicy RecordPolicy = CachePolicy::Default; + CachePolicy DefaultValuePolicy = CachePolicy::Default; + Ref<const Private::ICacheRecordPolicyShared> Shared; }; /** A cache record policy builder is used to construct a cache record policy. */ @@ -186,8 +186,8 @@ public: CacheRecordPolicy Build(); private: - CachePolicy BasePolicy = CachePolicy::Default; - RefPtr<Private::ICacheRecordPolicyShared> Shared; + CachePolicy BasePolicy = CachePolicy::Default; + Ref<Private::ICacheRecordPolicyShared> Shared; }; /** diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 3722a0d31..9e0561364 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -3,6 +3,7 @@ #pragma once #include <zencore/compactbinary.h> +#include <zencore/hashutils.h> #include <zencore/iohash.h> #include <zenstore/cache/cache.h> #include <zenstore/cache/cachedisklayer.h> @@ -307,7 +308,7 @@ private: ZenCacheNamespace* GetNamespace(std::string_view Namespace); void IterateNamespaces(const std::function<void(std::string_view Namespace, ZenCacheNamespace& Store)>& Callback) const; - typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>> NamespaceMap; + typedef std::unordered_map<std::string, std::unique_ptr<ZenCacheNamespace>, TransparentStringHash, std::equal_to<>> NamespaceMap; CacheStoreStats m_LastReportedMetrics; const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; diff --git a/src/zenstore/include/zenstore/cidstore.h b/src/zenstore/include/zenstore/cidstore.h index d54062476..c00e0449f 100644 --- a/src/zenstore/include/zenstore/cidstore.h +++ b/src/zenstore/include/zenstore/cidstore.h @@ -58,16 +58,14 @@ struct CidStoreConfiguration * */ -class CidStore final : public ChunkResolver, public StatsProvider +class CidStore final : public ChunkStore, public StatsProvider { public: CidStore(GcManager& Gc); ~CidStore(); - struct InsertResult - { - bool New = false; - }; + using InsertResult = ChunkStore::InsertResult; + enum class InsertMode { kCopyOnly, @@ -75,17 +73,17 @@ public: }; void Initialize(const CidStoreConfiguration& Config); - InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode = InsertMode::kMayBeMovedInPlace); - std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, - std::span<IoHash> RawHashes, - InsertMode Mode = InsertMode::kMayBeMovedInPlace); - virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash) override; + InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash, InsertMode Mode); + std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes) override; + std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, InsertMode Mode); + IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; bool IterateChunks(std::span<IoHash> DecompressedIds, const std::function<bool(size_t Index, const IoBuffer& Payload)>& AsyncCallback, WorkerThreadPool* OptionalWorkerPool, uint64_t LargeSizeLimit); - bool ContainsChunk(const IoHash& DecompressedId); - void FilterChunks(HashKeySet& InOutChunks); + bool ContainsChunk(const IoHash& DecompressedId) override; + void FilterChunks(HashKeySet& InOutChunks) override; void Flush(); CidStoreSize TotalSize() const; CidStoreStats Stats() const; diff --git a/src/zenstore/include/zenstore/memorycidstore.h b/src/zenstore/include/zenstore/memorycidstore.h new file mode 100644 index 000000000..0311274d5 --- /dev/null +++ b/src/zenstore/include/zenstore/memorycidstore.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "cidstore.h" +#include "zenstore.h" + +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/thread.h> + +#include <deque> +#include <span> +#include <thread> +#include <unordered_map> + +namespace zen { + +class HashKeySet; + +/** Memory-backed chunk store. + * + * Stores chunks in an in-memory hash map, optionally layered over a + * standard CidStore for write-through and read fallback. When a backing + * store is provided: + * + * - AddChunk writes to memory and asynchronously to the backing store. + * - FindChunkByCid checks memory first, then falls back to the backing store. + * - ContainsChunk and FilterChunks check memory first, then the backing store. + * + * The memory store does NOT cache read-through results from the backing store. + * Only chunks explicitly added via AddChunk/AddChunks are held in memory. + */ + +class MemoryCidStore : public ChunkStore +{ +public: + explicit MemoryCidStore(CidStore* BackingStore = nullptr); + ~MemoryCidStore(); + + InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash) override; + std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes) override; + IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + bool ContainsChunk(const IoHash& DecompressedId) override; + void FilterChunks(HashKeySet& InOutChunks) override; + +private: + RwLock m_Lock; + std::unordered_map<IoHash, IoBuffer, IoHash::Hasher> m_Chunks; + CidStore* m_BackingStore = nullptr; + + // Async write-through to backing store + struct PendingWrite + { + IoBuffer Data; + IoHash Hash; + }; + + std::mutex m_FlushLock; + std::vector<PendingWrite> m_FlushQueue; + Event m_FlushEvent; + std::thread m_FlushThread; + std::atomic<bool> m_FlushThreadEnabled{false}; + + void FlushThreadFunction(); +}; + +} // namespace zen diff --git a/src/zenstore/include/zenstore/projectstore.h b/src/zenstore/include/zenstore/projectstore.h index 100a82907..d05261967 100644 --- a/src/zenstore/include/zenstore/projectstore.h +++ b/src/zenstore/include/zenstore/projectstore.h @@ -305,11 +305,11 @@ public: std::unordered_set<IoHash, IoHash::Hasher> m_PendingPrepOpAttachments; GcClock::TimePoint m_PendingPrepOpAttachmentsRetainEnd; - RefPtr<OplogStorage> m_Storage; - uint64_t m_LogFlushPosition = 0; - bool m_IsLegacySnapshot = false; + Ref<OplogStorage> m_Storage; + uint64_t m_LogFlushPosition = 0; + bool m_IsLegacySnapshot = false; - RefPtr<OplogStorage> GetStorage(); + Ref<OplogStorage> GetStorage(); /** Scan oplog and register each entry, thus updating the in-memory tracking tables */ @@ -484,7 +484,7 @@ public: Project& Project, Oplog& Oplog, const std::unordered_set<std::string>& WantedFieldNames); - static CbObject GetChunkInfo(LoggerRef InLog, Project& Project, Oplog& Oplog, const Oid& ChunkId); + static CbObject GetChunkInfo(Project& Project, Oplog& Oplog, const Oid& ChunkId); struct GetChunkRangeResult { enum class EError : uint8_t @@ -502,8 +502,7 @@ public: uint64_t RawSize = 0; ZenContentType ContentType = ZenContentType::kUnknownContentType; }; - static GetChunkRangeResult GetChunkRange(LoggerRef InLog, - Project& Project, + static GetChunkRangeResult GetChunkRange(Project& Project, Oplog& Oplog, const Oid& ChunkId, uint64_t Offset, diff --git a/src/zenstore/include/zenstore/zenstore.h b/src/zenstore/include/zenstore/zenstore.h index bed219b4b..95ae33a4a 100644 --- a/src/zenstore/include/zenstore/zenstore.h +++ b/src/zenstore/include/zenstore/zenstore.h @@ -4,19 +4,56 @@ #include <zencore/zencore.h> +#include <span> +#include <vector> + #define ZENSTORE_API namespace zen { +class HashKeySet; class IoBuffer; struct IoHash; class ChunkResolver { public: + virtual ~ChunkResolver() = default; virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) = 0; }; +/** Abstract chunk store interface. + * + * Extends ChunkResolver with write and query operations. Both CidStore + * (disk-backed) and MemoryCidStore (in-memory) implement this interface, + * allowing callers to be agnostic about the storage backend. + */ +class ChunkStore : public ChunkResolver +{ +public: + struct InsertResult + { + bool New = false; + }; + + virtual InsertResult AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash) = 0; + virtual std::vector<InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes) = 0; + virtual bool ContainsChunk(const IoHash& DecompressedId) = 0; + virtual void FilterChunks(HashKeySet& InOutChunks) = 0; +}; + +/** Composite resolver that tries a primary store first, then a fallback. */ +class FallbackChunkResolver : public ChunkResolver +{ +public: + FallbackChunkResolver(ChunkResolver& Primary, ChunkResolver& Fallback); + IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + +private: + ChunkResolver& m_Primary; + ChunkResolver& m_Fallback; +}; + ZENSTORE_API void zenstore_forcelinktests(); } // namespace zen diff --git a/src/zenstore/memorycidstore.cpp b/src/zenstore/memorycidstore.cpp new file mode 100644 index 000000000..b4832029b --- /dev/null +++ b/src/zenstore/memorycidstore.cpp @@ -0,0 +1,143 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <zenstore/hashkeyset.h> +#include <zenstore/memorycidstore.h> + +namespace zen { + +MemoryCidStore::MemoryCidStore(CidStore* BackingStore) : m_BackingStore(BackingStore) +{ + if (m_BackingStore) + { + m_FlushThreadEnabled = true; + m_FlushThread = std::thread(&MemoryCidStore::FlushThreadFunction, this); + } +} + +MemoryCidStore::~MemoryCidStore() +{ + m_FlushThreadEnabled = false; + m_FlushEvent.Set(); + if (m_FlushThread.joinable()) + { + m_FlushThread.join(); + } +} + +MemoryCidStore::InsertResult +MemoryCidStore::AddChunk(const IoBuffer& ChunkData, const IoHash& RawHash) +{ + bool IsNew = false; + + m_Lock.WithExclusiveLock([&] { + auto [It, Inserted] = m_Chunks.try_emplace(RawHash, ChunkData); + IsNew = Inserted; + }); + + if (m_BackingStore) + { + std::lock_guard<std::mutex> Lock(m_FlushLock); + m_FlushQueue.push_back({.Data = ChunkData, .Hash = RawHash}); + m_FlushEvent.Set(); + } + + return {.New = IsNew}; +} + +std::vector<MemoryCidStore::InsertResult> +MemoryCidStore::AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes) +{ + std::vector<MemoryCidStore::InsertResult> Results; + Results.reserve(ChunkDatas.size()); + + for (size_t i = 0; i < ChunkDatas.size(); ++i) + { + Results.push_back(AddChunk(ChunkDatas[i], RawHashes[i])); + } + + return Results; +} + +IoBuffer +MemoryCidStore::FindChunkByCid(const IoHash& DecompressedId) +{ + IoBuffer Result; + + m_Lock.WithSharedLock([&] { + auto It = m_Chunks.find(DecompressedId); + if (It != m_Chunks.end()) + { + Result = It->second; + } + }); + + if (!Result && m_BackingStore) + { + Result = m_BackingStore->FindChunkByCid(DecompressedId); + } + + return Result; +} + +bool +MemoryCidStore::ContainsChunk(const IoHash& DecompressedId) +{ + bool Found = false; + + m_Lock.WithSharedLock([&] { Found = m_Chunks.find(DecompressedId) != m_Chunks.end(); }); + + if (!Found && m_BackingStore) + { + Found = m_BackingStore->ContainsChunk(DecompressedId); + } + + return Found; +} + +void +MemoryCidStore::FilterChunks(HashKeySet& InOutChunks) +{ + // Remove hashes that are present in our memory store + m_Lock.WithSharedLock([&] { InOutChunks.RemoveHashesIf([&](const IoHash& Hash) { return m_Chunks.find(Hash) != m_Chunks.end(); }); }); + + // Delegate remainder to backing store + if (m_BackingStore && !InOutChunks.IsEmpty()) + { + m_BackingStore->FilterChunks(InOutChunks); + } +} + +void +MemoryCidStore::FlushThreadFunction() +{ + SetCurrentThreadName("MemCidFlush"); + + while (m_FlushThreadEnabled) + { + m_FlushEvent.Wait(); + + std::vector<PendingWrite> Batch; + { + std::lock_guard<std::mutex> Lock(m_FlushLock); + Batch.swap(m_FlushQueue); + } + + for (PendingWrite& Write : Batch) + { + m_BackingStore->AddChunk(Write.Data, Write.Hash); + } + } + + // Drain remaining writes on shutdown + std::vector<PendingWrite> Remaining; + { + std::lock_guard<std::mutex> Lock(m_FlushLock); + Remaining.swap(m_FlushQueue); + } + for (PendingWrite& Write : Remaining) + { + m_BackingStore->AddChunk(Write.Data, Write.Hash); + } +} + +} // namespace zen diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp index 0c4abeb1e..8159f9f83 100644 --- a/src/zenstore/projectstore.cpp +++ b/src/zenstore/projectstore.cpp @@ -3403,11 +3403,11 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbPackage OpPackage) return EntryId; } -RefPtr<ProjectStore::OplogStorage> +Ref<ProjectStore::OplogStorage> ProjectStore::Oplog::GetStorage() { ZEN_MEMSCOPE(GetProjectstoreTag()); - RefPtr<OplogStorage> Storage; + Ref<OplogStorage> Storage; { RwLock::SharedLockScope _(m_OplogLock); Storage = m_Storage; @@ -3425,7 +3425,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObjectView Core) using namespace std::literals; - RefPtr<OplogStorage> Storage = GetStorage(); + Ref<OplogStorage> Storage = GetStorage(); if (!Storage) { return {}; @@ -3457,7 +3457,7 @@ ProjectStore::Oplog::AppendNewOplogEntries(std::span<CbObjectView> Cores) using namespace std::literals; - RefPtr<OplogStorage> Storage = GetStorage(); + Ref<OplogStorage> Storage = GetStorage(); if (!Storage) { return std::vector<ProjectStore::LogSequenceNumber>(Cores.size(), LogSequenceNumber{}); @@ -3516,7 +3516,18 @@ ProjectStore::Project::~Project() // Only write access times if we have not been explicitly deleted if (!m_OplogStoragePath.empty()) { - WriteAccessTimes(); + try + { + WriteAccessTimes(); + } + catch (const std::exception& Ex) + { + // RefCounted::Release() is noexcept, so a destructor that propagates an exception + // terminates the program. WriteAccessTimes() already catches I/O failures, but lock + // acquisition and allocations ahead of the internal try/catch can still throw, so + // we defend here as well. + ZEN_ERROR("project '{}': ~Project threw exception: '{}'", Identifier, Ex.what()); + } } } @@ -4739,7 +4750,7 @@ ProjectStore::GetProjectsList() CbObject ProjectStore::GetProjectFiles(LoggerRef InLog, Project& Project, Oplog& Oplog, const std::unordered_set<std::string>& WantedFieldNames) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -4894,7 +4905,7 @@ ProjectStore::GetProjectChunkInfos(LoggerRef InLog, Project& Project, Oplog& Opl ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("ProjectStore::GetProjectChunkInfos"); - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -5051,16 +5062,13 @@ ProjectStore::GetProjectChunkInfos(LoggerRef InLog, Project& Project, Oplog& Opl } CbObject -ProjectStore::GetChunkInfo(LoggerRef InLog, Project& Project, Oplog& Oplog, const Oid& ChunkId) +ProjectStore::GetChunkInfo(Project& Project, Oplog& Oplog, const Oid& ChunkId) { ZEN_MEMSCOPE(GetProjectstoreTag()); ZEN_TRACE_CPU("ProjectStore::GetChunkInfo"); using namespace std::literals; - auto Log = [&InLog]() { return InLog; }; - ZEN_UNUSED(Log); - IoBuffer Chunk = Oplog.FindChunk(Project.RootDir, ChunkId, nullptr); if (!Chunk) { @@ -5169,7 +5177,10 @@ ExtractRange(IoBuffer&& Chunk, uint64_t Offset, uint64_t Size, ZenContentType Ac const bool IsFullRange = (Offset == 0) && ((Size == ~(0ull)) || (Size == ChunkSize)); if (IsFullRange) { - Result.Chunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + if (ChunkSize > 0) + { + Result.Chunk = CompositeBuffer(SharedBuffer(std::move(Chunk))); + } Result.RawSize = 0; } else @@ -5206,8 +5217,7 @@ ExtractRange(IoBuffer&& Chunk, uint64_t Offset, uint64_t Size, ZenContentType Ac } ProjectStore::GetChunkRangeResult -ProjectStore::GetChunkRange(LoggerRef InLog, - Project& Project, +ProjectStore::GetChunkRange(Project& Project, Oplog& Oplog, const Oid& ChunkId, uint64_t Offset, @@ -5219,9 +5229,6 @@ ProjectStore::GetChunkRange(LoggerRef InLog, ZEN_TRACE_CPU("ProjectStore::GetChunkRange"); - auto Log = [&InLog]() { return InLog; }; - ZEN_UNUSED(Log); - uint64_t OldTag = OptionalInOutModificationTag == nullptr ? 0 : *OptionalInOutModificationTag; IoBuffer Chunk = Oplog.FindChunk(Project.RootDir, ChunkId, OptionalInOutModificationTag); if (!Chunk) @@ -5728,7 +5735,7 @@ public: ZEN_TRACE_CPU("Store::CompactStore"); ZEN_MEMSCOPE(GetProjectstoreTag()); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -5864,7 +5871,7 @@ ProjectStore::RemoveExpiredData(GcCtx& Ctx, GcStats& Stats) ZEN_TRACE_CPU("Store::RemoveExpiredData"); ZEN_MEMSCOPE(GetProjectstoreTag()); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -6017,7 +6024,7 @@ public: { ZEN_TRACE_CPU("Store::UpdateLockedState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; @@ -6094,7 +6101,7 @@ public: { ZEN_TRACE_CPU("Store::GetUnusedReferences"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); size_t InitialCount = IoCids.size(); size_t UsedCount = InitialCount; @@ -6118,6 +6125,7 @@ public: } private: + LoggerRef Log() { return m_ProjectStore.Log(); } ProjectStore& m_ProjectStore; std::vector<IoHash> m_References; }; @@ -6162,7 +6170,7 @@ public: { ZEN_TRACE_CPU("Store::Oplog::PreCache"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -6280,7 +6288,7 @@ public: { ZEN_TRACE_CPU("Store::Oplog::UpdateLockedState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); Stopwatch Timer; const auto _ = MakeGuard([&] { @@ -6388,7 +6396,7 @@ public: { ZEN_TRACE_CPU("Store::Oplog::GetUnusedReferences"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); const size_t InitialCount = IoCids.size(); size_t UsedCount = InitialCount; @@ -6414,6 +6422,7 @@ public: return UnusedReferences; } + LoggerRef Log() { return m_Project->Log(); } ProjectStore& m_ProjectStore; Ref<ProjectStore::Project> m_Project; std::string m_OplogId; @@ -6429,7 +6438,7 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) { ZEN_TRACE_CPU("Store::CreateReferenceCheckers"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); size_t ProjectCount = 0; size_t OplogCount = 0; @@ -6491,11 +6500,9 @@ ProjectStore::CreateReferenceCheckers(GcCtx& Ctx) std::vector<RwLock::SharedLockScope> ProjectStore::LockState(GcCtx& Ctx) { + ZEN_UNUSED(Ctx); ZEN_TRACE_CPU("Store::LockState"); - auto Log = [&Ctx]() { return Ctx.Logger; }; - ZEN_UNUSED(Log); - std::vector<RwLock::SharedLockScope> Locks; Locks.emplace_back(RwLock::SharedLockScope(m_ProjectsLock)); for (auto& ProjectIt : m_Projects) @@ -6527,7 +6534,7 @@ public: { ZEN_TRACE_CPU("Store::Validate"); - auto Log = [&Ctx]() { return Ctx.Logger; }; + ZEN_SCOPED_LOG(Ctx.Logger); ProjectStore::Oplog::ValidationResult Result; @@ -6626,9 +6633,6 @@ ProjectStore::CreateReferenceValidators(GcCtx& Ctx) return {}; } - auto Log = [&Ctx]() { return Ctx.Logger; }; - ZEN_UNUSED(Log); - DiscoverProjects(); std::vector<std::pair<std::string, std::string>> Oplogs; @@ -8243,8 +8247,7 @@ TEST_CASE("project.store.partial.read") { uint64_t ModificationTag = 0; - auto Result = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[1]][0].first, 0, @@ -8259,8 +8262,7 @@ TEST_CASE("project.store.partial.read") CompressedBuffer Attachment = CompressedBuffer::FromCompressed(Result.Chunk, RawHash, RawSize); CHECK(RawSize == Attachments[OpIds[1]][0].second.DecodeRawSize()); - auto Result2 = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result2 = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[1]][0].first, 0, @@ -8273,8 +8275,7 @@ TEST_CASE("project.store.partial.read") { uint64_t FullChunkModificationTag = 0; { - auto Result = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[2]][1].first, 0, @@ -8287,8 +8288,7 @@ TEST_CASE("project.store.partial.read") Attachments[OpIds[2]][1].second.DecodeRawSize()); } { - auto Result = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[2]][1].first, 0, @@ -8301,8 +8301,7 @@ TEST_CASE("project.store.partial.read") { uint64_t PartialChunkModificationTag = 0; { - auto Result = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[2]][1].first, 5, @@ -8325,8 +8324,7 @@ TEST_CASE("project.store.partial.read") } { - auto Result = ProjectStore.GetChunkRange(Log(), - *Project1, + auto Result = ProjectStore.GetChunkRange(*Project1, *Oplog1, Attachments[OpIds[2]][1].first, 0, diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index ad21bbc68..cfdcd294c 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -331,9 +331,6 @@ ScanFolder(LoggerRef InLog, const std::filesystem::path& Path, WorkerThreadPool& { ZEN_TRACE_CPU("workspaces::ScanFolderImpl"); - auto Log = [&InLog]() { return InLog; }; - ZEN_UNUSED(Log); - FolderScanner Data(InLog, WorkerPool, Path); Data.Traverse(); return std::make_unique<FolderStructure>(std::move(Data.FoundFiles), std::move(Data.FoundFileIds)); @@ -811,7 +808,7 @@ Workspaces::GetShareAlias(std::string_view Alias) const std::vector<Workspaces::WorkspaceConfiguration> Workspaces::ReadConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, std::string& OutError) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -835,7 +832,7 @@ Workspaces::WriteConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, const std::vector<WorkspaceConfiguration>& WorkspaceConfigurations) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -850,7 +847,7 @@ Workspaces::WriteConfig(const LoggerRef& InLog, std::vector<Workspaces::WorkspaceShareConfiguration> Workspaces::ReadWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, std::string& OutError) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -874,7 +871,7 @@ Workspaces::WriteWorkspaceConfig(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const std::vector<WorkspaceShareConfiguration>& WorkspaceShareConfigurations) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); using namespace std::literals; @@ -1049,9 +1046,6 @@ Workspaces::RemoveWorkspaceShare(const LoggerRef& Log, const std::filesystem::pa Workspaces::WorkspaceConfiguration Workspaces::FindWorkspace(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, const Oid& WorkspaceId) { - auto Log = [&InLog]() { return InLog; }; - ZEN_UNUSED(Log); - std::string Error; std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) @@ -1075,9 +1069,6 @@ Workspaces::FindWorkspace(const LoggerRef& InLog, const std::filesystem::path& WorkspaceStatePath, const std::filesystem::path& WorkspaceRoot) { - auto Log = [&InLog]() { return InLog; }; - ZEN_UNUSED(Log); - std::string Error; std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); if (!Error.empty()) @@ -1102,7 +1093,7 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, std::string_view ShareAlias, WorkspaceConfiguration& OutWorkspace) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); std::string Error; std::vector<WorkspaceConfiguration> Workspaces = ReadConfig(InLog, WorkspaceStatePath, Error); @@ -1151,7 +1142,7 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const Oid& WorkspaceShareId) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); std::string Error; std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) @@ -1174,7 +1165,7 @@ Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::pa Workspaces::WorkspaceShareConfiguration Workspaces::FindWorkspaceShare(const LoggerRef& InLog, const std::filesystem::path& WorkspaceRoot, const std::filesystem::path& SharePath) { - auto Log = [&InLog]() { return InLog; }; + ZEN_SCOPED_LOG(InLog); std::string Error; std::vector<WorkspaceShareConfiguration> Shares = ReadWorkspaceConfig(InLog, WorkspaceRoot, Error); if (!Error.empty()) diff --git a/src/zenstore/zenstore.cpp b/src/zenstore/zenstore.cpp index c563cc202..bf0c71211 100644 --- a/src/zenstore/zenstore.cpp +++ b/src/zenstore/zenstore.cpp @@ -2,6 +2,26 @@ #include "zenstore/zenstore.h" +#include <zencore/iobuffer.h> + +namespace zen { + +FallbackChunkResolver::FallbackChunkResolver(ChunkResolver& Primary, ChunkResolver& Fallback) : m_Primary(Primary), m_Fallback(Fallback) +{ +} + +IoBuffer +FallbackChunkResolver::FindChunkByCid(const IoHash& DecompressedId) +{ + if (IoBuffer Result = m_Primary.FindChunkByCid(DecompressedId)) + { + return Result; + } + return m_Fallback.FindChunkByCid(DecompressedId); +} + +} // namespace zen + #if ZEN_WITH_TESTS # include <zenstore/blockstore.h> |