diff options
| author | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
|---|---|---|
| committer | Liam Mitchell <[email protected]> | 2026-03-09 19:06:36 -0700 |
| commit | d1abc50ee9d4fb72efc646e17decafea741caa34 (patch) | |
| tree | e4288e00f2f7ca0391b83d986efcb69d3ba66a83 /src/zenstore | |
| parent | Allow requests with invalid content-types unless specified in command line or... (diff) | |
| parent | updated chunk–block analyser (#818) (diff) | |
| download | zen-d1abc50ee9d4fb72efc646e17decafea741caa34.tar.xz zen-d1abc50ee9d4fb72efc646e17decafea741caa34.zip | |
Merge branch 'main' into lm/restrict-content-type
Diffstat (limited to 'src/zenstore')
22 files changed, 201 insertions, 140 deletions
diff --git a/src/zenstore/blockstore.cpp b/src/zenstore/blockstore.cpp index 3ea91ead6..6197c7f24 100644 --- a/src/zenstore/blockstore.cpp +++ b/src/zenstore/blockstore.cpp @@ -1556,6 +1556,8 @@ BlockStore::GetMetaData(uint32_t BlockIndex) const #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.blockstore"); + TEST_CASE("blockstore.blockstoredisklocation") { BlockStoreLocation Zero = BlockStoreLocation{.BlockIndex = 0, .Offset = 0, .Size = 0}; @@ -2427,6 +2429,8 @@ TEST_CASE("blockstore.BlockStoreFileAppender") } } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 04a0781d3..dff1c3c61 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -266,13 +266,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobLookup.insert({BlobHash, NewBlobIndex}); } - m_LastAccessTimeUpdateCount++; if (m_TrackedBlobKeys) { m_TrackedBlobKeys->push_back(BlobHash); if (MetadataHash != IoHash::Zero) { - m_TrackedBlobKeys->push_back(BlobHash); + m_TrackedBlobKeys->push_back(MetadataHash); } } } @@ -374,8 +373,8 @@ BuildStore::PutMetadatas(std::span<const IoHash> BlobHashes, std::span<const IoB CompressedMetadataBuffers.resize(Metadatas.size()); if (OptionalWorkerPool) { - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); for (size_t Index = 0; Index < Metadatas.size(); Index++) { @@ -506,8 +505,8 @@ BuildStore::GetMetadatas(std::span<const IoHash> BlobHashes, WorkerThreadPool* O else { ZEN_WARN("Metadata {} for blob {} is malformed (not a compressed binary format)", - MetadataHashes[ResultIndex], - BlobHashes[ResultIndex]); + MetadataHashes[Index], + BlobHashes[MetaLocationResultIndexes[Index]]); } } } @@ -562,7 +561,7 @@ BuildStore::GetStorageStats() const RwLock::SharedLockScope _(m_Lock); Result.EntryCount = m_BlobLookup.size(); - for (auto LookupIt : m_BlobLookup) + for (const auto& LookupIt : m_BlobLookup) { const BlobIndex ReadBlobIndex = LookupIt.second; const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; @@ -635,7 +634,7 @@ BuildStore::CompactState() const size_t MetadataCount = m_MetadataEntries.size(); MetadataEntries.reserve(MetadataCount); - for (auto LookupIt : m_BlobLookup) + for (const auto& LookupIt : m_BlobLookup) { const IoHash& BlobHash = LookupIt.first; const BlobIndex ReadBlobIndex = LookupIt.second; @@ -956,7 +955,7 @@ BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::files std::vector<AccessTimeRecord> AccessRecords; AccessRecords.reserve(Header.AccessTimeCount); - for (auto It : m_BlobLookup) + for (const auto& It : m_BlobLookup) { const IoHash& Key = It.first; const BlobIndex Index = It.second; @@ -966,7 +965,7 @@ BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::files } uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount; TempFile.Write(AccessRecords.data(), RecordsSize, Offset); - Offset += sizeof(AccessTimesHeader) * Header.AccessTimeCount; + Offset += sizeof(AccessTimeRecord) * Header.AccessTimeCount; } if (TempFile.MoveTemporaryIntoPlace(AccessTimesPath, Ec); Ec) { @@ -1373,6 +1372,8 @@ BuildStore::LockState(GcCtx& Ctx) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.buildstore"); + TEST_CASE("BuildStore.Blobs") { ScopedTemporaryDirectory _; @@ -1822,6 +1823,8 @@ TEST_CASE("BuildStore.SizeLimit") } } +TEST_SUITE_END(); + void buildstore_forcelink() { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index ead7e4f3a..4640309d9 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -602,7 +602,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B if (FileSize < sizeof(BucketMetaHeader)) { - ZEN_WARN("Failed to read sidecar file '{}'. Minimum size {} expected, actual size: ", + ZEN_WARN("Failed to read sidecar file '{}'. Minimum size {} expected, actual size: {}", SidecarPath, sizeof(BucketMetaHeader), FileSize); @@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B return false; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData); if (Header.EntryCount > ExpectedEntryCount) { ZEN_WARN( @@ -654,6 +654,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B SidecarPath, sizeof(ManifestData), CurrentReadOffset); + break; } CurrentReadOffset += sizeof(ManifestData); @@ -1011,7 +1012,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(uint64_t LogPosi { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1057,7 +1058,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const return 0; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry); if (Header.EntryCount > ExpectedEntryCount) { return 0; @@ -1267,10 +1268,10 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco { RemoveMemCachedData(IndexLock, Payload); RemoveMetaData(IndexLock, Payload); + Location.Flags |= DiskLocation::kTombStone; + MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location}); } } - Location.Flags |= DiskLocation::kTombStone; - MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location}); } ZEN_ASSERT(!MissingEntries.empty()); @@ -2812,7 +2813,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c m_BucketDir, Ec.message(), RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + Sleep((3 - RetriesLeft) * 100); // Total 600 ms Ec.clear(); DataFile.MoveTemporaryIntoPlace(FsPath, Ec); RetriesLeft--; @@ -2866,11 +2867,12 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c { EntryIndex = It.value(); ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size())); - BucketPayload& Payload = m_Payloads[EntryIndex]; - uint64_t OldSize = Payload.Location.Size(); + BucketPayload& Payload = m_Payloads[EntryIndex]; + uint64_t OldSize = Payload.Location.Size(); + RemoveMemCachedData(IndexLock, Payload); + RemoveMetaData(IndexLock, Payload); Payload = BucketPayload{.Location = Loc}; m_AccessTimes[EntryIndex] = GcClock::TickCount(); - RemoveMemCachedData(IndexLock, Payload); m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed); } if ((Value.RawSize != 0 || Value.RawHash != IoHash::Zero) && Value.RawSize <= std::numeric_limits<std::uint32_t>::max()) @@ -3521,7 +3523,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, } else { - ZEN_WARN("Cache record {} payload is malformed. Reason: ", RawHash, ToString(Error)); + ZEN_WARN("Cache record {} payload is malformed. Reason: {}", RawHash, ToString(Error)); } return false; }; @@ -4282,8 +4284,8 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock SyncLock; WorkerThreadPool& Pool = GetLargeWorkerPool(EWorkloadType::Burst); - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4454,8 +4456,8 @@ ZenCacheDiskLayer::Flush() } { WorkerThreadPool& Pool = GetMediumWorkerPool(EWorkloadType::Burst); - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4496,8 +4498,8 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) RwLock::SharedLockScope _(m_Lock); - std::atomic<bool> Abort; - std::atomic<bool> Pause; + std::atomic<bool> Abort{false}; + std::atomic<bool> Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -4559,9 +4561,11 @@ ZenCacheDiskLayer::Stats() const ZenCacheDiskLayer::Info ZenCacheDiskLayer::GetInfo() const { - ZenCacheDiskLayer::Info Info = {.RootDir = m_RootDir, .Config = m_Configuration}; + ZenCacheDiskLayer::Info Info; + Info.RootDir = m_RootDir; { RwLock::SharedLockScope _(m_Lock); + Info.Config = m_Configuration; Info.BucketNames.reserve(m_Buckets.size()); for (auto& Kv : m_Buckets) { diff --git a/src/zenstore/cache/cachepolicy.cpp b/src/zenstore/cache/cachepolicy.cpp index ca8a95ca1..c1e7dc5b3 100644 --- a/src/zenstore/cache/cachepolicy.cpp +++ b/src/zenstore/cache/cachepolicy.cpp @@ -284,6 +284,9 @@ CacheRecordPolicyBuilder::Build() } #if ZEN_WITH_TESTS + +TEST_SUITE_BEGIN("store.cachepolicy"); + TEST_CASE("cachepolicy") { SUBCASE("atomics serialization") @@ -400,13 +403,13 @@ TEST_CASE("cacherecordpolicy") RecordPolicy.Save(Writer); CbObject Saved = Writer.Save()->AsObject(); CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved).Get(); - CHECK(!RecordPolicy.IsUniform()); - CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy); - CHECK(RecordPolicy.GetBasePolicy() == DefaultPolicy); - CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap); - CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap); - CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultValuePolicy); - CHECK(RecordPolicy.GetValuePolicies().size() == 2); + CHECK(!Loaded.IsUniform()); + CHECK(Loaded.GetRecordPolicy() == UnionPolicy); + CHECK(Loaded.GetBasePolicy() == DefaultPolicy); + CHECK(Loaded.GetValuePolicy(PartialOid) == PartialOverlap); + CHECK(Loaded.GetValuePolicy(NoOverlapOid) == NoOverlap); + CHECK(Loaded.GetValuePolicy(OtherOid) == DefaultValuePolicy); + CHECK(Loaded.GetValuePolicies().size() == 2); } } @@ -416,6 +419,8 @@ TEST_CASE("cacherecordpolicy") CHECK(Loaded.IsNull()); } } + +TEST_SUITE_END(); #endif void diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 94abcf547..90c5a5e60 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -866,8 +866,8 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Request.Complete = false; } } - Request.ElapsedTimeUs += Timer.GetElapsedTimeUs(); } + Request.ElapsedTimeUs += Timer.GetElapsedTimeUs(); }; m_UpstreamCache.GetCacheRecords(*Namespace, UpstreamRequests, std::move(OnCacheRecordGetComplete)); @@ -934,7 +934,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb *Namespace, Key.Bucket, Key.Hash, - Request.RecordObject ? ""sv : " (PARTIAL)"sv, + Request.RecordObject ? " (PARTIAL)"sv : ""sv, Request.Source ? Request.Source->Url : "LOCAL"sv, NiceLatencyNs(Request.ElapsedTimeUs * 1000)); m_CacheStats.MissCount++; @@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - ResponseObject.AddBool(true); + ResponseObject.AddBool(false); } } ResponseObject.EndArray(); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 52b494e45..cff0e9a35 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -608,7 +608,10 @@ ZenCacheStore::GetBatch::Commit() m_CacheStore.m_HitCount++; OpScope.SetBytes(Result.Value.GetSize()); } - m_CacheStore.m_MissCount++; + else + { + m_CacheStore.m_MissCount++; + } } } } @@ -683,8 +686,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context, return false; } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -719,8 +722,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context, } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -787,8 +790,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context, } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -813,7 +816,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) { std::function<void()> PostDropOp; { - RwLock::SharedLockScope _(m_NamespacesLock); + RwLock::ExclusiveLockScope _(m_NamespacesLock); if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) { ZenCacheNamespace& Namespace = *It->second; @@ -1392,6 +1395,8 @@ namespace testutils { } // namespace testutils +TEST_SUITE_BEGIN("store.structuredcachestore"); + TEST_CASE("cachestore.store") { ScopedTemporaryDirectory TempDir; @@ -1548,7 +1553,7 @@ TEST_CASE("cachestore.size") } } -TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) +TEST_CASE("cachestore.threadedinsert" * doctest::skip()) { // for (uint32_t i = 0; i < 100; ++i) { @@ -2741,6 +2746,8 @@ TEST_CASE("cachestore.newgc.basics") } } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index ed017988f..8855c87d8 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -153,7 +153,10 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) } for (std::future<void>& Result : Work) { - Result.get(); + if (Result.valid()) + { + Result.get(); + } } } } @@ -300,12 +303,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span<IoBuffer> Data, - std::span<IoHash> ChunkHashes, - std::span<size_t> Indexes, - std::vector<CasStore::InsertResult> Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span<IoBuffer> Data, + std::span<IoHash> ChunkHashes, + std::span<size_t> Indexes, + std::vector<CasStore::InsertResult>& Results) { for (size_t Index : Indexes) { @@ -426,7 +429,7 @@ CasImpl::IterateChunks(std::span<IoHash> DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool, LargeSizeLimit == 0 ? m_Config.HugeValueThreshold : Min(LargeSizeLimit, m_Config.HugeValueThreshold))) @@ -439,7 +442,7 @@ CasImpl::IterateChunks(std::span<IoHash> DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool, LargeSizeLimit == 0 ? m_Config.TinyValueThreshold : Min(LargeSizeLimit, m_Config.TinyValueThreshold))) @@ -452,7 +455,7 @@ CasImpl::IterateChunks(std::span<IoHash> DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool)) { @@ -512,6 +515,8 @@ CreateCasStore(GcManager& Gc) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.cas"); + TEST_CASE("CasStore") { ScopedTemporaryDirectory TempDir; @@ -553,6 +558,8 @@ TEST_CASE("CasStore") CHECK(Lookup2); } +TEST_SUITE_END(); + void CAS_forcelink() { diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 492ce9317..44664dac2 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -35,7 +35,7 @@ CasLogFile::~CasLogFile() } bool -CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) +CasLogFile::IsValid(const std::filesystem::path& FileName, size_t RecordSize) { if (!IsFile(FileName)) { @@ -71,7 +71,7 @@ CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) } void -CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode) +CasLogFile::Open(const std::filesystem::path& FileName, size_t RecordSize, Mode Mode) { m_RecordSize = RecordSize; @@ -205,7 +205,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntr m_File.Read(ReadBuffer.data(), BytesToRead, LogBaseOffset + ReadOffset); - for (int i = 0; i < int(EntriesToRead); ++i) + for (size_t i = 0; i < EntriesToRead; ++i) { Handler(ReadBuffer.data() + (i * m_RecordSize)); } diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index bedf91287..b20d8f565 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -48,13 +48,13 @@ struct CidStore::Impl std::vector<CidStore::InsertResult> AddChunks(std::span<IoBuffer> ChunkDatas, std::span<IoHash> RawHashes, CidStore::InsertMode Mode) { + ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); if (ChunkDatas.size() == 1) { std::vector<CidStore::InsertResult> Result(1); Result[0] = AddChunk(ChunkDatas[0], RawHashes[0], Mode); return Result; } - ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); std::vector<IoBuffer> Chunks; Chunks.reserve(ChunkDatas.size()); #if ZEN_BUILD_DEBUG @@ -81,6 +81,7 @@ struct CidStore::Impl m_CasStore.InsertChunks(Chunks, RawHashes, static_cast<CasStore::InsertMode>(Mode)); ZEN_ASSERT(CasResults.size() == ChunkDatas.size()); std::vector<CidStore::InsertResult> Result; + Result.reserve(CasResults.size()); for (const CasStore::InsertResult& CasResult : CasResults) { if (CasResult.New) diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 5d8f95c9e..b09892687 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -153,7 +153,7 @@ CasContainerStrategy::~CasContainerStrategy() } catch (const std::exception& Ex) { - ZEN_ERROR("~CasContainerStrategy failed with: ", Ex.what()); + ZEN_ERROR("~CasContainerStrategy failed with: {}", Ex.what()); } m_Gc.RemoveGcReferenceStore(*this); m_Gc.RemoveGcStorage(this); @@ -440,9 +440,9 @@ CasContainerStrategy::IterateChunks(std::span<const IoHash> ChunkHas return true; } - std::atomic<bool> AbortFlag; + std::atomic<bool> AbortFlag{false}; { - std::atomic<bool> PauseFlag; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -559,8 +559,8 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) std::vector<BlockStoreLocation> ChunkLocations; std::vector<IoHash> ChunkIndexToChunkHash; - std::atomic<bool> Abort; - std::atomic<bool> Pause; + std::atomic<bool> Abort{false}; + std::atomic<bool> Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -1007,7 +1007,7 @@ CasContainerStrategy::CompactIndex(RwLock::ExclusiveLockScope&) std::vector<BlockStoreDiskLocation> Locations; Locations.reserve(EntryCount); LocationMap.reserve(EntryCount); - for (auto It : m_LocationMap) + for (const auto& It : m_LocationMap) { size_t EntryIndex = Locations.size(); Locations.push_back(m_Locations[It.second]); @@ -1106,7 +1106,7 @@ CasContainerStrategy::MakeIndexSnapshot(bool ResetLog) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1136,7 +1136,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint uint64_t Size = ObjectIndexFile.FileSize(); if (Size >= sizeof(CasDiskIndexHeader)) { - uint64_t ExpectedEntryCount = (Size - sizeof(sizeof(CasDiskIndexHeader))) / sizeof(CasDiskIndexEntry); + uint64_t ExpectedEntryCount = (Size - sizeof(CasDiskIndexHeader)) / sizeof(CasDiskIndexEntry); CasDiskIndexHeader Header; ObjectIndexFile.Read(&Header, sizeof(Header), 0); if ((Header.Magic == CasDiskIndexHeader::ExpectedMagic) && (Header.Version == CasDiskIndexHeader::CurrentVersion) && @@ -1348,6 +1348,8 @@ CasContainerStrategy::OpenContainer(bool IsNewStore) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.compactcas"); + TEST_CASE("compactcas.hex") { uint32_t Value; @@ -2159,6 +2161,8 @@ TEST_CASE("compactcas.iteratechunks") } } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 31b3a68c4..0088afe6e 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -383,7 +383,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: HRESULT WriteRes = PayloadFile.Write(Cursor, Size); if (FAILED(WriteRes)) { - ThrowSystemException(hRes, fmt::format("failed to write {} bytes to shard file '{}'", ChunkSize, ChunkPath)); + ThrowSystemException(WriteRes, fmt::format("failed to write {} bytes to shard file '{}'", ChunkSize, ChunkPath)); } }; #else @@ -669,8 +669,8 @@ FileCasStrategy::IterateChunks(std::span<IoHash> ChunkHashes, return true; }; - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -823,8 +823,8 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_INFO("discovered {} files @ '{}' ({} not in index), scrubbing", m_Index.size(), m_RootDirectory, DiscoveredFilesNotInIndex); - std::atomic<bool> Abort; - std::atomic<bool> Pause; + std::atomic<bool> Abort{false}; + std::atomic<bool> Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -1016,7 +1016,7 @@ FileCasStrategy::MakeIndexSnapshot(bool ResetLog) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1052,7 +1052,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& uint64_t Size = ObjectIndexFile.FileSize(); if (Size >= sizeof(FileCasIndexHeader)) { - uint64_t ExpectedEntryCount = (Size - sizeof(sizeof(FileCasIndexHeader))) / sizeof(FileCasIndexEntry); + uint64_t ExpectedEntryCount = (Size - sizeof(FileCasIndexHeader)) / sizeof(FileCasIndexEntry); FileCasIndexHeader Header; ObjectIndexFile.Read(&Header, sizeof(Header), 0); if ((Header.Magic == FileCasIndexHeader::ExpectedMagic) && (Header.Version == FileCasIndexHeader::CurrentVersion) && @@ -1496,6 +1496,8 @@ FileCasStrategy::CreateReferencePruner(GcCtx& Ctx, GcReferenceStoreStats&) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.filecas"); + TEST_CASE("cas.chunk.mismatch") { } @@ -1793,6 +1795,8 @@ TEST_CASE("cas.file.move") # endif } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h index e93356927..41756b65f 100644 --- a/src/zenstore/filecas.h +++ b/src/zenstore/filecas.h @@ -74,7 +74,7 @@ private: { static const uint32_t kTombStone = 0x0000'0001; - bool IsFlagSet(const uint32_t Flag) const { return (Flags & kTombStone) == Flag; } + bool IsFlagSet(const uint32_t Flag) const { return (Flags & Flag) == Flag; } IoHash Key; uint32_t Flags = 0; diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 14caa5abf..b3450b805 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1494,7 +1494,8 @@ GcManager::CollectGarbage(const GcSettings& Settings) GcReferenceValidatorStats& Stats = Result.ReferenceValidatorStats[It.second].second; try { - // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or + // Go through all the ReferenceCheckers to see if the list of Cids the collector selected + // are referenced or not SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); ReferenceValidator->Validate(Ctx, Stats); } @@ -1952,7 +1953,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons Writer << "SingleThread"sv << Settings.SingleThread; Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent; Writer << "AttachmentRangeMin"sv << Settings.AttachmentRangeMin; - Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMin; + Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMax; Writer << "ForceStoreCacheAttachmentMetaData"sv << Settings.StoreCacheAttachmentMetaData; Writer << "ForceStoreProjectAttachmentMetaData"sv << Settings.StoreProjectAttachmentMetaData; Writer << "EnableValidation"sv << Settings.EnableValidation; @@ -2893,7 +2894,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { m_LastFullGCV2Result = Result; m_LastFullAttachmentRangeMin = AttachmentRangeMin; - m_LastFullAttachmentRangeMin = AttachmentRangeMax; + m_LastFullAttachmentRangeMax = AttachmentRangeMax; } Diff.DiskSize = Result.CompactStoresStatSum.RemovedDisk; Diff.MemorySize = Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory; @@ -3048,6 +3049,8 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("store.gc"); + TEST_CASE("gc.diskusagewindow") { DiskUsageWindow Stats; @@ -3379,6 +3382,8 @@ TEST_CASE("gc.attachmentrange") CHECK(AttachmentRangeMax == IoHash::Max); } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index 76cba05b9..ea2ef7f89 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -1,5 +1,5 @@ - // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include <zenstore/blockstore.h> @@ -223,7 +223,7 @@ private: uint64_t m_MetaLogFlushPosition = 0; std::unique_ptr<std::vector<IoHash>> m_TrackedBlobKeys; - std::atomic<uint64_t> m_LastAccessTimeUpdateCount; + std::atomic<uint64_t> m_LastAccessTimeUpdateCount{0}; friend class BuildStoreGcReferenceChecker; friend class BuildStoreGcReferencePruner; diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 3d684587d..393e289ac 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -153,14 +153,14 @@ public: struct BucketStats { - uint64_t DiskSize; - uint64_t MemorySize; - uint64_t DiskHitCount; - uint64_t DiskMissCount; - uint64_t DiskWriteCount; - uint64_t MemoryHitCount; - uint64_t MemoryMissCount; - uint64_t MemoryWriteCount; + uint64_t DiskSize = 0; + uint64_t MemorySize = 0; + uint64_t DiskHitCount = 0; + uint64_t DiskMissCount = 0; + uint64_t DiskWriteCount = 0; + uint64_t MemoryHitCount = 0; + uint64_t MemoryMissCount = 0; + uint64_t MemoryWriteCount = 0; metrics::RequestStatsSnapshot PutOps; metrics::RequestStatsSnapshot GetOps; }; @@ -174,8 +174,8 @@ public: struct DiskStats { std::vector<NamedBucketStats> BucketStats; - uint64_t DiskSize; - uint64_t MemorySize; + uint64_t DiskSize = 0; + uint64_t MemorySize = 0; }; struct PutResult @@ -395,12 +395,12 @@ public: TCasLogFile<DiskIndexEntry> m_SlogFile; uint64_t m_LogFlushPosition = 0; - std::atomic<uint64_t> m_DiskHitCount; - std::atomic<uint64_t> m_DiskMissCount; - std::atomic<uint64_t> m_DiskWriteCount; - std::atomic<uint64_t> m_MemoryHitCount; - std::atomic<uint64_t> m_MemoryMissCount; - std::atomic<uint64_t> m_MemoryWriteCount; + std::atomic<uint64_t> m_DiskHitCount{0}; + std::atomic<uint64_t> m_DiskMissCount{0}; + std::atomic<uint64_t> m_DiskWriteCount{0}; + std::atomic<uint64_t> m_MemoryHitCount{0}; + std::atomic<uint64_t> m_MemoryMissCount{0}; + std::atomic<uint64_t> m_MemoryWriteCount{0}; metrics::RequestStats m_PutOps; metrics::RequestStats m_GetOps; @@ -540,7 +540,7 @@ private: Configuration m_Configuration; std::atomic_uint64_t m_TotalMemCachedSize{}; std::atomic_bool m_IsMemCacheTrimming = false; - std::atomic<GcClock::Tick> m_NextAllowedTrimTick; + std::atomic<GcClock::Tick> m_NextAllowedTrimTick{}; mutable RwLock m_Lock; BucketMap_t m_Buckets; std::vector<std::unique_ptr<CacheBucket>> m_DroppedBuckets; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 791720589..8e9cd7fd7 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -40,12 +40,12 @@ struct CacheValueDetails { struct ValueDetails { - uint64_t Size; - uint64_t RawSize; + uint64_t Size = 0; + uint64_t RawSize = 0; IoHash RawHash; GcClock::Tick LastAccess{}; std::vector<IoHash> Attachments; - ZenContentType ContentType; + ZenContentType ContentType = ZenContentType::kBinary; }; struct BucketDetails diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 5a0a8b069..3722a0d31 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -70,9 +70,9 @@ public: struct NamespaceStats { - uint64_t HitCount; - uint64_t MissCount; - uint64_t WriteCount; + uint64_t HitCount = 0; + uint64_t MissCount = 0; + uint64_t WriteCount = 0; metrics::RequestStatsSnapshot PutOps; metrics::RequestStatsSnapshot GetOps; ZenCacheDiskLayer::DiskStats DiskStats; @@ -342,11 +342,11 @@ private: void LogWorker(); RwLock m_LogQueueLock; std::vector<AccessLogItem> m_LogQueue; - std::atomic_bool m_ExitLogging; + std::atomic_bool m_ExitLogging{false}; Event m_LogEvent; std::thread m_AsyncLoggingThread; - std::atomic_bool m_WriteLogEnabled; - std::atomic_bool m_AccessLogEnabled; + std::atomic_bool m_WriteLogEnabled{false}; + std::atomic_bool m_AccessLogEnabled{false}; friend class CacheStoreReferenceChecker; }; diff --git a/src/zenstore/include/zenstore/caslog.h b/src/zenstore/include/zenstore/caslog.h index f3dd32fb1..7967d9dae 100644 --- a/src/zenstore/include/zenstore/caslog.h +++ b/src/zenstore/include/zenstore/caslog.h @@ -20,8 +20,8 @@ public: kTruncate }; - static bool IsValid(std::filesystem::path FileName, size_t RecordSize); - void Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode); + static bool IsValid(const std::filesystem::path& FileName, size_t RecordSize); + void Open(const std::filesystem::path& FileName, size_t RecordSize, Mode Mode); void Append(const void* DataPointer, uint64_t DataSize); void Replay(std::function<void(const void*)>&& Handler, uint64_t SkipEntryCount); void Flush(); @@ -48,7 +48,7 @@ private: static_assert(sizeof(FileHeader) == 64); private: - void Open(std::filesystem::path FileName, size_t RecordSize, BasicFile::Mode Mode); + void Open(const std::filesystem::path& FileName, size_t RecordSize, BasicFile::Mode Mode); BasicFile m_File; FileHeader m_Header; @@ -60,8 +60,8 @@ template<typename T> class TCasLogFile : public CasLogFile { public: - static bool IsValid(std::filesystem::path FileName) { return CasLogFile::IsValid(FileName, sizeof(T)); } - void Open(std::filesystem::path FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); } + static bool IsValid(const std::filesystem::path& FileName) { return CasLogFile::IsValid(FileName, sizeof(T)); } + void Open(const std::filesystem::path& FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); } // This should be called before the Replay() is called to do some basic sanity checking bool Initialize() { return true; } diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 734d2e5a7..67cf852f9 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -238,7 +238,7 @@ bool FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector<IoHa /** * @brief An interface to implement a lock for Stop The World (from writing new data) * - * This interface is registered/unregistered to GcManager vua AddGcReferenceLocker() and RemoveGcReferenceLockerr() + * This interface is registered/unregistered to GcManager via AddGcReferenceLocker() and RemoveGcReferenceLocker() */ class GcReferenceLocker { @@ -443,8 +443,8 @@ struct GcSchedulerState uint64_t DiskFree = 0; GcClock::TimePoint LastFullGcTime{}; GcClock::TimePoint LastLightweightGcTime{}; - std::chrono::seconds RemainingTimeUntilLightweightGc; - std::chrono::seconds RemainingTimeUntilFullGc; + std::chrono::seconds RemainingTimeUntilLightweightGc{}; + std::chrono::seconds RemainingTimeUntilFullGc{}; uint64_t RemainingSpaceUntilFullGC = 0; std::chrono::milliseconds LastFullGcDuration{}; @@ -562,7 +562,7 @@ private: GcClock::TimePoint m_LastGcExpireTime{}; IoHash m_LastFullAttachmentRangeMin = IoHash::Zero; IoHash m_LastFullAttachmentRangeMax = IoHash::Max; - uint8_t m_AttachmentPassIndex; + uint8_t m_AttachmentPassIndex = 0; std::chrono::milliseconds m_LastFullGcDuration{}; GcStorageSize m_LastFullGCDiff; diff --git a/src/zenstore/include/zenstore/projectstore.h b/src/zenstore/include/zenstore/projectstore.h index 33ef996db..6f49cd024 100644 --- a/src/zenstore/include/zenstore/projectstore.h +++ b/src/zenstore/include/zenstore/projectstore.h @@ -67,8 +67,8 @@ public: struct OplogEntryAddress { - uint32_t Offset; // note: Multiple of m_OpsAlign! - uint32_t Size; + uint32_t Offset = 0; // note: Multiple of m_OpsAlign! + uint32_t Size = 0; }; struct OplogEntry @@ -80,11 +80,7 @@ public: uint32_t Reserved; inline bool IsTombstone() const { return OpCoreAddress.Offset == 0 && OpCoreAddress.Size == 0 && OpLsn.Number; } - inline void MakeTombstone() - { - OpLsn = {}; - OpCoreAddress.Offset = OpCoreAddress.Size = OpCoreHash = Reserved = 0; - } + inline void MakeTombstone() { OpCoreAddress.Offset = OpCoreAddress.Size = OpCoreHash = Reserved = 0; } }; static_assert(IsPow2(sizeof(OplogEntry))); diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp index 1ab2b317a..03086b473 100644 --- a/src/zenstore/projectstore.cpp +++ b/src/zenstore/projectstore.cpp @@ -1488,7 +1488,7 @@ ProjectStore::Oplog::Read() else { std::vector<OplogEntry> OpLogEntries; - uint64_t InvalidEntries; + uint64_t InvalidEntries = 0; m_Storage->ReadOplogEntriesFromLog(OpLogEntries, InvalidEntries, m_LogFlushPosition); for (const OplogEntry& OpEntry : OpLogEntries) { @@ -1750,8 +1750,8 @@ ProjectStore::Oplog::Validate(const std::filesystem::path& ProjectRootDir, } }; - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -2373,7 +2373,7 @@ ProjectStore::Oplog::IterateChunks(const std::filesystem::path& P else if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end()) { CidChunkIndexes.push_back(ChunkIndex); - CidChunkHashes.push_back(ChunkIt->second); + CidChunkHashes.push_back(MetaIt->second); } else if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) { @@ -2384,8 +2384,8 @@ ProjectStore::Oplog::IterateChunks(const std::filesystem::path& P } if (OptionalWorkerPool) { - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -3817,7 +3817,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo std::filesystem::path DeletePath; if (!RemoveOplog(OplogId, DeletePath)) { - ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath); + ZEN_WARN("Failed to clean up deleted oplog {}/{} at '{}'", Identifier, OplogId, OplogBasePath); } ReOpen = true; @@ -4053,8 +4053,8 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) RwLock::SharedLockScope _(m_ProjectLock); - std::atomic<bool> Abort; - std::atomic<bool> Pause; + std::atomic<bool> Abort{false}; + std::atomic<bool> Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -4360,7 +4360,7 @@ ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcMa , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker()) { ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath); - // m_Log.set_level(spdlog::level::debug); + // m_Log.SetLogLevel(zen::logging::Debug); m_Gc.AddGcStorage(this); m_Gc.AddGcReferencer(*this); m_Gc.AddGcReferenceLocker(*this); @@ -4433,8 +4433,8 @@ ProjectStore::Flush() } WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); - std::atomic<bool> AbortFlag; - std::atomic<bool> PauseFlag; + std::atomic<bool> AbortFlag{false}; + std::atomic<bool> PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4712,6 +4712,13 @@ ProjectStore::GetProjectsList() Response << "ProjectRootDir"sv << PathToUtf8(Prj.ProjectRootDir); Response << "EngineRootDir"sv << PathToUtf8(Prj.EngineRootDir); Response << "ProjectFilePath"sv << PathToUtf8(Prj.ProjectFilePath); + + const auto AccessTime = Prj.LastOplogAccessTime(""sv); + if (AccessTime != GcClock::TimePoint::min()) + { + Response << "LastAccessTime"sv << gsl::narrow<uint64_t>(AccessTime.time_since_epoch().count()); + } + Response.EndObject(); }); Response.EndArray(); @@ -4974,7 +4981,7 @@ ProjectStore::GetProjectChunkInfos(LoggerRef InLog, Project& Project, Oplog& Opl } if (WantsRawSizeField) { - ZEN_ASSERT_SLOW(Sizes[Index] == (uint64_t)-1); + ZEN_ASSERT_SLOW(RawSizes[Index] == (uint64_t)-1); RawSizes[Index] = Payload.GetSize(); } } @@ -5762,7 +5769,7 @@ public: } } - for (auto ProjectIt : m_ProjectStore.m_Projects) + for (const auto& ProjectIt : m_ProjectStore.m_Projects) { Ref<ProjectStore::Project> Project = ProjectIt.second; std::vector<std::string> OplogsToCompact = Project->GetOplogsToCompact(); @@ -6802,6 +6809,8 @@ namespace testutils { } // namespace testutils +TEST_SUITE_BEGIN("store.projectstore"); + TEST_CASE("project.opkeys") { using namespace std::literals; @@ -8473,6 +8482,8 @@ TEST_CASE("project.store.iterateoplog") } } +TEST_SUITE_END(); + #endif void diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index f0f975af4..ad21bbc68 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -383,7 +383,7 @@ Workspace::GetShares() const { std::vector<Ref<WorkspaceShare>> Shares; Shares.reserve(m_Shares.size()); - for (auto It : m_Shares) + for (const auto& It : m_Shares) { Shares.push_back(It.second); } @@ -435,7 +435,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) Workspace = FindWorkspace(Lock, WorkspaceId); if (Workspace) { - for (auto Share : Workspace->GetShares()) + for (const auto& Share : Workspace->GetShares()) { DeletedShares.insert(Share->GetConfig().Id); } @@ -482,6 +482,12 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) m_ShareAliases.erase(Share->GetConfig().Alias); } Workspace->SetShare(Configuration.Id, std::move(NewShare)); + if (!Configuration.Alias.empty()) + { + m_ShareAliases.insert_or_assign( + Configuration.Alias, + ShareAlias{.WorkspaceId = WorkspaceId, .ShareId = Configuration.Id}); + } } } else @@ -602,7 +608,7 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, { RequestedOffset = Size; } - if ((RequestedOffset + RequestedSize) > Size) + if (RequestedSize > Size - RequestedOffset) { RequestedSize = Size - RequestedOffset; } @@ -649,7 +655,7 @@ Workspaces::GetWorkspaces() const { std::vector<Oid> Workspaces; RwLock::SharedLockScope Lock(m_Lock); - for (auto It : m_Workspaces) + for (const auto& It : m_Workspaces) { Workspaces.push_back(It.first); } @@ -679,7 +685,7 @@ Workspaces::GetWorkspaceShares(const Oid& WorkspaceId) const if (Workspace) { std::vector<Oid> Shares; - for (auto Share : Workspace->GetShares()) + for (const auto& Share : Workspace->GetShares()) { Shares.push_back(Share->GetConfig().Id); } @@ -1356,6 +1362,8 @@ namespace { } // namespace +TEST_SUITE_BEGIN("store.workspaces"); + TEST_CASE("workspaces.scanfolder") { using namespace std::literals; @@ -1559,6 +1567,8 @@ TEST_CASE("workspace.share.alias") CHECK(!WS.GetShareAlias("my_share").has_value()); } +TEST_SUITE_END(); + #endif void |