diff options
Diffstat (limited to 'zenserver/cache/structuredcachestore.cpp')
| -rw-r--r-- | zenserver/cache/structuredcachestore.cpp | 94 |
1 files changed, 49 insertions, 45 deletions
diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 1fab3b312..9773c2ed2 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -552,8 +552,17 @@ void ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value) { { + BucketValue IndexValue(Value.Value, GcClock::TickCount()); RwLock::ExclusiveLockScope _(m_BucketLock); - m_CacheMap.insert_or_assign(HashKey, BucketValue(Value.Value, GcClock::TickCount())); + if (auto It = m_CacheMap.find(HashKey); It != m_CacheMap.end()) + { + m_TotalSize.fetch_sub(It->second.Payload.GetSize(), std::memory_order::relaxed); + It.value() = std::move(IndexValue); + } + else + { + m_CacheMap.insert_or_assign(HashKey, std::move(IndexValue)); + } } m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed); @@ -564,6 +573,7 @@ ZenCacheMemoryLayer::CacheBucket::Drop() { RwLock::ExclusiveLockScope _(m_BucketLock); m_CacheMap.clear(); + m_TotalSize.store(0); } ////////////////////////////////////////////////////////////////////////// @@ -820,7 +830,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is { m_BucketDir = BucketDir; - m_TotalSize = 0; + m_TotalStandaloneSize = 0; m_Index.clear(); @@ -846,9 +856,9 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is for (const auto& Entry : m_Index) { const DiskLocation& Location = Entry.second.Location; - m_TotalSize.fetch_add(Location.Size(), std::memory_order::relaxed); if (Location.IsFlagSet(DiskLocation::kStandaloneFile)) { + m_TotalStandaloneSize.fetch_add(Location.Size(), std::memory_order::relaxed); continue; } const BlockStoreLocation& BlockLocation = Location.GetBlockLocation(m_PayloadAlignment); @@ -1153,6 +1163,23 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx) m_Index.erase(BadKey); } } + for (const DiskIndexEntry& Entry : LogEntries) + { + if (Entry.Location.IsFlagSet(DiskLocation::kStandaloneFile)) + { + ExtendablePathBuilder<256> Path; + BuildPath(Path, Entry.Key); + fs::path FilePath = Path.ToPath(); + RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key)); + if (fs::is_regular_file(FilePath)) + { + ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8()); + std::error_code Ec; + fs::remove(FilePath, Ec); // We don't care if we fail, we are no longer tracking this file... + } + m_TotalStandaloneSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed); + } + } m_SlogFile.Append(LogEntries); } } @@ -1277,7 +1304,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) uint64_t ReadBlockLongestTimeUs = 0; uint64_t TotalChunkCount = 0; uint64_t DeletedSize = 0; - uint64_t OldTotalSize = m_TotalSize.load(std::memory_order::relaxed); + uint64_t OldTotalSize = TotalSize(); uint64_t DeletedCount = 0; uint64_t MovedCount = 0; @@ -1359,6 +1386,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) for (const auto& Entry : ExpiredStandaloneEntries) { m_Index.erase(Entry.Key); + m_TotalStandaloneSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed); } m_SlogFile.Append(ExpiredStandaloneEntries); } @@ -1422,10 +1450,9 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) } m_SlogFile.Append(DiskIndexEntry{.Key = Key, .Location = RestoreLocation}); m_Index.insert({Key, {Loc, GcClock::TickCount()}}); - m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed); + m_TotalStandaloneSize.fetch_add(RestoreLocation.Size(), std::memory_order::relaxed); continue; } - m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed); DeletedSize += Entry.Location.Size(); DeletedCount++; } @@ -1478,13 +1505,13 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) if (!PerformDelete) { m_BlockStore.ReclaimSpace(BlockStoreState, ChunkLocations, KeepChunkIndexes, m_PayloadAlignment, true); - uint64_t TotalSize = m_TotalSize.load(std::memory_order_relaxed); + uint64_t CurrentTotalSize = TotalSize(); ZEN_INFO("garbage collect from '{}' DISABLED, found #{} {} chunks of total #{} {}", m_BucketDir / m_BucketName, DeleteCount, 0, // NiceBytes(TotalSize - NewTotalSize), - TotalChunkCount, - NiceBytes(TotalSize)); + CurrentTotalSize, + NiceBytes(CurrentTotalSize)); return; } @@ -1533,8 +1560,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx) if (Entry.Location.GetFlags() & DiskLocation::kTombStone) { m_Index.erase(Entry.Key); - uint64_t ChunkSize = Entry.Location.GetBlockLocation(m_PayloadAlignment).Size; - m_TotalSize.fetch_sub(ChunkSize); continue; } m_Index[Entry.Key].Location = Entry.Location; @@ -1706,29 +1731,21 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c DiskLocation Loc(NewFileSize, EntryFlags); IndexEntry Entry = IndexEntry(Loc, GcClock::TickCount()); - uint64_t OldFileSize = 0; RwLock::ExclusiveLockScope _(m_IndexLock); if (auto It = m_Index.find(HashKey); It == m_Index.end()) { // Previously unknown object - m_Index.insert({HashKey, Entry}); + m_Index.insert_or_assign(HashKey, std::move(Entry)); } else { // TODO: should check if write is idempotent and bail out if it is? - OldFileSize = It.value().Location.Size(); - It.value() = Entry; + m_TotalStandaloneSize.fetch_sub(It.value().Location.Size(), std::memory_order::relaxed); + It.value() = std::move(Entry); } m_SlogFile.Append({.Key = HashKey, .Location = Loc}); - if (OldFileSize <= NewFileSize) - { - m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed); - } - else - { - m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed); - } + m_TotalStandaloneSize.fetch_add(NewFileSize, std::memory_order::relaxed); } void @@ -1764,7 +1781,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const m_Index.insert({HashKey, {Location, GcClock::TickCount()}}); } }); - m_TotalSize.fetch_add(Value.Value.Size(), std::memory_order::relaxed); } ////////////////////////////////////////////////////////////////////////// @@ -2010,9 +2026,9 @@ ZenCacheDiskLayer::TotalSize() const static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc"; ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration) -: GcStorage(Gc) -, GcContributor(Gc) -, m_Gc(Gc) +//: GcStorage(Gc) +//, GcContributor(Gc) +: m_Gc(Gc) , m_Configuration(Configuration) { CreateDirectories(m_Configuration.BasePath); @@ -2170,18 +2186,6 @@ ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Names } } -void -ZenCacheStore::GatherReferences(GcContext& GcCtx) -{ - IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.GatherReferences(GcCtx); }); -} - -void -ZenCacheStore::CollectGarbage(GcContext& GcCtx) -{ - IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.CollectGarbage(GcCtx); }); -} - GcStorageSize ZenCacheStore::StorageSize() const { @@ -2297,8 +2301,8 @@ TEST_CASE("z$.size") } CacheSize = Zcs.StorageSize(); - CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.DiskSize); - CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.MemorySize); + CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize); + CHECK_LE(CacheValue.GetSize() * Count, CacheSize.MemorySize); } { @@ -2307,7 +2311,7 @@ TEST_CASE("z$.size") const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); - CHECK_EQ(SerializedSize.DiskSize, CacheSize.DiskSize); + CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize); for (size_t Bucket = 0; Bucket < 4; ++Bucket) { @@ -2340,7 +2344,7 @@ TEST_CASE("z$.size") } CacheSize = Zcs.StorageSize(); - CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.DiskSize); + CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize); CHECK_EQ(0, CacheSize.MemorySize); } @@ -2350,7 +2354,7 @@ TEST_CASE("z$.size") const GcStorageSize SerializedSize = Zcs.StorageSize(); CHECK_EQ(SerializedSize.MemorySize, 0); - CHECK_EQ(SerializedSize.DiskSize, CacheSize.DiskSize); + CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize); for (size_t Bucket = 0; Bucket < 4; ++Bucket) { @@ -2606,7 +2610,7 @@ TEST_CASE("z$.threadedinsert") // * doctest::skip(true)) } const uint64_t TotalSize = Zcs.StorageSize().DiskSize; - CHECK_EQ(kChunkSize * Chunks.size(), TotalSize); + CHECK_LE(kChunkSize * Chunks.size(), TotalSize); { std::atomic<size_t> WorkCompleted = 0; |