Fix disk usage stats (#194)

* Improve tracking of used disk space for filecas and compactcas
Add tracking of used disk space for project store
Remove ZenCacheStore as GcStorage/GcContributor
 - underlying ZenCacheNamespace instances register themselves directly
 - removing this also fixes double reporting of GcStorageSize for namespaces
* changelog

1 files changed, 49 insertions, 45 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 1fab3b312..9773c2ed2 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -552,8 +552,17 @@ void
 ZenCacheMemoryLayer::CacheBucket::Put(const IoHash& HashKey, const ZenCacheValue& Value)
 {
 	{
+		BucketValue				   IndexValue(Value.Value, GcClock::TickCount());
 		RwLock::ExclusiveLockScope _(m_BucketLock);
-		m_CacheMap.insert_or_assign(HashKey, BucketValue(Value.Value, GcClock::TickCount()));
+		if (auto It = m_CacheMap.find(HashKey); It != m_CacheMap.end())
+		{
+			m_TotalSize.fetch_sub(It->second.Payload.GetSize(), std::memory_order::relaxed);
+			It.value() = std::move(IndexValue);
+		}
+		else
+		{
+			m_CacheMap.insert_or_assign(HashKey, std::move(IndexValue));
+		}
 	}
 
 	m_TotalSize.fetch_add(Value.Value.GetSize(), std::memory_order::relaxed);
@@ -564,6 +573,7 @@ ZenCacheMemoryLayer::CacheBucket::Drop()
 {
 	RwLock::ExclusiveLockScope _(m_BucketLock);
 	m_CacheMap.clear();
+	m_TotalSize.store(0);
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -820,7 +830,7 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
 {
 	m_BucketDir = BucketDir;
 
-	m_TotalSize = 0;
+	m_TotalStandaloneSize = 0;
 
 	m_Index.clear();
 
@@ -846,9 +856,9 @@ ZenCacheDiskLayer::CacheBucket::OpenLog(const fs::path& BucketDir, const bool Is
 	for (const auto& Entry : m_Index)
 	{
 		const DiskLocation& Location = Entry.second.Location;
-		m_TotalSize.fetch_add(Location.Size(), std::memory_order::relaxed);
 		if (Location.IsFlagSet(DiskLocation::kStandaloneFile))
 		{
+			m_TotalStandaloneSize.fetch_add(Location.Size(), std::memory_order::relaxed);
 			continue;
 		}
 		const BlockStoreLocation& BlockLocation = Location.GetBlockLocation(m_PayloadAlignment);
@@ -1153,6 +1163,23 @@ ZenCacheDiskLayer::CacheBucket::Scrub(ScrubContext& Ctx)
 					m_Index.erase(BadKey);
 				}
 			}
+			for (const DiskIndexEntry& Entry : LogEntries)
+			{
+				if (Entry.Location.IsFlagSet(DiskLocation::kStandaloneFile))
+				{
+					ExtendablePathBuilder<256> Path;
+					BuildPath(Path, Entry.Key);
+					fs::path				   FilePath = Path.ToPath();
+					RwLock::ExclusiveLockScope ValueLock(LockForHash(Entry.Key));
+					if (fs::is_regular_file(FilePath))
+					{
+						ZEN_DEBUG("deleting bad standalone cache file '{}'", Path.ToUtf8());
+						std::error_code Ec;
+						fs::remove(FilePath, Ec);  // We don't care if we fail, we are no longer tracking this file...
+					}
+					m_TotalStandaloneSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed);
+				}
+			}
 			m_SlogFile.Append(LogEntries);
 		}
 	}
@@ -1277,7 +1304,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 	uint64_t  ReadBlockLongestTimeUs  = 0;
 	uint64_t  TotalChunkCount		  = 0;
 	uint64_t  DeletedSize			  = 0;
-	uint64_t  OldTotalSize			  = m_TotalSize.load(std::memory_order::relaxed);
+	uint64_t  OldTotalSize			  = TotalSize();
 
 	uint64_t DeletedCount = 0;
 	uint64_t MovedCount	  = 0;
@@ -1359,6 +1386,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 			for (const auto& Entry : ExpiredStandaloneEntries)
 			{
 				m_Index.erase(Entry.Key);
+				m_TotalStandaloneSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed);
 			}
 			m_SlogFile.Append(ExpiredStandaloneEntries);
 		}
@@ -1422,10 +1450,9 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 				}
 				m_SlogFile.Append(DiskIndexEntry{.Key = Key, .Location = RestoreLocation});
 				m_Index.insert({Key, {Loc, GcClock::TickCount()}});
-				m_TotalSize.fetch_add(Entry.Location.Size(), std::memory_order::relaxed);
+				m_TotalStandaloneSize.fetch_add(RestoreLocation.Size(), std::memory_order::relaxed);
 				continue;
 			}
-			m_TotalSize.fetch_sub(Entry.Location.Size(), std::memory_order::relaxed);
 			DeletedSize += Entry.Location.Size();
 			DeletedCount++;
 		}
@@ -1478,13 +1505,13 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 	if (!PerformDelete)
 	{
 		m_BlockStore.ReclaimSpace(BlockStoreState, ChunkLocations, KeepChunkIndexes, m_PayloadAlignment, true);
-		uint64_t TotalSize = m_TotalSize.load(std::memory_order_relaxed);
+		uint64_t CurrentTotalSize = TotalSize();
 		ZEN_INFO("garbage collect from '{}' DISABLED, found #{} {} chunks of total #{} {}",
 				 m_BucketDir / m_BucketName,
 				 DeleteCount,
 				 0,	 // NiceBytes(TotalSize - NewTotalSize),
-				 TotalChunkCount,
-				 NiceBytes(TotalSize));
+				 CurrentTotalSize,
+				 NiceBytes(CurrentTotalSize));
 		return;
 	}
 
@@ -1533,8 +1560,6 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 					if (Entry.Location.GetFlags() & DiskLocation::kTombStone)
 					{
 						m_Index.erase(Entry.Key);
-						uint64_t ChunkSize = Entry.Location.GetBlockLocation(m_PayloadAlignment).Size;
-						m_TotalSize.fetch_sub(ChunkSize);
 						continue;
 					}
 					m_Index[Entry.Key].Location = Entry.Location;
@@ -1706,29 +1731,21 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 	DiskLocation Loc(NewFileSize, EntryFlags);
 	IndexEntry	 Entry = IndexEntry(Loc, GcClock::TickCount());
 
-	uint64_t				   OldFileSize = 0;
 	RwLock::ExclusiveLockScope _(m_IndexLock);
 	if (auto It = m_Index.find(HashKey); It == m_Index.end())
 	{
 		// Previously unknown object
-		m_Index.insert({HashKey, Entry});
+		m_Index.insert_or_assign(HashKey, std::move(Entry));
 	}
 	else
 	{
 		// TODO: should check if write is idempotent and bail out if it is?
-		OldFileSize = It.value().Location.Size();
-		It.value()	= Entry;
+		m_TotalStandaloneSize.fetch_sub(It.value().Location.Size(), std::memory_order::relaxed);
+		It.value() = std::move(Entry);
 	}
 
 	m_SlogFile.Append({.Key = HashKey, .Location = Loc});
-	if (OldFileSize <= NewFileSize)
-	{
-		m_TotalSize.fetch_add(NewFileSize - OldFileSize, std::memory_order::relaxed);
-	}
-	else
-	{
-		m_TotalSize.fetch_sub(OldFileSize - NewFileSize, std::memory_order::relaxed);
-	}
+	m_TotalStandaloneSize.fetch_add(NewFileSize, std::memory_order::relaxed);
 }
 
 void
@@ -1764,7 +1781,6 @@ ZenCacheDiskLayer::CacheBucket::PutInlineCacheValue(const IoHash& HashKey, const
 			m_Index.insert({HashKey, {Location, GcClock::TickCount()}});
 		}
 	});
-	m_TotalSize.fetch_add(Value.Value.Size(), std::memory_order::relaxed);
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -2010,9 +2026,9 @@ ZenCacheDiskLayer::TotalSize() const
 static constexpr std::string_view UE4DDCNamespaceName = "ue4.ddc";
 
 ZenCacheStore::ZenCacheStore(GcManager& Gc, const Configuration& Configuration)
-: GcStorage(Gc)
-, GcContributor(Gc)
-, m_Gc(Gc)
+//: GcStorage(Gc)
+//, GcContributor(Gc)
+: m_Gc(Gc)
 , m_Configuration(Configuration)
 {
 	CreateDirectories(m_Configuration.BasePath);
@@ -2170,18 +2186,6 @@ ZenCacheStore::IterateNamespaces(const std::function<void(std::string_view Names
 	}
 }
 
-void
-ZenCacheStore::GatherReferences(GcContext& GcCtx)
-{
-	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.GatherReferences(GcCtx); });
-}
-
-void
-ZenCacheStore::CollectGarbage(GcContext& GcCtx)
-{
-	IterateNamespaces([&](std::string_view, ZenCacheNamespace& Store) { Store.CollectGarbage(GcCtx); });
-}
-
 GcStorageSize
 ZenCacheStore::StorageSize() const
 {
@@ -2297,8 +2301,8 @@ TEST_CASE("z$.size")
 			}
 
 			CacheSize = Zcs.StorageSize();
-			CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.DiskSize);
-			CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.MemorySize);
+			CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
+			CHECK_LE(CacheValue.GetSize() * Count, CacheSize.MemorySize);
 		}
 
 		{
@@ -2307,7 +2311,7 @@ TEST_CASE("z$.size")
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
-			CHECK_EQ(SerializedSize.DiskSize, CacheSize.DiskSize);
+			CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize);
 
 			for (size_t Bucket = 0; Bucket < 4; ++Bucket)
 			{
@@ -2340,7 +2344,7 @@ TEST_CASE("z$.size")
 			}
 
 			CacheSize = Zcs.StorageSize();
-			CHECK_EQ(CacheValue.GetSize() * Count, CacheSize.DiskSize);
+			CHECK_LE(CacheValue.GetSize() * Count, CacheSize.DiskSize);
 			CHECK_EQ(0, CacheSize.MemorySize);
 		}
 
@@ -2350,7 +2354,7 @@ TEST_CASE("z$.size")
 
 			const GcStorageSize SerializedSize = Zcs.StorageSize();
 			CHECK_EQ(SerializedSize.MemorySize, 0);
-			CHECK_EQ(SerializedSize.DiskSize, CacheSize.DiskSize);
+			CHECK_LE(SerializedSize.DiskSize, CacheSize.DiskSize);
 
 			for (size_t Bucket = 0; Bucket < 4; ++Bucket)
 			{
@@ -2606,7 +2610,7 @@ TEST_CASE("z$.threadedinsert")	// * doctest::skip(true))
 		}
 
 		const uint64_t TotalSize = Zcs.StorageSize().DiskSize;
-		CHECK_EQ(kChunkSize * Chunks.size(), TotalSize);
+		CHECK_LE(kChunkSize * Chunks.size(), TotalSize);
 
 		{
 			std::atomic<size_t> WorkCompleted = 0;