Added z$ GC tests.

1 files changed, 140 insertions, 28 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index 89214ded6..b6f91857c 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -37,8 +37,9 @@ ZEN_THIRD_PARTY_INCLUDES_END
 
 namespace zen {
 
-namespace fs = std::filesystem;
 using namespace fmt::literals;
+using PathBuilder = WideStringBuilder<256>;
+namespace fs	  = std::filesystem;
 
 static CbObject
 LoadCompactBinaryObject(const fs::path& Path)
@@ -652,7 +653,7 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, Zen
 bool
 ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue)
 {
-	WideStringBuilder<128> DataFilePath;
+	PathBuilder DataFilePath;
 	BuildPath(DataFilePath, HashKey);
 
 	RwLock::SharedLockScope ValueLock(LockForHash(HashKey));
@@ -916,6 +917,11 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 
 	RwLock::ExclusiveLockScope _(m_IndexLock);
 
+	if (m_Index.empty())
+	{
+		return;
+	}
+
 	std::vector<DiskIndexEntry>					  DiskEntries;
 	std::vector<std::pair<size_t, GcClock::Tick>> Timestamps;
 	uint64_t									  TotalSize{};
@@ -946,8 +952,8 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 
 	// Remove all standalone file(s)
 	{
-		std::error_code		   Ec;
-		WideStringBuilder<128> Path;
+		std::error_code Ec;
+		PathBuilder		Path;
 
 		for (size_t Idx = 0; Idx < FirstValid; ++Idx)
 		{
@@ -972,7 +978,7 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		}
 	}
 
-	if (GcCtx.IsContainerGcEnabled())
+	if (GcCtx.IsContainerGcEnabled() && Count != NewCount)
 	{
 		// super naive GC implementation for small object container file
 
@@ -1006,38 +1012,40 @@ ZenCacheDiskLayer::CacheBucket::CollectGarbage(GcContext& GcCtx)
 		std::filesystem::path		TmpSobsPath{m_BucketDir / "zen.sobs.tmp"};
 		std::filesystem::path		TmpSlogPath{m_BucketDir / "zen.slog.tmp"};
 		TCasLogFile<DiskIndexEntry> TmpLog;
-		BasicFile					TmpSobs;
 		IndexMap					TmpIndex;
 		uint64_t					TmpCursor{};
 		uint64_t					TmpTotalSize{};
 
-		TmpSobs.Open(TmpSobsPath, true);
-		std::vector<uint8_t> Chunk;
-
-		for (size_t Idx = FirstValid; Idx != Count; ++Idx)
 		{
-			const DiskIndexEntry& Entry = DiskEntries[Idx];
-
-			DiskLocation NewLoc;
+			BasicFile TmpSobs;
+			TmpSobs.Open(TmpSobsPath, true);
+			std::vector<uint8_t> Chunk;
 
-			if (Entry.Location.IsFlagSet(DiskLocation::kStandaloneFile))
-			{
-				NewLoc = DiskLocation(0, Entry.Location.Size(), 0, DiskLocation::kStandaloneFile);
-			}
-			else
+			for (size_t Idx = FirstValid; Idx != Count; ++Idx)
 			{
-				Chunk.resize(Entry.Location.Size());
-				m_SobsFile.Read(Chunk.data(), Chunk.size(), Entry.Location.Offset());
+				const DiskIndexEntry& Entry = DiskEntries[Idx];
 
-				NewLoc = DiskLocation(TmpCursor, Chunk.size(), 0, 0);
-				TmpSobs.Write(Chunk.data(), Chunk.size(), TmpCursor);
-				TmpCursor = RoundUp(TmpCursor + Chunk.size(), 16);
-			}
+				DiskLocation NewLoc;
 
-			TmpLog.Append({.Key = Entry.Key, .Location = NewLoc});
-			TmpIndex.insert({Entry.Key, {NewLoc, GcClock::TickCount()}});
+				if (Entry.Location.IsFlagSet(DiskLocation::kStandaloneFile))
+				{
+					NewLoc = DiskLocation(0, Entry.Location.Size(), 0, DiskLocation::kStandaloneFile);
+				}
+				else
+				{
+					Chunk.resize(Entry.Location.Size());
+					m_SobsFile.Read(Chunk.data(), Chunk.size(), Entry.Location.Offset());
 
-			TmpTotalSize += NewLoc.Size();
+					NewLoc = DiskLocation(TmpCursor, Chunk.size(), 0, 0);
+					TmpSobs.Write(Chunk.data(), Chunk.size(), TmpCursor);
+					TmpCursor = RoundUp(TmpCursor + Chunk.size(), 16);
+				}
+
+				TmpLog.Append({.Key = Entry.Key, .Location = NewLoc});
+				TmpIndex.insert({Entry.Key, {NewLoc, GcClock::TickCount()}});
+
+				TmpTotalSize += NewLoc.Size();
+			}
 		}
 
 		// Swap state
@@ -1112,7 +1120,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 {
 	RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey));
 
-	WideStringBuilder<128> DataFilePath;
+	PathBuilder DataFilePath;
 	BuildPath(DataFilePath, HashKey);
 
 	TemporaryFile DataFile;
@@ -1452,6 +1460,16 @@ namespace testutils {
 
 	IoHash CreateKey(size_t KeyValue) { return IoHash::HashBuffer(&KeyValue, sizeof(size_t)); }
 
+	IoBuffer CreateBinaryCacheValue(uint64_t Size)
+	{
+		std::vector<uint32_t> Data(size_t(Size / sizeof uint32_t));
+		std::generate(Data.begin(), Data.end(), [Idx = 0]() mutable { return Idx++; });
+
+		IoBuffer Buf(IoBuffer::Clone, Data.data(), Data.size() * sizeof(uint32_t));
+		Buf.SetContentType(ZenContentType::kBinary);
+		return Buf;
+	};
+
 }  // namespace testutils
 
 TEST_CASE("zcache.store")
@@ -1668,6 +1686,100 @@ TEST_CASE("zcache.gc")
 			}
 		}
 	}
+
+	SUBCASE("gc removes standalone values")
+	{
+		ScopedTemporaryDirectory TempDir;
+		CasGc					 Gc;
+		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		const auto				 Bucket		 = "fortysixandtwo"sv;
+		const GcClock::TimePoint CurrentTime = GcClock::Now();
+
+		std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
+
+		for (const auto& Key : Keys)
+		{
+			IoBuffer Value = testutils::CreateBinaryCacheValue(128 << 10);
+			Zcs.Put(Bucket, Key, {.Value = Value});
+		}
+
+		{
+			GcContext GcCtx;
+			GcCtx.MaxCacheDuration(std::chrono::hours(46));
+
+			Zcs.CollectGarbage(GcCtx);
+
+			for (const auto& Key : Keys)
+			{
+				ZenCacheValue CacheValue;
+				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
+				CHECK(Exists);
+			}
+		}
+
+		// Move forward in time and collect again
+		{
+			GcContext GcCtx(CurrentTime + std::chrono::hours(46));
+			GcCtx.MaxCacheDuration(std::chrono::minutes(2));
+
+			Zcs.CollectGarbage(GcCtx);
+
+			for (const auto& Key : Keys)
+			{
+				ZenCacheValue CacheValue;
+				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
+				CHECK(!Exists);
+			}
+		}
+	}
+
+	SUBCASE("gc removes small objects (container)")
+	{
+		ScopedTemporaryDirectory TempDir;
+		CasGc					 Gc;
+		ZenCacheStore			 Zcs(Gc, TempDir.Path() / "cache");
+		const auto				 Bucket		 = "rightintwo"sv;
+		const GcClock::TimePoint CurrentTime = GcClock::Now();
+
+		std::vector<IoHash> Keys{CreateKey(1), CreateKey(2), CreateKey(3)};
+
+		for (const auto& Key : Keys)
+		{
+			IoBuffer Value = testutils::CreateBinaryCacheValue(128);
+			Zcs.Put(Bucket, Key, {.Value = Value});
+		}
+
+		{
+			GcContext GcCtx;
+			GcCtx.MaxCacheDuration(std::chrono::hours(2));
+			GcCtx.SetContainerGcEnabled(true);
+
+			Zcs.CollectGarbage(GcCtx);
+
+			for (const auto& Key : Keys)
+			{
+				ZenCacheValue CacheValue;
+				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
+				CHECK(Exists);
+			}
+		}
+
+		// Move forward in time and collect again
+		{
+			GcContext GcCtx(CurrentTime + std::chrono::hours(2));
+			GcCtx.MaxCacheDuration(std::chrono::minutes(2));
+			GcCtx.SetContainerGcEnabled(true);
+
+			Zcs.CollectGarbage(GcCtx);
+
+			for (const auto& Key : Keys)
+			{
+				ZenCacheValue CacheValue;
+				const bool	  Exists = Zcs.Get(Bucket, Key, CacheValue);
+				CHECK(!Exists);
+			}
+		}
+	}
 }
 
 #endif