z$: hooked up bucket traversal for gc

added some tests
added "test" mode for zenserver, to run any tests embedded in the zenserver (example: `zenserver test -tx=z$.*`)

5 files changed, 148 insertions, 23 deletions

diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp
index e7b840ed8..02ec43115 100644
--- a/zenserver/cache/structuredcachestore.cpp
+++ b/zenserver/cache/structuredcachestore.cpp
@@ -6,19 +6,27 @@
 #include <zencore/windows.h>
 
 #include <zencore/compactbinary.h>
+#include <zencore/compactbinarybuilder.h>
+#include <zencore/compactbinarypackage.h>
+#include <zencore/compactbinaryvalidation.h>
+#include <zencore/compress.h>
 #include <zencore/filesystem.h>
 #include <zencore/fmtutils.h>
 #include <zencore/iobuffer.h>
 #include <zencore/logging.h>
 #include <zencore/string.h>
+#include <zencore/testing.h>
+#include <zencore/testutils.h>
 #include <zencore/thread.h>
 #include <zenstore/basicfile.h>
 #include <zenstore/cas.h>
 #include <zenstore/caslog.h>
+#include <zenstore/cidstore.h>
 
 #include <concepts>
 #include <filesystem>
 #include <unordered_map>
+#include <ranges>
 
 ZEN_THIRD_PARTY_INCLUDES_START
 #include <fmt/core.h>
@@ -31,7 +39,7 @@ namespace zen {
 
 using namespace fmt::literals;
 
-ZenCacheStore::ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir) : m_DiskLayer{Cas, RootDir}
+ZenCacheStore::ZenCacheStore(const std::filesystem::path& RootDir) : m_DiskLayer{RootDir}
 {
 	ZEN_INFO("initializing structured cache at '{}'", RootDir);
 	CreateDirectories(RootDir);
@@ -125,7 +133,8 @@ ZenCacheStore::Scrub(ScrubContext& Ctx)
 void
 ZenCacheStore::GarbageCollect(GcContext& GcCtx)
 {
-	ZEN_UNUSED(GcCtx);
+	m_DiskLayer.GarbageCollect(GcCtx);
+	m_MemLayer.GarbageCollect(GcCtx);
 }
 
 //////////////////////////////////////////////////////////////////////////
@@ -213,7 +222,12 @@ ZenCacheMemoryLayer::Scrub(ScrubContext& Ctx)
 void
 ZenCacheMemoryLayer::GarbageCollect(GcContext& GcCtx)
 {
-	ZEN_UNUSED(GcCtx);
+	RwLock::SharedLockScope _(m_Lock);
+
+	for (auto& Kv : m_Buckets)
+	{
+		Kv.second.GarbageCollect(GcCtx);
+	}
 }
 
 void
@@ -358,7 +372,7 @@ static_assert(sizeof(DiskIndexEntry) == 36);
 
 struct ZenCacheDiskLayer::CacheBucket
 {
-	CacheBucket(CasStore& Cas);
+	CacheBucket();
 	~CacheBucket();
 
 	void		OpenOrCreate(std::filesystem::path BucketDir, bool AllowCreate = true);
@@ -374,7 +388,6 @@ struct ZenCacheDiskLayer::CacheBucket
 	inline bool IsOk() const { return m_IsOk; }
 
 private:
-	CasStore&			  m_CasStore;
 	std::filesystem::path m_BucketDir;
 	Oid					  m_BucketId;
 	bool				  m_IsOk				 = false;
@@ -405,7 +418,7 @@ private:
 	inline RwLock& LockForHash(const IoHash& Hash) { return m_ShardedLocks[Hash.Hash[19]]; }
 };
 
-ZenCacheDiskLayer::CacheBucket::CacheBucket(CasStore& Cas) : m_CasStore(Cas)
+ZenCacheDiskLayer::CacheBucket::CacheBucket()
 {
 }
 
@@ -839,7 +852,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c
 
 //////////////////////////////////////////////////////////////////////////
 
-ZenCacheDiskLayer::ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir) : m_RootDir(RootDir), m_CasStore(Cas)
+ZenCacheDiskLayer::ZenCacheDiskLayer(const std::filesystem::path& RootDir) : m_RootDir(RootDir)
 {
 }
 
@@ -873,7 +886,7 @@ ZenCacheDiskLayer::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 		}
 		else
 		{
-			auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+			auto It = m_Buckets.try_emplace(std::string(InBucket));
 			Bucket	= &It.first->second;
 
 			std::filesystem::path BucketPath = m_RootDir;
@@ -916,7 +929,7 @@ ZenCacheDiskLayer::Put(std::string_view InBucket, const IoHash& HashKey, const Z
 		}
 		else
 		{
-			auto It = m_Buckets.try_emplace(std::string(InBucket), m_CasStore);
+			auto It = m_Buckets.try_emplace(std::string(InBucket));
 			Bucket	= &It.first->second;
 
 			std::filesystem::path bucketPath = m_RootDir;
@@ -972,7 +985,7 @@ ZenCacheDiskLayer::DiscoverBuckets()
 		}
 		else
 		{
-			auto InsertResult = m_Buckets.try_emplace(BucketName8, m_CasStore);
+			auto InsertResult = m_Buckets.try_emplace(BucketName8);
 
 			std::filesystem::path BucketPath = m_RootDir;
 			BucketPath /= BucketName8;
@@ -1050,7 +1063,100 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx)
 void
 ZenCacheDiskLayer::GarbageCollect(GcContext& GcCtx)
 {
-	ZEN_UNUSED(GcCtx);
+	RwLock::SharedLockScope _(m_Lock);
+
+	for (auto& Kv : m_Buckets)
+	{
+		Kv.second.GarbageCollect(GcCtx);
+	}
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+#if ZEN_WITH_TESTS
+
+TEST_CASE("z$.store")
+{
+	using namespace fmt::literals;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+
+	ZenCacheStore Zcs(TempDir.Path() / "cache");
+
+	const int kIterationCount = 100;
+
+	for (int i = 0; i < kIterationCount; ++i)
+	{
+		const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+		CbObjectWriter Cbo;
+		Cbo << "hey" << i;
+		CbObject Obj = Cbo.Save();
+
+		ZenCacheValue Value;
+		Value.Value = Obj.GetBuffer().AsIoBuffer();
+		Value.Value.SetContentType(ZenContentType::kCbObject);
+
+		Zcs.Put("test_bucket"sv, Key, Value);
+	}
+
+	for (int i = 0; i < kIterationCount; ++i)
+	{
+		const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+		ZenCacheValue Value;
+		Zcs.Get("test_bucket"sv, Key, /* out */ Value);
+
+		REQUIRE(Value.Value);
+		CHECK(Value.Value.GetContentType() == ZenContentType::kCbObject);
+		CHECK_EQ(ValidateCompactBinary(Value.Value, CbValidateMode::All), CbValidateError::None);
+		CbObject Obj = LoadCompactBinaryObject(Value.Value);
+		CHECK_EQ(Obj["hey"].AsInt32(), i);
+	}
+}
+
+TEST_CASE("z$.gc")
+{
+	using namespace fmt::literals;
+	using namespace std::literals;
+
+	ScopedTemporaryDirectory TempDir;
+
+	CasStoreConfiguration	  Config{.RootDirectory = TempDir.Path()};
+	std::unique_ptr<CasStore> Cas{CreateCasStore()};
+	Cas->Initialize(Config);
+
+	CidStore Cid(*Cas, TempDir.Path());
+
+	ZenCacheStore Zcs(TempDir.Path() / "cache");
+
+	const int kIterationCount = 100;
+
+	for (int i = 0; i < kIterationCount; ++i)
+	{
+		const IoHash Key = IoHash::HashBuffer(&i, sizeof i);
+
+		CompressedBuffer CompBuf = CompressedBuffer::Compress(SharedBuffer::MakeView("abcd"sv));
+		Cid.AddChunk(CompBuf);
+
+		CbObjectWriter Cbo;
+		Cbo << "hey" << i << "ref" << CbAttachment(CompBuf);
+		CbObject Obj = Cbo.Save();
+
+		ZenCacheValue Value;
+		Value.Value = Obj.GetBuffer().AsIoBuffer();
+		Value.Value.SetContentType(ZenContentType::kCbObject);
+
+		Zcs.Put("test_bucket"sv, Key, Value);
+	}
+}
+
+#endif
+
+void
+z$_forcelink()
+{
 }
 
 }  // namespace zen
diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h
index 4753af627..760f4995c 100644
--- a/zenserver/cache/structuredcachestore.h
+++ b/zenserver/cache/structuredcachestore.h
@@ -101,7 +101,7 @@ private:
 class ZenCacheDiskLayer
 {
 public:
-	ZenCacheDiskLayer(CasStore& Cas, const std::filesystem::path& RootDir);
+	explicit ZenCacheDiskLayer(const std::filesystem::path& RootDir);
 	~ZenCacheDiskLayer();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -119,7 +119,6 @@ private:
 	  */
 	struct CacheBucket;
 
-	CasStore&									 m_CasStore;
 	std::filesystem::path						 m_RootDir;
 	RwLock										 m_Lock;
 	std::unordered_map<std::string, CacheBucket> m_Buckets;	 // TODO: make this case insensitive
@@ -128,7 +127,7 @@ private:
 class ZenCacheStore
 {
 public:
-	ZenCacheStore(CasStore& Cas, const std::filesystem::path& RootDir);
+	explicit ZenCacheStore(const std::filesystem::path& RootDir);
 	~ZenCacheStore();
 
 	bool Get(std::string_view Bucket, const IoHash& HashKey, ZenCacheValue& OutValue);
@@ -146,4 +145,6 @@ private:
 	uint64_t			  m_LastScrubTime		   = 0;
 };
 
+void z$_forcelink();
+
 }  // namespace zen
diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp
index 00abed513..1a04aa39b 100644
--- a/zenserver/zenserver.cpp
+++ b/zenserver/zenserver.cpp
@@ -596,7 +596,7 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig)
 	auto ValueOrDefault = [](std::string_view Value, std::string_view Default) { return Value.empty() ? Default : Value; };
 
 	ZEN_INFO("instantiating structured cache service");
-	m_CacheStore = std::make_unique<ZenCacheStore>(*m_CasStore, m_DataRoot / "cache");
+	m_CacheStore = std::make_unique<ZenCacheStore>(m_DataRoot / "cache");
 
 	std::unique_ptr<zen::UpstreamCache> UpstreamCache;
 	if (ServiceConfig.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled)
@@ -842,6 +842,22 @@ ZenWindowsService::Run()
 	return 0;
 }
 
+#if ZEN_WITH_TESTS
+int
+test_main(int argc, char** argv)
+{
+	zen::zencore_forcelinktests();
+	zen::zenhttp_forcelinktests();
+	zen::zenstore_forcelinktests();
+
+	zen::logging::InitializeLogging();
+
+	spdlog::set_level(spdlog::level::debug);
+
+	return doctest::Context(argc, argv).run();
+}
+#endif
+
 int
 main(int argc, char* argv[])
 {
@@ -851,6 +867,16 @@ main(int argc, char* argv[])
 	mi_version();
 #endif
 
+#if ZEN_WITH_TESTS
+	if (argc >= 2)
+	{
+		if (argv[1] == "test"sv)
+		{
+			return test_main(argc, argv);
+		}
+	}
+#endif
+
 	try
 	{
 		ZenServerOptions GlobalOptions;
diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj
index b670582e7..d954d3f8d 100644
--- a/zenserver/zenserver.vcxproj
+++ b/zenserver/zenserver.vcxproj
@@ -120,7 +120,6 @@
     <ClInclude Include="testing\httptest.h" />
     <ClInclude Include="upstream\jupiter.h" />
     <ClInclude Include="projectstore.h" />
-    <ClInclude Include="cache\cacheagent.h" />
     <ClInclude Include="testing\launch.h" />
     <ClInclude Include="casstore.h" />
     <ClInclude Include="diag\diagsvcs.h" />
@@ -142,7 +141,6 @@
     <ClCompile Include="monitoring\httpstats.cpp" />
     <ClCompile Include="monitoring\httpstatus.cpp" />
     <ClCompile Include="projectstore.cpp" />
-    <ClCompile Include="cache\cacheagent.cpp" />
     <ClCompile Include="sos\sos.cpp" />
     <ClCompile Include="testing\httptest.cpp" />
     <ClCompile Include="upstream\jupiter.cpp" />
diff --git a/zenserver/zenserver.vcxproj.filters b/zenserver/zenserver.vcxproj.filters
index b87fa0016..04c6267ba 100644
--- a/zenserver/zenserver.vcxproj.filters
+++ b/zenserver/zenserver.vcxproj.filters
@@ -5,9 +5,6 @@
     <ClInclude Include="projectstore.h" />
     <ClInclude Include="casstore.h" />
     <ClInclude Include="testing\launch.h" />
-    <ClInclude Include="cache\cacheagent.h">
-      <Filter>cache</Filter>
-    </ClInclude>
     <ClInclude Include="diag\diagsvcs.h">
       <Filter>diag</Filter>
     </ClInclude>
@@ -49,9 +46,6 @@
     <ClCompile Include="zenserver.cpp" />
     <ClCompile Include="projectstore.cpp" />
     <ClCompile Include="casstore.cpp" />
-    <ClCompile Include="cache\cacheagent.cpp">
-      <Filter>cache</Filter>
-    </ClCompile>
     <ClCompile Include="experimental\usnjournal.cpp">
       <Filter>experimental</Filter>
     </ClCompile>