Remove ZEN_CACHE_TRACKER etc

this was code which was originally intended for use with GC but it's no longer useful

4 files changed, 0 insertions, 437 deletions

diff --git a/src/zenserver/cache/cachetracking.cpp b/src/zenserver/cache/cachetracking.cpp
deleted file mode 100644
index 9119e3122..000000000
--- a/src/zenserver/cache/cachetracking.cpp
+++ /dev/null
@@ -1,376 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#include "cachetracking.h"
-
-#if ZEN_USE_CACHE_TRACKER
-
-#	include <zencore/compactbinarybuilder.h>
-#	include <zencore/compactbinaryvalue.h>
-#	include <zencore/endian.h>
-#	include <zencore/filesystem.h>
-#	include <zencore/logging.h>
-#	include <zencore/scopeguard.h>
-#	include <zencore/string.h>
-
-#	include <zencore/testing.h>
-#	include <zencore/testutils.h>
-
-ZEN_THIRD_PARTY_INCLUDES_START
-#	pragma comment(lib, "Rpcrt4.lib")	// RocksDB made me do this
-#	include <fmt/format.h>
-#	include <rocksdb/db.h>
-#	include <tsl/robin_map.h>
-#	include <tsl/robin_set.h>
-#	include <gsl/gsl-lite.hpp>
-ZEN_THIRD_PARTY_INCLUDES_END
-
-namespace zen {
-
-namespace rocksdb = ROCKSDB_NAMESPACE;
-
-static constinit auto Epoch = std::chrono::time_point<std::chrono::system_clock>{};
-
-static uint64_t
-GetCurrentCacheTimeStamp()
-{
-	auto	 Duration = std::chrono::system_clock::now() - Epoch;
-	uint64_t Millis	  = std::chrono::duration_cast<std::chrono::milliseconds>(Duration).count();
-
-	return Millis;
-}
-
-struct CacheAccessSnapshot
-{
-public:
-	void TrackAccess(std::string_view BucketSegment, const IoHash& HashKey)
-	{
-		BucketTracker* Tracker = GetBucket(std::string(BucketSegment));
-
-		Tracker->Track(HashKey);
-	}
-
-	bool SerializeSnapshot(CbObjectWriter& Cbo)
-	{
-		bool					   Serialized = false;
-		RwLock::ExclusiveLockScope _(m_Lock);
-
-		for (const auto& Kv : m_BucketMapping)
-		{
-			if (m_Buckets[Kv.second]->Size())
-			{
-				Cbo.BeginArray(Kv.first);
-				m_Buckets[Kv.second]->SerializeSnapshotAndClear(Cbo);
-				Cbo.EndArray();
-				Serialized = true;
-			}
-		}
-
-		return Serialized;
-	}
-
-private:
-	struct BucketTracker
-	{
-		mutable RwLock		   Lock;
-		tsl::robin_set<IoHash> AccessedKeys;
-
-		void Track(const IoHash& HashKey)
-		{
-			if (RwLock::SharedLockScope _(Lock); AccessedKeys.contains(HashKey))
-			{
-				return;
-			}
-
-			RwLock::ExclusiveLockScope _(Lock);
-
-			AccessedKeys.insert(HashKey);
-		}
-
-		void SerializeSnapshotAndClear(CbObjectWriter& Cbo)
-		{
-			RwLock::ExclusiveLockScope _(Lock);
-
-			for (const IoHash& Hash : AccessedKeys)
-			{
-				Cbo.AddHash(Hash);
-			}
-
-			AccessedKeys.clear();
-		}
-
-		size_t Size() const
-		{
-			RwLock::SharedLockScope _(Lock);
-			return AccessedKeys.size();
-		}
-	};
-
-	BucketTracker* GetBucket(const std::string& BucketName)
-	{
-		RwLock::SharedLockScope _(m_Lock);
-
-		if (auto It = m_BucketMapping.find(BucketName); It == m_BucketMapping.end())
-		{
-			_.ReleaseNow();
-
-			return AddNewBucket(BucketName);
-		}
-		else
-		{
-			return m_Buckets[It->second].get();
-		}
-	}
-
-	BucketTracker* AddNewBucket(const std::string& BucketName)
-	{
-		RwLock::ExclusiveLockScope _(m_Lock);
-
-		if (auto It = m_BucketMapping.find(BucketName); It == m_BucketMapping.end())
-		{
-			const uint32_t BucketIndex = gsl::narrow<uint32_t>(m_Buckets.size());
-			m_Buckets.emplace_back(std::make_unique<BucketTracker>());
-			m_BucketMapping[BucketName] = BucketIndex;
-
-			return m_Buckets[BucketIndex].get();
-		}
-		else
-		{
-			return m_Buckets[It->second].get();
-		}
-	}
-
-	RwLock										m_Lock;
-	std::vector<std::unique_ptr<BucketTracker>> m_Buckets;
-	tsl::robin_map<std::string, uint32_t>		m_BucketMapping;
-};
-
-struct ZenCacheTracker::Impl
-{
-	Impl(std::filesystem::path StateDirectory)
-	{
-		std::filesystem::path StatsDbPath{StateDirectory / ".zdb"};
-
-		std::string RocksdbPath = StatsDbPath.string();
-
-		ZEN_DEBUG("opening tracker db at '{}'", RocksdbPath);
-
-		rocksdb::DB*	   Db = nullptr;
-		rocksdb::DBOptions Options;
-		Options.create_if_missing = true;
-
-		std::vector<std::string> ExistingColumnFamilies;
-		rocksdb::Status			 Status = rocksdb::DB::ListColumnFamilies(Options, RocksdbPath, &ExistingColumnFamilies);
-
-		std::vector<rocksdb::ColumnFamilyDescriptor> ColumnDescriptors;
-
-		if (Status.IsPathNotFound())
-		{
-			ColumnDescriptors.emplace_back(rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName, {}});
-		}
-		else if (Status.ok())
-		{
-			for (const std::string& Column : ExistingColumnFamilies)
-			{
-				rocksdb::ColumnFamilyDescriptor ColumnFamily;
-				ColumnFamily.name = Column;
-				ColumnDescriptors.push_back(ColumnFamily);
-			}
-		}
-		else
-		{
-			throw std::runtime_error(fmt::format("column family iteration failed for '{}': '{}'", RocksdbPath, Status.getState()).c_str());
-		}
-
-		Status = rocksdb::DB::Open(Options, RocksdbPath, ColumnDescriptors, &m_RocksDbColumnHandles, &Db);
-
-		if (!Status.ok())
-		{
-			throw std::runtime_error(fmt::format("database open failed for '{}': '{}'", RocksdbPath, Status.getState()).c_str());
-		}
-
-		m_RocksDb.reset(Db);
-	}
-
-	~Impl()
-	{
-		for (auto* Column : m_RocksDbColumnHandles)
-		{
-			delete Column;
-		}
-
-		m_RocksDbColumnHandles.clear();
-	}
-
-	struct KeyStruct
-	{
-		uint64_t TimestampLittleEndian;
-	};
-
-	void TrackAccess(std::string_view BucketSegment, const IoHash& HashKey) { m_CurrentSnapshot.TrackAccess(BucketSegment, HashKey); }
-
-	void SaveSnapshot()
-	{
-		CbObjectWriter Cbo;
-
-		if (m_CurrentSnapshot.SerializeSnapshot(Cbo))
-		{
-			IoBuffer SnapshotBuffer = Cbo.Save().GetBuffer().AsIoBuffer();
-
-			const KeyStruct Key{.TimestampLittleEndian = ToNetworkOrder(GetCurrentCacheTimeStamp())};
-			rocksdb::Slice	KeySlice{(const char*)&Key, sizeof Key};
-			rocksdb::Slice	ValueSlice{(char*)SnapshotBuffer.Data(), SnapshotBuffer.Size()};
-
-			rocksdb::WriteOptions Wo;
-			m_RocksDb->Put(Wo, KeySlice, ValueSlice);
-		}
-	}
-
-	void IterateSnapshots(std::function<void(uint64_t TimeStamp, CbObject Snapshot)>&& Callback)
-	{
-		rocksdb::ManagedSnapshot Snap(m_RocksDb.get());
-
-		rocksdb::ReadOptions Ro;
-		Ro.snapshot = Snap.snapshot();
-
-		std::unique_ptr<rocksdb::Iterator> It{m_RocksDb->NewIterator(Ro)};
-
-		const KeyStruct ZeroKey{.TimestampLittleEndian = 0};
-		rocksdb::Slice	ZeroKeySlice{(const char*)&ZeroKey, sizeof ZeroKey};
-
-		It->Seek(ZeroKeySlice);
-
-		while (It->Valid())
-		{
-			rocksdb::Slice KeySlice	  = It->key();
-			rocksdb::Slice ValueSlice = It->value();
-
-			if (KeySlice.size() == sizeof(KeyStruct))
-			{
-				IoBuffer ValueBuffer(IoBuffer::Wrap, ValueSlice.data(), ValueSlice.size());
-
-				CbObject Value = LoadCompactBinaryObject(ValueBuffer);
-
-				uint64_t Key = FromNetworkOrder(*reinterpret_cast<const uint64_t*>(KeySlice.data()));
-
-				Callback(Key, Value);
-			}
-
-			It->Next();
-		}
-	}
-
-	std::unique_ptr<rocksdb::DB>			  m_RocksDb;
-	std::vector<rocksdb::ColumnFamilyHandle*> m_RocksDbColumnHandles;
-	CacheAccessSnapshot						  m_CurrentSnapshot;
-};
-
-ZenCacheTracker::ZenCacheTracker(std::filesystem::path StateDirectory) : m_Impl(new Impl(StateDirectory))
-{
-}
-
-ZenCacheTracker::~ZenCacheTracker()
-{
-	delete m_Impl;
-}
-
-void
-ZenCacheTracker::TrackAccess(std::string_view BucketSegment, const IoHash& HashKey)
-{
-	m_Impl->TrackAccess(BucketSegment, HashKey);
-}
-
-void
-ZenCacheTracker::SaveSnapshot()
-{
-	m_Impl->SaveSnapshot();
-}
-
-void
-ZenCacheTracker::IterateSnapshots(std::function<void(uint64_t TimeStamp, CbObject Snapshot)>&& Callback)
-{
-	m_Impl->IterateSnapshots(std::move(Callback));
-}
-
-#	if ZEN_WITH_TESTS
-
-TEST_CASE("z$.tracker")
-{
-	using namespace std::literals;
-
-	const uint64_t t0 = GetCurrentCacheTimeStamp();
-
-	ScopedTemporaryDirectory TempDir;
-
-	ZenCacheTracker Zcs(TempDir.Path());
-
-	tsl::robin_set<IoHash> KeyHashes;
-
-	for (int i = 0; i < 10000; ++i)
-	{
-		IoHash KeyHash = IoHash::HashBuffer(&i, sizeof i);
-
-		KeyHashes.insert(KeyHash);
-
-		Zcs.TrackAccess("foo"sv, KeyHash);
-	}
-
-	for (int i = 0; i < 10000; ++i)
-	{
-		IoHash KeyHash = IoHash::HashBuffer(&i, sizeof i);
-
-		Zcs.TrackAccess("foo"sv, KeyHash);
-	}
-
-	Zcs.SaveSnapshot();
-
-	for (int n = 0; n < 10; ++n)
-	{
-		for (int i = 0; i < 1000; ++i)
-		{
-			const int Index	  = i + n * 1000;
-			IoHash	  KeyHash = IoHash::HashBuffer(&Index, sizeof Index);
-
-			Zcs.TrackAccess("foo"sv, KeyHash);
-		}
-
-		Zcs.SaveSnapshot();
-	}
-
-	Zcs.SaveSnapshot();
-
-	const uint64_t t1 = GetCurrentCacheTimeStamp();
-
-	int SnapshotCount = 0;
-
-	Zcs.IterateSnapshots([&](uint64_t TimeStamp, CbObject Snapshot) {
-		CHECK(TimeStamp >= t0);
-		CHECK(TimeStamp <= t1);
-
-		for (auto& Field : Snapshot)
-		{
-			CHECK_EQ(Field.GetName(), "foo"sv);
-
-			const CbArray& Array = Field.AsArray();
-
-			for (const auto& Element : Array)
-			{
-				CHECK(KeyHashes.contains(Element.GetValue().AsHash()));
-			}
-		}
-
-		++SnapshotCount;
-	});
-
-	CHECK_EQ(SnapshotCount, 11);
-}
-
-#	endif
-
-void
-cachetracker_forcelink()
-{
-}
-
-}  // namespace zen
-
-#endif	// ZEN_USE_CACHE_TRACKER
diff --git a/src/zenserver/cache/cachetracking.h b/src/zenserver/cache/cachetracking.h
deleted file mode 100644
index fdfe1a4c7..000000000
--- a/src/zenserver/cache/cachetracking.h
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright Epic Games, Inc. All Rights Reserved.
-
-#pragma once
-
-#include <zencore/iohash.h>
-
-#include <stdint.h>
-#include <filesystem>
-#include <functional>
-
-namespace zen {
-
-#define ZEN_USE_CACHE_TRACKER 0
-#if ZEN_USE_CACHE_TRACKER
-
-class CbObject;
-
-/**
- */
-
-class ZenCacheTracker
-{
-public:
-	ZenCacheTracker(std::filesystem::path StateDirectory);
-	~ZenCacheTracker();
-
-	void TrackAccess(std::string_view BucketSegment, const IoHash& HashKey);
-	void SaveSnapshot();
-	void IterateSnapshots(std::function<void(uint64_t TimeStamp, CbObject Snapshot)>&& Callback);
-
-private:
-	struct Impl;
-
-	Impl* m_Impl = nullptr;
-};
-
-void cachetracker_forcelink();
-
-#endif	// ZEN_USE_CACHE_TRACKER
-
-}  // namespace zen
diff --git a/src/zenserver/cache/structuredcachestore.cpp b/src/zenserver/cache/structuredcachestore.cpp
index 7368d45da..59a047c08 100644
--- a/src/zenserver/cache/structuredcachestore.cpp
+++ b/src/zenserver/cache/structuredcachestore.cpp
@@ -194,10 +194,6 @@ ZenCacheNamespace::ZenCacheNamespace(GcManager& Gc, const std::filesystem::path&
 	CreateDirectories(RootDir);
 
 	m_DiskLayer.DiscoverBuckets();
-
-#if ZEN_USE_CACHE_TRACKER
-	m_AccessTracker.reset(new ZenCacheTracker(RootDir));
-#endif
 }
 
 ZenCacheNamespace::~ZenCacheNamespace()
@@ -211,15 +207,6 @@ ZenCacheNamespace::Get(std::string_view InBucket, const IoHash& HashKey, ZenCach
 
 	bool Ok = m_MemLayer.Get(InBucket, HashKey, OutValue);
 
-#if ZEN_USE_CACHE_TRACKER
-	auto _ = MakeGuard([&] {
-		if (!Ok)
-			return;
-
-		m_AccessTracker->TrackAccess(InBucket, HashKey);
-	});
-#endif
-
 	if (Ok)
 	{
 		ZEN_ASSERT(OutValue.Value.Size());
diff --git a/src/zenserver/cache/structuredcachestore.h b/src/zenserver/cache/structuredcachestore.h
index e4f71a6cf..f94babb64 100644
--- a/src/zenserver/cache/structuredcachestore.h
+++ b/src/zenserver/cache/structuredcachestore.h
@@ -20,13 +20,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
 #include <filesystem>
 #include <unordered_map>
 
-#define ZEN_USE_CACHE_TRACKER 0
-
 namespace zen {
 
 class PathBuilderBase;
 class GcManager;
-class ZenCacheTracker;
 class ScrubContext;
 
 /******************************************************************************
@@ -464,10 +461,6 @@ private:
 	uint64_t			  m_DiskLayerSizeThreshold = 1 * 1024;
 	uint64_t			  m_LastScrubTime		   = 0;
 
-#if ZEN_USE_CACHE_TRACKER
-	std::unique_ptr<ZenCacheTracker> m_AccessTracker;
-#endif
-
 	ZenCacheNamespace(const ZenCacheNamespace&) = delete;
 	ZenCacheNamespace& operator=(const ZenCacheNamespace&) = delete;
 };